def test_parse_create(tmppath):
"""Test opendir."""
rooturl = mkurl(tmppath)
fs, path = opener.parse(rooturl + "/non-existing")
assert not fs.exists(path)
fs, path = opener.parse(rooturl + "/non-existing", create_dir=True)
assert fs.exists(path)
def test_parse_create(tmppath):
"""Test opendir."""
rooturl = mkurl(tmppath)
fs, path = opener.parse(rooturl + "/non-existing")
assert not fs.exists(path)
fs, path = opener.parse(rooturl + "/non-existing", create_dir=True)
assert fs.exists(path)
def test_parse(tmppath):
"""Test parse."""
rooturl = mkurl(tmppath)
fs, path = opener.parse(rooturl + "/data")
assert path == "data"
assert fs
fs, path = opener.parse(rooturl + "/data/")
assert path == ""
assert fs
def copy(self, dst, **kwargs):
"""Copy file to a new destination.
Returns JSON Patch with proposed change pointing to new copy.
"""
_fs, filename = opener.parse(self.uri)
_fs_dst, filename_dst = opener.parse(dst)
copyfile(_fs, filename, _fs_dst, filename_dst, **kwargs)
return [{'op': 'replace', 'path': self.pointer, 'value': dst}]
def copy(self, dst, **kwargs):
"""Copy file to a new destination.
Returns JSON Patch with proposed change pointing to new copy.
"""
_fs, filename = opener.parse(self.uri)
_fs_dst, filename_dst = opener.parse(dst)
copyfile(_fs, filename, _fs_dst, filename_dst, **kwargs)
return [{'op': 'replace', 'path': self.pointer, 'value': dst}]
def test_parse(tmppath):
"""Test parse."""
rooturl = mkurl(tmppath)
fs, path = opener.parse(rooturl + "/data")
assert path == "data"
assert fs
fs, path = opener.parse(rooturl + "/data/")
assert path == ""
assert fs
def setcontents(self, source, name, chunk_size=65536):
"""Create a new file from a string or file-like object.
:note: All paths has to be absolute or specified in full URI format.
:param data: .
:param name: File URI or filename generator taking `self` as argument.
"""
if isinstance(source, six.string_types):
self['source'] = source
f = opener.open(source, 'rb')
else:
f = source
if callable(name):
name = name(self)
else:
name = fs.path.abspath(name)
signals.document_before_content_set.send(self, name=name)
if self.get('source', '') != name:
data = f.read()
_fs, filename = opener.parse(name)
_fs.setcontents(filename, data, chunk_size)
_fs.close()
signals.document_after_content_set.send(self, name=name)
if hasattr(f, 'close'):
f.close()
self['uri'] = name
self.commit()
def __init__(self, url_or_fs, path=None):
"""
:param url_or_fs:
:param path:
:return:
"""
from fs.opener import opener
if path:
self._fs, self._path = url_or_fs, path
else:
self._fs, self._path = opener.parse(url_or_fs)
self._writer = None
self._reader = None
self._compress = True
self._process = None # Process name for report_progress
self._start_time = 0
if not self._path.endswith(self.EXTENSION):
self._path = self._path + self.EXTENSION
def __init__(self, url_or_fs, path=None):
"""
:param url_or_fs:
:param path:
:return:
"""
from fs.opener import opener
if path:
self._fs, self._path = url_or_fs, path
else:
self._fs, self._path = opener.parse(url_or_fs)
self._writer = None
self._reader = None
self._compress = True
self._process = None # Process name for report_progress
self._start_time = 0
if not self._path.endswith(self.EXTENSION):
self._path = self._path + self.EXTENSION
def save_to_uri(self, bytes, uri, save_metadata=True):
# Have to use a two-step process to write to the file: open the
# filesystem, then open the file. Have to open the filesystem
# as writeable in case this is a virtual filesystem (like ZipFS),
# otherwise the write to the actual file will fail with a read-
# only filesystem error.
if uri.startswith("file://"):
# FIXME: workaround to allow opening of file:// URLs with the
# ! character
uri = uri.replace("file://", "")
fs, relpath = opener.parse(uri, writeable=True)
fh = fs.open(relpath, 'wb')
log.debug("saving to %s" % uri)
fh.write(bytes)
fh.close()
if save_metadata:
metadata_dict = dict()
self.get_extra_metadata(metadata_dict)
if metadata_dict:
relpath += ".omnivore"
log.debug("saving extra metadata to %s" % relpath)
jsonpickle.set_encoder_options("json", sort_keys=True, indent=4)
bytes = jsonpickle.dumps(metadata_dict)
text = jsonutil.collapse_json(bytes)
header = self.get_extra_metadata_header()
fh = fs.open(relpath, 'wb')
fh.write(header)
fh.write(text)
fh.close()
self.metadata_dirty = False
fs.close()
def __init__(self, url_or_fs, path=None):
"""
Args:
url_or_fs (str or filesystem):
path (str):
"""
from fs.opener import opener
if path:
self._fs, self._path = url_or_fs, path
else:
self._fs, self._path = opener.parse(url_or_fs)
if not self._fs.hassyspath(''):
# Pytables requirement.
raise HDFError('HDFPartition requires filesystem having sys path.')
self._writer = None
self._reader = None
self._process = None # Process name for report_progress
self._start_time = 0
if not self._path.endswith(self.EXTENSION):
self._path = self._path + self.EXTENSION
def __init__(self, url_or_fs, path=None):
"""
Args:
url_or_fs (str or filesystem):
path (str):
"""
from fs.opener import opener
if path:
self._fs, self._path = url_or_fs, path
else:
self._fs, self._path = opener.parse(url_or_fs)
if not self._fs.hassyspath(''):
# Pytables requirement.
raise HDFError('HDFPartition requires filesystem having sys path.')
self._writer = None
self._reader = None
self._process = None # Process name for report_progress
self._start_time = 0
if not self._path.endswith(self.EXTENSION):
self._path = self._path + self.EXTENSION
def setcontents(self, source, name, chunk_size=65536):
"""Create a new file from a string or file-like object.
:note: All paths has to be absolute or specified in full URI format.
:param data: .
:param name: File URI or filename generator taking `self` as argument.
"""
if isinstance(source, six.string_types):
self['source'] = source
f = opener.open(source, 'rb')
else:
f = source
if callable(name):
name = name(self)
else:
name = fs.path.abspath(name)
signals.document_before_content_set.send(self, name=name)
if self.get('source', '') != name:
data = f.read()
_fs, filename = opener.parse(name)
_fs.setcontents(filename, data, chunk_size)
_fs.close()
signals.document_after_content_set.send(self, name=name)
if hasattr(f, 'close'):
f.close()
self['uri'] = name
self.commit()
def get_fs(uri):
if uri.startswith("file://"):
# FIXME: workaround to allow opening of file:// URLs with the
# ! character
uri = uri.replace("file://", "")
fs, relpath = opener.parse(uri)
log.debug("Filesystem: %s" % fs)
fh = fs.open(relpath, "rb")
return fh, fs, relpath
def remove(self, force=False):
"""Remove file reference from record.
If force is True it removes the file from filesystem
"""
if force:
_fs, filename = opener.parse(self.uri)
_fs.remove(filename)
self.uri = None
def remove(self, force=False):
"""Remove file reference from record.
If force is True it removes the file from filesystem
"""
if force:
_fs, filename = opener.parse(self.uri)
_fs.remove(filename)
self.uri = None
def get_fs(self, uri=None):
if uri is None:
uri = self.metadata.uri
if uri.startswith("file://"):
# FIXME: workaround to allow opening of file:// URLs with the
# ! character
uri = uri.replace("file://", "")
fs, relpath = opener.parse(uri)
log.debug("Filesystem: %s" % fs)
fh = fs.open(relpath, "rb")
return fh, fs, relpath
def test_handles_preexisting_keyring(self):
from fs.opener import opener
fs, path = opener.parse(self.keyring_filename, writeable=True)
keyring_file = fs.open(path, 'wb')
keyring_file.write("""[svc1]
user1 = cHdkMQ==
""")
keyring_file.close()
pyf_keyring = keyring.backends.pyfs.PlaintextKeyring(
filename=self.keyring_filename)
self.assertEquals('pwd1', pyf_keyring.get_password('svc1', 'user1'))
def compute_checksum(self, progress_callback=None):
"""Compute checksum of file."""
fs, path = opener.parse(self.file.uri)
fp = fs.open(path, 'rb')
try:
value = self._compute_checksum(
fp, progress_callback=progress_callback)
except StorageError:
fp.close()
raise
return value
def test_handles_preexisting_keyring(self):
from fs.opener import opener
fs, path = opener.parse(self.keyring_filename, writeable=True)
keyring_file = fs.open(path, 'wb')
keyring_file.write(
"""[svc1]
user1 = cHdkMQ==
""")
keyring_file.close()
pyf_keyring = keyring.backends.pyfs.PlaintextKeyring(
filename=self.keyring_filename)
self.assertEquals('pwd1', pyf_keyring.get_password('svc1', 'user1'))
def send_file(self, restricted=False):
"""Send file to the client."""
try:
fs, path = opener.parse(self.file.uri)
fp = fs.open(path, 'rb')
return send_stream(
fp, path, self.file.size,
time.mktime(self.file.updated.timetuple()),
restricted=restricted, etag=self.file.checksum,
content_md5=self.file.checksum)
except Exception as e:
raise StorageError('Could not send file: {}'.format(e))
def test_handles_preexisting_keyring(self):
from fs.opener import opener
fs, path = opener.parse(self.keyring_filename, writeable=True)
keyring_file = fs.open(path, 'w')
file_data = textwrap.dedent("""
[svc1]
user1 = cHdkMQ==
""").lstrip()
keyring_file.write(file_data)
keyring_file.close()
pyf_keyring = pyfs.PlaintextKeyring(filename=self.keyring_filename)
self.assertEquals('pwd1', pyf_keyring.get_password('svc1', 'user1'))
def test_handles_preexisting_keyring(self):
from fs.opener import opener
fs, path = opener.parse(self.keyring_filename, writeable=True)
keyring_file = fs.open(path, 'w')
file_data = textwrap.dedent("""
[svc1]
user1 = cHdkMQ==
""").lstrip()
keyring_file.write(file_data)
keyring_file.close()
pyf_keyring = pyfs.PlaintextKeyring(
filename=self.keyring_filename)
self.assertEquals('pwd1', pyf_keyring.get_password('svc1', 'user1'))
def delete(self, force=False):
"""Deletes the instance of document.
:param force: If it is True then the document is deleted including
attached files and metadata.
"""
self['deleted'] = True
if force and self.get('uri') is not None:
signals.document_before_file_delete.send(self)
fs, filename = opener.parse(self['uri'])
fs.remove(filename)
self['uri'] = None
self.commit()
def delete(self, force=False):
"""Deletes the instance of document.
:param force: If it is True then the document is deleted including
attached files and metadata.
"""
self['deleted'] = True
if force and self.get('uri') is not None:
signals.document_before_file_delete.send(self)
fs, filename = opener.parse(self['uri'])
fs.remove(filename)
self['uri'] = None
self.commit()
def file_from_zip(zip_file_path, filename, mode='r'):
# eg rootpath =
# FastQC.out/15-02380-CE11-T13-L1_AACCAG_L001_R1_001_fastqc.zip
# ie fastq_filename + '_fastqc.zip'
# fs.opener.opener magic detects a filesystem type if we give it
# a URI-like string, eg zip://foo/bla/file.zip
# (eg https://commons.apache.org/proper/commons-vfs/filesystems.html)
# So we munge the path to a zip file to become a compatible URI
# (an http, ftp, webdav url could also be used)
if splitext(zip_file_path)[1] == '.zip':
zip_file_path = 'zip://' + zip_file_path
with opener.parse(zip_file_path)[0] as vfs:
for fn in vfs.walkfiles():
if os.path.basename(fn) == filename:
return vfs.open(fn, mode)
def file_from_zip(zip_file_path, filename, mode='r'):
# eg rootpath =
# FastQC.out/15-02380-CE11-T13-L1_AACCAG_L001_R1_001_fastqc.zip
# ie fastq_filename + '_fastqc.zip'
# fs.opener.opener magic detects a filesystem type if we give it
# a URI-like string, eg zip://foo/bla/file.zip
# (eg https://commons.apache.org/proper/commons-vfs/filesystems.html)
# So we munge the path to a zip file to become a compatible URI
# (an http, ftp, webdav url could also be used)
if splitext(zip_file_path)[1] == '.zip':
zip_file_path = 'zip://' + zip_file_path
with opener.parse(zip_file_path)[0] as vfs:
for fn in vfs.walkfiles():
if os.path.basename(fn) == filename:
return vfs.open(fn, mode)
def setcontents(self, source, **kwargs):
"""Create a new file from a string or file-like object."""
if isinstance(source, six.string_types):
_file = opener.open(source, 'rb')
else:
_file = source
# signals.document_before_content_set.send(self)
data = _file.read()
_fs, filename = opener.parse(self.uri)
_fs.setcontents(filename, data, **kwargs)
_fs.close()
# signals.document_after_content_set.send(self)
if isinstance(source, six.string_types) and hasattr(_file, 'close'):
_file.close()
def setcontents(self, source, **kwargs):
"""Create a new file from a string or file-like object."""
if isinstance(source, six.string_types):
_file = opener.open(source, 'rb')
else:
_file = source
# signals.document_before_content_set.send(self)
data = _file.read()
_fs, filename = opener.parse(self.uri)
_fs.setcontents(filename, data, **kwargs)
_fs.close()
# signals.document_after_content_set.send(self)
if isinstance(source, six.string_types) and hasattr(_file, 'close'):
_file.close()
def save_to_uri(self, uri, editor, saver=None, save_metadata=True):
# Have to use a two-step process to write to the file: open the
# filesystem, then open the file. Have to open the filesystem
# as writeable in case this is a virtual filesystem (like ZipFS),
# otherwise the write to the actual file will fail with a read-
# only filesystem error.
if saver is None:
bytes = self.bytes.tostring()
else:
bytes = saver(self, editor)
if uri.startswith("file://"):
# FIXME: workaround to allow opening of file:// URLs with the
# ! character
uri = uri.replace("file://", "")
fs, relpath = opener.parse(uri, writeable=True)
fh = fs.open(relpath, 'wb')
log.debug("saving to %s" % uri)
fh.write(bytes)
fh.close()
if save_metadata:
mdict = self.init_extra_metadata_dict(editor)
task_metadata = dict()
editor.to_metadata_dict(task_metadata, self)
self.store_task_specific_metadata(editor, mdict, task_metadata)
if mdict:
relpath += ".omnivore"
log.debug("saving extra metadata to %s" % relpath)
jsonpickle.set_encoder_options("json",
sort_keys=True,
indent=4)
bytes = jsonpickle.dumps(mdict)
text = jsonutil.collapse_json(bytes, 8,
self.json_expand_keywords)
header = editor.get_extra_metadata_header()
fh = fs.open(relpath, 'wb')
fh.write(header)
fh.write(text)
fh.close()
fs.close()
def test_handles_preexisting_keyring(self):
if FS_VER < version.parse("2.0.0"):
from fs.opener import opener
fs, path = opener.parse(self.keyring_filename, writeable=True)
else:
import fs.opener
fs, path = fs.opener.open(self.keyring_root,
writeable=True,
create=True)
path = os.path.basename(self.keyring_filename)
keyring_file = fs.open(path, 'w')
file_data = textwrap.dedent("""
[svc1]
user1 = cHdkMQ==
""").lstrip()
keyring_file.write(file_data)
keyring_file.close()
pyf_keyring = pyfs.PlaintextKeyring(filename=self.keyring_filename)
assert 'pwd1' == pyf_keyring.get_password('svc1', 'user1')
def normalize_uri(uri):
if uri.startswith("file://"):
# FIXME: workaround to allow opening of file:// URLs with the
# ! character
uri = uri.replace("file://", "")
if uri:
fs, relpath = opener.parse(uri)
if fs.haspathurl(relpath):
uri = fs.getpathurl(relpath)
elif fs.hassyspath(relpath):
abspath = fs.getsyspath(relpath)
if abspath.startswith("\\\\?\\UNC\\"):
# Leave long UNC paths as raw paths because there is no
# accepted way to convert to URI
uri = abspath
else:
if abspath.startswith("\\\\?\\") and len(abspath) < 260:
# on windows, pyfilesystem returns extended path notation to
# allow paths greater than 256 characters. If the path is
# short, change slashes to normal and remove the prefix
abspath = abspath[4:].replace("\\", "/")
uri = "file://" + abspath
return uri
def normalize_uri(uri):
if uri.startswith("file://"):
# FIXME: workaround to allow opening of file:// URLs with the
# ! character
uri = uri.replace("file://", "")
if uri:
fs, relpath = opener.parse(uri)
if fs.haspathurl(relpath):
uri = fs.getpathurl(relpath)
elif fs.hassyspath(relpath):
abspath = fs.getsyspath(relpath)
if abspath.startswith("\\\\?\\UNC\\"):
# Leave long UNC paths as raw paths because there is no
# accepted way to convert to URI
uri = abspath
else:
if abspath.startswith("\\\\?\\") and len(abspath) < 260:
# on windows, pyfilesystem returns extended path notation to
# allow paths greater than 256 characters. If the path is
# short, change slashes to normal and remove the prefix
abspath = abspath[4:].replace("\\", "/")
uri = "file://" + abspath
return uri
def open_fs(self, fs_url, writeable=False, create_dir=False):
fs, path = opener.parse(fs_url, writeable=writeable, create_dir=create_dir)
fs.cache_hint(True)
return fs, path
def open(self, mode='r', **kwargs):
"""Open file ``uri`` under the pointer."""
_fs, filename = opener.parse(self.uri)
return _fs.open(filename, mode=mode, **kwargs)
def syspath(self):
fs, relpath = opener.parse(self.uri)
self.read_only = fs.getmeta('read_only')
if fs.hassyspath(relpath):
return fs.getsyspath(relpath)
raise TypeError("No system path for %s" % self.uri)
def run(self):
args = self.args
from fs.opener import opener
fs, fspath = opener.parse(args.path)
from ...loggingconf import init_logging
if os.path.exists(args.logging):
init_logging(args.logging)
archive = Archive()
lib = archive.create_library(long_name="moya.run", namespace=namespaces.run)
if not lib.import_document(fs, fspath):
for failed_doc in lib.failed_documents:
self.console.document_error(failed_doc.msg,
failed_doc.path,
failed_doc.code,
failed_doc.line,
failed_doc.col)
return -1
app = archive.create_app('run', 'moya.run')
c = Context()
c['.console'] = self.console
c['.app'] = app
c['.now'] = ExpressionDateTime.utcnow()
try:
c['tz'] = Timezone(os.environ['TZ'])
except KeyError:
pass
import locale
_locale, encoding = locale.getdefaultlocale()
c['.locale'] = _locale
params = {}
if args.params:
for p in args.params:
if '=' not in p:
sys.stderr.write("{} is not in the form <name>=<expression>\n")
return -1
k, v = p.split('=', 1)
params[k] = v
if args.templatedir is not None:
archive.init_template_engine('moya', {})
archive.init_templates('default', args.templatedir, 100)
console = self.console
try:
archive.build_libs()
except errors.ParseError as e:
line, col = e.position
console.document_error(text_type(e),
e.path,
e._code,
line,
col)
return None
except errors.ElementError as element_error:
line = element_error.source_line
col = None
console.document_error(text_type(element_error),
element_error.element._location,
element_error.element._code,
line,
col)
raise errors.StartupFailedError('Failed to build project')
# call = archive.get_callable('moya.run#main',
# 'moya.run',
# breakpoint=args.breakpoint)
call = archive.get_callable_from_document(fspath,
args.elementref,
fs=fs,
breakpoint=args.breakpoint,
archive=archive,
lib=lib)
if call is None:
raise ValueError("Element reference '%s' not found in document" % args.elementref)
try:
with pilot.manage_request(None, c):
if args.timer:
with timer():
call(c, **params)
else:
call(c, **params)
except Exception as e:
console.obj(c, e)
def move(self, dst, **kwargs):
"""Move file to a new destination and update ``uri``."""
_fs, filename = opener.parse(self.uri)
_fs_dst, filename_dst = opener.parse(dst)
movefile(_fs, filename, _fs_dst, filename_dst, **kwargs)
self.uri = dst
def connect():
"""Connect to the EOS server thingy."""
remote_root = "root://eosuser.cern.ch"
remote_dir = "//eos/user/o/otrondru/"
return opener.parse(remote_root + remote_dir)
def open_fs(self, fs_url, writeable=False, create_dir=False):
fs, path = opener.parse(fs_url,
writeable=writeable,
create_dir=create_dir)
fs.cache_hint(True)
return fs, path
def move(self, dst, **kwargs):
"""Move file to a new destination and update ``uri``."""
_fs, filename = opener.parse(self.uri)
_fs_dst, filename_dst = opener.parse(dst)
movefile(_fs, filename, _fs_dst, filename_dst, **kwargs)
self.uri = dst
def connect(url=None):
url = url or _full_url()
return opener.parse(url)
def open(self, mode='r', **kwargs):
"""Open a the 'uri' as a file-like object."""
_fs, filename = opener.parse(self['uri'])
return _fs.open(filename, mode=mode, **kwargs)
def run(self):
args = self.args
from fs.opener import opener
fs, fspath = opener.parse(args.path)
from ...loggingconf import init_logging
if args.logging and os.path.exists(args.logging):
init_logging(args.logging)
archive = Archive()
lib = archive.create_library(long_name="moya.run", namespace=namespaces.run)
if not lib.import_document(fs, fspath):
for failed_doc in lib.failed_documents:
self.console.document_error(failed_doc.msg,
failed_doc.path,
failed_doc.code,
failed_doc.line,
failed_doc.col)
return -1
app = archive.create_app('run', 'moya.run')
c = Context()
c['.console'] = self.console
c['.app'] = app
c['.now'] = ExpressionDateTime.utcnow()
try:
c['tz'] = Timezone(os.environ['TZ'])
except KeyError:
pass
import locale
_locale, encoding = locale.getdefaultlocale()
c['.locale'] = _locale
params = {}
if args.params:
for p in args.params:
if '=' not in p:
sys.stderr.write("{} is not in the form <name>=<expression>\n")
return -1
k, v = p.split('=', 1)
params[k] = v
if args.templatedir is not None:
archive.init_template_engine('moya', {})
archive.init_templates('default', args.templatedir, 100)
console = self.console
try:
archive.build_libs()
except errors.ParseError as e:
line, col = e.position
console.document_error(text_type(e),
e.path,
e._code,
line,
col)
return None
except errors.ElementError as element_error:
line = element_error.source_line
col = None
console.document_error(text_type(element_error),
element_error.element._location,
element_error.element._code,
line,
col)
raise errors.StartupFailedError('Failed to build project')
call = archive.get_callable_from_document(fspath,
args.elementref,
fs=fs,
breakpoint=args.breakpoint,
archive=archive,
lib=lib)
if call is None:
raise ValueError("Element reference '%s' not found in document" % args.elementref)
try:
with pilot.manage_request(None, c):
if args.timer:
with timer():
call(c, **params)
else:
call(c, **params)
except Exception as e:
console.obj(c, e)
def open(self, mode='r', **kwargs):
"""Open a the 'uri' as a file-like object."""
_fs, filename = opener.parse(self['uri'])
return _fs.open(filename, mode=mode, **kwargs)
def open(self, mode='r', **kwargs):
"""Open file ``uri`` under the pointer."""
_fs, filename = opener.parse(self.uri)
return _fs.open(filename, mode=mode, **kwargs)
def check_read_only(self):
fs, relpath = opener.parse(self.uri)
self.read_only = fs.getmeta('read_only')
return self.read_only