def download_objects(self, dest, objects):
def is_parent_dir(possible_parent, possible_child):
"""Checks if possible_child is a sub-path of possible_parent"""
if not possible_parent.resource:
return possible_child.resource and \
possible_child.project == possible_parent.project
return possible_child.startswith(
utils.with_trailing_slash(possible_parent))
source = self
if source == (self.drive +
self.project): # need to convert dx://proj to dx://proj:
source = DXPath(self + ':')
for obj in objects:
if utils.is_dx_path(obj) and not is_parent_dir(
source, DXPath(obj)):
raise ValueError('"%s" must be child of download path "%s"' %
(obj, self))
# Convert requested download objects to full object paths
objs_to_download = {
obj: DXPath(obj) if utils.is_dx_path(obj) else source / obj
for obj in objects
}
results = {}
for obj, dx_obj in objs_to_download.items():
dest_resource = dx_obj[len(source):].lstrip('/')
dest_obj = PosixPath(dest) / dest_resource
dx_obj.copy(dest_obj)
results[obj] = dest_obj
return results
def __new__(cls, path):
if cls is Path:
if not hasattr(path, 'startswith'):
raise TypeError('must be a string like')
if utils.is_dx_path(path):
cls = utils.find_dx_class(path)
elif utils.is_swift_path(path):
from stor.swift import SwiftPath
cls = SwiftPath
elif utils.is_s3_path(path):
from stor.s3 import S3Path
cls = S3Path
elif os.path == ntpath:
from stor.windows import WindowsPath
cls = WindowsPath
elif os.path == posixpath:
from stor.posix import PosixPath
cls = PosixPath
else: # pragma: no cover
assert False, 'path is not compatible with stor'
return text_type.__new__(cls, path)
def _wrapped_list(path, **kwargs):
"""Use iterative walkfiles for DX paths, rather than trying to generate full list first"""
if utils.is_dx_path(path):
func = stor.walkfiles
else:
func = stor.list
return func(path, **kwargs)
def copytree(self, dest, raise_if_same_project=False, **kwargs):
"""Copies a source directory to a destination directory.
This is not an atomic operation.
If the destination path already exists as a directory, the source tree
including the root folder is copied over as a subfolder of the destination.
If the source and destination directories belong to the same project,
the tree is moved instead of copied. Also, in such cases, the root folder
of the project cannot be the source path. Please listdir the root folder
and copy/copytree individual items if needed.
For example, assume the following file hierarchy::
project1/
- b/
- - 1.txt
project2/
Doing a copytree from ``project1:/b/`` to a new dx destination of
``project2:/c`` is performed with::
Path('dx://project1:/b').copytree('dx://project2:/c')
The end result for project2 looks like::
project2/
- c/
- - 1.txt
If the destination path directory already exists, the folder is copied
as a subfolder of the destination. If this new destination also exists,
a TargetExistsError is raised.
If the source is a root folder, and is cloned to an existing destination directory
or if the destination is also a root folder, the tree is moved under project name.
Refer to ``dx`` docs for detailed information.
Args:
dest (Path|str): The directory to copy to. Must not exist if
its a posix directory
raise_if_same_project (bool, default False): Allows moving files within project
instead of cloning. If True, raises an error to prevent moving the directory.
Only takes effect when both source and destination directory are
within the same DX Project
Raises:
DNAnexusError: Attempt to clone within same project and raise_if_same_project=True
TargetExistsError: All possible destinations for source directory already exist
NotFoundError: source directory path doesn't exist
"""
dest = Path(dest)
if utils.is_dx_path(dest):
if self.isdir():
if dest.canonical_project == self.canonical_project:
if not raise_if_same_project:
self._movetree(dest)
else:
raise DNAnexusError(
'Source and destination are in same project. '
'Set raise_if_same_project=False to allow this.')
else:
self._clonetree(dest)
else:
raise stor_exceptions.NotFoundError(
'No project or directory was found at path ({})'.format(
self))
else:
super(DXPath, self).copytree(
dest) # for other filesystems, delegate to utils.copytree
def copy(self, dest, raise_if_same_project=False, **kwargs):
"""Copies data object to destination path.
If dest already exists as a directory on the DX platform, the file is copied
underneath dest directory with original name.
If the target destination already exists as a file, it is first deleted before
the copy is attempted.
For example, assume the following file hierarchy::
dxProject/
- a/
- - 1.txt
anotherDxProject/
Doing a copy of ``1.txt`` to a new destination of ``b.txt`` is
performed with::
Path('dx://dxProject:/a/1.txt').copy('dx://anotherDxProject/b.txt')
The end result for anotherDxProject looks like::
anotherDxProject/
- b.txt
And, if the destination already exists as a directory, i.e. we have::
dxProject/
- a/
- - 1.txt
anotherDxProject/
- b.txt/
Performing copy with following command::
Path('dx://dxProject:/a/1.txt').copy('dx://anotherDxProject/b.txt')
Will yield the resulting structure to be::
anotherDxProject/
- b.txt/
- - 1.txt
If the source file and destination belong to the same project, the files are
moved instead of copied, if the raise_if_same_project flag is False; because
the same underlying file cannot appear in two locations in the same project.
If the final destination for the file already is an existing file,
that file is deleted before the file is copied.
Args:
dest (Path|str): The destination file or directory.
raise_if_same_project (bool, default False): Controls moving file within project
instead of cloning. If True, raises an error to prevent this move.
Only takes effect when both source and destination are
within the same DX Project
Raises:
DNAnexusError: When copying within same project with raise_if_same_project=False
NotFoundError: When the source file path doesn't exist
"""
dest = Path(dest)
if utils.is_dx_path(dest):
if self.isfile():
if dest.canonical_project == self.canonical_project:
if not raise_if_same_project:
self._move(dest)
else:
raise DNAnexusError(
'Source and destination are in same project. '
'Set raise_if_same_project=False to allow this.')
else:
self._clone(dest)
else:
raise stor_exceptions.NotFoundError(
'No data object was found for the given path on DNAnexus')
else:
super(DXPath, self).copy(
dest) # for other filesystems, delegate to utils.copy