def canonical_resource(self):
"""The dxid of the file at this path
Raises:
MultipleObjectsSameNameError: if filename is not unique
NotFoundError: if resource is not found on DX platform
ValueError: if path looks like a folder path (i.e., ends with trailing slash)
"""
if not self.resource:
return None
if utils.has_trailing_slash(self):
raise ValueError(
'Invalid operation ({method}) on folder path ({path})'.format(
path=self, method=sys._getframe(2).f_code.co_name))
objects = [{
'name': self.name,
'folder': ('/' + self.resource).parent,
'project': self.canonical_project,
'batchsize': 2
}]
with _wrap_dx_calls():
results = dxpy.resolve_data_objects(objects=objects)[0]
if len(results) > 1:
raise MultipleObjectsSameNameError(
'Multiple objects found at path ({}). '
'Try using a canonical ID instead'.format(self))
elif len(results) == 1:
return results[0]['id']
else:
raise stor_exceptions.NotFoundError(
'No data object was found for the given path ({}) on DNAnexus'.
format(self))
def _parse_s3_error(exc, **kwargs):
"""
Parses botocore.exception.ClientError exceptions to throw a more
informative exception.
"""
http_status = exc.response.get('ResponseMetadata',
{}).get('HTTPStatusCode')
msg = exc.response['Error'].get('Message', 'Unknown')
code = exc.response['Error'].get('Code')
operation_name = exc.operation_name
msg += ' Bucket: ' + kwargs.get('Bucket') if 'Bucket' in kwargs else ''
msg += ' Key: ' + kwargs.get('Key') if 'Key' in kwargs else ''
if http_status == 403:
if 'storage class' in msg and code == 'InvalidObjectState':
if operation_name == 'GetObject':
return exceptions.ObjectInColdStorageError(msg, exc)
elif operation_name == 'RestoreObject':
return exceptions.AlreadyRestoredError(msg, exc)
else: # pragma: no cover
return exceptions.UnauthorizedError(msg, exc)
else:
return exceptions.UnauthorizedError(msg, exc)
elif http_status == 404:
return exceptions.NotFoundError(msg, exc)
elif http_status == 503:
return exceptions.UnavailableError(msg, exc)
elif http_status == 409:
if 'Object restore is already in progress' in msg:
return exceptions.RestoreAlreadyInProgressError(msg, exc)
else: # pragma: no cover
return exceptions.ConflictError(msg, exc)
else:
return exceptions.RemoteError(msg, exc)
def upload(self, to_upload, **kwargs):
"""Upload a list of files and directories to a directory.
This is not a batch level operation.
If some file errors, the files uploaded before will remain present.
Args:
to_upload (List[Union[str, OBSUploadObject]]): A list of posix file names,
directory names, or OBSUploadObject objects to upload.
Raises:
ValueError: When source path is not a directory
TargetExistsError: When destination directory already exists
"""
dx_upload_objects = [
name for name in to_upload if isinstance(name, OBSUploadObject)
]
all_files_to_upload = utils.walk_files_and_dirs([
name for name in to_upload
if not isinstance(name, OBSUploadObject)
])
dx_upload_objects.extend([
OBSUploadObject(
f,
object_name=('/' +
self.resource if self.resource else Path('')) /
utils.file_name_to_object_name(f)) for f in all_files_to_upload
])
for upload_obj in dx_upload_objects:
upload_obj.object_name = Path(upload_obj.object_name)
upload_obj.source = Path(upload_obj.source)
dest_file = Path('{drive}{project}:{path}'.format(
drive=self.drive,
project=self.canonical_project,
path=upload_obj.object_name))
if upload_obj.source.isfile():
dest_is_file = dest_file.isfile()
if dest_is_file: # only occurs if upload is called directly with existing objects
logger.warning(
'Destination path ({}) already exists, will not cause '
'duplicate file objects on the platform. Skipping...'.
format(dest_file))
else:
with _wrap_dx_calls():
dxpy.upload_local_file(
filename=upload_obj.source,
project=self.canonical_project,
folder='/' + (dest_file.parent.resource or ''),
parents=True,
name=dest_file.name)
elif upload_obj.source.isdir():
dest_file.makedirs_p()
else:
raise stor_exceptions.NotFoundError(
'Source path ({}) does not exist. Please provide a valid source'
.format(upload_obj.source))
def _parse_s3_error(exc, **kwargs):
"""
Parses botocore.exception.ClientError exceptions to throw a more
informative exception.
"""
http_status = exc.response.get('ResponseMetadata',
{}).get('HTTPStatusCode')
msg = exc.response['Error'].get('Message', 'Unknown')
# Give some more info about the error's origins
if 'Bucket' in kwargs:
msg += ' Bucket: ' + kwargs.get('Bucket')
if 'Key' in kwargs:
msg += ', Key: ' + kwargs.get('Key')
if http_status == 403:
return exceptions.UnauthorizedError(msg, exc)
elif http_status == 404:
return exceptions.NotFoundError(msg, exc)
elif http_status == 503:
return exceptions.UnavailableError(msg, exc)
return exceptions.RemoteError(msg, exc)
def rmtree(self):
"""
Removes a resource and all of its contents.
The path should point to a project or directory.
Raises:
NotFoundError: The path points to a nonexistent directory
"""
proj_handler = dxpy.DXProject(self.canonical_project)
if not self.resource:
folders = self.listdir(only='folders')
files = self.listdir(only='objects')
for folder_p in folders:
folder_p.rmtree()
for file_p in files:
file_p.remove()
return
try:
proj_handler.remove_folder('/' + self.resource, recurse=True)
except dxpy.exceptions.ResourceNotFound as e:
raise stor_exceptions.NotFoundError(
'No folders were found with the given path ({})'.format(self),
e)
self.clear_cached_properties()
def test_list_not_found(self, mock_list):
mock_list.side_effect = exceptions.NotFoundError('not found')
with self.assertOutputMatches(exit_status='1', stderr='s3://bucket/path'):
self.parse_args('stor list s3://bucket/path')
def copytree(self, dest, raise_if_same_project=False, **kwargs):
"""Copies a source directory to a destination directory.
This is not an atomic operation.
If the destination path already exists as a directory, the source tree
including the root folder is copied over as a subfolder of the destination.
If the source and destination directories belong to the same project,
the tree is moved instead of copied. Also, in such cases, the root folder
of the project cannot be the source path. Please listdir the root folder
and copy/copytree individual items if needed.
For example, assume the following file hierarchy::
project1/
- b/
- - 1.txt
project2/
Doing a copytree from ``project1:/b/`` to a new dx destination of
``project2:/c`` is performed with::
Path('dx://project1:/b').copytree('dx://project2:/c')
The end result for project2 looks like::
project2/
- c/
- - 1.txt
If the destination path directory already exists, the folder is copied
as a subfolder of the destination. If this new destination also exists,
a TargetExistsError is raised.
If the source is a root folder, and is cloned to an existing destination directory
or if the destination is also a root folder, the tree is moved under project name.
Refer to ``dx`` docs for detailed information.
Args:
dest (Path|str): The directory to copy to. Must not exist if
its a posix directory
raise_if_same_project (bool, default False): Allows moving files within project
instead of cloning. If True, raises an error to prevent moving the directory.
Only takes effect when both source and destination directory are
within the same DX Project
Raises:
DNAnexusError: Attempt to clone within same project and raise_if_same_project=True
TargetExistsError: All possible destinations for source directory already exist
NotFoundError: source directory path doesn't exist
"""
dest = Path(dest)
if utils.is_dx_path(dest):
if self.isdir():
if dest.canonical_project == self.canonical_project:
if not raise_if_same_project:
self._movetree(dest)
else:
raise DNAnexusError(
'Source and destination are in same project. '
'Set raise_if_same_project=False to allow this.')
else:
self._clonetree(dest)
else:
raise stor_exceptions.NotFoundError(
'No project or directory was found at path ({})'.format(
self))
else:
super(DXPath, self).copytree(
dest) # for other filesystems, delegate to utils.copytree
def copy(self, dest, raise_if_same_project=False, **kwargs):
"""Copies data object to destination path.
If dest already exists as a directory on the DX platform, the file is copied
underneath dest directory with original name.
If the target destination already exists as a file, it is first deleted before
the copy is attempted.
For example, assume the following file hierarchy::
dxProject/
- a/
- - 1.txt
anotherDxProject/
Doing a copy of ``1.txt`` to a new destination of ``b.txt`` is
performed with::
Path('dx://dxProject:/a/1.txt').copy('dx://anotherDxProject/b.txt')
The end result for anotherDxProject looks like::
anotherDxProject/
- b.txt
And, if the destination already exists as a directory, i.e. we have::
dxProject/
- a/
- - 1.txt
anotherDxProject/
- b.txt/
Performing copy with following command::
Path('dx://dxProject:/a/1.txt').copy('dx://anotherDxProject/b.txt')
Will yield the resulting structure to be::
anotherDxProject/
- b.txt/
- - 1.txt
If the source file and destination belong to the same project, the files are
moved instead of copied, if the raise_if_same_project flag is False; because
the same underlying file cannot appear in two locations in the same project.
If the final destination for the file already is an existing file,
that file is deleted before the file is copied.
Args:
dest (Path|str): The destination file or directory.
raise_if_same_project (bool, default False): Controls moving file within project
instead of cloning. If True, raises an error to prevent this move.
Only takes effect when both source and destination are
within the same DX Project
Raises:
DNAnexusError: When copying within same project with raise_if_same_project=False
NotFoundError: When the source file path doesn't exist
"""
dest = Path(dest)
if utils.is_dx_path(dest):
if self.isfile():
if dest.canonical_project == self.canonical_project:
if not raise_if_same_project:
self._move(dest)
else:
raise DNAnexusError(
'Source and destination are in same project. '
'Set raise_if_same_project=False to allow this.')
else:
self._clone(dest)
else:
raise stor_exceptions.NotFoundError(
'No data object was found for the given path on DNAnexus')
else:
super(DXPath, self).copy(
dest) # for other filesystems, delegate to utils.copy
def test_list_not_found(self, mock_list):
mock_list.side_effect = exceptions.NotFoundError('not found')
with self.assertRaisesRegexp(SystemExit, '1'):
self.parse_args('stor list s3://bucket/path')
self.assertIn('s3://bucket/path', sys.stderr.getvalue())