def _prep_for_copy(self, dest):
"""Handles logic, for finalizing target destination, making parent folders
and deleting existing target, common to _clone and _move"""
dest_is_dir = dest.isdir()
target_dest = dest
if dest_is_dir or utils.has_trailing_slash(dest):
target_dest = dest / self.name
if not dest_is_dir and target_dest.parent.resource:
target_dest.parent.makedirs_p()
if target_dest.isfile():
target_dest.remove()
should_rename = not dest_is_dir and not utils.has_trailing_slash(dest)
return target_dest, should_rename
def _prep_for_copytree(self, dest):
"""Handles logic, for finalizing target destination, making parent folders
and checking for clashes, common to _clonetree and _movetree"""
source = utils.remove_trailing_slash(self)
dest_is_dir = dest.isdir()
should_rename = True
target_dest = dest
if dest_is_dir or utils.has_trailing_slash(dest):
target_dest = dest / (source.name if source.resource else
source.virtual_project)
if target_dest.isdir():
raise stor_exceptions.TargetExistsError(
'Destination path ({}) already exists, will not cause '
'duplicate folders to exist. Remove the original first'.
format(target_dest))
should_rename = False
if not source.resource:
target_dest.makedirs_p()
elif not dest_is_dir and target_dest.parent.resource: # don't call makedirs_p on project
target_dest.parent.makedirs_p()
moved_folder_path = target_dest.parent / source.name
return target_dest, should_rename, moved_folder_path
def download_object(self, dest, config=None, **kwargs):
"""
Downloads a file from S3 to a destination file.
Args:
dest (str): The destination path to download file to.
Notes:
- The destination directory will be created automatically if it doesn't exist.
- This method downloads to paths relative to the current
directory.
"""
result = {'source': self, 'dest': dest, 'success': True}
if utils.has_trailing_slash(self):
# Handle directory markers separately
utils.make_dest_dir(str(dest))
return result
dl_kwargs = {
'bucket': self.bucket,
'key': str(self.resource),
'filename': str(dest),
'config': config
}
utils.make_dest_dir(self.parts_class(dest).parent)
try:
self._make_s3_transfer('download_file', **dl_kwargs)
except exceptions.RemoteError as e:
result['success'] = False
result['error'] = e
return result
def canonical_resource(self):
"""The dxid of the file at this path
Raises:
MultipleObjectsSameNameError: if filename is not unique
NotFoundError: if resource is not found on DX platform
ValueError: if path looks like a folder path (i.e., ends with trailing slash)
"""
if not self.resource:
return None
if utils.has_trailing_slash(self):
raise ValueError(
'Invalid operation ({method}) on folder path ({path})'.format(
path=self, method=sys._getframe(2).f_code.co_name))
objects = [{
'name': self.name,
'folder': ('/' + self.resource).parent,
'project': self.canonical_project,
'batchsize': 2
}]
with _wrap_dx_calls():
results = dxpy.resolve_data_objects(objects=objects)[0]
if len(results) > 1:
raise MultipleObjectsSameNameError(
'Multiple objects found at path ({}). '
'Try using a canonical ID instead'.format(self))
elif len(results) == 1:
return results[0]['id']
else:
raise stor_exceptions.NotFoundError(
'No data object was found for the given path ({}) on DNAnexus'.
format(self))
def isfile(self):
"""Determine an object exists at the specified path
Returns:
bool: True if path points to an existing file
"""
if not self.resource or utils.has_trailing_slash(self):
return False
try:
self.stat()
return True
except stor_exceptions.NotFoundError:
return False
def _upload_object(self, upload_obj, config=None):
"""Upload a single object given an OBSUploadObject."""
if utils.has_trailing_slash(upload_obj.object_name):
# Handle empty directories separately
ul_kwargs = {
'Bucket': self.bucket,
'Key': utils.with_trailing_slash(str(upload_obj.object_name))
}
if upload_obj.options and 'headers' in upload_obj.options:
ul_kwargs.update(upload_obj.options['headers'])
s3_call = self._s3_client_call
method = 'put_object'
else:
ul_kwargs = {
'bucket': self.bucket,
'key': str(upload_obj.object_name),
'filename': upload_obj.source,
'config': config
}
if upload_obj.options and 'headers' in upload_obj.options:
ul_kwargs['extra_args'] = upload_obj.options['headers']
s3_call = self._make_s3_transfer
method = 'upload_file'
result = {
'source':
upload_obj.source,
'dest':
S3Path(self.drive + self.bucket) /
(ul_kwargs.get('key') or ul_kwargs.get('Key')),
'success':
True
}
try:
s3_call(method, **ul_kwargs)
except exceptions.RemoteError as e:
result['success'] = False
result['error'] = e
return result
def isfile(self):
try:
return self.stat() and not utils.has_trailing_slash(self)
except (exceptions.NotFoundError, ValueError):
return False
def update_progress(self, result):
"""Keep track of total uploaded bytes by referencing the object sizes"""
self.uploaded_bytes += (os.path.getsize(result['source'])
if not utils.has_trailing_slash(result['dest'])
else 0)
def list(
self,
starts_with=None,
limit=None,
condition=None,
use_manifest=False,
# hidden args
list_as_dir=False,
ignore_dir_markers=False,
**kwargs):
"""
List contents using the resource of the path as a prefix.
Args:
starts_with (str): Allows for an additional search path to be
appended to the current swift path. The current path will be
treated as a directory.
limit (int): Limit the amount of results returned.
condition (function(results) -> bool): The method will only return
when the results matches the condition.
use_manifest (bool): Perform the list and use the data manfest file to validate
the list.
Returns:
List[S3Path]: Every path in the listing
Raises:
RemoteError: An s3 client error occurred.
ConditionNotMetError: Results were returned, but they did not meet the condition.
"""
bucket = self.bucket
prefix = self.resource
utils.validate_condition(condition)
if use_manifest:
object_names = utils.get_data_manifest_contents(self)
manifest_cond = partial(utils.validate_manifest_list, object_names)
condition = (utils.join_conditions(condition, manifest_cond)
if condition else manifest_cond)
if starts_with:
prefix = prefix / starts_with if prefix else starts_with
else:
prefix = prefix or ''
list_kwargs = {
'Bucket': bucket,
'Prefix': prefix,
'PaginationConfig': {}
}
if limit:
list_kwargs['PaginationConfig']['MaxItems'] = limit
if list_as_dir:
# Ensure the the prefix has a trailing slash if there is a prefix
list_kwargs['Prefix'] = utils.with_trailing_slash(
prefix) if prefix else ''
list_kwargs['Delimiter'] = '/'
path_prefix = S3Path('%s%s' % (self.drive, bucket))
results = self._get_s3_iterator('list_objects_v2', **list_kwargs)
list_results = []
try:
for page in results:
if 'Contents' in page:
list_results.extend([
path_prefix / result['Key']
for result in page['Contents']
if not ignore_dir_markers or (
ignore_dir_markers
and not utils.has_trailing_slash(result['Key']))
])
if list_as_dir and 'CommonPrefixes' in page:
list_results.extend([
path_prefix / result['Prefix']
for result in page['CommonPrefixes']
])
except botocore_exceptions.ClientError as e:
raise _parse_s3_error(e) from e
utils.check_condition(condition, list_results)
return list_results
def update_progress(self, result):
"""Tracks number of bytes downloaded."""
self.downloaded_bytes += (os.path.getsize(
result['dest']) if not utils.has_trailing_slash(result['source'])
else 0)
def test_has_trailing_slash_false(self):
self.assertFalse(utils.has_trailing_slash('no/slash'))
def test_has_trailing_slash_true(self):
self.assertTrue(utils.has_trailing_slash('has/slash/'))
def test_has_trailing_slash_none(self):
self.assertFalse(utils.has_trailing_slash(None))
def test_has_trailing_slash(self):
self.assertFalse(utils.has_trailing_slash(''))
