def _prep_for_copy(self, dest): """Handles logic, for finalizing target destination, making parent folders and deleting existing target, common to _clone and _move""" dest_is_dir = dest.isdir() target_dest = dest if dest_is_dir or utils.has_trailing_slash(dest): target_dest = dest / self.name if not dest_is_dir and target_dest.parent.resource: target_dest.parent.makedirs_p() if target_dest.isfile(): target_dest.remove() should_rename = not dest_is_dir and not utils.has_trailing_slash(dest) return target_dest, should_rename
def _prep_for_copytree(self, dest): """Handles logic, for finalizing target destination, making parent folders and checking for clashes, common to _clonetree and _movetree""" source = utils.remove_trailing_slash(self) dest_is_dir = dest.isdir() should_rename = True target_dest = dest if dest_is_dir or utils.has_trailing_slash(dest): target_dest = dest / (source.name if source.resource else source.virtual_project) if target_dest.isdir(): raise stor_exceptions.TargetExistsError( 'Destination path ({}) already exists, will not cause ' 'duplicate folders to exist. Remove the original first'. format(target_dest)) should_rename = False if not source.resource: target_dest.makedirs_p() elif not dest_is_dir and target_dest.parent.resource: # don't call makedirs_p on project target_dest.parent.makedirs_p() moved_folder_path = target_dest.parent / source.name return target_dest, should_rename, moved_folder_path
def download_object(self, dest, config=None, **kwargs): """ Downloads a file from S3 to a destination file. Args: dest (str): The destination path to download file to. Notes: - The destination directory will be created automatically if it doesn't exist. - This method downloads to paths relative to the current directory. """ result = {'source': self, 'dest': dest, 'success': True} if utils.has_trailing_slash(self): # Handle directory markers separately utils.make_dest_dir(str(dest)) return result dl_kwargs = { 'bucket': self.bucket, 'key': str(self.resource), 'filename': str(dest), 'config': config } utils.make_dest_dir(self.parts_class(dest).parent) try: self._make_s3_transfer('download_file', **dl_kwargs) except exceptions.RemoteError as e: result['success'] = False result['error'] = e return result
def canonical_resource(self): """The dxid of the file at this path Raises: MultipleObjectsSameNameError: if filename is not unique NotFoundError: if resource is not found on DX platform ValueError: if path looks like a folder path (i.e., ends with trailing slash) """ if not self.resource: return None if utils.has_trailing_slash(self): raise ValueError( 'Invalid operation ({method}) on folder path ({path})'.format( path=self, method=sys._getframe(2).f_code.co_name)) objects = [{ 'name': self.name, 'folder': ('/' + self.resource).parent, 'project': self.canonical_project, 'batchsize': 2 }] with _wrap_dx_calls(): results = dxpy.resolve_data_objects(objects=objects)[0] if len(results) > 1: raise MultipleObjectsSameNameError( 'Multiple objects found at path ({}). ' 'Try using a canonical ID instead'.format(self)) elif len(results) == 1: return results[0]['id'] else: raise stor_exceptions.NotFoundError( 'No data object was found for the given path ({}) on DNAnexus'. format(self))
def isfile(self): """Determine an object exists at the specified path Returns: bool: True if path points to an existing file """ if not self.resource or utils.has_trailing_slash(self): return False try: self.stat() return True except stor_exceptions.NotFoundError: return False
def _upload_object(self, upload_obj, config=None): """Upload a single object given an OBSUploadObject.""" if utils.has_trailing_slash(upload_obj.object_name): # Handle empty directories separately ul_kwargs = { 'Bucket': self.bucket, 'Key': utils.with_trailing_slash(str(upload_obj.object_name)) } if upload_obj.options and 'headers' in upload_obj.options: ul_kwargs.update(upload_obj.options['headers']) s3_call = self._s3_client_call method = 'put_object' else: ul_kwargs = { 'bucket': self.bucket, 'key': str(upload_obj.object_name), 'filename': upload_obj.source, 'config': config } if upload_obj.options and 'headers' in upload_obj.options: ul_kwargs['extra_args'] = upload_obj.options['headers'] s3_call = self._make_s3_transfer method = 'upload_file' result = { 'source': upload_obj.source, 'dest': S3Path(self.drive + self.bucket) / (ul_kwargs.get('key') or ul_kwargs.get('Key')), 'success': True } try: s3_call(method, **ul_kwargs) except exceptions.RemoteError as e: result['success'] = False result['error'] = e return result
def isfile(self): try: return self.stat() and not utils.has_trailing_slash(self) except (exceptions.NotFoundError, ValueError): return False
def update_progress(self, result): """Keep track of total uploaded bytes by referencing the object sizes""" self.uploaded_bytes += (os.path.getsize(result['source']) if not utils.has_trailing_slash(result['dest']) else 0)
def list( self, starts_with=None, limit=None, condition=None, use_manifest=False, # hidden args list_as_dir=False, ignore_dir_markers=False, **kwargs): """ List contents using the resource of the path as a prefix. Args: starts_with (str): Allows for an additional search path to be appended to the current swift path. The current path will be treated as a directory. limit (int): Limit the amount of results returned. condition (function(results) -> bool): The method will only return when the results matches the condition. use_manifest (bool): Perform the list and use the data manfest file to validate the list. Returns: List[S3Path]: Every path in the listing Raises: RemoteError: An s3 client error occurred. ConditionNotMetError: Results were returned, but they did not meet the condition. """ bucket = self.bucket prefix = self.resource utils.validate_condition(condition) if use_manifest: object_names = utils.get_data_manifest_contents(self) manifest_cond = partial(utils.validate_manifest_list, object_names) condition = (utils.join_conditions(condition, manifest_cond) if condition else manifest_cond) if starts_with: prefix = prefix / starts_with if prefix else starts_with else: prefix = prefix or '' list_kwargs = { 'Bucket': bucket, 'Prefix': prefix, 'PaginationConfig': {} } if limit: list_kwargs['PaginationConfig']['MaxItems'] = limit if list_as_dir: # Ensure the the prefix has a trailing slash if there is a prefix list_kwargs['Prefix'] = utils.with_trailing_slash( prefix) if prefix else '' list_kwargs['Delimiter'] = '/' path_prefix = S3Path('%s%s' % (self.drive, bucket)) results = self._get_s3_iterator('list_objects_v2', **list_kwargs) list_results = [] try: for page in results: if 'Contents' in page: list_results.extend([ path_prefix / result['Key'] for result in page['Contents'] if not ignore_dir_markers or ( ignore_dir_markers and not utils.has_trailing_slash(result['Key'])) ]) if list_as_dir and 'CommonPrefixes' in page: list_results.extend([ path_prefix / result['Prefix'] for result in page['CommonPrefixes'] ]) except botocore_exceptions.ClientError as e: raise _parse_s3_error(e) from e utils.check_condition(condition, list_results) return list_results
def update_progress(self, result): """Tracks number of bytes downloaded.""" self.downloaded_bytes += (os.path.getsize( result['dest']) if not utils.has_trailing_slash(result['source']) else 0)
def test_has_trailing_slash_false(self): self.assertFalse(utils.has_trailing_slash('no/slash'))
def test_has_trailing_slash_true(self): self.assertTrue(utils.has_trailing_slash('has/slash/'))
def test_has_trailing_slash_none(self): self.assertFalse(utils.has_trailing_slash(None))
def test_has_trailing_slash(self): self.assertFalse(utils.has_trailing_slash(''))