def _get_pwd(service=None):
"""
Returns the present working directory for the given service,
or all services if none specified.
"""
parser = _get_env()
if service:
try:
return utils.with_trailing_slash(parser.get('env', service))
except configparser.NoOptionError as e:
six.raise_from(ValueError('%s is an invalid service' % service), e)
return [utils.with_trailing_slash(value) for name, value in parser.items('env')]
def is_parent_dir(possible_parent, possible_child):
"""Checks if possible_child is a sub-path of possible_parent"""
if not possible_parent.resource:
return possible_child.resource and \
possible_child.project == possible_parent.project
return possible_child.startswith(
utils.with_trailing_slash(possible_parent))
def rmtree(self):
"""
Removes a resource and all of its contents. The path should point to a directory.
If the specified resource is an object, nothing will happen.
"""
# Ensure there is a trailing slash (path is a dir)
delete_path = utils.with_trailing_slash(self)
delete_list = delete_path.list()
while len(delete_list) > 0:
# boto3 only allows deletion of up to 1000 objects at a time
len_range = min(len(delete_list), 1000)
objects = {
'Objects': [{
'Key': delete_list.pop(0).resource
} for i in range(len_range)]
}
response = self._s3_client_call('delete_objects',
Bucket=self.bucket,
Delete=objects)
if 'Errors' in response:
raise exceptions.RemoteError(
'an error occurred while using rmtree: %s, Key: %s' %
(response['Errors'][0].get('Message'),
response['Errors'][0].get('Key')), response['Errors'])
def _get_pwd(service=None):
"""
Returns the present working directory for the given service,
or all services if none specified.
"""
def to_text(data):
if six.PY2: # pragma: no cover
data = data.decode(locale.getpreferredencoding(False))
return data
parser = _get_env()
if service:
try:
return to_text(
utils.with_trailing_slash(parser.get('env', service)))
except configparser.NoOptionError as e:
six.raise_from(ValueError('%s is an invalid service' % service), e)
return [
to_text(utils.with_trailing_slash(value))
for name, value in parser.items('env')
]
def isdir(self):
"""
Any S3 object whose name ends with a ``/`` is considered to be an empty directory marker.
These objects will not be downloaded and instead an empty directory will be created.
This follows Amazon's convention as described in the
`S3 User Guide <http://docs.aws.amazon.com/AmazonS3/latest/UG/FolderOperations.html>`_.
"""
# Handle buckets separately (in case the bucket is empty)
if not self.resource:
try:
return bool(self._s3_client_call('head_bucket', Bucket=self.bucket))
except exceptions.NotFoundError:
return False
try:
return bool(utils.with_trailing_slash(self).list(limit=1))
except exceptions.NotFoundError:
return False
def _upload_object(self, upload_obj, config=None):
"""Upload a single object given an OBSUploadObject."""
if utils.has_trailing_slash(upload_obj.object_name):
# Handle empty directories separately
ul_kwargs = {
'Bucket': self.bucket,
'Key': utils.with_trailing_slash(str(upload_obj.object_name))
}
if upload_obj.options and 'headers' in upload_obj.options:
ul_kwargs.update(upload_obj.options['headers'])
s3_call = self._s3_client_call
method = 'put_object'
else:
ul_kwargs = {
'bucket': self.bucket,
'key': str(upload_obj.object_name),
'filename': upload_obj.source,
'config': config
}
if upload_obj.options and 'headers' in upload_obj.options:
ul_kwargs['extra_args'] = upload_obj.options['headers']
s3_call = self._make_s3_transfer
method = 'upload_file'
result = {
'source':
upload_obj.source,
'dest':
S3Path(self.drive + self.bucket) /
(ul_kwargs.get('key') or ul_kwargs.get('Key')),
'success':
True
}
try:
s3_call(method, **ul_kwargs)
except exceptions.RemoteError as e:
result['success'] = False
result['error'] = e
return result
def exists(self):
"""
Checks existence of the path.
Returns:
bool: True if the path exists, False otherwise.
Raises:
RemoteError: A non-404 error occurred.
"""
if not self.resource:
try:
return bool(self._s3_client_call('head_bucket', Bucket=self.bucket))
except exceptions.NotFoundError:
return False
try:
return bool(self.stat())
except exceptions.NotFoundError:
pass
try:
return bool(utils.with_trailing_slash(self).list(limit=1))
except exceptions.NotFoundError:
return False
def upload(self,
source,
condition=None,
use_manifest=False,
headers=None,
**kwargs):
"""Uploads a list of files and directories to s3.
Note that the S3Path is treated as a directory.
Note that for user-provided OBSUploadObjects, an empty directory's destination
must have a trailing slash.
Args:
source (List[str|OBSUploadObject]): A list of source files, directories, and
OBSUploadObjects to upload to S3.
condition (function(results) -> bool): The method will only return
when the results of upload matches the condition.
use_manifest (bool): Generate a data manifest and validate the upload results
are in the manifest.
headers (dict): A dictionary of object headers to apply to the object.
Headers will not be applied to OBSUploadObjects and any headers
specified by an OBSUploadObject will override these headers.
Headers should be specified as key-value pairs,
e.g. {'ContentLanguage': 'en'}
Returns:
List[S3Path]: A list of the uploaded files as S3Paths.
Notes:
- This method uploads to paths relative to the current
directory.
"""
if use_manifest and not (len(source) == 1
and os.path.isdir(source[0])):
raise ValueError(
'can only upload one directory with use_manifest=True')
utils.validate_condition(condition)
files_to_convert = utils.walk_files_and_dirs(
[name for name in source if not isinstance(name, OBSUploadObject)])
files_to_upload = [
obj for obj in source if isinstance(obj, OBSUploadObject)
]
manifest_file_name = (Path(source[0]) / utils.DATA_MANIFEST_FILE_NAME
if use_manifest else None)
resource_base = self.resource or Path('')
files_to_upload.extend([
OBSUploadObject(
name,
resource_base /
(utils.with_trailing_slash(
utils.file_name_to_object_name(name)) if
Path(name).isdir() else utils.file_name_to_object_name(name)),
options={'headers': headers} if headers else None)
for name in files_to_convert if name != manifest_file_name
])
if use_manifest:
# Generate the data manifest and save it remotely
object_names = [o.object_name for o in files_to_upload]
utils.generate_and_save_data_manifest(source[0], object_names)
manifest_obj_name = resource_base / utils.file_name_to_object_name(
manifest_file_name)
manifest_obj = OBSUploadObject(
str(manifest_file_name),
manifest_obj_name,
options={'headers': headers} if headers else None)
self._upload_object(manifest_obj)
# Make a condition for validating the upload
manifest_cond = partial(utils.validate_manifest_list, object_names)
condition = (utils.join_conditions(condition, manifest_cond)
if condition else manifest_cond)
options = settings.get()['s3:upload']
segment_size = utils.str_to_bytes(options.get('segment_size'))
transfer_config = {
'multipart_threshold': segment_size,
'max_concurrency': options.get('segment_threads'),
'multipart_chunksize': segment_size
}
upload_w_config = partial(self._upload_object, config=transfer_config)
uploaded = {'completed': [], 'failed': []}
with S3UploadLogger(len(files_to_upload)) as ul:
pool = ThreadPool(options['object_threads'])
try:
result_iter = pool.imap_unordered(upload_w_config,
files_to_upload)
while True:
try:
result = result_iter.next(0xFFFF)
if result['success']:
ul.add_result(result)
uploaded['completed'].append(result)
else:
uploaded['failed'].append(result)
except StopIteration:
break
pool.close()
except BaseException:
pool.terminate()
raise
finally:
pool.join()
if uploaded['failed']:
raise exceptions.FailedUploadError(
'an error occurred while uploading', uploaded)
utils.check_condition(condition,
[r['dest'] for r in uploaded['completed']])
return uploaded
def download(self, dest, condition=None, use_manifest=False, **kwargs):
"""Downloads a directory from S3 to a destination directory.
Args:
dest (str): The destination path to download file to. If downloading to a directory,
there must be a trailing slash. The directory will be created if it doesn't exist.
condition (function(results) -> bool): The method will only return
when the results of download matches the condition.
Returns:
List[S3Path]: A list of the downloaded objects.
Notes:
- The destination directory will be created automatically if it doesn't exist.
- This method downloads to paths relative to the current directory.
"""
utils.validate_condition(condition)
if use_manifest:
object_names = utils.get_data_manifest_contents(self)
manifest_cond = partial(utils.validate_manifest_list, object_names)
condition = (utils.join_conditions(condition, manifest_cond)
if condition else manifest_cond)
source = utils.with_trailing_slash(self)
files_to_download = [{
'source': file,
'dest': dest
} for file in source.list()]
options = settings.get()['s3:download']
segment_size = utils.str_to_bytes(options.get('segment_size'))
transfer_config = {
'multipart_threshold': segment_size,
'max_concurrency': options.get('segment_threads'),
'multipart_chunksize': segment_size
}
download_w_config = partial(self._download_object_worker,
config=transfer_config)
downloaded = {'completed': [], 'failed': []}
with S3DownloadLogger(len(files_to_download)) as dl:
pool = ThreadPool(options['object_threads'])
try:
result_iter = pool.imap_unordered(download_w_config,
files_to_download)
while True:
try:
result = result_iter.next(0xFFFF)
if result['success']:
dl.add_result(result)
downloaded['completed'].append(result)
else:
downloaded['failed'].append(result)
except StopIteration:
break
pool.close()
except BaseException:
pool.terminate()
raise
finally:
pool.join()
if downloaded['failed']:
raise exceptions.FailedDownloadError(
'an error occurred while downloading', downloaded)
utils.check_condition(condition,
[r['source'] for r in downloaded['completed']])
return downloaded
def _download_object_worker(self, obj_params, config=None):
"""Downloads a single object. Helper for threaded download."""
name = self.parts_class(
obj_params['source'][len(utils.with_trailing_slash(self)):])
return obj_params['source'].download_object(obj_params['dest'] / name,
config=config)
def list(
self,
starts_with=None,
limit=None,
condition=None,
use_manifest=False,
# hidden args
list_as_dir=False,
ignore_dir_markers=False,
**kwargs):
"""
List contents using the resource of the path as a prefix.
Args:
starts_with (str): Allows for an additional search path to be
appended to the current swift path. The current path will be
treated as a directory.
limit (int): Limit the amount of results returned.
condition (function(results) -> bool): The method will only return
when the results matches the condition.
use_manifest (bool): Perform the list and use the data manfest file to validate
the list.
Returns:
List[S3Path]: Every path in the listing
Raises:
RemoteError: An s3 client error occurred.
ConditionNotMetError: Results were returned, but they did not meet the condition.
"""
bucket = self.bucket
prefix = self.resource
utils.validate_condition(condition)
if use_manifest:
object_names = utils.get_data_manifest_contents(self)
manifest_cond = partial(utils.validate_manifest_list, object_names)
condition = (utils.join_conditions(condition, manifest_cond)
if condition else manifest_cond)
if starts_with:
prefix = prefix / starts_with if prefix else starts_with
else:
prefix = prefix or ''
list_kwargs = {
'Bucket': bucket,
'Prefix': prefix,
'PaginationConfig': {}
}
if limit:
list_kwargs['PaginationConfig']['MaxItems'] = limit
if list_as_dir:
# Ensure the the prefix has a trailing slash if there is a prefix
list_kwargs['Prefix'] = utils.with_trailing_slash(
prefix) if prefix else ''
list_kwargs['Delimiter'] = '/'
path_prefix = S3Path('%s%s' % (self.drive, bucket))
results = self._get_s3_iterator('list_objects_v2', **list_kwargs)
list_results = []
try:
for page in results:
if 'Contents' in page:
list_results.extend([
path_prefix / result['Key']
for result in page['Contents']
if not ignore_dir_markers or (
ignore_dir_markers
and not utils.has_trailing_slash(result['Key']))
])
if list_as_dir and 'CommonPrefixes' in page:
list_results.extend([
path_prefix / result['Prefix']
for result in page['CommonPrefixes']
])
except botocore_exceptions.ClientError as e:
raise _parse_s3_error(e) from e
utils.check_condition(condition, list_results)
return list_results
def test_no_trailing_slash(self):
path = SwiftPath('swift://AUTH_stor_test/container')
utils.is_writeable(path) # no trailing slash
self.mock_copy.assert_called_with(self.filename,
utils.with_trailing_slash(path),
swift_retry_options=None)
