def list_objects(self,
bucket_name,
prefix=None,
delimiter=None,
all_versions=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsListRequest)
global_params = None
if fields_scope == cloud_api.FieldsScope.SHORT:
global_params = self.messages.StandardQueryParameters()
global_params.fields = (
'prefixes,items/name,items/size,items/generation,nextPageToken'
)
object_list = None
while True:
apitools_request = self.messages.StorageObjectsListRequest(
bucket=bucket_name,
prefix=prefix,
delimiter=delimiter,
versions=all_versions,
projection=projection,
pageToken=object_list.nextPageToken if object_list else None,
maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE)
try:
object_list = self.client.objects.List(
apitools_request, global_params=global_params)
except apitools_exceptions.HttpError as e:
core_exceptions.reraise(
cloud_errors.translate_error(e, _ERROR_TRANSLATION))
# Yield objects.
# TODO(b/160238394) Decrypt metadata fields if necessary.
for object_metadata in object_list.items:
object_metadata.bucket = bucket_name
yield gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
# Yield prefixes.
for prefix_string in object_list.prefixes:
yield resource_reference.PrefixResource(storage_url.CloudUrl(
scheme=storage_url.ProviderPrefix.GCS,
bucket_name=bucket_name,
object_name=prefix_string),
prefix=prefix_string)
if not object_list.nextPageToken:
break
def _list_objects_side_effect(self, *unused_args, **kwargs):
"""Mock the list_objects API method sourcing data from TEST_OBJECT_NAMES.
Args:
**kwargs (dict): Contains arguments dict for list_objects.
Yields:
resource_reference.Resource instances consisting of list of
ObjectResource and/or PrefixResource instanes
filtered based on the request prefix and request delimiter.
"""
objects = []
prefixes = set([])
request_prefix = kwargs['prefix'] or ''
request_delimiter = kwargs['delimiter']
filtered_object_suffixes = [
object_name[len(request_prefix):]
for object_name in TEST_OBJECT_NAMES
if object_name.startswith(request_prefix)
]
for object_suffix in filtered_object_suffixes:
if request_delimiter:
name, _, suffix = object_suffix.partition(request_delimiter)
if not suffix: # Leaf object.
objects.append(
self.messages.Object(name=request_prefix +
object_suffix,
bucket=TEST_BUCKET_NAME))
else:
prefixes.add('%s%s%s' %
(request_prefix, name, request_delimiter))
else:
objects.append(
self.messages.Object(name=request_prefix + object_suffix,
bucket=TEST_BUCKET_NAME))
prefixes = sorted(list(prefixes))
objects = self.messages.Objects(items=objects, prefixes=prefixes)
for o in objects.items:
yield gcs_api._object_resource_from_metadata(o)
for prefix_string in objects.prefixes:
yield resource_reference.PrefixResource(
storage_url_object=storage_url.CloudUrl(
scheme=cloud_api.DEFAULT_PROVIDER,
bucket_name=kwargs['bucket_name'],
object_name=prefix_string),
prefix=prefix_string)
def _get_prefix_resource_from_s3_response(self, prefix_dict, bucket_name):
"""Creates resource_reference.PrefixResource from S3 API response.
Args:
prefix_dict (dict): The S3 API response representing a prefix.
bucket_name (str): Bucket for the prefix.
Returns:
A resource_reference.PrefixResource instance.
"""
prefix = prefix_dict['Prefix']
return resource_reference.PrefixResource(storage_url.CloudUrl(
scheme=self.scheme, bucket_name=bucket_name, object_name=prefix),
prefix=prefix)
def get_prefix_resource(scheme, bucket, prefix): url = storage_url.CloudUrl(scheme, bucket_name=bucket, object_name=prefix) return resource_reference.PrefixResource(url, prefix)
def test_gets_prefix_resource(self):
url_string = 'gs://bucket/prefix/'
parsed_url = storage_url.storage_url_from_string(url_string)
resource = resource_reference.PrefixResource(parsed_url, 'prefix/')
self.assertEqual(test_resources.from_url_string(url_string), resource)
