def list_objects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" projection = self._get_projection( fields_scope, self.messages.StorageObjectsListRequest) global_params = None if fields_scope == cloud_api.FieldsScope.SHORT: global_params = self.messages.StandardQueryParameters() global_params.fields = ( 'prefixes,items/name,items/size,items/generation,nextPageToken' ) object_list = None while True: apitools_request = self.messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, pageToken=object_list.nextPageToken if object_list else None, maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE) try: object_list = self.client.objects.List( apitools_request, global_params=global_params) except apitools_exceptions.HttpError as e: core_exceptions.reraise( cloud_errors.translate_error(e, _ERROR_TRANSLATION)) # Yield objects. # TODO(b/160238394) Decrypt metadata fields if necessary. for object_metadata in object_list.items: object_metadata.bucket = bucket_name yield gcs_metadata_util.get_object_resource_from_metadata( object_metadata) # Yield prefixes. for prefix_string in object_list.prefixes: yield resource_reference.PrefixResource(storage_url.CloudUrl( scheme=storage_url.ProviderPrefix.GCS, bucket_name=bucket_name, object_name=prefix_string), prefix=prefix_string) if not object_list.nextPageToken: break
def _list_objects_side_effect(self, *unused_args, **kwargs): """Mock the list_objects API method sourcing data from TEST_OBJECT_NAMES. Args: **kwargs (dict): Contains arguments dict for list_objects. Yields: resource_reference.Resource instances consisting of list of ObjectResource and/or PrefixResource instanes filtered based on the request prefix and request delimiter. """ objects = [] prefixes = set([]) request_prefix = kwargs['prefix'] or '' request_delimiter = kwargs['delimiter'] filtered_object_suffixes = [ object_name[len(request_prefix):] for object_name in TEST_OBJECT_NAMES if object_name.startswith(request_prefix) ] for object_suffix in filtered_object_suffixes: if request_delimiter: name, _, suffix = object_suffix.partition(request_delimiter) if not suffix: # Leaf object. objects.append( self.messages.Object(name=request_prefix + object_suffix, bucket=TEST_BUCKET_NAME)) else: prefixes.add('%s%s%s' % (request_prefix, name, request_delimiter)) else: objects.append( self.messages.Object(name=request_prefix + object_suffix, bucket=TEST_BUCKET_NAME)) prefixes = sorted(list(prefixes)) objects = self.messages.Objects(items=objects, prefixes=prefixes) for o in objects.items: yield gcs_api._object_resource_from_metadata(o) for prefix_string in objects.prefixes: yield resource_reference.PrefixResource( storage_url_object=storage_url.CloudUrl( scheme=cloud_api.DEFAULT_PROVIDER, bucket_name=kwargs['bucket_name'], object_name=prefix_string), prefix=prefix_string)
def _get_prefix_resource_from_s3_response(self, prefix_dict, bucket_name): """Creates resource_reference.PrefixResource from S3 API response. Args: prefix_dict (dict): The S3 API response representing a prefix. bucket_name (str): Bucket for the prefix. Returns: A resource_reference.PrefixResource instance. """ prefix = prefix_dict['Prefix'] return resource_reference.PrefixResource(storage_url.CloudUrl( scheme=self.scheme, bucket_name=bucket_name, object_name=prefix), prefix=prefix)
def get_prefix_resource(scheme, bucket, prefix): url = storage_url.CloudUrl(scheme, bucket_name=bucket, object_name=prefix) return resource_reference.PrefixResource(url, prefix)
def test_gets_prefix_resource(self): url_string = 'gs://bucket/prefix/' parsed_url = storage_url.storage_url_from_string(url_string) resource = resource_reference.PrefixResource(parsed_url, 'prefix/') self.assertEqual(test_resources.from_url_string(url_string), resource)