def _get_object_resource_from_s3_response(object_dict, bucket_name, object_name=None): """Creates resource_reference.S3ObjectResource from S3 API response. Args: object_dict (dict): Dictionary representing S3 API response. bucket_name (str): Bucket response is relevant to. object_name (str | None): Object if relevant to query. Returns: resource_reference.S3ObjectResource populated with data. """ object_url = _get_object_url_from_s3_response( object_dict, bucket_name, object_name or object_dict['Key']) etag = None if 'ETag' in object_dict: etag = object_dict['ETag'] elif 'CopyObjectResult' in object_dict: etag = object_dict['CopyObjectResult']['ETag'] size = object_dict.get('Size') if size is None: size = object_dict.get('ContentLength') return s3_resource_reference.S3ObjectResource(object_url, etag=etag, metadata=object_dict, size=size)
def get_object_resource_from_s3_response(object_dict, bucket_name, object_name=None): """Creates resource_reference.S3ObjectResource from S3 API response. Args: object_dict (dict): Dictionary representing S3 API response. bucket_name (str): Bucket response is relevant to. object_name (str|None): Object if relevant to query. Returns: resource_reference.S3ObjectResource populated with data. """ object_url = _get_object_url_from_s3_response( object_dict, bucket_name, object_name or object_dict['Key']) if 'Size' in object_dict: size = object_dict.get('Size') else: size = object_dict.get('ContentLength') etag = _get_etag(object_dict) return s3_resource_reference.S3ObjectResource( object_url, content_type=object_dict.get('ContentType'), etag=etag, md5_hash=_get_md5_hash_from_etag(etag, object_url), metadata=object_dict, size=size)
def test_get_object_metadata_populates_acls_with_full_fieldscope(self): expected_request_params = {'Bucket': BUCKET_NAME, 'Key': OBJECT_NAME} self.stubber.add_response( 'head_object', service_response={}, expected_params=expected_request_params) self.stubber.add_response( 'get_object_acl', service_response={ 'Grants': [], 'Owner': { 'ID': '1' }, 'ResponseMetadata': {}, }, expected_params=expected_request_params) with self.stubber: object_resource = self.s3_api.get_object_metadata( BUCKET_NAME, OBJECT_NAME, fields_scope=cloud_api.FieldsScope.FULL) expected_resource = s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, OBJECT_NAME), metadata={ 'ACL': { 'Grants': [], 'Owner': { 'ID': '1' }, }, }) self.assertEqual(object_resource, expected_resource)
def test_list_objects_with_full_fields_scope(self): # Test with more than one object to ensure that get_object_metadata call # is being made for each object. objects = [{'Key': 'obj1'}, {'Key': 'obj2'}] self.stubber.add_response( 'list_objects_v2', service_response={ 'Name': BUCKET_NAME, 'Contents': objects, }, expected_params={ 'Bucket': BUCKET_NAME, 'Prefix': '', 'Delimiter': '', }) expected_resources = [] for object_dict in objects: # For each object, we expect a call to the head_object and get_object_acl # API methods. self.stubber.add_response( 'head_object', service_response={'ETag': 'e'}, expected_params={ 'Bucket': BUCKET_NAME, 'Key': object_dict['Key'] }) self.stubber.add_response( 'get_object_acl', service_response={'Grants': []}, expected_params={ 'Bucket': BUCKET_NAME, 'Key': object_dict['Key'] }) expected_resources.append( s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, object_dict['Key']), etag='e', metadata={ 'ETag': 'e', 'ACL': { 'Grants': [] } })) with self.stubber: observed = self.s3_api.list_objects( BUCKET_NAME, fields_scope=cloud_api.FieldsScope.FULL) self.assertEqual(list(observed), expected_resources)
def test_get_object_metadata_creates_object_resource(self): response = {'ETag': 'e', 'ContentLength': 5} self.stubber.add_response( 'head_object', service_response=response, expected_params={ 'Bucket': BUCKET_NAME, 'Key': OBJECT_NAME, }) expected_cloud_url = storage_url.CloudUrl( SCHEME, bucket_name=BUCKET_NAME, object_name=OBJECT_NAME) expected_resource = s3_resource_reference.S3ObjectResource( expected_cloud_url, etag='e', size=5, metadata=response) with self.stubber: object_resource = self.s3_api.get_object_metadata(BUCKET_NAME, OBJECT_NAME) self.assertEqual(object_resource, expected_resource)
def test_get_object_metadata_populates_acls_with_error_message(self): self.stubber.add_response( 'head_object', service_response={}, expected_params={ 'Bucket': BUCKET_NAME, 'Key': OBJECT_NAME }) self.stubber.add_client_error('get_object_acl') with self.stubber: object_resource = self.s3_api.get_object_metadata( BUCKET_NAME, OBJECT_NAME, fields_scope=cloud_api.FieldsScope.FULL) expected_resource = s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, OBJECT_NAME), metadata={ 'ACL': errors.S3ApiError('An error occurred () when calling the' ' GetObjectAcl operation: ') }) self.assertEqual(object_resource, expected_resource)
def test_list_objects_with_all_versions(self): object_metadata_with_version = {'Key': 'obj', 'VersionId': 'v1'} self.stubber.add_response( 'list_object_versions', service_response={ 'Name': BUCKET_NAME, 'Versions': [object_metadata_with_version], }, expected_params={ 'Bucket': BUCKET_NAME, 'Prefix': '', 'Delimiter': '', }) expected_resources = [ s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj', 'v1'), metadata=object_metadata_with_version) ] with self.stubber: observed = self.s3_api.list_objects(BUCKET_NAME, all_versions=True) self.assertCountEqual(observed, expected_resources)
def test_dumps_object_metadata_with_errors_in_correct_format(self): time = datetime.datetime(1111, 1, 1) cors_error = errors.S3ApiError( 'An error occurred () when calling the GetBucketCors operation: ') resource = s3_resource_reference.S3ObjectResource( storage_url.CloudUrl( storage_url.ProviderPrefix.S3, bucket_name='bucket'), metadata={ 'Error': cors_error, 'List': [{'TimeInListInDict': time}], 'Nested': {'ZeroInt': 0, 'DoubleNested': {'NestedTime': time}}, 'String': 'abc', 'Time': time}) expected_dump = textwrap.dedent("""\ { "url": "s3://bucket", "type": "cloud_object", "metadata": { "Error": "An error occurred () when calling the GetBucketCors operation: ", "List": [ { "TimeInListInDict": "1111-01-01T00:00:00" } ], "Nested": { "DoubleNested": { "NestedTime": "1111-01-01T00:00:00" }, "ZeroInt": 0 }, "String": "abc", "Time": "1111-01-01T00:00:00" } }""") self.assertEqual(resource.get_metadata_dump(), expected_dump)
class ListObjectsTest(test_base.StorageTestBase, parameterized.TestCase): _OBJECT_RESPONSE_METADATA = { 'Key': 'obj', 'LastModified': datetime.datetime(1, 1, 1), 'ETag': 'etag_1', 'Size': 1, 'StorageClass': 'class_1', } def SetUp(self): self.s3_api = s3_api.S3Api() self.stubber = stub.Stubber(self.s3_api.client) @parameterized.named_parameters([{ 'testcase_name': '_with_multiple_objects_and_prefixes_present', 'response': { 'Name': BUCKET_NAME, 'Contents': [ { 'Key': 'obj1' }, { 'Key': 'obj2' }, ], 'CommonPrefixes': [ { 'Prefix': 'dir1/' }, { 'Prefix': 'dir2/' }, ], }, 'expected_resources': [ s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj1'), metadata={'Key': 'obj1'}), s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj2'), metadata={'Key': 'obj2'}), test_resources.get_prefix_resource(SCHEME, BUCKET_NAME, 'dir1/'), test_resources.get_prefix_resource(SCHEME, BUCKET_NAME, 'dir2/'), ], }, { 'testcase_name': '_with_missing_prefixes', 'response': { 'Name': BUCKET_NAME, 'Contents': [{ 'Key': 'obj1' }], }, 'expected_resources': [ s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj1'), metadata={'Key': 'obj1'}) ], }, { 'testcase_name': '_with_missing_objects', 'response': { 'Name': BUCKET_NAME, 'CommonPrefixes': [{ 'Prefix': 'dir1/' }], }, 'expected_resources': [ test_resources.get_prefix_resource(SCHEME, BUCKET_NAME, 'dir1/'), ], }, { 'testcase_name': '_with_both_objects_and_prefixes_missing', 'response': { 'Name': BUCKET_NAME }, 'expected_resources': [], }, { 'testcase_name': '_with_correct_metadata_populated', 'response': { 'Name': BUCKET_NAME, 'Contents': [_OBJECT_RESPONSE_METADATA], }, 'expected_resources': [ s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj'), etag='etag_1', metadata=_OBJECT_RESPONSE_METADATA) ], }]) def test_list_objects_returns_correct_object_and_prefix_resources( self, response, expected_resources): self.stubber.add_response( 'list_objects_v2', service_response=response, expected_params={ 'Bucket': BUCKET_NAME, 'Prefix': '', 'Delimiter': '', }) with self.stubber: observed = self.s3_api.list_objects(BUCKET_NAME) self.assertCountEqual(observed, expected_resources) def test_list_objects_with_all_versions(self): object_metadata_with_version = {'Key': 'obj', 'VersionId': 'v1'} self.stubber.add_response( 'list_object_versions', service_response={ 'Name': BUCKET_NAME, 'Versions': [object_metadata_with_version], }, expected_params={ 'Bucket': BUCKET_NAME, 'Prefix': '', 'Delimiter': '', }) expected_resources = [ s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj', 'v1'), metadata=object_metadata_with_version) ] with self.stubber: observed = self.s3_api.list_objects(BUCKET_NAME, all_versions=True) self.assertCountEqual(observed, expected_resources) def test_list_objects_with_full_fields_scope(self): # Test with more than one object to ensure that get_object_metadata call # is being made for each object. objects = [{'Key': 'obj1'}, {'Key': 'obj2'}] self.stubber.add_response( 'list_objects_v2', service_response={ 'Name': BUCKET_NAME, 'Contents': objects, }, expected_params={ 'Bucket': BUCKET_NAME, 'Prefix': '', 'Delimiter': '', }) expected_resources = [] for object_dict in objects: # For each object, we expect a call to the head_object and get_object_acl # API methods. self.stubber.add_response( 'head_object', service_response={'ETag': 'e'}, expected_params={ 'Bucket': BUCKET_NAME, 'Key': object_dict['Key'] }) self.stubber.add_response( 'get_object_acl', service_response={'Grants': []}, expected_params={ 'Bucket': BUCKET_NAME, 'Key': object_dict['Key'] }) expected_resources.append( s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, object_dict['Key']), etag='e', metadata={ 'ETag': 'e', 'ACL': { 'Grants': [] } })) with self.stubber: observed = self.s3_api.list_objects( BUCKET_NAME, fields_scope=cloud_api.FieldsScope.FULL) self.assertEqual(list(observed), expected_resources)