def _get_object_resource_from_s3_response(object_dict,
bucket_name,
object_name=None):
"""Creates resource_reference.S3ObjectResource from S3 API response.
Args:
object_dict (dict): Dictionary representing S3 API response.
bucket_name (str): Bucket response is relevant to.
object_name (str | None): Object if relevant to query.
Returns:
resource_reference.S3ObjectResource populated with data.
"""
object_url = _get_object_url_from_s3_response(
object_dict, bucket_name, object_name or object_dict['Key'])
etag = None
if 'ETag' in object_dict:
etag = object_dict['ETag']
elif 'CopyObjectResult' in object_dict:
etag = object_dict['CopyObjectResult']['ETag']
size = object_dict.get('Size')
if size is None:
size = object_dict.get('ContentLength')
return s3_resource_reference.S3ObjectResource(object_url,
etag=etag,
metadata=object_dict,
size=size)
def get_object_resource_from_s3_response(object_dict,
bucket_name,
object_name=None):
"""Creates resource_reference.S3ObjectResource from S3 API response.
Args:
object_dict (dict): Dictionary representing S3 API response.
bucket_name (str): Bucket response is relevant to.
object_name (str|None): Object if relevant to query.
Returns:
resource_reference.S3ObjectResource populated with data.
"""
object_url = _get_object_url_from_s3_response(
object_dict, bucket_name, object_name or object_dict['Key'])
if 'Size' in object_dict:
size = object_dict.get('Size')
else:
size = object_dict.get('ContentLength')
etag = _get_etag(object_dict)
return s3_resource_reference.S3ObjectResource(
object_url,
content_type=object_dict.get('ContentType'),
etag=etag,
md5_hash=_get_md5_hash_from_etag(etag, object_url),
metadata=object_dict,
size=size)
def test_get_object_metadata_populates_acls_with_full_fieldscope(self):
expected_request_params = {'Bucket': BUCKET_NAME, 'Key': OBJECT_NAME}
self.stubber.add_response(
'head_object',
service_response={},
expected_params=expected_request_params)
self.stubber.add_response(
'get_object_acl',
service_response={
'Grants': [],
'Owner': {
'ID': '1'
},
'ResponseMetadata': {},
},
expected_params=expected_request_params)
with self.stubber:
object_resource = self.s3_api.get_object_metadata(
BUCKET_NAME, OBJECT_NAME, fields_scope=cloud_api.FieldsScope.FULL)
expected_resource = s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, OBJECT_NAME),
metadata={
'ACL': {
'Grants': [],
'Owner': {
'ID': '1'
},
},
})
self.assertEqual(object_resource, expected_resource)
def test_list_objects_with_full_fields_scope(self):
# Test with more than one object to ensure that get_object_metadata call
# is being made for each object.
objects = [{'Key': 'obj1'}, {'Key': 'obj2'}]
self.stubber.add_response(
'list_objects_v2',
service_response={
'Name': BUCKET_NAME,
'Contents': objects,
},
expected_params={
'Bucket': BUCKET_NAME,
'Prefix': '',
'Delimiter': '',
})
expected_resources = []
for object_dict in objects:
# For each object, we expect a call to the head_object and get_object_acl
# API methods.
self.stubber.add_response(
'head_object',
service_response={'ETag': 'e'},
expected_params={
'Bucket': BUCKET_NAME,
'Key': object_dict['Key']
})
self.stubber.add_response(
'get_object_acl',
service_response={'Grants': []},
expected_params={
'Bucket': BUCKET_NAME,
'Key': object_dict['Key']
})
expected_resources.append(
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, object_dict['Key']),
etag='e',
metadata={
'ETag': 'e',
'ACL': {
'Grants': []
}
}))
with self.stubber:
observed = self.s3_api.list_objects(
BUCKET_NAME, fields_scope=cloud_api.FieldsScope.FULL)
self.assertEqual(list(observed), expected_resources)
def test_get_object_metadata_creates_object_resource(self):
response = {'ETag': 'e', 'ContentLength': 5}
self.stubber.add_response(
'head_object',
service_response=response,
expected_params={
'Bucket': BUCKET_NAME,
'Key': OBJECT_NAME,
})
expected_cloud_url = storage_url.CloudUrl(
SCHEME, bucket_name=BUCKET_NAME, object_name=OBJECT_NAME)
expected_resource = s3_resource_reference.S3ObjectResource(
expected_cloud_url, etag='e', size=5, metadata=response)
with self.stubber:
object_resource = self.s3_api.get_object_metadata(BUCKET_NAME,
OBJECT_NAME)
self.assertEqual(object_resource, expected_resource)
def test_get_object_metadata_populates_acls_with_error_message(self):
self.stubber.add_response(
'head_object',
service_response={},
expected_params={
'Bucket': BUCKET_NAME,
'Key': OBJECT_NAME
})
self.stubber.add_client_error('get_object_acl')
with self.stubber:
object_resource = self.s3_api.get_object_metadata(
BUCKET_NAME, OBJECT_NAME, fields_scope=cloud_api.FieldsScope.FULL)
expected_resource = s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, OBJECT_NAME),
metadata={
'ACL':
errors.S3ApiError('An error occurred () when calling the'
' GetObjectAcl operation: ')
})
self.assertEqual(object_resource, expected_resource)
def test_list_objects_with_all_versions(self):
object_metadata_with_version = {'Key': 'obj', 'VersionId': 'v1'}
self.stubber.add_response(
'list_object_versions',
service_response={
'Name': BUCKET_NAME,
'Versions': [object_metadata_with_version],
},
expected_params={
'Bucket': BUCKET_NAME,
'Prefix': '',
'Delimiter': '',
})
expected_resources = [
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj', 'v1'),
metadata=object_metadata_with_version)
]
with self.stubber:
observed = self.s3_api.list_objects(BUCKET_NAME, all_versions=True)
self.assertCountEqual(observed, expected_resources)
def test_dumps_object_metadata_with_errors_in_correct_format(self):
time = datetime.datetime(1111, 1, 1)
cors_error = errors.S3ApiError(
'An error occurred () when calling the GetBucketCors operation: ')
resource = s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(
storage_url.ProviderPrefix.S3, bucket_name='bucket'),
metadata={
'Error': cors_error,
'List': [{'TimeInListInDict': time}],
'Nested': {'ZeroInt': 0, 'DoubleNested': {'NestedTime': time}},
'String': 'abc',
'Time': time})
expected_dump = textwrap.dedent("""\
{
"url": "s3://bucket",
"type": "cloud_object",
"metadata": {
"Error": "An error occurred () when calling the GetBucketCors operation: ",
"List": [
{
"TimeInListInDict": "1111-01-01T00:00:00"
}
],
"Nested": {
"DoubleNested": {
"NestedTime": "1111-01-01T00:00:00"
},
"ZeroInt": 0
},
"String": "abc",
"Time": "1111-01-01T00:00:00"
}
}""")
self.assertEqual(resource.get_metadata_dump(), expected_dump)
class ListObjectsTest(test_base.StorageTestBase, parameterized.TestCase):
_OBJECT_RESPONSE_METADATA = {
'Key': 'obj',
'LastModified': datetime.datetime(1, 1, 1),
'ETag': 'etag_1',
'Size': 1,
'StorageClass': 'class_1',
}
def SetUp(self):
self.s3_api = s3_api.S3Api()
self.stubber = stub.Stubber(self.s3_api.client)
@parameterized.named_parameters([{
'testcase_name':
'_with_multiple_objects_and_prefixes_present',
'response': {
'Name': BUCKET_NAME,
'Contents': [
{
'Key': 'obj1'
},
{
'Key': 'obj2'
},
],
'CommonPrefixes': [
{
'Prefix': 'dir1/'
},
{
'Prefix': 'dir2/'
},
],
},
'expected_resources': [
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj1'),
metadata={'Key': 'obj1'}),
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj2'),
metadata={'Key': 'obj2'}),
test_resources.get_prefix_resource(SCHEME, BUCKET_NAME, 'dir1/'),
test_resources.get_prefix_resource(SCHEME, BUCKET_NAME, 'dir2/'),
],
}, {
'testcase_name':
'_with_missing_prefixes',
'response': {
'Name': BUCKET_NAME,
'Contents': [{
'Key': 'obj1'
}],
},
'expected_resources': [
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj1'),
metadata={'Key': 'obj1'})
],
}, {
'testcase_name':
'_with_missing_objects',
'response': {
'Name': BUCKET_NAME,
'CommonPrefixes': [{
'Prefix': 'dir1/'
}],
},
'expected_resources': [
test_resources.get_prefix_resource(SCHEME, BUCKET_NAME, 'dir1/'),
],
}, {
'testcase_name': '_with_both_objects_and_prefixes_missing',
'response': {
'Name': BUCKET_NAME
},
'expected_resources': [],
}, {
'testcase_name':
'_with_correct_metadata_populated',
'response': {
'Name': BUCKET_NAME,
'Contents': [_OBJECT_RESPONSE_METADATA],
},
'expected_resources': [
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj'),
etag='etag_1',
metadata=_OBJECT_RESPONSE_METADATA)
],
}])
def test_list_objects_returns_correct_object_and_prefix_resources(
self, response, expected_resources):
self.stubber.add_response(
'list_objects_v2',
service_response=response,
expected_params={
'Bucket': BUCKET_NAME,
'Prefix': '',
'Delimiter': '',
})
with self.stubber:
observed = self.s3_api.list_objects(BUCKET_NAME)
self.assertCountEqual(observed, expected_resources)
def test_list_objects_with_all_versions(self):
object_metadata_with_version = {'Key': 'obj', 'VersionId': 'v1'}
self.stubber.add_response(
'list_object_versions',
service_response={
'Name': BUCKET_NAME,
'Versions': [object_metadata_with_version],
},
expected_params={
'Bucket': BUCKET_NAME,
'Prefix': '',
'Delimiter': '',
})
expected_resources = [
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, 'obj', 'v1'),
metadata=object_metadata_with_version)
]
with self.stubber:
observed = self.s3_api.list_objects(BUCKET_NAME, all_versions=True)
self.assertCountEqual(observed, expected_resources)
def test_list_objects_with_full_fields_scope(self):
# Test with more than one object to ensure that get_object_metadata call
# is being made for each object.
objects = [{'Key': 'obj1'}, {'Key': 'obj2'}]
self.stubber.add_response(
'list_objects_v2',
service_response={
'Name': BUCKET_NAME,
'Contents': objects,
},
expected_params={
'Bucket': BUCKET_NAME,
'Prefix': '',
'Delimiter': '',
})
expected_resources = []
for object_dict in objects:
# For each object, we expect a call to the head_object and get_object_acl
# API methods.
self.stubber.add_response(
'head_object',
service_response={'ETag': 'e'},
expected_params={
'Bucket': BUCKET_NAME,
'Key': object_dict['Key']
})
self.stubber.add_response(
'get_object_acl',
service_response={'Grants': []},
expected_params={
'Bucket': BUCKET_NAME,
'Key': object_dict['Key']
})
expected_resources.append(
s3_resource_reference.S3ObjectResource(
storage_url.CloudUrl(SCHEME, BUCKET_NAME, object_dict['Key']),
etag='e',
metadata={
'ETag': 'e',
'ACL': {
'Grants': []
}
}))
with self.stubber:
observed = self.s3_api.list_objects(
BUCKET_NAME, fields_scope=cloud_api.FieldsScope.FULL)
self.assertEqual(list(observed), expected_resources)