def get_object_metadata(self, bucket_name, object_name, generation=None, fields_scope=None): """See super class.""" request = {'Bucket': bucket_name, 'Key': object_name} # The VersionId keyword argument to head_object is not nullable if it is # present, so only include it in the function call if it has a value. if generation is not None: request['VersionId'] = generation try: object_dict = self.client.head_object(**request) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error)) # User requested ACL's with FieldsScope.FULL. if fields_scope is cloud_api.FieldsScope.FULL: try: acl_response = self.client.get_object_acl(**request) acl_response.pop('ResponseMetadata', None) object_dict['ACL'] = acl_response except botocore.exceptions.ClientError as error: object_dict['ACL'] = errors.S3ApiError(error) return self._get_object_resource_from_s3_response( object_dict, bucket_name, object_name)
def get_bucket(self, bucket_name, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" metadata = {'Name': bucket_name} # TODO (b/168716392): As new commands are implemented, they may want # specific error handling for different methods. try: # Low-bandwidth way to determine if bucket exists for FieldsScope.SHORT. metadata.update( self.client.get_bucket_location(Bucket=bucket_name)) except botocore.exceptions.ClientError as error: _raise_if_not_found_error(error, bucket_name) metadata['LocationConstraint'] = errors.S3ApiError(error) if fields_scope is not cloud_api.FieldsScope.SHORT: # Data for FieldsScope.NO_ACL. for key, api_call, result_has_key in [ ('CORSRules', self.client.get_bucket_cors, True), ('ServerSideEncryptionConfiguration', self.client.get_bucket_encryption, True), ('LifecycleConfiguration', self.client.get_bucket_lifecycle_configuration, False), ('LoggingEnabled', self.client.get_bucket_logging, True), ('Payer', self.client.get_bucket_request_payment, True), ('Versioning', self.client.get_bucket_versioning, False), ('Website', self.client.get_bucket_website, False), ]: try: api_result = api_call(Bucket=bucket_name) # Some results are wrapped in dictionaries with keys matching "key". metadata[key] = api_result.get( key) if result_has_key else api_result except botocore.exceptions.ClientError as error: metadata[key] = errors.S3ApiError(error) # User requested ACL's with FieldsScope.FULL. if fields_scope is cloud_api.FieldsScope.FULL: try: metadata['ACL'] = self.client.get_bucket_acl( Bucket=bucket_name) except botocore.exceptions.ClientError as error: metadata['ACL'] = errors.S3ApiError(error) return s3_resource_reference.S3BucketResource(storage_url.CloudUrl( storage_url.ProviderPrefix.S3, bucket_name), metadata=metadata)
def upload_object(self, upload_stream, upload_resource, progress_callback=None, request_config=None): """See super class.""" # TODO(b/160998556): Implement resumable upload. del progress_callback kwargs = { 'Bucket': upload_resource.storage_url.bucket_name, 'Key': upload_resource.storage_url.object_name, 'Body': upload_stream.read() } if request_config and request_config.predefined_acl_string: kwargs['ACL'] = _translate_predefined_acl_string_to_s3( request_config.predefined_acl_string) try: response = self.client.put_object(**kwargs) return self._get_object_resource_from_s3_response( response, upload_resource.storage_url.bucket_name, upload_resource.storage_url.object_name) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def CopyObject(self, source_resource, destination_resource, progress_callback=None, request_config=None): """See super class.""" del progress_callback source_kwargs = {'Bucket': source_resource.storage_url.bucket_name, 'Key': source_resource.storage_url.object_name} if source_resource.storage_url.generation: source_kwargs['VersionId'] = source_resource.storage_url.generation kwargs = {'Bucket': destination_resource.storage_url.bucket_name, 'Key': destination_resource.storage_url.object_name, 'CopySource': source_kwargs} if request_config and request_config.predefined_acl_string: kwargs['ACL'] = _TranslatePredefinedAclStringToS3( request_config.predefined_acl_string) try: response = self.client.copy_object(**kwargs) return self._GetObjectResourceFromS3Response( response, kwargs['Bucket'], kwargs['Key']) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def test_gets_bucket_with_individual_calls_failing(self, failing_method_name): # Method name, method name in error, metadata key, API response, and # flag describing if API response has key mirroring metadata key. get_bucket_api_call_data = [ ( 'get_bucket_location', 'GetBucketLocation', 'LocationConstraint', {'LocationConstraint': ''}, True ), ( 'get_bucket_cors', 'GetBucketCors', 'CORSRules', {'CORSRules': []}, True ), ( 'get_bucket_lifecycle_configuration', 'GetBucketLifecycleConfiguration', 'LifecycleConfiguration', {'Rules': []}, False ), ( 'get_bucket_logging', 'GetBucketLogging', 'LoggingEnabled', {'LoggingEnabled': { 'TargetBucket': '', 'TargetGrants': [], 'TargetPrefix': ''}}, True ), ( 'get_bucket_request_payment', 'GetBucketRequestPayment', 'Payer', {'Payer': 'BucketOwner'}, True ), ( 'get_bucket_versioning', 'GetBucketVersioning', 'Versioning', {}, False ), ( 'get_bucket_website', 'GetBucketWebsite', 'Website', {}, False ), ( 'get_bucket_acl', 'GetBucketAcl', 'ACL', {'Owner': {}, 'Grants': []}, False ), ] expected_metadata = {'Name': BUCKET_NAME} for (method_name, error_string_of_method_name, metadata_key, response, result_has_key) in get_bucket_api_call_data: if failing_method_name == method_name: self.stubber.add_client_error(method=method_name) expected_metadata[metadata_key] = errors.S3ApiError( 'An error occurred () when calling the {} operation: ' .format(error_string_of_method_name)) else: self.stubber.add_response(method=method_name, expected_params={'Bucket': BUCKET_NAME}, service_response=response) expected_metadata[metadata_key] = ( response[metadata_key] if result_has_key else response) expected_resource = s3_resource_reference.S3BucketResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME), metadata=expected_metadata) with self.stubber: observed_resource = self.s3_api.get_bucket( BUCKET_NAME, fields_scope=cloud_api.FieldsScope.FULL) self.assertEqual(observed_resource, expected_resource)
def test_gets_bucket_receives_all_api_errors(self): get_bucket_api_method_names_and_metadata_key = [ ('get_bucket_location', 'GetBucketLocation', 'LocationConstraint'), ('get_bucket_cors', 'GetBucketCors', 'CORSRules'), ('get_bucket_lifecycle_configuration', 'GetBucketLifecycleConfiguration', 'LifecycleConfiguration'), ('get_bucket_logging', 'GetBucketLogging', 'LoggingEnabled'), ('get_bucket_request_payment', 'GetBucketRequestPayment', 'Payer'), ('get_bucket_versioning', 'GetBucketVersioning', 'Versioning'), ('get_bucket_website', 'GetBucketWebsite', 'Website'), ('get_bucket_acl', 'GetBucketAcl', 'ACL'), ] expected_metadata = {'Name': BUCKET_NAME} for (method_name, error_string_of_method_name, metadata_key) in get_bucket_api_method_names_and_metadata_key: self.stubber.add_client_error(method=method_name) expected_metadata[metadata_key] = errors.S3ApiError( 'An error occurred () when calling the {} operation: ' .format(error_string_of_method_name)) expected_resource = s3_resource_reference.S3BucketResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME), metadata=expected_metadata) with self.stubber: observed_resource = self.s3_api.get_bucket( BUCKET_NAME, fields_scope=cloud_api.FieldsScope.FULL) self.assertEqual(observed_resource, expected_resource)
def _make_patch_request(self, bucket_resource, patch_function, patch_kwargs): patch_kwargs['Bucket'] = bucket_resource.storage_url.bucket_name try: patch_function(**patch_kwargs) except botocore.exceptions.ClientError as error: _raise_if_not_found_error(error, bucket_resource.storage_url.bucket_name) log.error(errors.S3ApiError(error))
def ListBuckets(self, fields_scope=None): """See super class.""" try: response = self.client.list_buckets() for bucket in response['Buckets']: yield resource_reference.BucketResource( storage_url.CloudUrl(storage_url.ProviderPrefix.S3, bucket['Name']), metadata={'Bucket': bucket, 'Owner': response['Owner']}) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" try: response = self.client.list_buckets() for bucket in response['Buckets']: if fields_scope == cloud_api.FieldsScope.FULL: yield self.get_bucket(bucket['Name'], fields_scope) else: yield s3_resource_reference.S3BucketResource( storage_url.CloudUrl( storage_url.ProviderPrefix.S3, bucket['Name']), metadata={'Bucket': bucket, 'Owner': response['Owner']}) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def upload_object(self, source_stream, destination_resource, progress_callback=None, request_config=None, serialization_data=None, tracker_callback=None, upload_strategy=cloud_api.UploadStrategy.SIMPLE): """See super class.""" del progress_callback, serialization_data, tracker_callback if upload_strategy != cloud_api.UploadStrategy.SIMPLE: raise command_errors.Error('Invalid upload strategy: {}.'.format( upload_strategy.value)) if request_config is None: request_config = cloud_api.RequestConfig() # All fields common to both put_object and upload_fileobj are added # to the extra_args dict. extra_args = {} if request_config.predefined_acl_string: extra_args['ACL'] = _translate_predefined_acl_string_to_s3( request_config.predefined_acl_string) if request_config.md5_hash: # The upload_fileobj method can perform multipart uploads, so it cannot # validate with user-provided MD5 hashes. Hence we use the put_object API # method if MD5 validation is requested. if request_config.size > MAX_PUT_OBJECT_SIZE: raise errors.S3ApiError( 'Cannot upload to destination: {url} because MD5 validation can' ' only be performed for file size <= {maxsize} Bytes. Current file' ' size is {filesize} Bytes. You can remove the MD5 validation' ' requirement to complete the upload'.format( url=destination_resource.storage_url.url_string, maxsize=MAX_PUT_OBJECT_SIZE, filesize=request_config.size)) extra_args['ContentMD5'] = request_config.md5_hash return self._upload_using_put_object(source_stream, destination_resource, extra_args) else: # We default to calling the upload_fileobj method provided by boto3 which # is a managed-transfer utility that can perform mulitpart uploads # automatically. It can be used for non-seekable source_streams as well. return self._upload_using_managed_transfer_utility( source_stream, destination_resource, extra_args)
def ListObjects(self, bucket_name, prefix='', delimiter='', all_versions=None, fields_scope=None): """See super class.""" try: paginator = self.client.get_paginator('list_objects_v2') page_iterator = paginator.paginate( Bucket=bucket_name, Prefix=prefix, Delimiter=delimiter) for page in page_iterator: for object_dict in page.get('Contents', []): yield self._GetObjectResourceFromS3Response(object_dict, bucket_name) for prefix_dict in page.get('CommonPrefixes', []): prefix = prefix_dict['Prefix'] yield self._GetPrefixResourceFromS3Response(prefix_dict, bucket_name) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def list_objects(self, bucket_name, prefix=None, delimiter=None, all_versions=False, fields_scope=None): """See super class.""" if all_versions: api_method_name = 'list_object_versions' objects_key = 'Versions' else: api_method_name = 'list_objects_v2' objects_key = 'Contents' try: paginator = self.client.get_paginator(api_method_name) page_iterator = paginator.paginate( Bucket=bucket_name, Prefix=prefix if prefix is not None else '', Delimiter=delimiter if delimiter is not None else '') for page in page_iterator: for object_dict in page.get(objects_key, []): if fields_scope is cloud_api.FieldsScope.FULL: # The metadata present in the list_objects_v2 response or the # list_object_versions response is not enough # for a FULL scope. Hence, calling the GetObjectMetadata method # to get the additonal metadata and ACLs information. yield self.get_object_metadata( bucket_name=bucket_name, object_name=object_dict['Key'], request_config=request_config_factory. get_request_config( storage_url.CloudUrl( scheme=storage_url.ProviderPrefix.S3)), generation=object_dict.get('VersionId'), fields_scope=fields_scope) else: yield s3_metadata_util.get_object_resource_from_s3_response( object_dict, bucket_name) for prefix_dict in page.get('CommonPrefixes', []): yield s3_metadata_util.get_prefix_resource_from_s3_response( prefix_dict, bucket_name) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def GetObjectMetadata(self, bucket_name, object_name, generation=None, fields_scope=None): """See super class.""" request = {'Bucket': bucket_name, 'Key': object_name} # The VersionId keyword argument to head_object is not nullable if it is # present, so only include it in the function call if it has a value. if generation is not None: request['VersionId'] = generation try: object_dict = self.client.head_object(**request) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error)) return self._GetObjectResourceFromS3Response( object_dict, bucket_name, object_name)
def test_get_object_metadata_populates_acls_with_error_message(self): self.stubber.add_response( 'head_object', service_response={}, expected_params={ 'Bucket': BUCKET_NAME, 'Key': OBJECT_NAME }) self.stubber.add_client_error('get_object_acl') with self.stubber: object_resource = self.s3_api.get_object_metadata( BUCKET_NAME, OBJECT_NAME, fields_scope=cloud_api.FieldsScope.FULL) expected_resource = s3_resource_reference.S3ObjectResource( storage_url.CloudUrl(SCHEME, BUCKET_NAME, OBJECT_NAME), metadata={ 'ACL': errors.S3ApiError('An error occurred () when calling the' ' GetObjectAcl operation: ') }) self.assertEqual(object_resource, expected_resource)
def get_object_metadata(self, bucket_name, object_name, request_config=None, generation=None, fields_scope=None): """See super class.""" del request_config request = {'Bucket': bucket_name, 'Key': object_name} # The VersionId keyword argument to head_object is not nullable if it is # present, so only include it in the function call if it has a value. if generation is not None: request['VersionId'] = generation try: object_dict = self.client.head_object(**request) except botocore.exceptions.ClientError as e: _raise_if_not_found_error( e, storage_url.CloudUrl(storage_url.ProviderPrefix.S3, bucket_name, object_name, generation).url_string) raise e # User requested ACL's with FieldsScope.FULL. if fields_scope is cloud_api.FieldsScope.FULL: try: acl_response = self.client.get_object_acl(**request) acl_response.pop('ResponseMetadata', None) object_dict['ACL'] = acl_response except botocore.exceptions.ClientError as error: object_dict['ACL'] = errors.S3ApiError(error) return s3_metadata_util.get_object_resource_from_s3_response( object_dict, bucket_name, object_name)
def DownloadObject(self, bucket_name, object_name, download_stream, compressed_encoding=False, decryption_wrapper=None, digesters=None, download_strategy=cloud_api.DownloadStrategy.ONE_SHOT, generation=None, object_size=None, progress_callback=None, serialization_data=None, start_byte=0, end_byte=None): """See super class.""" kwargs = {'Bucket': bucket_name, 'Key': object_name} if generation: kwargs['VersionId'] = generation try: response = self.client.get_object(**kwargs) download_stream.write(response['Body'].read()) return response.get('ContentEncoding', None) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def test_dumps_object_metadata_with_errors_in_correct_format(self): time = datetime.datetime(1111, 1, 1) cors_error = errors.S3ApiError( 'An error occurred () when calling the GetBucketCors operation: ') resource = s3_resource_reference.S3ObjectResource( storage_url.CloudUrl( storage_url.ProviderPrefix.S3, bucket_name='bucket'), metadata={ 'Error': cors_error, 'List': [{'TimeInListInDict': time}], 'Nested': {'ZeroInt': 0, 'DoubleNested': {'NestedTime': time}}, 'String': 'abc', 'Time': time}) expected_dump = textwrap.dedent("""\ { "url": "s3://bucket", "type": "cloud_object", "metadata": { "Error": "An error occurred () when calling the GetBucketCors operation: ", "List": [ { "TimeInListInDict": "1111-01-01T00:00:00" } ], "Nested": { "DoubleNested": { "NestedTime": "1111-01-01T00:00:00" }, "ZeroInt": 0 }, "String": "abc", "Time": "1111-01-01T00:00:00" } }""") self.assertEqual(resource.get_metadata_dump(), expected_dump)
def get_object_metadata(self, bucket_name, object_name, generation=None, fields_scope=None): """See super class.""" request = {'Bucket': bucket_name, 'Key': object_name} # The VersionId keyword argument to head_object is not nullable if it is # present, so only include it in the function call if it has a value. if generation is not None: request['VersionId'] = generation try: object_dict = self.client.head_object(**request) except botocore.exceptions.ClientError as e: if e.response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404: # Allows custom error handling. raise errors.NotFoundError('Object not found: {}'.format( storage_url.CloudUrl(storage_url.ProviderPrefix.S3, bucket_name, object_name, generation).url_string)) raise e # User requested ACL's with FieldsScope.FULL. if fields_scope is cloud_api.FieldsScope.FULL: try: acl_response = self.client.get_object_acl(**request) acl_response.pop('ResponseMetadata', None) object_dict['ACL'] = acl_response except botocore.exceptions.ClientError as error: object_dict['ACL'] = errors.S3ApiError(error) return _get_object_resource_from_s3_response(object_dict, bucket_name, object_name)
def patch_bucket(self, bucket_resource, request_config=None, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" del fields_scope, request_config # Unused. if ('FullACLConfiguration' in bucket_resource.metadata or 'ACL' in bucket_resource.metadata): try: if 'FullACLConfiguration' in bucket_resource.metadata: # Can contain canned ACL and other settings. # Takes priority over 'ACL' metadata key. kwargs = bucket_resource.metadata['FullACLConfiguration'] else: # Data returned by get_bucket_acl. kwargs = { 'AccessControlPolicy': bucket_resource.metadata['ACL'] } kwargs['Bucket'] = bucket_resource.name self.client.put_bucket_acl(**kwargs) except botocore.exceptions.ClientError as error: _raise_if_not_found_error(error, bucket_resource.name) # Don't return any ACL information in case the failure affected both # metadata keys. bucket_resource.metadata.pop('FullACLConfiguration', None) bucket_resource.metadata.pop('ACL', None) log.error(errors.S3ApiError(error)) patchable_metadata = { # Key -> (client function, function kwargs). 'CORSRules': ( self.client.put_bucket_cors, {'CORSConfiguration': { 'CORSRules': bucket_resource.metadata.get('CORSRules'), }}), 'ServerSideEncryptionConfiguration': ( self.client.put_bucket_encryption, {'ServerSideEncryptionConfiguration': bucket_resource.metadata.get( 'ServerSideEncryptionConfiguration'), }), 'LifecycleConfiguration': ( self.client.put_bucket_lifecycle_configuration, {'LifecycleConfiguration': bucket_resource.metadata.get( 'LifecycleConfiguration'), }), 'LoggingEnabled': ( self.client.put_bucket_logging, {'BucketLoggingStatus': { 'LoggingEnabled': bucket_resource.metadata.get( 'LoggingEnabled'), }}), 'Payer': ( self.client.put_bucket_request_payment, {'RequestPaymentConfiguration': { 'Payer': bucket_resource.metadata.get('Payer'), }}), 'Versioning': ( self.client.put_bucket_versioning, {'VersioningConfiguration': bucket_resource.metadata.get( 'Versioning'), }), 'Website': ( self.client.put_bucket_website, {'WebsiteConfiguration': bucket_resource.metadata.get('Website')}), } for metadata_key, (patch_function, patch_kwargs) in patchable_metadata.items(): if metadata_key not in bucket_resource.metadata: continue patch_kwargs['Bucket'] = bucket_resource.name try: patch_function(**patch_kwargs) except botocore.exceptions.ClientError as error: _raise_if_not_found_error(error, bucket_resource.name) log.error(errors.S3ApiError(error)) del bucket_resource.metadata[metadata_key] return bucket_resource