def _upload_object(self, file_obj, object_key):
"""Upload objects to S3 in streaming fashion.
:param file file_obj: A file like object to upload. At a minimum, it
must implement the read method, and must return bytes.
:param str object_key: The destination key where to upload the object.
:raise S3DestinationError: if failed to upload object.
"""
remote_name = "s3://{bucket}/{name}".format(bucket=self.bucket,
name=object_key)
LOG.debug("Generating S3 transfer config")
s3_transfer_config = self.get_transfer_config()
LOG.debug("Starting to stream to %s", remote_name)
try:
self.s3_client.upload_fileobj(file_obj,
self.bucket,
object_key,
Config=s3_transfer_config)
LOG.debug("Successfully streamed to %s", remote_name)
except ClientError as err:
raise S3DestinationError(err)
return self._validate_upload(object_key)
def validate_client_response(response):
"""Validates the response returned by the client. Raises an exception
if the response code is not 200 or 204
:param response: The response that needs to be validated.
:type response: dict
:raise S3DestinationError: if response from S3 is invalid.
"""
try:
http_status_code = response['ResponseMetadata']['HTTPStatusCode']
except KeyError as err:
raise S3DestinationError(
'S3 client returned invalid response: %s' % err)
if http_status_code not in [200, 204]:
raise S3DestinationError('S3 client returned error code: %s' %
http_status_code)
def list_files(self,
prefix,
recursive=False,
pattern=None,
files_only=False):
"""
List files in the destination that have common prefix.
:param prefix: Common prefix. May include the bucket name.
(e.g. ``s3://my_bucket/foo/``) or simply a prefix in the bucket
(e.g. ``foo/``).
:type prefix: str
:param recursive: Does nothing for this class.
:return: sorted list of file names.
:param pattern: files must match with this regexp if specified.
:type pattern: str
:param files_only: Does nothing for this class.
:return: Full S3 url in form ``s3://bucket/path/to/file``.
:rtype: list(str)
:raise S3DestinationError: if failed to list files.
"""
s3client = boto3.resource('s3')
bucket = s3client.Bucket(self.bucket)
LOG.debug('Listing bucket %s', self.bucket)
LOG.debug('prefix = %s', prefix)
norm_prefix = prefix.replace('s3://%s' % bucket.name, '')
norm_prefix = norm_prefix.lstrip('/')
LOG.debug('normal prefix = %s', norm_prefix)
# Try to list the bucket several times
# because of intermittent error NoSuchBucket:
# https://travis-ci.org/twindb/backup/jobs/204053690
expire = time.time() + S3_READ_TIMEOUT
retry_interval = 2
while time.time() < expire:
try:
files = []
all_objects = bucket.objects.filter(Prefix=norm_prefix)
for file_object in all_objects:
if pattern:
if re.search(pattern, file_object.key):
files.append('s3://{bucket}/{key}'.format(
bucket=self.bucket, key=file_object.key))
else:
files.append('s3://{bucket}/{key}'.format(
bucket=self.bucket, key=file_object.key))
return sorted(files)
except ClientError as err:
LOG.warning('%s. Will retry in %d seconds.', err,
retry_interval)
time.sleep(retry_interval)
retry_interval *= 2
raise S3DestinationError('Failed to list files.')
def find_files(self, prefix, run_type):
"""
Find files with common prefix and given run type.
:param prefix: Common prefix.
:type prefix: str
:param run_type: daily, hourly, etc
:type run_type: str
:return: list of file names
:rtype: list(str)
:raise S3DestinationError: if failed to find files.
"""
s3client = boto3.resource('s3')
bucket = s3client.Bucket(self.bucket)
LOG.debug('Listing %s in bucket %s', prefix, bucket)
# Try to list the bucket several times
# because of intermittent error NoSuchBucket:
# https://travis-ci.org/twindb/backup/jobs/204066704
retry_timeout = time.time() + S3_READ_TIMEOUT
retry_interval = 2
while time.time() < retry_timeout:
try:
files = []
all_objects = bucket.objects.filter(Prefix='')
for file_object in all_objects:
if "/" + run_type + "/" in file_object.key:
files.append("s3://%s/%s" %
(self.bucket, file_object.key))
return sorted(files)
except ClientError as err:
LOG.warning('%s. Will retry in %d seconds.', err,
retry_interval)
time.sleep(retry_interval)
retry_interval *= 2
except Exception as err:
LOG.error('Failed to list objects in bucket %s: %s',
self.bucket, err)
raise
raise S3DestinationError('Failed to find files.')
def list_files(self, prefix, recursive=False):
"""
List files in the destination that have common prefix.
:param prefix: Common prefix.
:type prefix: str
:param recursive: Does nothing for this class.
:return: sorted list of file names
:rtype: list(str)
:raise S3DestinationError: if failed to list files.
"""
s3client = boto3.resource('s3')
bucket = s3client.Bucket(self.bucket)
LOG.debug('Listing %s in bucket %s', prefix, self.bucket)
norm_prefix = prefix.replace('s3://%s/' % bucket.name, '')
LOG.debug('norm_prefix = %s', norm_prefix)
# Try to list the bucket several times
# because of intermittent error NoSuchBucket:
# https://travis-ci.org/twindb/backup/jobs/204053690
retry_timeout = time.time() + S3_READ_TIMEOUT
retry_interval = 2
while time.time() < retry_timeout:
try:
files = []
all_objects = bucket.objects.filter(Prefix=norm_prefix)
for file_object in all_objects:
files.append(file_object.key)
return sorted(files)
except ClientError as err:
LOG.warning('%s. Will retry in %d seconds.', err,
retry_interval)
time.sleep(retry_interval)
retry_interval *= 2
raise S3DestinationError('Failed to list files.')