def make_s3_resource(self, region_name=None):
"""Create a :py:mod:`boto3` S3 resource, with its client
wrapped in a :py:class:`mrjob.retry.RetryWrapper`
:param region: region to use to choose S3 endpoint
It's best to use :py:meth:`get_bucket` because it chooses the
appropriate S3 endpoint automatically. If you are trying to get
bucket metadata, use :py:meth:`make_s3_client`.
.. versionadded:: 0.6.0
"""
# give a non-cryptic error message if boto3 isn't installed
if boto3 is None:
raise ImportError('You must install boto3 to connect to S3')
kwargs = self._client_kwargs(region_name)
log.debug(
'creating S3 resource (%s)' %
(kwargs['endpoint_url'] or kwargs['region_name'] or 'default'))
s3_resource = boto3.resource('s3', **kwargs)
s3_resource.meta.client = _wrap_aws_client(s3_resource.meta.client)
return s3_resource
def test_min_backoff(self):
self.wrapped_client = _wrap_aws_client(self.client, min_backoff=1000)
self.add_transient_error(socket.error(110, 'Connection timed out'))
self.assertEqual(self.wrapped_client.list_buckets(), dict(Buckets=[]))
self.sleep.assert_called_with(1000)
self.assertTrue(self.log.info.called)
def make_s3_client(self, region_name=None):
"""Create a :py:mod:`boto3` S3 client,
wrapped in a :py:class:`mrjob.retry.RetryWrapper`
:param region: region to use to choose S3 endpoint.
"""
# give a non-cryptic error message if boto3 isn't installed
if boto3 is None:
raise ImportError('You must install boto3 to connect to S3')
kwargs = self._client_kwargs(region_name or self._s3_region)
return _wrap_aws_client(boto3.client('s3', **kwargs))
def setUp(self):
super(WrapAWSClientTestCase, self).setUp()
# don't actually wait between retries
self.sleep = self.start(patch('time.sleep'))
self.log = self.start(patch('mrjob.retry.log'))
self.list_buckets = self.start(
patch('tests.mock_boto3.s3.MockS3Client.list_buckets',
side_effect=[dict(Buckets=[])]))
self.client = self.client('s3')
self.wrapped_client = _wrap_aws_client(self.client)
def make_s3_client(self, region_name=None):
"""Create a :py:mod:`boto3` S3 client,
wrapped in a :py:class:`mrjob.retry.RetryWrapper`
:param region: region to use to choose S3 endpoint.
.. versionadded:: 0.6.0
"""
# give a non-cryptic error message if boto3 isn't installed
if boto3 is None:
raise ImportError('You must install boto3 to connect to S3')
kwargs = self._client_kwargs(region_name or self._s3_region)
log.debug(
'creating S3 client (%s)' %
(kwargs['endpoint_url'] or kwargs['region_name'] or 'default'))
return _wrap_aws_client(boto3.client('s3', **kwargs))