def getEnvironment(self, profile=None):
"""Return environment variables that should be set for the profile."""
eventHooks = HierarchicalEmitter()
session = Session(event_hooks=eventHooks)
if profile:
session.set_config_variable('profile', profile)
awscli_initialize(eventHooks)
session.emit('session-initialized', session=session)
creds = session.get_credentials()
env = {}
def set(key, value):
if value:
env[key] = value
set('AWS_ACCESS_KEY_ID', creds.access_key)
set('AWS_SECRET_ACCESS_KEY', creds.secret_key)
# AWS_SESSION_TOKEN is the ostensibly the standard:
# http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs
# http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment
set('AWS_SESSION_TOKEN', creds.token)
# ...but boto expects AWS_SECURITY_TOKEN. Set both for compatibility.
# https://github.com/boto/boto/blob/b016c07d834df5bce75141c4b9d2f3d30352e1b8/boto/connection.py#L438
set('AWS_SECURITY_TOKEN', creds.token)
set('AWS_DEFAULT_REGION', session.get_config_variable('region'))
return env
def factory(context, request):
"""
:type context: object
:type request: pyramid.request.Request
:rtype: boto3.Session
"""
session = None
if cache is not None:
session = getattr(cache, session_name, None)
if session is None:
core_session = None
if core_settings:
core_session = CoreSession()
for k, v in core_settings.items():
core_session.set_config_variable(k, v)
session = Session(botocore_session=core_session, **settings)
if cache is not None:
setattr(cache, session_name, session)
return session
def test_assume_role_uses_correct_region(self):
config = ('[profile A]\n'
'role_arn = arn:aws:iam::123456789:role/RoleA\n'
'source_profile = B\n\n'
'[profile B]\n'
'aws_access_key_id = abc123\n'
'aws_secret_access_key = def456\n')
self.write_config(config)
session = Session(profile='A')
# Verify that when we configure the session with a specific region
# that we use that region when creating the sts client.
session.set_config_variable('region', 'cn-north-1')
create_client, expected_creds = self.create_stubbed_sts_client(session)
session.create_client = create_client
resolver = create_credential_resolver(session)
provider = resolver.get_provider('assume-role')
creds = provider.load()
self.assert_creds_equal(creds, expected_creds)
self.assertEqual(self.actual_client_region, 'cn-north-1')
def test_assume_role_uses_correct_region(self):
config = (
'[profile A]\n'
'role_arn = arn:aws:iam::123456789:role/RoleA\n'
'source_profile = B\n\n'
'[profile B]\n'
'aws_access_key_id = abc123\n'
'aws_secret_access_key = def456\n'
)
self.write_config(config)
session = Session(profile='A')
# Verify that when we configure the session with a specific region
# that we use that region when creating the sts client.
session.set_config_variable('region', 'cn-north-1')
create_client, expected_creds = self.create_stubbed_sts_client(session)
session.create_client = create_client
resolver = create_credential_resolver(session)
provider = resolver.get_provider('assume-role')
creds = provider.load()
self.assert_creds_equal(creds, expected_creds)
self.assertEqual(self.actual_client_region, 'cn-north-1')
def getEnvironment(self, profile=None):
"""Return environment variables that should be set for the profile."""
eventHooks = HierarchicalEmitter()
session = Session(event_hooks=eventHooks)
if profile:
session.set_config_variable('profile', profile)
eventHooks.register('session-initialized',
inject_assume_role_provider_cache,
unique_id='inject_assume_role_cred_provider_cache')
session.emit('session-initialized', session=session)
creds = session.get_credentials()
env = {}
def set(key, value):
if value:
env[key] = value
set('AWS_ACCESS_KEY_ID', creds.access_key)
set('AWS_SECRET_ACCESS_KEY', creds.secret_key)
# AWS_SESSION_TOKEN is the ostensibly the standard:
# http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs
# http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment
set('AWS_SESSION_TOKEN', creds.token)
# ...but boto expects AWS_SECURITY_TOKEN. Set both for compatibility.
# https://github.com/boto/boto/blob/b016c07d834df5bce75141c4b9d2f3d30352e1b8/boto/connection.py#L438
set('AWS_SECURITY_TOKEN', creds.token)
set('AWS_DEFAULT_REGION', session.get_config_variable('region'))
return env
def _get_session():
session = Session()
session.set_credentials('foo', 'bar')
session.set_config_variable('region', 'us-west-2')
session.config_filename = 'no-exist-foo'
return session
class TestBotocoreCRTRequestSerializer(unittest.TestCase):
def setUp(self):
self.region = 'us-west-2'
self.session = Session()
self.session.set_config_variable('region', self.region)
self.request_serializer = s3transfer.crt.BotocoreCRTRequestSerializer(
self.session)
self.bucket = "test_bucket"
self.key = "test_key"
self.files = FileCreator()
self.filename = self.files.create_file('myfile', 'my content')
self.expected_path = "/" + self.bucket + "/" + self.key
self.expected_host = "s3.%s.amazonaws.com" % (self.region)
def tearDown(self):
self.files.remove_all()
def test_upload_request(self):
callargs = CallArgs(bucket=self.bucket,
key=self.key,
fileobj=self.filename,
extra_args={},
subscribers=[])
coordinator = s3transfer.crt.CRTTransferCoordinator()
future = s3transfer.crt.CRTTransferFuture(
s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator)
crt_request = self.request_serializer.serialize_http_request(
"put_object", future)
self.assertEqual("PUT", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self.assertIsNone(crt_request.headers.get("Authorization"))
def test_download_request(self):
callargs = CallArgs(bucket=self.bucket,
key=self.key,
fileobj=self.filename,
extra_args={},
subscribers=[])
coordinator = s3transfer.crt.CRTTransferCoordinator()
future = s3transfer.crt.CRTTransferFuture(
s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator)
crt_request = self.request_serializer.serialize_http_request(
"get_object", future)
self.assertEqual("GET", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self.assertIsNone(crt_request.headers.get("Authorization"))
def test_delete_request(self):
callargs = CallArgs(bucket=self.bucket,
key=self.key,
extra_args={},
subscribers=[])
coordinator = s3transfer.crt.CRTTransferCoordinator()
future = s3transfer.crt.CRTTransferFuture(
s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator)
crt_request = self.request_serializer.serialize_http_request(
"delete_object", future)
self.assertEqual("DELETE", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self.assertIsNone(crt_request.headers.get("Authorization"))
def _get_session():
session = Session()
session.set_credentials('foo', 'bar')
session.set_config_variable('region', 'us-west-2')
session.config_filename = 'no-exist-foo'
return session
class TestCRTTransferManager(unittest.TestCase):
def setUp(self):
self.region = 'us-west-2'
self.bucket = "test_bucket"
self.key = "test_key"
self.files = FileCreator()
self.filename = self.files.create_file('myfile', 'my content')
self.expected_path = "/" + self.bucket + "/" + self.key
self.expected_host = "s3.%s.amazonaws.com" % (self.region)
self.s3_request = mock.Mock(awscrt.s3.S3Request)
self.s3_crt_client = mock.Mock(awscrt.s3.S3Client)
self.s3_crt_client.make_request.return_value = self.s3_request
self.session = Session()
self.session.set_config_variable('region', self.region)
self.request_serializer = s3transfer.crt.BotocoreCRTRequestSerializer(
self.session)
self.transfer_manager = s3transfer.crt.CRTTransferManager(
crt_s3_client=self.s3_crt_client,
crt_request_serializer=self.request_serializer)
self.record_subscriber = RecordingSubscriber()
def tearDown(self):
self.files.remove_all()
def _assert_subscribers_called(self, expected_future=None):
self.assertTrue(self.record_subscriber.on_queued_called)
self.assertTrue(self.record_subscriber.on_done_called)
if expected_future:
self.assertIs(self.record_subscriber.on_queued_future,
expected_future)
self.assertIs(self.record_subscriber.on_done_future,
expected_future)
def _invoke_done_callbacks(self, **kwargs):
callargs = self.s3_crt_client.make_request.call_args
callargs_kwargs = callargs[1]
on_done = callargs_kwargs["on_done"]
on_done(error=None)
def _simulate_file_download(self, recv_filepath):
self.files.create_file(recv_filepath, "fake resopnse")
def _simulate_make_request_side_effect(self, **kwargs):
if kwargs.get('recv_filepath'):
self._simulate_file_download(kwargs['recv_filepath'])
self._invoke_done_callbacks()
return mock.DEFAULT
def test_upload(self):
self.s3_crt_client.make_request.side_effect = self._simulate_make_request_side_effect
future = self.transfer_manager.upload(self.filename, self.bucket,
self.key, {},
[self.record_subscriber])
future.result()
callargs = self.s3_crt_client.make_request.call_args
callargs_kwargs = callargs[1]
self.assertEqual(callargs_kwargs["send_filepath"], self.filename)
self.assertIsNone(callargs_kwargs["recv_filepath"])
self.assertEqual(callargs_kwargs["type"],
awscrt.s3.S3RequestType.PUT_OBJECT)
crt_request = callargs_kwargs["request"]
self.assertEqual("PUT", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self._assert_subscribers_called(future)
def test_download(self):
self.s3_crt_client.make_request.side_effect = self._simulate_make_request_side_effect
future = self.transfer_manager.download(self.bucket, self.key,
self.filename, {},
[self.record_subscriber])
future.result()
callargs = self.s3_crt_client.make_request.call_args
callargs_kwargs = callargs[1]
# the recv_filepath will be set to a temporary file path with some
# random suffix
self.assertTrue(
re.match(self.filename + ".*", callargs_kwargs["recv_filepath"]))
self.assertIsNone(callargs_kwargs["send_filepath"])
self.assertEqual(callargs_kwargs["type"],
awscrt.s3.S3RequestType.GET_OBJECT)
crt_request = callargs_kwargs["request"]
self.assertEqual("GET", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self._assert_subscribers_called(future)
with open(self.filename, 'rb') as f:
# Check the fake response overwrites the file because of download
self.assertEqual(f.read(), b'fake resopnse')
def test_delete(self):
self.s3_crt_client.make_request.side_effect = self._simulate_make_request_side_effect
future = self.transfer_manager.delete(self.bucket, self.key, {},
[self.record_subscriber])
future.result()
callargs = self.s3_crt_client.make_request.call_args
callargs_kwargs = callargs[1]
self.assertIsNone(callargs_kwargs["send_filepath"])
self.assertIsNone(callargs_kwargs["recv_filepath"])
self.assertEqual(callargs_kwargs["type"],
awscrt.s3.S3RequestType.DEFAULT)
crt_request = callargs_kwargs["request"]
self.assertEqual("DELETE", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self._assert_subscribers_called(future)
def test_blocks_when_max_requests_processes_reached(self):
futures = []
callargs = (self.bucket, self.key, self.filename, {}, [])
max_request_processes = 128 # the hard coded max processes
all_concurrent = max_request_processes + 1
threads = []
for i in range(0, all_concurrent):
thread = submitThread(self.transfer_manager, futures, callargs)
thread.start()
threads.append(thread)
self.assertLessEqual(self.s3_crt_client.make_request.call_count,
max_request_processes)
# Release lock
callargs = self.s3_crt_client.make_request.call_args
callargs_kwargs = callargs[1]
on_done = callargs_kwargs["on_done"]
on_done(error=None)
for thread in threads:
thread.join()
self.assertEqual(self.s3_crt_client.make_request.call_count,
all_concurrent)
def _cancel_function(self):
self.cancel_called = True
self.s3_request.finished_future.set_exception(
awscrt.exceptions.from_code(0))
self._invoke_done_callbacks()
def test_cancel(self):
self.s3_request.finished_future = Future()
self.cancel_called = False
self.s3_request.cancel = self._cancel_function
try:
with self.transfer_manager:
future = self.transfer_manager.upload(self.filename,
self.bucket, self.key,
{}, [])
raise KeyboardInterrupt()
except KeyboardInterrupt:
pass
with self.assertRaises(awscrt.exceptions.AwsCrtError):
future.result()
self.assertTrue(self.cancel_called)
def test_serializer_error_handling(self):
class SerializationException(Exception):
pass
class ExceptionRaisingSerializer(
s3transfer.crt.BaseCRTRequestSerializer):
def serialize_http_request(self, transfer_type, future):
raise SerializationException()
not_impl_serializer = ExceptionRaisingSerializer()
transfer_manager = s3transfer.crt.CRTTransferManager(
crt_s3_client=self.s3_crt_client,
crt_request_serializer=not_impl_serializer)
future = transfer_manager.upload(self.filename, self.bucket, self.key,
{}, [])
with self.assertRaises(SerializationException):
future.result()
def test_crt_s3_client_error_handling(self):
self.s3_crt_client.make_request.side_effect = awscrt.exceptions.from_code(
0)
future = self.transfer_manager.upload(self.filename, self.bucket,
self.key, {}, [])
with self.assertRaises(awscrt.exceptions.AwsCrtError):
future.result()