def getEnvironment(self, profile=None): """Return environment variables that should be set for the profile.""" eventHooks = HierarchicalEmitter() session = Session(event_hooks=eventHooks) if profile: session.set_config_variable('profile', profile) awscli_initialize(eventHooks) session.emit('session-initialized', session=session) creds = session.get_credentials() env = {} def set(key, value): if value: env[key] = value set('AWS_ACCESS_KEY_ID', creds.access_key) set('AWS_SECRET_ACCESS_KEY', creds.secret_key) # AWS_SESSION_TOKEN is the ostensibly the standard: # http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs # http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment set('AWS_SESSION_TOKEN', creds.token) # ...but boto expects AWS_SECURITY_TOKEN. Set both for compatibility. # https://github.com/boto/boto/blob/b016c07d834df5bce75141c4b9d2f3d30352e1b8/boto/connection.py#L438 set('AWS_SECURITY_TOKEN', creds.token) set('AWS_DEFAULT_REGION', session.get_config_variable('region')) return env
def factory(context, request): """ :type context: object :type request: pyramid.request.Request :rtype: boto3.Session """ session = None if cache is not None: session = getattr(cache, session_name, None) if session is None: core_session = None if core_settings: core_session = CoreSession() for k, v in core_settings.items(): core_session.set_config_variable(k, v) session = Session(botocore_session=core_session, **settings) if cache is not None: setattr(cache, session_name, session) return session
def test_assume_role_uses_correct_region(self): config = ('[profile A]\n' 'role_arn = arn:aws:iam::123456789:role/RoleA\n' 'source_profile = B\n\n' '[profile B]\n' 'aws_access_key_id = abc123\n' 'aws_secret_access_key = def456\n') self.write_config(config) session = Session(profile='A') # Verify that when we configure the session with a specific region # that we use that region when creating the sts client. session.set_config_variable('region', 'cn-north-1') create_client, expected_creds = self.create_stubbed_sts_client(session) session.create_client = create_client resolver = create_credential_resolver(session) provider = resolver.get_provider('assume-role') creds = provider.load() self.assert_creds_equal(creds, expected_creds) self.assertEqual(self.actual_client_region, 'cn-north-1')
def test_assume_role_uses_correct_region(self): config = ( '[profile A]\n' 'role_arn = arn:aws:iam::123456789:role/RoleA\n' 'source_profile = B\n\n' '[profile B]\n' 'aws_access_key_id = abc123\n' 'aws_secret_access_key = def456\n' ) self.write_config(config) session = Session(profile='A') # Verify that when we configure the session with a specific region # that we use that region when creating the sts client. session.set_config_variable('region', 'cn-north-1') create_client, expected_creds = self.create_stubbed_sts_client(session) session.create_client = create_client resolver = create_credential_resolver(session) provider = resolver.get_provider('assume-role') creds = provider.load() self.assert_creds_equal(creds, expected_creds) self.assertEqual(self.actual_client_region, 'cn-north-1')
def getEnvironment(self, profile=None): """Return environment variables that should be set for the profile.""" eventHooks = HierarchicalEmitter() session = Session(event_hooks=eventHooks) if profile: session.set_config_variable('profile', profile) eventHooks.register('session-initialized', inject_assume_role_provider_cache, unique_id='inject_assume_role_cred_provider_cache') session.emit('session-initialized', session=session) creds = session.get_credentials() env = {} def set(key, value): if value: env[key] = value set('AWS_ACCESS_KEY_ID', creds.access_key) set('AWS_SECRET_ACCESS_KEY', creds.secret_key) # AWS_SESSION_TOKEN is the ostensibly the standard: # http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs # http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment set('AWS_SESSION_TOKEN', creds.token) # ...but boto expects AWS_SECURITY_TOKEN. Set both for compatibility. # https://github.com/boto/boto/blob/b016c07d834df5bce75141c4b9d2f3d30352e1b8/boto/connection.py#L438 set('AWS_SECURITY_TOKEN', creds.token) set('AWS_DEFAULT_REGION', session.get_config_variable('region')) return env
def _get_session(): session = Session() session.set_credentials('foo', 'bar') session.set_config_variable('region', 'us-west-2') session.config_filename = 'no-exist-foo' return session
class TestBotocoreCRTRequestSerializer(unittest.TestCase): def setUp(self): self.region = 'us-west-2' self.session = Session() self.session.set_config_variable('region', self.region) self.request_serializer = s3transfer.crt.BotocoreCRTRequestSerializer( self.session) self.bucket = "test_bucket" self.key = "test_key" self.files = FileCreator() self.filename = self.files.create_file('myfile', 'my content') self.expected_path = "/" + self.bucket + "/" + self.key self.expected_host = "s3.%s.amazonaws.com" % (self.region) def tearDown(self): self.files.remove_all() def test_upload_request(self): callargs = CallArgs(bucket=self.bucket, key=self.key, fileobj=self.filename, extra_args={}, subscribers=[]) coordinator = s3transfer.crt.CRTTransferCoordinator() future = s3transfer.crt.CRTTransferFuture( s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator) crt_request = self.request_serializer.serialize_http_request( "put_object", future) self.assertEqual("PUT", crt_request.method) self.assertEqual(self.expected_path, crt_request.path) self.assertEqual(self.expected_host, crt_request.headers.get("host")) self.assertIsNone(crt_request.headers.get("Authorization")) def test_download_request(self): callargs = CallArgs(bucket=self.bucket, key=self.key, fileobj=self.filename, extra_args={}, subscribers=[]) coordinator = s3transfer.crt.CRTTransferCoordinator() future = s3transfer.crt.CRTTransferFuture( s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator) crt_request = self.request_serializer.serialize_http_request( "get_object", future) self.assertEqual("GET", crt_request.method) self.assertEqual(self.expected_path, crt_request.path) self.assertEqual(self.expected_host, crt_request.headers.get("host")) self.assertIsNone(crt_request.headers.get("Authorization")) def test_delete_request(self): callargs = CallArgs(bucket=self.bucket, key=self.key, extra_args={}, subscribers=[]) coordinator = s3transfer.crt.CRTTransferCoordinator() future = s3transfer.crt.CRTTransferFuture( s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator) crt_request = self.request_serializer.serialize_http_request( "delete_object", future) self.assertEqual("DELETE", crt_request.method) self.assertEqual(self.expected_path, crt_request.path) self.assertEqual(self.expected_host, crt_request.headers.get("host")) self.assertIsNone(crt_request.headers.get("Authorization"))
class TestCRTTransferManager(unittest.TestCase): def setUp(self): self.region = 'us-west-2' self.bucket = "test_bucket" self.key = "test_key" self.files = FileCreator() self.filename = self.files.create_file('myfile', 'my content') self.expected_path = "/" + self.bucket + "/" + self.key self.expected_host = "s3.%s.amazonaws.com" % (self.region) self.s3_request = mock.Mock(awscrt.s3.S3Request) self.s3_crt_client = mock.Mock(awscrt.s3.S3Client) self.s3_crt_client.make_request.return_value = self.s3_request self.session = Session() self.session.set_config_variable('region', self.region) self.request_serializer = s3transfer.crt.BotocoreCRTRequestSerializer( self.session) self.transfer_manager = s3transfer.crt.CRTTransferManager( crt_s3_client=self.s3_crt_client, crt_request_serializer=self.request_serializer) self.record_subscriber = RecordingSubscriber() def tearDown(self): self.files.remove_all() def _assert_subscribers_called(self, expected_future=None): self.assertTrue(self.record_subscriber.on_queued_called) self.assertTrue(self.record_subscriber.on_done_called) if expected_future: self.assertIs(self.record_subscriber.on_queued_future, expected_future) self.assertIs(self.record_subscriber.on_done_future, expected_future) def _invoke_done_callbacks(self, **kwargs): callargs = self.s3_crt_client.make_request.call_args callargs_kwargs = callargs[1] on_done = callargs_kwargs["on_done"] on_done(error=None) def _simulate_file_download(self, recv_filepath): self.files.create_file(recv_filepath, "fake resopnse") def _simulate_make_request_side_effect(self, **kwargs): if kwargs.get('recv_filepath'): self._simulate_file_download(kwargs['recv_filepath']) self._invoke_done_callbacks() return mock.DEFAULT def test_upload(self): self.s3_crt_client.make_request.side_effect = self._simulate_make_request_side_effect future = self.transfer_manager.upload(self.filename, self.bucket, self.key, {}, [self.record_subscriber]) future.result() callargs = self.s3_crt_client.make_request.call_args callargs_kwargs = callargs[1] self.assertEqual(callargs_kwargs["send_filepath"], self.filename) self.assertIsNone(callargs_kwargs["recv_filepath"]) self.assertEqual(callargs_kwargs["type"], awscrt.s3.S3RequestType.PUT_OBJECT) crt_request = callargs_kwargs["request"] self.assertEqual("PUT", crt_request.method) self.assertEqual(self.expected_path, crt_request.path) self.assertEqual(self.expected_host, crt_request.headers.get("host")) self._assert_subscribers_called(future) def test_download(self): self.s3_crt_client.make_request.side_effect = self._simulate_make_request_side_effect future = self.transfer_manager.download(self.bucket, self.key, self.filename, {}, [self.record_subscriber]) future.result() callargs = self.s3_crt_client.make_request.call_args callargs_kwargs = callargs[1] # the recv_filepath will be set to a temporary file path with some # random suffix self.assertTrue( re.match(self.filename + ".*", callargs_kwargs["recv_filepath"])) self.assertIsNone(callargs_kwargs["send_filepath"]) self.assertEqual(callargs_kwargs["type"], awscrt.s3.S3RequestType.GET_OBJECT) crt_request = callargs_kwargs["request"] self.assertEqual("GET", crt_request.method) self.assertEqual(self.expected_path, crt_request.path) self.assertEqual(self.expected_host, crt_request.headers.get("host")) self._assert_subscribers_called(future) with open(self.filename, 'rb') as f: # Check the fake response overwrites the file because of download self.assertEqual(f.read(), b'fake resopnse') def test_delete(self): self.s3_crt_client.make_request.side_effect = self._simulate_make_request_side_effect future = self.transfer_manager.delete(self.bucket, self.key, {}, [self.record_subscriber]) future.result() callargs = self.s3_crt_client.make_request.call_args callargs_kwargs = callargs[1] self.assertIsNone(callargs_kwargs["send_filepath"]) self.assertIsNone(callargs_kwargs["recv_filepath"]) self.assertEqual(callargs_kwargs["type"], awscrt.s3.S3RequestType.DEFAULT) crt_request = callargs_kwargs["request"] self.assertEqual("DELETE", crt_request.method) self.assertEqual(self.expected_path, crt_request.path) self.assertEqual(self.expected_host, crt_request.headers.get("host")) self._assert_subscribers_called(future) def test_blocks_when_max_requests_processes_reached(self): futures = [] callargs = (self.bucket, self.key, self.filename, {}, []) max_request_processes = 128 # the hard coded max processes all_concurrent = max_request_processes + 1 threads = [] for i in range(0, all_concurrent): thread = submitThread(self.transfer_manager, futures, callargs) thread.start() threads.append(thread) self.assertLessEqual(self.s3_crt_client.make_request.call_count, max_request_processes) # Release lock callargs = self.s3_crt_client.make_request.call_args callargs_kwargs = callargs[1] on_done = callargs_kwargs["on_done"] on_done(error=None) for thread in threads: thread.join() self.assertEqual(self.s3_crt_client.make_request.call_count, all_concurrent) def _cancel_function(self): self.cancel_called = True self.s3_request.finished_future.set_exception( awscrt.exceptions.from_code(0)) self._invoke_done_callbacks() def test_cancel(self): self.s3_request.finished_future = Future() self.cancel_called = False self.s3_request.cancel = self._cancel_function try: with self.transfer_manager: future = self.transfer_manager.upload(self.filename, self.bucket, self.key, {}, []) raise KeyboardInterrupt() except KeyboardInterrupt: pass with self.assertRaises(awscrt.exceptions.AwsCrtError): future.result() self.assertTrue(self.cancel_called) def test_serializer_error_handling(self): class SerializationException(Exception): pass class ExceptionRaisingSerializer( s3transfer.crt.BaseCRTRequestSerializer): def serialize_http_request(self, transfer_type, future): raise SerializationException() not_impl_serializer = ExceptionRaisingSerializer() transfer_manager = s3transfer.crt.CRTTransferManager( crt_s3_client=self.s3_crt_client, crt_request_serializer=not_impl_serializer) future = transfer_manager.upload(self.filename, self.bucket, self.key, {}, []) with self.assertRaises(SerializationException): future.result() def test_crt_s3_client_error_handling(self): self.s3_crt_client.make_request.side_effect = awscrt.exceptions.from_code( 0) future = self.transfer_manager.upload(self.filename, self.bucket, self.key, {}, []) with self.assertRaises(awscrt.exceptions.AwsCrtError): future.result()