def setUp(self):
super(TestKinesisListStreams, self).setUp()
self.stream_name = "kinesis-test-stream"
self.region = "us-east-1"
self.client = self.session.create_client("kinesis", self.region)
self.http_stubber = ClientHTTPStubber(self.client)
self.http_stubber.add_response()
class TestDocDBPresignUrlInjection(BaseSessionTest):
def setUp(self):
super(TestDocDBPresignUrlInjection, self).setUp()
self.client = self.session.create_client('docdb', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
def assert_presigned_url_injected_in_request(self, body):
self.assertIn('PreSignedUrl', body)
self.assertNotIn('SourceRegion', body)
def test_copy_db_cluster_snapshot(self):
params = {
'SourceDBClusterSnapshotIdentifier': 'source-db',
'TargetDBClusterSnapshotIdentifier': 'target-db',
'SourceRegion': 'us-east-1'
}
response_body = (
b'<CopyDBClusterSnapshotResponse>'
b'<CopyDBClusterSnapshotResult>'
b'</CopyDBClusterSnapshotResult>'
b'</CopyDBClusterSnapshotResponse>'
)
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.copy_db_cluster_snapshot(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def setUp(self):
super(TestRegionRedirect, self).setUp()
self.client = self.session.create_client(
's3',
'us-west-2',
config=Config(
signature_version='s3v4',
s3={'addressing_style': 'path'},
))
self.http_stubber = ClientHTTPStubber(self.client)
self.redirect_response = {
'status':
301,
'headers': {
'x-amz-bucket-region': 'eu-central-1'
},
'body':
(b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<Error>'
b' <Code>PermanentRedirect</Code>'
b' <Message>The bucket you are attempting to access must be'
b' addressed using the specified endpoint. Please send '
b' all future requests to this endpoint.'
b' </Message>'
b' <Bucket>foo</Bucket>'
b' <Endpoint>foo.s3.eu-central-1.amazonaws.com</Endpoint>'
b'</Error>')
}
self.bad_signing_region_response = {
'status':
400,
'headers': {
'x-amz-bucket-region': 'eu-central-1'
},
'body': (b'<?xml version="1.0" encoding="UTF-8"?>'
b'<Error>'
b' <Code>AuthorizationHeaderMalformed</Code>'
b' <Message>the region us-west-2 is wrong; '
b'expecting eu-central-1</Message>'
b' <Region>eu-central-1</Region>'
b' <RequestId>BD9AA1730D454E39</RequestId>'
b' <HostId></HostId>'
b'</Error>')
}
self.success_response = {
'status':
200,
'headers': {},
'body': (b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<ListBucketResult>'
b' <Name>foo</Name>'
b' <Prefix></Prefix>'
b' <Marker></Marker>'
b' <MaxKeys>1000</MaxKeys>'
b' <EncodingType>url</EncodingType>'
b' <IsTruncated>false</IsTruncated>'
b'</ListBucketResult>')
}
class TestCopySnapshotCustomization(BaseSessionTest):
def setUp(self):
super(TestCopySnapshotCustomization, self).setUp()
self.session = botocore.session.get_session()
self.client = self.session.create_client('ec2', 'us-east-1')
self.http_stubber = ClientHTTPStubber(self.client)
self.snapshot_id = 'snap-0123abc'
self.copy_response = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<CopySnapshotResponse>\n'
'<snapshotId>%s</snapshotId>\n'
'</CopySnapshotResponse>\n')
self.now = datetime.datetime(2011, 9, 9, 23, 36)
self.datetime_patch = mock.patch.object(
botocore.auth.datetime, 'datetime',
mock.Mock(wraps=datetime.datetime))
self.mocked_datetime = self.datetime_patch.start()
self.mocked_datetime.utcnow.return_value = self.now
def tearDown(self):
super(TestCopySnapshotCustomization, self).tearDown()
self.datetime_patch.stop()
def add_copy_snapshot_response(self, snapshot_id):
body = (self.copy_response % snapshot_id).encode('utf-8')
self.http_stubber.add_response(body=body)
def test_copy_snapshot_injects_presigned_url(self):
self.add_copy_snapshot_response(self.snapshot_id)
with self.http_stubber:
result = self.client.copy_snapshot(
SourceRegion='us-west-2',
SourceSnapshotId=self.snapshot_id,
)
self.assertEqual(result['SnapshotId'], self.snapshot_id)
self.assertEqual(len(self.http_stubber.requests), 1)
snapshot_request = self.http_stubber.requests[0]
body = parse_qs(snapshot_request.body)
self.assertIn('PresignedUrl', body)
presigned_url = urlparse(body['PresignedUrl'][0])
self.assertEqual(presigned_url.scheme, 'https')
self.assertEqual(presigned_url.netloc, 'ec2.us-west-2.amazonaws.com')
query_args = parse_qs(presigned_url.query)
self.assertEqual(query_args['Action'], ['CopySnapshot'])
self.assertEqual(query_args['Version'], ['2016-11-15'])
self.assertEqual(query_args['SourceRegion'], ['us-west-2'])
self.assertEqual(query_args['DestinationRegion'], ['us-east-1'])
self.assertEqual(query_args['SourceSnapshotId'], [self.snapshot_id])
self.assertEqual(query_args['X-Amz-Algorithm'], ['AWS4-HMAC-SHA256'])
expected_credential = 'access_key/20110909/us-west-2/ec2/aws4_request'
self.assertEqual(query_args['X-Amz-Credential'], [expected_credential])
self.assertEqual(query_args['X-Amz-Date'], ['20110909T233600Z'])
self.assertEqual(query_args['X-Amz-Expires'], ['3600'])
self.assertEqual(query_args['X-Amz-SignedHeaders'], ['host'])
expected_signature = (
'a94a6b52afdf3daa34c2e2a38a62b72c8dac129c9904c61aa1a5d86e38628537')
self.assertEqual(query_args['X-Amz-Signature'], [expected_signature])
def test_content_sha256_not_set_if_config_value_is_false(self):
config = Config(signature_version='s3v4',
s3={'payload_signing_enabled': False})
self.client = self.session.create_client('s3',
self.region,
config=config)
self.http_stubber = ClientHTTPStubber(self.client)
self.http_stubber.add_response()
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
sha_header = sent_headers.get('x-amz-content-sha256')
self.assertEqual(sha_header, b'UNSIGNED-PAYLOAD')
def test_500_error_with_non_xml_body(self):
# Note: This exact test case may not be applicable from
# an integration standpoint if the issue is fixed in the future.
#
# The issue is that:
# S3 returns a 200 response but the received response from urllib3 has
# a 500 status code and the headers are in the body of the
# the response. Botocore will try to parse out the error body as xml,
# but the body is invalid xml because it is full of headers.
# So instead of blowing up on an XML parsing error, we
# should at least use the 500 status code because that can be
# retried.
#
# We are unsure of what exactly causes the response to be mangled
# but we expect it to be how 100 continues are handled.
non_xml_content = ('x-amz-id-2: foo\r\n'
'x-amz-request-id: bar\n'
'Date: Tue, 06 Oct 2015 03:20:38 GMT\r\n'
'ETag: "a6d856bc171fc6aa1b236680856094e2"\r\n'
'Content-Length: 0\r\n'
'Server: AmazonS3\r\n').encode('utf-8')
s3 = self.session.create_client('s3')
with ClientHTTPStubber(s3) as http_stubber:
http_stubber.add_response(status=500, body=non_xml_content)
http_stubber.add_response()
response = s3.put_object(Bucket='mybucket',
Key='mykey',
Body=b'foo')
# The first response should have been retried even though the xml is
# invalid and eventually return the 200 response.
self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'],
200)
self.assertEqual(len(http_stubber.requests), 2)
def _verify_expected_endpoint_url(region,
bucket,
key,
s3_config,
is_secure=True,
customer_provided_endpoint=None,
expected_url=None,
signature_version=None):
environ = {}
with mock.patch('os.environ', environ):
environ['AWS_ACCESS_KEY_ID'] = 'access_key'
environ['AWS_SECRET_ACCESS_KEY'] = 'secret_key'
environ['AWS_CONFIG_FILE'] = 'no-exist-foo'
environ['AWS_SHARED_CREDENTIALS_FILE'] = 'no-exist-foo'
session = create_session()
session.config_filename = 'no-exist-foo'
config = Config(signature_version=signature_version, s3=s3_config)
s3 = session.create_client('s3',
region_name=region,
use_ssl=is_secure,
config=config,
endpoint_url=customer_provided_endpoint)
with ClientHTTPStubber(s3) as http_stubber:
http_stubber.add_response()
s3.put_object(Bucket=bucket, Key=key, Body=b'bar')
assert_equal(http_stubber.requests[0].url, expected_url)
def setUp(self):
super(TestCopySnapshotCustomization, self).setUp()
self.session = botocore.session.get_session()
self.client = self.session.create_client('ec2', 'us-east-1')
self.http_stubber = ClientHTTPStubber(self.client)
self.snapshot_id = 'snap-0123abc'
self.copy_response = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<CopySnapshotResponse>\n'
'<snapshotId>%s</snapshotId>\n'
'</CopySnapshotResponse>\n')
self.now = datetime.datetime(2011, 9, 9, 23, 36)
self.datetime_patch = mock.patch.object(
botocore.auth.datetime, 'datetime',
mock.Mock(wraps=datetime.datetime))
self.mocked_datetime = self.datetime_patch.start()
self.mocked_datetime.utcnow.return_value = self.now
def _test_amz_sdk_request_header_with_test_case(self, responses,
utcnow_side_effects,
expected_headers,
client_config):
datetime_patcher = mock.patch.object(
ibm_botocore.endpoint.datetime, 'datetime',
mock.Mock(wraps=datetime.datetime))
mocked_datetime = datetime_patcher.start()
mocked_datetime.utcnow.side_effect = utcnow_side_effects
client = self.session.create_client('dynamodb',
self.region,
config=client_config)
with ClientHTTPStubber(client) as http_stubber:
for response in responses:
http_stubber.add_response(headers=response[1],
status=response[0],
body=b'{}')
client.list_tables()
amz_sdk_request_headers = [
request.headers['amz-sdk-request']
for request in http_stubber.requests
]
self.assertListEqual(amz_sdk_request_headers, expected_headers)
datetime_patcher.stop()
def test_old_model_continues_to_work():
# This test ensures that botocore can load the service models as they exist
# today. There's a directory in tests/functional/models that is a
# snapshot of a service model. This test ensures that we can continue
# to stub an API call using this model. That way if the models ever
# change we have a mechanism to ensure that the existing models continue
# to work with botocore. The test should not change (with the exception
# of potential changes to the ClientHTTPStubber), and the files in
# tests/functional/models should not change!
session = Session()
loader = session.get_component('data_loader')
# We're adding our path to the existing search paths so we don't have to
# copy additional data files such as _retry.json to our TEST_MODELS_DIR.
# We only care about the service model and endpoints file not changing.
# This also prevents us from having to make any changes to this models dir
# if we end up adding a new data file that's needed to create clients.
# We're adding our TEST_MODELS_DIR as the first element in the list to
# ensure we load the endpoints.json file from TEST_MODELS_DIR. For the
# service model we have an extra safety net where we can choose a custom
# client name.
loader.search_paths.insert(0, TEST_MODELS_DIR)
# The model dir we copied was renamed to 'custom-lambda'
# to ensure we're loading our version of the model and not
# the built in one.
client = session.create_client(
'custom-acm',
region_name='us-west-2',
aws_access_key_id='foo',
aws_secret_access_key='bar',
)
with ClientHTTPStubber(client) as stubber:
stubber.add_response(url='https://acm.us-west-2.amazonaws.com/',
headers={
'x-amzn-RequestId': 'abcd',
'Date': 'Fri, 26 Oct 2018 01:46:30 GMT',
'Content-Length': '29',
'Content-Type': 'application/x-amz-json-1.1'
},
body=b'{"CertificateSummaryList":[]}')
response = client.list_certificates()
assert_equal(
response, {
'CertificateSummaryList': [],
'ResponseMetadata': {
'HTTPHeaders': {
'content-length': '29',
'content-type': 'application/x-amz-json-1.1',
'date': 'Fri, 26 Oct 2018 01:46:30 GMT',
'x-amzn-requestid': 'abcd'
},
'HTTPStatusCode': 200,
'RequestId': 'abcd',
'RetryAttempts': 0
}
})
# Also verify we can use the paginators as well.
assert_equal(client.can_paginate('list_certificates'), True)
assert_equal(client.waiter_names, ['certificate_validated'])
class UnsignedOperationTestCase(object):
def __init__(self, client, operation_name, parameters):
self._client = client
self._operation_name = operation_name
self._parameters = parameters
self._http_stubber = ClientHTTPStubber(self._client)
def run(self):
operation = getattr(self._client, self._operation_name)
self._http_stubber.add_response(body=b'{}')
with self._http_stubber:
operation(**self._parameters)
request = self._http_stubber.requests[0]
assert_false('authorization' in request.headers,
'authorization header found in unsigned operation')
class TestNeptunePresignUrlInjection(BaseSessionTest):
def setUp(self):
super(TestNeptunePresignUrlInjection, self).setUp()
self.client = self.session.create_client('neptune', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
def assert_presigned_url_injected_in_request(self, body):
self.assertIn('PreSignedUrl', body)
self.assertNotIn('SourceRegion', body)
def test_create_db_cluster(self):
params = {
'DBClusterIdentifier': 'my-cluster',
'Engine': 'neptune',
'SourceRegion': 'us-east-1'
}
response_body = (
b'<CreateDBClusterResponse>'
b'<CreateDBClusterResult>'
b'</CreateDBClusterResult>'
b'</CreateDBClusterResponse>'
)
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.create_db_cluster(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def test_copy_db_cluster_snapshot(self):
params = {
'SourceDBClusterSnapshotIdentifier': 'source-db',
'TargetDBClusterSnapshotIdentifier': 'target-db',
'SourceRegion': 'us-east-1'
}
response_body = (
b'<CopyDBClusterSnapshotResponse>'
b'<CopyDBClusterSnapshotResult>'
b'</CopyDBClusterSnapshotResult>'
b'</CopyDBClusterSnapshotResponse>'
)
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.copy_db_cluster_snapshot(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
class TestCloudsearchdomain(BaseSessionTest):
def setUp(self):
super(TestCloudsearchdomain, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'cloudsearchdomain', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_search(self):
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
self.client.search(query='foo')
request = self.http_stubber.requests[0]
self.assertIn('q=foo', request.body)
self.assertEqual(request.method, 'POST')
content_type = b'application/x-www-form-urlencoded'
self.assertEqual(request.headers.get('Content-Type'), content_type)
class TestCloudsearchdomain(BaseSessionTest):
def setUp(self):
super(TestCloudsearchdomain, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('cloudsearchdomain',
self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_search(self):
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
self.client.search(query='foo')
request = self.http_stubber.requests[0]
self.assertIn('q=foo', request.body)
self.assertEqual(request.method, 'POST')
content_type = b'application/x-www-form-urlencoded'
self.assertEqual(request.headers.get('Content-Type'), content_type)
class TestMachineLearning(BaseSessionTest):
def setUp(self):
super(TestMachineLearning, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('machinelearning',
self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_predict(self):
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
custom_endpoint = 'https://myendpoint.amazonaws.com/'
self.client.predict(MLModelId='ml-foo',
Record={'Foo': 'Bar'},
PredictEndpoint=custom_endpoint)
sent_request = self.http_stubber.requests[0]
self.assertEqual(sent_request.url, custom_endpoint)
class TestRDSPresignUrlInjection(BaseSessionTest):
def setUp(self):
super(TestRDSPresignUrlInjection, self).setUp()
self.client = self.session.create_client('rds', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
def assert_presigned_url_injected_in_request(self, body):
self.assertIn('PreSignedUrl', body)
self.assertNotIn('SourceRegion', body)
def test_copy_snapshot(self):
params = {
'SourceDBSnapshotIdentifier': 'source-db',
'TargetDBSnapshotIdentifier': 'target-db',
'SourceRegion': 'us-east-1'
}
response_body = (
b'<CopyDBSnapshotResponse>'
b'<CopyDBSnapshotResult></CopyDBSnapshotResult>'
b'</CopyDBSnapshotResponse>'
)
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.copy_db_snapshot(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def test_create_db_instance_read_replica(self):
params = {
'SourceDBInstanceIdentifier': 'source-db',
'DBInstanceIdentifier': 'target-db',
'SourceRegion': 'us-east-1'
}
response_body = (
b'<CreateDBInstanceReadReplicaResponse>'
b'<CreateDBInstanceReadReplicaResult>'
b'</CreateDBInstanceReadReplicaResult>'
b'</CreateDBInstanceReadReplicaResponse>'
)
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.create_db_instance_read_replica(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def get_prepared_request(self, operation, params, force_hmacv1=False):
if force_hmacv1:
self.session.register('choose-signer', self.enable_hmacv1)
client = self.session.create_client('s3', self.region_name)
with ClientHTTPStubber(client) as http_stubber:
http_stubber.add_response()
getattr(client, operation)(**params)
# Return the request that was sent over the wire.
return http_stubber.requests[0]
def assert_will_retry_n_times(self, client, num_retries):
num_responses = num_retries + 1
with ClientHTTPStubber(client) as http_stubber:
for _ in range(num_responses):
http_stubber.add_response(status=500, body=b'{}')
with self.assertRaisesRegexp(
ClientError, 'reached max retries: %s' % num_retries):
yield
self.assertEqual(len(http_stubber.requests), num_responses)
class TestMachineLearning(BaseSessionTest):
def setUp(self):
super(TestMachineLearning, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'machinelearning', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_predict(self):
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
custom_endpoint = 'https://myendpoint.amazonaws.com/'
self.client.predict(
MLModelId='ml-foo',
Record={'Foo': 'Bar'},
PredictEndpoint=custom_endpoint
)
sent_request = self.http_stubber.requests[0]
self.assertEqual(sent_request.url, custom_endpoint)
class UnsignedOperationTestCase(object):
def __init__(self, client, operation_name, parameters):
self._client = client
self._operation_name = operation_name
self._parameters = parameters
self._http_stubber = ClientHTTPStubber(self._client)
def run(self):
operation = getattr(self._client, self._operation_name)
self._http_stubber.add_response(body=b'{}')
with self._http_stubber:
operation(**self._parameters)
request = self._http_stubber.requests[0]
assert_false(
'authorization' in request.headers,
'authorization header found in unsigned operation'
)
class TestLex(BaseSessionTest):
def setUp(self):
super(TestLex, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('lex-runtime', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_unsigned_payload(self):
params = {
'botName': 'foo',
'botAlias': 'bar',
'userId': 'baz',
'contentType': 'application/octet-stream',
'inputStream': b''
}
timestamp = datetime(2017, 3, 22, 0, 0)
with mock.patch('botocore.auth.datetime') as _datetime:
_datetime.datetime.utcnow.return_value = timestamp
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
self.client.post_content(**params)
request = self.http_stubber.requests[0]
# The payload gets added to the string to sign, and then part of the
# signature. The signature will be part of the authorization header.
# Since we don't have direct access to the payload signature,
# we compare the authorization instead.
authorization = request.headers.get('authorization')
expected_authorization = (
b'AWS4-HMAC-SHA256 '
b'Credential=access_key/20170322/us-west-2/lex/aws4_request, '
b'SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date,'
b' Signature='
b'7f93fde5c36163dce6ee116fcfebab13474ab903782fea04c00bb1dedc3fc4cc'
)
self.assertEqual(authorization, expected_authorization)
content_header = request.headers.get('x-amz-content-sha256')
self.assertEqual(content_header, b'UNSIGNED-PAYLOAD')
class TestLex(BaseSessionTest):
def setUp(self):
super(TestLex, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('lex-runtime', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_unsigned_payload(self):
params = {
'botName': 'foo',
'botAlias': 'bar',
'userId': 'baz',
'contentType': 'application/octet-stream',
'inputStream': b''
}
timestamp = datetime(2017, 3, 22, 0, 0)
with mock.patch('botocore.auth.datetime') as _datetime:
_datetime.datetime.utcnow.return_value = timestamp
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
self.client.post_content(**params)
request = self.http_stubber.requests[0]
# The payload gets added to the string to sign, and then part of the
# signature. The signature will be part of the authorization header.
# Since we don't have direct access to the payload signature,
# we compare the authorization instead.
authorization = request.headers.get('authorization')
expected_authorization = (
b'AWS4-HMAC-SHA256 '
b'Credential=access_key/20170322/us-west-2/lex/aws4_request, '
b'SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date,'
b' Signature='
b'7f93fde5c36163dce6ee116fcfebab13474ab903782fea04c00bb1dedc3fc4cc'
)
self.assertEqual(authorization, expected_authorization)
content_header = request.headers.get('x-amz-content-sha256')
self.assertEqual(content_header, b'UNSIGNED-PAYLOAD')
def _make_client_call_with_errors(client, operation_name, kwargs):
operation = getattr(client, xform_name(operation_name))
exception = ConnectionClosedError(endpoint_url='https://mock.eror')
with ClientHTTPStubber(client) as http_stubber:
http_stubber.responses.append(exception)
http_stubber.responses.append(None)
try:
response = operation(**kwargs)
except ClientError as e:
assert False, ('Request was not retried properly, '
'received error:\n%s' % pformat(e))
def _make_client_call_with_errors(client, operation_name, kwargs):
operation = getattr(client, xform_name(operation_name))
exception = ConnectionClosedError(endpoint_url='https://mock.eror')
with ClientHTTPStubber(client, strict=False) as http_stubber:
http_stubber.responses.append(exception)
try:
response = operation(**kwargs)
except ClientError as e:
assert False, ('Request was not retried properly, '
'received error:\n%s' % pformat(e))
# Ensure we used the stubber as we're not using it in strict mode
assert len(http_stubber.responses) == 0, 'Stubber was not used!'
def assert_will_retry_n_times(self, client, num_retries,
status=500, body=b'{}'):
num_responses = num_retries + 1
if not isinstance(body, bytes):
body = json.dumps(body).encode()
with ClientHTTPStubber(client) as http_stubber:
for _ in range(num_responses):
http_stubber.add_response(status=status, body=body)
with self.assertRaisesRegexp(
ClientError, 'reached max retries: %s' % num_retries):
yield
self.assertEqual(len(http_stubber.requests), num_responses)
def test_unsigned_operations(operation_name, parameters):
environ = {
'AWS_ACCESS_KEY_ID': 'access_key',
'AWS_SECRET_ACCESS_KEY': 'secret_key',
'AWS_CONFIG_FILE': 'no-exist-foo',
}
with mock.patch('os.environ', environ):
session = create_session()
session.config_filename = 'no-exist-foo'
client = session.create_client('cognito-idp', 'us-west-2')
http_stubber = ClientHTTPStubber(client)
operation = getattr(client, operation_name)
http_stubber.add_response(body=b'{}')
with http_stubber:
operation(**parameters)
request = http_stubber.requests[0]
assert 'authorization' not in request.headers, (
'authorization header found in unsigned operation')
def setUp(self):
super(TestRecordStatementsInjections, self).setUp()
self.client = self.session.create_client('s3', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
self.s3_response_body = (
'<ListAllMyBucketsResult '
' xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
' <Owner>'
' <ID>d41d8cd98f00b204e9800998ecf8427e</ID>'
' <DisplayName>foo</DisplayName>'
' </Owner>'
' <Buckets>'
' <Bucket>'
' <Name>bar</Name>'
' <CreationDate>1912-06-23T22:57:02.000Z</CreationDate>'
' </Bucket>'
' </Buckets>'
'</ListAllMyBucketsResult>').encode('utf-8')
self.recording_handler = RecordingHandler()
history_recorder = get_global_history_recorder()
history_recorder.enable()
history_recorder.add_handler(self.recording_handler)
class TestApiGateway(BaseSessionTest):
def setUp(self):
super(TestApiGateway, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('apigateway', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_get_export(self):
params = {
'restApiId': 'foo',
'stageName': 'bar',
'exportType': 'swagger',
'accepts': 'application/yaml'
}
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
self.client.get_export(**params)
request = self.http_stubber.requests[0]
self.assertEqual(request.method, 'GET')
self.assertEqual(request.headers.get('Accept'),
b'application/yaml')
class TestApiGateway(BaseSessionTest):
def setUp(self):
super(TestApiGateway, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'apigateway', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def test_get_export(self):
params = {
'restApiId': 'foo',
'stageName': 'bar',
'exportType': 'swagger',
'accepts': 'application/yaml'
}
self.http_stubber.add_response(body=b'{}')
with self.http_stubber:
self.client.get_export(**params)
request = self.http_stubber.requests[0]
self.assertEqual(request.method, 'GET')
self.assertEqual(request.headers.get('Accept'), b'application/yaml')
class TestRDSPresignUrlInjection(BaseSessionTest):
def setUp(self):
super(TestRDSPresignUrlInjection, self).setUp()
self.client = self.session.create_client('rds', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
def assert_presigned_url_injected_in_request(self, body):
self.assertIn('PreSignedUrl', body)
self.assertNotIn('SourceRegion', body)
def test_copy_snapshot(self):
params = {
'SourceDBSnapshotIdentifier': 'source-db',
'TargetDBSnapshotIdentifier': 'target-db',
'SourceRegion': 'us-east-1'
}
response_body = (b'<CopyDBSnapshotResponse>'
b'<CopyDBSnapshotResult></CopyDBSnapshotResult>'
b'</CopyDBSnapshotResponse>')
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.copy_db_snapshot(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def test_create_db_instance_read_replica(self):
params = {
'SourceDBInstanceIdentifier': 'source-db',
'DBInstanceIdentifier': 'target-db',
'SourceRegion': 'us-east-1'
}
response_body = (b'<CreateDBInstanceReadReplicaResponse>'
b'<CreateDBInstanceReadReplicaResult>'
b'</CreateDBInstanceReadReplicaResult>'
b'</CreateDBInstanceReadReplicaResponse>')
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.create_db_instance_read_replica(**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def test_start_db_instance_automated_backups_replication(self):
params = {
'SourceDBInstanceArn':
'arn:aws:rds:us-east-1:123456789012:db:source-db-instance',
'SourceRegion': 'us-east-1',
}
response_body = (
b'<StartDBInstanceAutomatedBackupsReplicationResponse>'
b'<StartDBInstanceAutomatedBackupsReplicationResult>'
b'</StartDBInstanceAutomatedBackupsReplicationResult>'
b'</StartDBInstanceAutomatedBackupsReplicationResponse>')
self.http_stubber.add_response(body=response_body)
with self.http_stubber:
self.client.start_db_instance_automated_backups_replication(
**params)
sent_request = self.http_stubber.requests[0]
self.assert_presigned_url_injected_in_request(sent_request.body)
def create_client(self,
service_name='dynamodb',
region=None,
config=None,
endpoint_url=None):
if region is None:
region = self.region
client = self.session.create_client(service_name,
region,
config=config,
endpoint_url=endpoint_url)
http_stubber = ClientHTTPStubber(client)
return client, http_stubber
def test_resign_request_in_us_east_1_fails(self):
region_headers = {'x-amz-bucket-region': 'eu-central-1'}
# Verify that the final 400 response is propagated
# back to the user.
client = self.session.create_client('s3', 'us-east-1')
with ClientHTTPStubber(client) as http_stubber:
http_stubber.add_response(status=400)
http_stubber.add_response(status=400, headers=region_headers)
http_stubber.add_response(headers=region_headers)
# The final request still fails with a 400.
http_stubber.add_response(status=400)
with self.assertRaises(ClientError) as e:
client.head_object(Bucket='foo', Key='bar')
self.assertEqual(len(http_stubber.requests), 4)
def test_public_apis_will_not_be_signed(client, operation, kwargs):
with ClientHTTPStubber(client) as http_stubber:
http_stubber.responses.append(EarlyExit())
try:
operation(**kwargs)
except EarlyExit:
pass
request = http_stubber.requests[0]
sig_v2_disabled = 'SignatureVersion=2' not in request.url
assert sig_v2_disabled, "SigV2 is incorrectly enabled"
sig_v3_disabled = 'X-Amzn-Authorization' not in request.headers
assert sig_v3_disabled, "SigV3 is incorrectly enabled"
sig_v4_disabled = 'Authorization' not in request.headers
assert sig_v4_disabled, "SigV4 is incorrectly enabled"
def test_can_exhaust_default_retry_quota(self):
# Quota of 500 / 5 retry costs == 100 retry attempts
# 100 retry attempts / 2 retries per API call == 50 client calls
client = self.create_client_with_retry_mode('dynamodb',
retry_mode='standard')
for i in range(50):
with self.assert_will_retry_n_times(client, 2, status=502):
client.list_tables()
# Now on the 51th attempt we should see quota errors, which we can
# verify by looking at the request metadata.
with ClientHTTPStubber(client) as http_stubber:
http_stubber.add_response(status=502, body=b'{}')
with self.assertRaises(ClientError) as e:
client.list_tables()
self.assertTrue(
e.exception.response['ResponseMetadata'].get('RetryQuotaReached'))
def test_int_values_with_sigv4(self):
s3 = self.session.create_client(
's3', config=Config(signature_version='s3v4'))
with ClientHTTPStubber(s3) as http_stubber:
http_stubber.add_response()
s3.upload_part(Bucket='foo',
Key='bar',
Body=b'foo',
UploadId='bar',
PartNumber=1,
ContentLength=3)
headers = http_stubber.requests[0].headers
# Verify that the request integer value of 3 has been converted to
# string '3'. This also means we've made it pass the signer which
# expects string values in order to sign properly.
self.assertEqual(headers['Content-Length'], b'3')
class TestS3SigV4(BaseS3OperationTest):
def setUp(self):
super(TestS3SigV4, self).setUp()
self.client = self.session.create_client(
's3', self.region, config=Config(signature_version='s3v4'))
self.http_stubber = ClientHTTPStubber(self.client)
self.http_stubber.add_response()
def get_sent_headers(self):
return self.http_stubber.requests[0].headers
def test_content_md5_set(self):
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
self.assertIn('content-md5', self.get_sent_headers())
def test_content_sha256_set_if_config_value_is_true(self):
config = Config(signature_version='s3v4',
s3={'payload_signing_enabled': True})
self.client = self.session.create_client('s3',
self.region,
config=config)
self.http_stubber = ClientHTTPStubber(self.client)
self.http_stubber.add_response()
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
sha_header = sent_headers.get('x-amz-content-sha256')
self.assertNotEqual(sha_header, b'UNSIGNED-PAYLOAD')
def test_content_sha256_not_set_if_config_value_is_false(self):
config = Config(signature_version='s3v4',
s3={'payload_signing_enabled': False})
self.client = self.session.create_client('s3',
self.region,
config=config)
self.http_stubber = ClientHTTPStubber(self.client)
self.http_stubber.add_response()
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
sha_header = sent_headers.get('x-amz-content-sha256')
self.assertEqual(sha_header, b'UNSIGNED-PAYLOAD')
def test_content_sha256_set_if_md5_is_unavailable(self):
with mock.patch('ibm_botocore.auth.MD5_AVAILABLE', False):
with mock.patch('ibm_botocore.handlers.MD5_AVAILABLE', False):
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
unsigned = 'UNSIGNED-PAYLOAD'
self.assertNotEqual(sent_headers['x-amz-content-sha256'], unsigned)
self.assertNotIn('content-md5', sent_headers)
def test_multiple_transitions_returns_one(self):
response_body = ('<?xml version="1.0" ?>'
'<LifecycleConfiguration xmlns="http://s3.amazonaws.'
'com/doc/2006-03-01/">'
' <Rule>'
' <ID>transitionRule</ID>'
' <Prefix>foo</Prefix>'
' <Status>Enabled</Status>'
' <Transition>'
' <Days>40</Days>'
' <StorageClass>STANDARD_IA</StorageClass>'
' </Transition>'
' <Transition>'
' <Days>70</Days>'
' <StorageClass>GLACIER</StorageClass>'
' </Transition>'
' </Rule>'
' <Rule>'
' <ID>noncurrentVersionRule</ID>'
' <Prefix>bar</Prefix>'
' <Status>Enabled</Status>'
' <NoncurrentVersionTransition>'
' <NoncurrentDays>40</NoncurrentDays>'
' <StorageClass>STANDARD_IA</StorageClass>'
' </NoncurrentVersionTransition>'
' <NoncurrentVersionTransition>'
' <NoncurrentDays>70</NoncurrentDays>'
' <StorageClass>GLACIER</StorageClass>'
' </NoncurrentVersionTransition>'
' </Rule>'
'</LifecycleConfiguration>').encode('utf-8')
s3 = self.session.create_client('s3')
with ClientHTTPStubber(s3) as http_stubber:
http_stubber.add_response(body=response_body)
response = s3.get_bucket_lifecycle(Bucket='mybucket')
# Each Transition member should have at least one of the
# transitions provided.
self.assertEqual(response['Rules'][0]['Transition'], {
'Days': 40,
'StorageClass': 'STANDARD_IA'
})
self.assertEqual(response['Rules'][1]['NoncurrentVersionTransition'], {
'NoncurrentDays': 40,
'StorageClass': 'STANDARD_IA'
})
def setUp(self):
super(TestRecordStatementsInjections, self).setUp()
self.client = self.session.create_client('s3', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
self.s3_response_body = (
'<ListAllMyBucketsResult '
' xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
' <Owner>'
' <ID>d41d8cd98f00b204e9800998ecf8427e</ID>'
' <DisplayName>foo</DisplayName>'
' </Owner>'
' <Buckets>'
' <Bucket>'
' <Name>bar</Name>'
' <CreationDate>1912-06-23T22:57:02.000Z</CreationDate>'
' </Bucket>'
' </Buckets>'
'</ListAllMyBucketsResult>'
).encode('utf-8')
self.recording_handler = RecordingHandler()
history_recorder = get_global_history_recorder()
history_recorder.enable()
history_recorder.add_handler(self.recording_handler)
class TestRecordStatementsInjections(BaseSessionTest):
def setUp(self):
super(TestRecordStatementsInjections, self).setUp()
self.client = self.session.create_client('s3', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
self.s3_response_body = (
'<ListAllMyBucketsResult '
' xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
' <Owner>'
' <ID>d41d8cd98f00b204e9800998ecf8427e</ID>'
' <DisplayName>foo</DisplayName>'
' </Owner>'
' <Buckets>'
' <Bucket>'
' <Name>bar</Name>'
' <CreationDate>1912-06-23T22:57:02.000Z</CreationDate>'
' </Bucket>'
' </Buckets>'
'</ListAllMyBucketsResult>'
).encode('utf-8')
self.recording_handler = RecordingHandler()
history_recorder = get_global_history_recorder()
history_recorder.enable()
history_recorder.add_handler(self.recording_handler)
def _get_all_events_of_type(self, event_type):
recorded_calls = self.recording_handler.recorded_calls
matching = [call for call in recorded_calls
if call[0] == event_type]
return matching
def test_does_record_api_call(self):
self.http_stubber.add_response(body=self.s3_response_body)
with self.http_stubber:
self.client.list_buckets()
api_call_events = self._get_all_events_of_type('API_CALL')
self.assertEqual(len(api_call_events), 1)
event = api_call_events[0]
event_type, payload, source = event
self.assertEqual(payload, {
'operation': u'ListBuckets',
'params': {},
'service': 's3'
})
self.assertEqual(source, 'BOTOCORE')
def test_does_record_http_request(self):
self.http_stubber.add_response(body=self.s3_response_body)
with self.http_stubber:
self.client.list_buckets()
http_request_events = self._get_all_events_of_type('HTTP_REQUEST')
self.assertEqual(len(http_request_events), 1)
event = http_request_events[0]
event_type, payload, source = event
method = payload['method']
self.assertEqual(method, u'GET')
# The header values vary too much per request to verify them here.
# Instead just check the presense of each expected header.
headers = payload['headers']
for expected_header in ['Authorization', 'User-Agent', 'X-Amz-Date',
'X-Amz-Content-SHA256']:
self.assertIn(expected_header, headers)
body = payload['body']
self.assertIsNone(body)
streaming = payload['streaming']
self.assertEquals(streaming, False)
url = payload['url']
self.assertEquals(url, 'https://s3.us-west-2.amazonaws.com/')
self.assertEqual(source, 'BOTOCORE')
def test_does_record_http_response(self):
self.http_stubber.add_response(body=self.s3_response_body)
with self.http_stubber:
self.client.list_buckets()
http_response_events = self._get_all_events_of_type('HTTP_RESPONSE')
self.assertEqual(len(http_response_events), 1)
event = http_response_events[0]
event_type, payload, source = event
self.assertEqual(payload, {
'status_code': 200,
'headers': {},
'streaming': False,
'body': self.s3_response_body,
'context': {'operation_name': 'ListBuckets'}
}
)
self.assertEqual(source, 'BOTOCORE')
def test_does_record_parsed_response(self):
self.http_stubber.add_response(body=self.s3_response_body)
with self.http_stubber:
self.client.list_buckets()
parsed_response_events = self._get_all_events_of_type(
'PARSED_RESPONSE')
self.assertEqual(len(parsed_response_events), 1)
event = parsed_response_events[0]
event_type, payload, source = event
# Given that the request contains headers with a user agent string
# a date and a signature we need to disassemble the call and manually
# assert the interesting bits since mock can only assert if the args
# all match exactly.
owner = payload['Owner']
self.assertEqual(owner, {
'DisplayName': 'foo',
'ID': 'd41d8cd98f00b204e9800998ecf8427e'
})
buckets = payload['Buckets']
self.assertEqual(len(buckets), 1)
bucket = buckets[0]
self.assertEqual(bucket['Name'], 'bar')
metadata = payload['ResponseMetadata']
self.assertEqual(metadata, {
'HTTPHeaders': {},
'HTTPStatusCode': 200,
'RetryAttempts': 0
})
def setUp(self):
super(TestRDSPresignUrlInjection, self).setUp()
self.client = self.session.create_client('rds', 'us-west-2')
self.http_stubber = ClientHTTPStubber(self.client)
def __init__(self, client, operation_name, parameters):
self._client = client
self._operation_name = operation_name
self._parameters = parameters
self._http_stubber = ClientHTTPStubber(self._client)
def setUp(self):
super(TestLex, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('lex-runtime', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def setUp(self):
super(TestMachineLearning, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'machinelearning', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def setUp(self):
super(TestCloudsearchdomain, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'cloudsearchdomain', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
def setUp(self):
super(TestApiGateway, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'apigateway', self.region)
self.http_stubber = ClientHTTPStubber(self.client)
