def create_client_sts_stub(service, *args, **kwargs):
client = _original_create_client(service, *args, **kwargs)
stub = Stubber(client)
response = self.create_assume_role_response(expected_creds)
self.actual_client_region = client.meta.region_name
stub.add_response('assume_role', response)
stub.activate()
return client
class TestCloudwatchLogsPagination(BaseSessionTest):
def setUp(self):
super(TestCloudwatchLogsPagination, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('logs',
self.region,
aws_secret_access_key='foo',
aws_access_key_id='bar',
aws_session_token='baz')
self.stubber = Stubber(self.client)
self.stubber.activate()
def test_token_with_triple_underscores(self):
response = {
'events': [{
'logStreamName': 'foobar',
'timestamp': 1560195817,
'message': 'a thing happened',
'ingestionTime': 1560195817,
'eventId': 'foo',
}],
'searchedLogStreams': [{
'logStreamName': 'foobar',
'searchedCompletely': False,
}],
}
group_name = 'foo'
token = 'foo___bar'
expected_args = {
'logGroupName': group_name,
'nextToken': token,
}
self.stubber.add_response('filter_log_events', response, expected_args)
paginator = self.client.get_paginator('filter_log_events')
pages = paginator.paginate(
PaginationConfig={
'MaxItems': 1,
'StartingToken': token,
},
logGroupName=group_name,
)
result = pages.build_full_result()
self.assertEqual(len(result['events']), 1)
class TestS3ObjectSummary(unittest.TestCase):
def setUp(self):
self.session = ibm_boto3.session.Session(
aws_access_key_id='foo',
aws_secret_access_key='bar',
region_name='us-west-2',
)
self.s3 = self.session.resource('s3')
self.obj_summary = self.s3.ObjectSummary('my_bucket', 'my_key')
self.obj_summary_size = 12
self.stubber = Stubber(self.s3.meta.client)
self.stubber.activate()
self.stubber.add_response(
method='head_object',
service_response={
'ContentLength': self.obj_summary_size,
'ETag': 'my-etag',
'ContentType': 'binary',
},
expected_params={
'Bucket': 'my_bucket',
'Key': 'my_key'
},
)
def tearDown(self):
self.stubber.deactivate()
def test_has_load(self):
# Validate load was injected onto ObjectSummary.
assert hasattr(self.obj_summary, 'load')
def test_autoloads_correctly(self):
# In HeadObject the parameter returned is ContentLength, this
# should get mapped to Size of ListObject since the resource uses
# the shape returned to by ListObjects.
assert self.obj_summary.size == self.obj_summary_size
def test_cannot_access_other_non_related_parameters(self):
# Even though an HeadObject was used to load this, it should
# only expose the attributes from its shape defined in ListObjects.
assert not hasattr(self.obj_summary, 'content_length')
class StubbedClient(object):
def __init__(self):
self._client = ibm_botocore.session.get_session().create_client(
's3', 'us-west-2', aws_access_key_id='foo',
aws_secret_access_key='bar')
self._stubber = Stubber(self._client)
self._stubber.activate()
self._caught_stubber_errors = []
def get_object(self, **kwargs):
return self._client.get_object(**kwargs)
def head_object(self, **kwargs):
return self._client.head_object(**kwargs)
def add_response(self, *args, **kwargs):
self._stubber.add_response(*args, **kwargs)
def add_client_error(self, *args, **kwargs):
self._stubber.add_client_error(*args, **kwargs)
class TestRDSPagination(BaseSessionTest):
def setUp(self):
super(TestRDSPagination, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('rds', self.region)
self.stubber = Stubber(self.client)
def test_can_specify_zero_marker(self):
service_response = {
'LogFileData': 'foo',
'Marker': '2',
'AdditionalDataPending': True
}
expected_params = {
'DBInstanceIdentifier': 'foo',
'LogFileName': 'bar',
'NumberOfLines': 2,
'Marker': '0'
}
function_name = 'download_db_log_file_portion'
# The stubber will assert that the function is called with the expected
# parameters.
self.stubber.add_response(function_name, service_response,
expected_params)
self.stubber.activate()
try:
paginator = self.client.get_paginator(function_name)
result = paginator.paginate(DBInstanceIdentifier='foo',
LogFileName='bar',
NumberOfLines=2,
PaginationConfig={
'StartingToken': '0',
'MaxItems': 3
}).build_full_result()
self.assertEqual(result['LogFileData'], 'foo')
self.assertIn('NextToken', result)
except StubAssertionError as e:
self.fail(str(e))
class TestAutoscalingPagination(BaseSessionTest):
def setUp(self):
super(TestAutoscalingPagination, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client('autoscaling',
self.region,
aws_secret_access_key='foo',
aws_access_key_id='bar',
aws_session_token='baz')
self.stubber = Stubber(self.client)
self.stubber.activate()
def _setup_scaling_pagination(self,
page_size=200,
max_items=100,
total_items=600):
"""
Add to the stubber to test paginating describe_scaling_activities.
WARNING: This only handles cases where max_items cleanly divides
page_size.
"""
requests_per_page = page_size / max_items
if requests_per_page != ceil(requests_per_page):
raise NotImplementedError(
"This only handles setup where max_items is less than "
"page_size and where max_items evenly divides page_size.")
requests_per_page = int(requests_per_page)
num_pages = int(ceil(total_items / page_size))
previous_next_token = None
for i in range(num_pages):
page = self.create_describe_scaling_response(page_size=page_size)
# Don't create a next_token for the final page
if i + 1 == num_pages:
next_token = None
else:
next_token = random_chars(10)
expected_args = {}
if previous_next_token:
expected_args['StartingToken'] = previous_next_token
# The same page may be accessed multiple times because we are
# truncating it at max_items
for _ in range(requests_per_page - 1):
# The page is copied because the paginator will modify the
# response object, causing issues when using the stubber.
self.stubber.add_response('describe_scaling_activities',
page.copy())
if next_token is not None:
page['NextToken'] = next_token
# Copying the page here isn't necessary because it is about to
# be blown away anyway.
self.stubber.add_response('describe_scaling_activities', page)
previous_next_token = next_token
def create_describe_scaling_response(self, page_size=200):
"""Create a valid describe_scaling_activities response."""
page = []
date = datetime.now()
for _ in range(page_size):
page.append({
'AutoScalingGroupName': 'test',
'ActivityId': random_chars(10),
'Cause': 'test',
'StartTime': date,
'StatusCode': '200',
})
return {'Activities': page}
def test_repeated_build_full_results(self):
# This ensures that we can cleanly paginate using build_full_results.
max_items = 100
total_items = 600
self._setup_scaling_pagination(max_items=max_items,
total_items=total_items,
page_size=200)
paginator = self.client.get_paginator('describe_scaling_activities')
conf = {'MaxItems': max_items}
pagination_tokens = []
result = paginator.paginate(PaginationConfig=conf).build_full_result()
all_results = result['Activities']
while 'NextToken' in result:
starting_token = result['NextToken']
# We should never get a duplicate pagination token.
self.assertNotIn(starting_token, pagination_tokens)
pagination_tokens.append(starting_token)
conf['StartingToken'] = starting_token
pages = paginator.paginate(PaginationConfig=conf)
result = pages.build_full_result()
all_results.extend(result['Activities'])
self.assertEqual(len(all_results), total_items)
class TestStubber(unittest.TestCase):
def setUp(self):
self.event_emitter = hooks.HierarchicalEmitter()
self.client = mock.Mock()
self.client.meta.events = self.event_emitter
self.client.meta.method_to_api_mapping.get.return_value = 'foo'
self.stubber = Stubber(self.client)
self.validate_parameters_mock = mock.Mock()
self.validate_parameters_patch = mock.patch(
'ibm_botocore.stub.validate_parameters',
self.validate_parameters_mock)
self.validate_parameters_patch.start()
def tearDown(self):
self.validate_parameters_patch.stop()
def emit_get_response_event(self,
model=None,
request_dict=None,
signer=None,
context=None):
if model is None:
model = mock.Mock()
model.name = 'foo'
handler, response = self.event_emitter.emit_until_response(
event_name='before-call.myservice.foo',
model=model,
params=request_dict,
request_signer=signer,
context=context)
return response
def test_stubber_registers_events(self):
self.event_emitter = mock.Mock()
self.client.meta.events = self.event_emitter
self.stubber.activate()
# This just ensures that we register at the correct event
# and nothing more
self.event_emitter.register_first.assert_called_with(
'before-parameter-build.*.*', mock.ANY, unique_id=mock.ANY)
self.event_emitter.register.assert_called_with('before-call.*.*',
mock.ANY,
unique_id=mock.ANY)
def test_stubber_unregisters_events(self):
self.event_emitter = mock.Mock()
self.client.meta.events = self.event_emitter
self.stubber.activate()
self.stubber.deactivate()
self.event_emitter.unregister.assert_any_call(
'before-parameter-build.*.*', mock.ANY, unique_id=mock.ANY)
self.event_emitter.unregister.assert_any_call('before-call.*.*',
mock.ANY,
unique_id=mock.ANY)
def test_context_manager(self):
self.event_emitter = mock.Mock()
self.client.meta.events = self.event_emitter
with self.stubber:
# Ensure events are registered in context
self.event_emitter.register_first.assert_called_with(
'before-parameter-build.*.*', mock.ANY, unique_id=mock.ANY)
self.event_emitter.register.assert_called_with('before-call.*.*',
mock.ANY,
unique_id=mock.ANY)
# Ensure events are no longer registered once we leave the context
self.event_emitter.unregister.assert_any_call(
'before-parameter-build.*.*', mock.ANY, unique_id=mock.ANY)
self.event_emitter.unregister.assert_any_call('before-call.*.*',
mock.ANY,
unique_id=mock.ANY)
def test_add_response(self):
response = {'foo': 'bar'}
self.stubber.add_response('foo', response)
with self.assertRaises(AssertionError):
self.stubber.assert_no_pending_responses()
def test_add_response_fails_when_missing_client_method(self):
del self.client.foo
with self.assertRaises(ValueError):
self.stubber.add_response('foo', {})
def test_validates_service_response(self):
self.stubber.add_response('foo', {})
self.assertTrue(self.validate_parameters_mock.called)
def test_validate_ignores_response_metadata(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
service_model = ServiceModel({
'documentation': '',
'operations': {
'foo': {
'name': 'foo',
'input': {
'shape': 'StringShape'
},
'output': {
'shape': 'StringShape'
}
}
},
'shapes': {
'StringShape': {
'type': 'string'
}
}
})
op_name = service_model.operation_names[0]
output_shape = service_model.operation_model(op_name).output_shape
self.client.meta.service_model = service_model
self.stubber.add_response('TestOperation', service_response)
self.validate_parameters_mock.assert_called_with({}, output_shape)
# Make sure service response hasn't been mutated
self.assertEqual(service_response,
{'ResponseMetadata': {
'foo': 'bar'
}})
def test_validates_on_empty_output_shape(self):
service_model = ServiceModel({
'documentation': '',
'operations': {
'foo': {
'name': 'foo'
}
}
})
self.client.meta.service_model = service_model
with self.assertRaises(ParamValidationError):
self.stubber.add_response('TestOperation', {'foo': 'bar'})
def test_get_response(self):
service_response = {'bar': 'baz'}
self.stubber.add_response('foo', service_response)
self.stubber.activate()
response = self.emit_get_response_event()
self.assertEqual(response[1], service_response)
self.assertEqual(response[0].status_code, 200)
def test_get_client_error_response(self):
error_code = "foo"
service_message = "bar"
self.stubber.add_client_error('foo', error_code, service_message)
self.stubber.activate()
response = self.emit_get_response_event()
self.assertEqual(response[1]['Error']['Message'], service_message)
self.assertEqual(response[1]['Error']['Code'], error_code)
def test_get_client_error_with_extra_error_meta(self):
error_code = "foo"
error_message = "bar"
error_meta = {
"Endpoint": "https://foo.bar.baz",
}
self.stubber.add_client_error('foo',
error_code,
error_message,
http_status_code=301,
service_error_meta=error_meta)
with self.stubber:
response = self.emit_get_response_event()
error = response[1]['Error']
self.assertIn('Endpoint', error)
self.assertEqual(error['Endpoint'], "https://foo.bar.baz")
def test_get_client_error_with_extra_response_meta(self):
error_code = "foo"
error_message = "bar"
stub_response_meta = {
"RequestId": "79104EXAMPLEB723",
}
self.stubber.add_client_error('foo',
error_code,
error_message,
http_status_code=301,
response_meta=stub_response_meta)
with self.stubber:
response = self.emit_get_response_event()
actual_response_meta = response[1]['ResponseMetadata']
self.assertIn('RequestId', actual_response_meta)
self.assertEqual(actual_response_meta['RequestId'], "79104EXAMPLEB723")
def test_get_response_errors_with_no_stubs(self):
self.stubber.activate()
with self.assertRaises(UnStubbedResponseError):
self.emit_get_response_event()
def test_assert_no_responses_remaining(self):
self.stubber.add_response('foo', {})
with self.assertRaises(AssertionError):
self.stubber.assert_no_pending_responses()
class TestStubber(unittest.TestCase):
def setUp(self):
session = ibm_botocore.session.get_session()
config = ibm_botocore.config.Config(signature_version=ibm_botocore.UNSIGNED)
self.client = session.create_client('s3', config=config)
self.stubber = Stubber(self.client)
def test_stubber_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
self.stubber.activate()
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_context_manager_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
with self.stubber:
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_activated_stubber_errors_with_no_registered_stubs(self):
self.stubber.activate()
# Params one per line for readability.
with self.assertRaisesRegexp(StubResponseError,
"'Bucket': 'asdfasdfasdfasdf',\n"):
self.client.list_objects(
Bucket='asdfasdfasdfasdf',
Delimiter='asdfasdfasdfasdf',
Prefix='asdfasdfasdfasdf',
EncodingType='url')
def test_stubber_errors_when_stubs_are_used_up(self):
self.stubber.add_response('list_objects', {})
self.stubber.activate()
self.client.list_objects(Bucket='foo')
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
def test_client_error_response(self):
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_can_add_expected_params_to_client_error(self):
self.stubber.add_client_error(
'list_objects', 'Error', 'error',
expected_params={'Bucket': 'foo'}
)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_can_expected_param_fails_in_client_error(self):
self.stubber.add_client_error(
'list_objects', 'Error', 'error',
expected_params={'Bucket': 'foo'}
)
self.stubber.activate()
# We expect an AssertionError instead of a ClientError
# because we're calling the operation with the wrong
# param value.
with self.assertRaises(AssertionError):
self.client.list_objects(Bucket='wrong-argument-value')
def test_expected_params_success(self):
service_response = {}
expected_params = {'Bucket': 'foo'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should be called successfully with no errors being thrown
# for mismatching expected params.
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should call should raise an for mismatching expected params.
with self.assertRaisesRegexp(StubResponseError,
"{'Bucket': 'bar'},\n"):
self.client.list_objects(Bucket='foo')
def test_expected_params_mixed_with_errors_responses(self):
# Add an error response
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
# Add a response with incorrect expected params
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# The first call should throw and error as expected.
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
# The second call should throw an error for unexpected parameters
with self.assertRaisesRegexp(StubResponseError, 'Expected parameters'):
self.client.list_objects(Bucket='foo')
def test_can_continue_to_call_after_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for unexpected parameters
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
# The stubber should still have the responses queued up
# even though the original parameters did not match the expected ones.
self.client.list_objects(Bucket='bar')
self.stubber.assert_no_pending_responses()
def test_still_relies_on_param_validation_with_expected_params(self):
service_response = {}
expected_params = {'Buck': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(ParamValidationError):
self.client.list_objects(Buck='bar')
def test_any_ignores_param_for_validation(self):
service_response = {}
expected_params = {'Bucket': stub.ANY}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.add_response(
'list_objects', service_response, expected_params)
try:
with self.stubber:
self.client.list_objects(Bucket='foo')
self.client.list_objects(Bucket='bar')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_mixed_any_and_concrete_params(self):
service_response = {}
expected_params = {'Bucket': stub.ANY, 'Key': 'foo.txt'}
self.stubber.add_response(
'head_object', service_response, expected_params)
self.stubber.add_response(
'head_object', service_response, expected_params)
try:
with self.stubber:
self.client.head_object(Bucket='foo', Key='foo.txt')
self.client.head_object(Bucket='bar', Key='foo.txt')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_nested_any_param(self):
service_response = {}
expected_params = {
'Bucket': 'foo',
'Key': 'bar.txt',
'Metadata': {
'MyMeta': stub.ANY,
}
}
self.stubber.add_response(
'put_object', service_response, expected_params)
self.stubber.add_response(
'put_object', service_response, expected_params)
try:
with self.stubber:
self.client.put_object(
Bucket='foo',
Key='bar.txt',
Metadata={
'MyMeta': 'Foo',
}
)
self.client.put_object(
Bucket='foo',
Key='bar.txt',
Metadata={
'MyMeta': 'Bar',
}
)
except StubAssertionError:
self.fail(
"stub.ANY failed to ignore nested parameter for validation.")
def test_ANY_repr(self):
self.assertEqual(repr(stub.ANY), '<ANY>')
def test_none_param(self):
service_response = {}
expected_params = {'Buck': None}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(StubAssertionError):
self.client.list_objects(Buck='bar')
def test_many_expected_params(self):
service_response = {}
expected_params = {
'Bucket': 'mybucket',
'Prefix': 'myprefix',
'Delimiter': '/',
'EncodingType': 'url'
}
self.stubber.add_response(
'list_objects', service_response, expected_params)
try:
with self.stubber:
self.client.list_objects(**expected_params)
except StubAssertionError:
self.fail(
"Stubber inappropriately raised error for same parameters.")
class BaseTransferTest(unittest.TestCase):
def setUp(self):
self.session = ibm_boto3.session.Session(
aws_access_key_id='foo', aws_secret_access_key='bar',
region_name='us-west-2')
self.s3 = self.session.resource('s3')
self.stubber = Stubber(self.s3.meta.client)
self.bucket = 'mybucket'
self.key = 'mykey'
self.upload_id = 'uploadid'
self.etag = '"example0etag"'
self.progress = 0
self.progress_times_called = 0
def stub_head(self, content_length=4, expected_params=None):
head_response = {
'AcceptRanges': 'bytes',
'ContentLength': content_length,
'ContentType': 'binary/octet-stream',
'ETag': self.etag,
'Metadata': {},
'ResponseMetadata': {
'HTTPStatusCode': 200,
}
}
if expected_params is None:
expected_params = {
'Bucket': self.bucket,
'Key': self.key
}
self.stubber.add_response(
method='head_object', service_response=head_response,
expected_params=expected_params)
def stub_create_multipart_upload(self):
# Add the response and assert params for CreateMultipartUpload
create_upload_response = {
"Bucket": self.bucket,
"Key": self.key,
"UploadId": self.upload_id
}
expected_params = {
"Bucket": self.bucket,
"Key": self.key,
}
self.stubber.add_response(
method='create_multipart_upload',
service_response=create_upload_response,
expected_params=expected_params)
def stub_complete_multipart_upload(self, parts):
complete_upload_response = {
"Location": "us-west-2",
"Bucket": self.bucket,
"Key": self.key,
"ETag": self.etag
}
expected_params = {
"Bucket": self.bucket,
"Key": self.key,
"MultipartUpload": {
"Parts": parts
},
"UploadId": self.upload_id
}
self.stubber.add_response(
method='complete_multipart_upload',
service_response=complete_upload_response,
expected_params=expected_params)