def test_validate_ignores_response_metadata(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
service_model = ServiceModel({
'documentation': '',
'operations': {
'foo': {
'name': 'foo',
'input': {'shape': 'StringShape'},
'output': {'shape': 'StringShape'}
}
},
'shapes': {
'StringShape': {'type': 'string'}
}
})
op_name = service_model.operation_names[0]
output_shape = service_model.operation_model(op_name).output_shape
self.client.meta.service_model = service_model
self.stubber.add_response('TestOperation', service_response)
self.validate_parameters_mock.assert_called_with(
{}, output_shape)
# Make sure service response hasn't been mutated
self.assertEqual(
service_response, {'ResponseMetadata': {'foo': 'bar'}})
def _load_service_model(self, service_name, api_version=None):
json_model = self._loader.load_service_model(service_name,
'service-2',
api_version=api_version)
service_model = ServiceModel(json_model, service_name=service_name)
self._register_retries(service_model)
return service_model
def setUp(self):
self.model = {
'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01',
'jsonVersion': '1.1', 'targetPrefix': 'foo'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'TimestampType'},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
class TestCLIArgument(unittest.TestCase):
def setUp(self):
self.service_name = "baz"
self.service_model = ServiceModel(
{
"metadata": {"endpointPrefix": "bad"},
"operations": {"SampleOperation": {"name": "SampleOperation", "input": {"shape": "Input"}}},
"shapes": {
"StringShape": {"type": "string"},
"Input": {"type": "structure", "members": {"Foo": {"shape": "StringShape"}}},
},
},
self.service_name,
)
self.operation_model = self.service_model.operation_model("SampleOperation")
self.argument_model = self.operation_model.input_shape.members["Foo"]
self.event_emitter = mock.Mock()
def create_argument(self):
return arguments.CLIArgument(
self.argument_model.name, self.argument_model, self.operation_model, self.event_emitter
)
def test_unpack_uses_service_name_in_event(self):
self.event_emitter.emit.return_value = ["value"]
argument = self.create_argument()
params = {}
argument.add_to_params(params, "value")
expected_event_name = "process-cli-arg.%s.%s" % (self.service_name, "sample-operation")
actual_event_name = self.event_emitter.emit.call_args[0][0]
self.assertEqual(actual_event_name, expected_event_name)
class TestBinaryTypes(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Blob': {'shape': 'BlobType'},
}
},
'BlobType': {
'type': 'blob',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def assert_serialized_blob_equals(self, request, blob_bytes):
# This method handles all the details of the base64 decoding.
encoded = base64.b64encode(blob_bytes)
# Now the serializers actually have the base64 encoded contents
# as str types so we need to decode back. We know that this is
# ascii so it's safe to use the ascii encoding.
expected = encoded.decode('ascii')
self.assertEqual(request['body']['Blob'], expected)
def test_blob_accepts_bytes_type(self):
body = b'bytes body'
request = self.serialize_to_request(input_params={'Blob': body})
def test_blob_accepts_str_type(self):
body = u'ascii text'
request = self.serialize_to_request(input_params={'Blob': body})
self.assert_serialized_blob_equals(
request, blob_bytes=body.encode('ascii'))
def test_blob_handles_unicode_chars(self):
body = u'\u2713'
request = self.serialize_to_request(input_params={'Blob': body})
self.assert_serialized_blob_equals(
request, blob_bytes=body.encode('utf-8'))
def setUp(self):
self.model = {
'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'TimestampHeader': {
'shape': 'TimestampType',
'location': 'header',
'locationName': 'x-timestamp'
},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
def setUp(self):
self.model = {
'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Foo': {
'shape': 'FooShape',
'locationName': 'Foo'
},
},
'payload': 'Foo'
},
'FooShape': {
'type': 'list',
'member': {'shape': 'StringShape'}
},
'StringShape': {
'type': 'string',
}
}
}
self.service_model = ServiceModel(self.model)
def test_route53_resource_id(self):
event = self.session.create_event('before-parameter-build', 'route53',
'GetHostedZone')
params = {
'Id': '/hostedzone/ABC123',
'HostedZoneId': '/hostedzone/ABC123',
'ResourceId': '/hostedzone/DEF456',
'DelegationSetId': '/hostedzone/GHI789',
'Other': '/hostedzone/foo'
}
operation_def = {
'name': 'GetHostedZone',
'input': {
'shape': 'GetHostedZoneInput'
}
}
service_def = {
'metadata': {},
'shapes': {
'GetHostedZoneInput': {
'type': 'structure',
'members': {
'Id': {
'shape': 'ResourceId'
},
'HostedZoneId': {
'shape': 'ResourceId'
},
'ResourceId': {
'shape': 'ResourceId'
},
'DelegationSetId': {
'shape': 'DelegationSetId'
},
'Other': {
'shape': 'String'
}
}
},
'ResourceId': {
'type': 'string'
},
'DelegationSetId': {
'type': 'string'
},
'String': {
'type': 'string'
}
}
}
model = OperationModel(operation_def, ServiceModel(service_def))
self.session.emit(event, params=params, model=model)
self.assertEqual(params['Id'], 'ABC123')
self.assertEqual(params['HostedZoneId'], 'ABC123')
self.assertEqual(params['ResourceId'], 'DEF456')
self.assertEqual(params['DelegationSetId'], 'GHI789')
# This one should have been left alone
self.assertEqual(params['Other'], '/hostedzone/foo')
def get_model_location(session, service_definition, service_name=None):
"""Gets the path of where a service-2.json file should go in ~/.aws/models
:type session: botocore.session.Session
:param session: A session object
:type service_definition: dict
:param service_definition: The json loaded service definition
:type service_name: str
:param service_name: The service name to use. If this not provided,
this will be determined from a combination of available services
and the service definition.
:returns: The path to where are model should be placed based on
the service defintion and the current services in botocore.
"""
# Add the ServiceModel abstraction over the service json definition to
# make it easier to work with.
service_model = ServiceModel(service_definition)
# Determine the service_name if not provided
if service_name is None:
endpoint_prefix = service_model.endpoint_prefix
service_name = _get_service_name(session, endpoint_prefix)
api_version = service_model.api_version
# For the model location we only want the custom data path (~/.aws/models
# not the one set by AWS_DATA_PATH)
data_path = session.get_component('data_loader').CUSTOMER_DATA_PATH
# Use the version of the model to determine the file's naming convention.
service_model_name = ('service-%d.json' %
int(float(service_definition.get('version', '2.0'))))
return os.path.join(data_path, service_name, api_version,
service_model_name)
class TestCLIArgument(unittest.TestCase):
def setUp(self):
self.service_name = 'baz'
self.service_model = ServiceModel(
{
'metadata': {
'endpointPrefix': 'bad',
},
'operations': {
'SampleOperation': {
'name': 'SampleOperation',
'input': {
'shape': 'Input'
}
}
},
'shapes': {
'StringShape': {
'type': 'string'
},
'Input': {
'type': 'structure',
'members': {
'Foo': {
'shape': 'StringShape'
}
}
}
}
}, self.service_name)
self.operation_model = self.service_model.operation_model(
'SampleOperation')
self.argument_model = self.operation_model.input_shape.members['Foo']
self.event_emitter = mock.Mock()
def create_argument(self):
return arguments.CLIArgument(self.argument_model.name,
self.argument_model, self.operation_model,
self.event_emitter)
def test_unpack_uses_service_name_in_event(self):
self.event_emitter.emit.return_value = ['value']
argument = self.create_argument()
params = {}
argument.add_to_params(params, 'value')
expected_event_name = 'process-cli-arg.%s.%s' % (self.service_name,
'sample-operation')
actual_event_name = self.event_emitter.emit.call_args[0][0]
self.assertEqual(actual_event_name, expected_event_name)
def test_list_type_has_correct_nargs_value(self):
# We don't actually care about the values, we just need a ListArgument
# type.
arg = arguments.ListArgument(argument_model=self.argument_model,
event_emitter=self.event_emitter,
is_required=True,
name='test-nargs',
operation_model=None,
serialized_name='TestNargs')
self.assertEqual(arg.nargs, '*')
def setUp(self):
self.model = {
'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'StringTestType'},
}
},
'StringTestType': {
'type': 'string',
'min': 15
}
}
}
self.service_model = ServiceModel(self.model)
def get_service_model(self, service, api_version=None):
"""Get the service model for the service."""
with mock.patch('botocore.loaders.Loader.list_available_services',
return_value=[service]):
return ServiceModel(self.loader.load_service_model(
service, type_name='service-2', api_version=api_version),
service_name=service)
def setUp(self):
self.model = {
'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'ContentLength': {
'shape': 'Integer',
'location': 'header',
'locationName': 'Content-Length'
},
}
},
'Integer': {
'type': 'integer'
},
}
}
self.service_model = ServiceModel(self.model)
def test_resource_loads_collections(self, mock_model):
model = {
'hasMany': {
u'Queues': {
'request': {
'operation': 'ListQueues'
},
'resource': {
'type': 'Queue'
}
}
}
}
defs = {
'Queue': {}
}
service_model = ServiceModel({})
mock_model.return_value.name = 'queues'
resource = self.load('test', 'test', model, defs, service_model)()
self.assertTrue(hasattr(resource, 'queues'),
'Resource should expose queues collection')
self.assertIsInstance(resource.queues, CollectionManager,
'Queues collection should be a collection manager')
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = mock.Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = mock.Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(self.client,
cache=cache,
current_time=time)
def test_glacier_version_header_added(self):
request_dict = {
'headers': {}
}
model = ServiceModel({'metadata': {'apiVersion': '2012-01-01'}})
handlers.add_glacier_version(model, request_dict)
self.assertEqual(request_dict['headers']['x-amz-glacier-version'],
'2012-01-01')
def test_route53_resource_id_missing_input_shape(self):
event = 'before-parameter-build.route53.GetHostedZone'
params = {'HostedZoneId': '/hostedzone/ABC123'}
operation_def = {'name': 'GetHostedZone'}
service_def = {'metadata': {}, 'shapes': {}}
model = OperationModel(operation_def, ServiceModel(service_def))
self.session.emit(event, params=params, model=model)
self.assertEqual(params['HostedZoneId'], '/hostedzone/ABC123')
def load_service(service: str,
version: str = None,
model_type="service-2") -> ServiceModel:
"""
For example: load_service("sqs", "2012-11-05")
"""
service_description = loader.load_service_model(service, model_type,
version)
return ServiceModel(service_description, service)
class TestRestXMLUnicodeSerialization(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {
'protocol': 'rest-xml',
'apiVersion': '2014-01-01'
},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {
'shape': 'InputShape'
},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Foo': {
'shape': 'FooShape',
'locationName': 'Foo'
},
},
'payload': 'Foo'
},
'FooShape': {
'type': 'list',
'member': {
'shape': 'StringShape'
}
},
'StringShape': {
'type': 'string',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_restxml_serializes_unicode(self):
params = {'Foo': [u'\u65e5\u672c\u8a9e\u3067\u304a\uff4b']}
try:
self.serialize_to_request(params)
except UnicodeEncodeError:
self.fail("RestXML serializer failed to serialize unicode text.")
class TestJSONTimestampSerialization(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01',
'jsonVersion': '1.1', 'targetPrefix': 'foo'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'TimestampType'},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_accepts_iso_8601_format(self):
body = json.loads(self.serialize_to_request(
{'Timestamp': '1970-01-01T00:00:00'})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
def test_accepts_epoch(self):
body = json.loads(self.serialize_to_request(
{'Timestamp': '0'})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
# Can also be an integer 0.
body = json.loads(self.serialize_to_request(
{'Timestamp': 0})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
def test_accepts_partial_iso_format(self):
body = json.loads(self.serialize_to_request(
{'Timestamp': '1970-01-01'})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
def setUp(self):
self.model = {
"metadata": {
'endpointPrefix': 'myservice',
'serviceFullName': 'MyService',
},
'operations': {
'OperationName': {
'name':
'OperationName',
'errors': [
{
'shape': 'ExceptionMissingCode'
},
{
'shape': 'ExceptionWithModeledCode'
},
],
},
'AnotherOperationName': {
'name':
'AnotherOperationName',
'errors': [
{
'shape': 'ExceptionForAnotherOperation'
},
{
'shape': 'ExceptionWithModeledCode'
},
],
}
},
'shapes': {
'ExceptionWithModeledCode': {
'type': 'structure',
'members': {},
'error': {
'code': 'ModeledCode'
},
'exception': True,
},
'ExceptionMissingCode': {
'type': 'structure',
'members': {},
'exception': True,
},
'ExceptionForAnotherOperation': {
'type': 'structure',
'members': {},
'exception': True,
}
}
}
self.service_model = ServiceModel(self.model)
self.exceptions_factory = ClientExceptionsFactory()
class TestJSONTimestampSerialization(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01',
'jsonVersion': '1.1', 'targetPrefix': 'foo'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'TimestampType'},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_accepts_iso_8601_format(self):
body = json.loads(self.serialize_to_request(
{'Timestamp': '1970-01-01T00:00:00'})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
def test_accepts_epoch(self):
body = json.loads(self.serialize_to_request(
{'Timestamp': '0'})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
# Can also be an integer 0.
body = json.loads(self.serialize_to_request(
{'Timestamp': 0})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
def test_accepts_partial_iso_format(self):
body = json.loads(self.serialize_to_request(
{'Timestamp': '1970-01-01'})['body'].decode('utf-8'))
self.assertEqual(body['Timestamp'], 0)
def setUp(self):
self.model = {
'metadata': {
'protocol': 'rest-xml',
'apiVersion': '2014-01-01'
},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {
'shape': 'InputShape'
},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Foo': {
'shape': 'FooShape',
'locationName': 'Foo'
},
},
'payload': 'Foo'
},
'FooShape': {
'type': 'list',
'member': {
'shape': 'StringShape'
}
},
'StringShape': {
'type': 'string',
}
}
}
self.service_model = ServiceModel(self.model)
def test_decode_json_policy(self):
parsed = {
'Document': '{"foo": "foobarbaz"}',
'Other': 'bar',
}
service_def = {
'operations': {
'Foo': {
'output': {
'shape': 'PolicyOutput'
},
}
},
'shapes': {
'PolicyOutput': {
'type': 'structure',
'members': {
'Document': {
'shape': 'policyDocumentType'
},
'Other': {
'shape': 'stringType'
}
}
},
'policyDocumentType': {
'type': 'string'
},
'stringType': {
'type': 'string'
},
}
}
model = ServiceModel(service_def)
op_model = model.operation_model('Foo')
handlers.json_decode_policies(parsed, op_model)
self.assertEqual(parsed['Document'], {'foo': 'foobarbaz'})
no_document = {'Other': 'bar'}
handlers.json_decode_policies(no_document, op_model)
self.assertEqual(no_document, {'Other': 'bar'})
def setUp(self):
super(TestCollectionFactory, self).setUp()
self.client = mock.Mock()
self.client.can_paginate.return_value = False
self.parent = mock.Mock()
self.parent.meta = ResourceMeta('test', client=self.client)
self.resource_factory = ResourceFactory()
self.service_model = ServiceModel({})
self.factory = CollectionFactory()
self.load = self.factory.load_from_definition
def __init__(self, service: ServiceModel) -> None:
super().__init__(service)
# When parsing a request, we need to lookup the operation based on the HTTP method and URI.
# Therefore we create a mapping when the parser is initialized.
self.operation_lookup = defaultdict(lambda: defaultdict(OperationModel))
for operation in service.operation_names:
operation_model = service.operation_model(operation)
http = operation_model.http
if len(http) > 0:
method = http.get("method")
request_uri = http.get("requestUri")
self.operation_lookup[method][request_uri] = operation_model
def handle_shapes(service_model: ServiceModel, class_name, shapes_path):
top_level_shapes = [(service_model.shape_for(name), class_name)
for name in service_model.shape_names]
if not top_level_shapes:
return
docs_shapes_path = f'docs/{shapes_path}'
create_new_file(docs_shapes_path)
service_name = get_service_name(service_model)
all_shapes = find_all_shapes(top_level_shapes)
shape_docs = [get_shape_doc(shapes_path, shape) for shape in all_shapes]
write_lines(docs_shapes_path,
[f'# {service_name} data types'] + shape_docs)
def test_decode_json_policy(self):
parsed = {"Document": '{"foo": "foobarbaz"}', "Other": "bar"}
service_def = {
"operations": {"Foo": {"output": {"shape": "PolicyOutput"}}},
"shapes": {
"PolicyOutput": {
"type": "structure",
"members": {"Document": {"shape": "policyDocumentType"}, "Other": {"shape": "stringType"}},
},
"policyDocumentType": {"type": "string"},
"stringType": {"type": "string"},
},
}
model = ServiceModel(service_def)
op_model = model.operation_model("Foo")
handlers.json_decode_policies(parsed, op_model)
self.assertEqual(parsed["Document"], {"foo": "foobarbaz"})
no_document = {"Other": "bar"}
handlers.json_decode_policies(no_document, op_model)
self.assertEqual(no_document, {"Other": "bar"})
def __init__(self, session: Session, service_name: ServiceName):
loader = session._loader
botocore_session: BotocoreSession = session._session
service_data = botocore_session.get_service_data(
service_name.boto3_name)
self.service_name = service_name
self.service_model = ServiceModel(service_data,
service_name.boto3_name)
self._typed_dict_map: dict[str, TypeTypedDict] = {}
self._waiters_shape: Mapping[str, Any] | None = None
try:
self._waiters_shape = loader.load_service_model(
service_name.boto3_name, "waiters-2")
except UnknownServiceError:
pass
self._paginators_shape: Mapping[str, Any] | None = None
try:
self._paginators_shape = loader.load_service_model(
service_name.boto3_name, "paginators-1")
except UnknownServiceError:
pass
self._resources_shape: Mapping[str, Any] | None = None
try:
self._resources_shape = loader.load_service_model(
service_name.boto3_name, "resources-1")
except UnknownServiceError:
pass
self.logger = get_logger()
self.response_metadata_typed_dict = TypeTypedDict(
"ResponseMetadataTypeDef",
[
TypedDictAttribute("RequestId", Type.str, True),
TypedDictAttribute("HostId", Type.str, True),
TypedDictAttribute("HTTPStatusCode", Type.int, True),
TypedDictAttribute("HTTPHeaders", Type.DictStrStr, True),
TypedDictAttribute("RetryAttempts", Type.int, True),
],
)
self.proxy_operation_model = OperationModel({}, self.service_model)
def generate_service_types(output, service: ServiceModel, doc=True):
output.write("import sys\n")
output.write("from typing import Dict, List, Optional\n")
output.write("from datetime import datetime\n")
output.write("if sys.version_info >= (3, 8):\n")
output.write(" from typing import TypedDict\n")
output.write("else:\n")
output.write(" from typing_extensions import TypedDict\n")
output.write("\n")
output.write(
"from localstack.aws.api import handler, RequestContext, ServiceException, ServiceRequest"
)
output.write("\n")
# ==================================== print type declarations
nodes: Dict[str, ShapeNode] = {}
for shape_name in service.shape_names:
shape = service.shape_for(shape_name)
nodes[shape_name] = ShapeNode(service, shape)
# output.write("__all__ = [\n")
# for name in nodes.keys():
# output.write(f' "{name}",\n')
# output.write("]\n")
printed: Set[str] = set()
visited: Set[str] = set()
stack: List[str] = list(nodes.keys())
stack = sorted(stack, key=lambda name: nodes[name].get_order())
stack.reverse()
while stack:
name = stack.pop()
if name in printed:
continue
node = nodes[name]
dependencies = [dep for dep in node.dependencies if dep not in printed]
if not dependencies:
node.print_declaration(output, doc=doc)
printed.add(name)
elif name in visited:
# break out of circular dependencies
node.print_declaration(output, doc=doc, quote_types=True)
printed.add(name)
else:
stack.append(name)
stack.extend(dependencies)
visited.add(name)
class TestRestXMLUnicodeSerialization(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Foo': {
'shape': 'FooShape',
'locationName': 'Foo'
},
},
'payload': 'Foo'
},
'FooShape': {
'type': 'list',
'member': {'shape': 'StringShape'}
},
'StringShape': {
'type': 'string',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_restxml_serializes_unicode(self):
params = {
'Foo': [u'\u65e5\u672c\u8a9e\u3067\u304a\uff4b']
}
try:
self.serialize_to_request(params)
except UnicodeEncodeError:
self.fail("RestXML serializer failed to serialize unicode text.")
def __init__(self, session: Session, service_name: ServiceName):
loader = session._loader # pylint: disable=protected-access
botocore_session: BotocoreSession = session._session # pylint: disable=protected-access
service_data = botocore_session.get_service_data(
service_name.boto3_name)
self.service_name = service_name
self.service_model = ServiceModel(service_data,
service_name.boto3_name)
self._typed_dict_map: Dict[str, TypeTypedDict] = {}
self._waiters_shape: Optional[Shape] = None
try:
self._waiters_shape = loader.load_service_model(
service_name.boto3_name, "waiters-2")
except UnknownServiceError:
pass
self._paginators_shape: Optional[Shape] = None
try:
self._paginators_shape = loader.load_service_model(
service_name.boto3_name, "paginators-1")
except UnknownServiceError:
pass
self._resources_shape: Optional[Shape] = None
try:
self._resources_shape = loader.load_service_model(
service_name.boto3_name, "resources-1")
except UnknownServiceError:
pass
self.logger = get_logger()
self.response_metadata_typed_dict = TypeTypedDict(
"ResponseMetadata",
[
TypedDictAttribute("RequestId", Type.str, True),
TypedDictAttribute("HostId", Type.str, True),
TypedDictAttribute("HTTPStatusCode", Type.int, True),
TypedDictAttribute("HTTPHeaders", Type.DictStrAny, True),
TypedDictAttribute("RetryAttempts", Type.int, True),
],
)
def test_validates_on_empty_output_shape(self):
service_model = ServiceModel({
'documentation': '',
'operations': {
'foo': {
'name': 'foo'
}
}
})
self.client.meta.service_model = service_model
with self.assertRaises(ParamValidationError):
self.stubber.add_response('TestOperation', {'foo': 'bar'})
def test_decode_json_policy(self):
parsed = {
'Document': '{"foo": "foobarbaz"}',
'Other': 'bar',
}
service_def = {
'operations': {
'Foo': {
'output': {'shape': 'PolicyOutput'},
}
},
'shapes': {
'PolicyOutput': {
'type': 'structure',
'members': {
'Document': {
'shape': 'policyDocumentType'
},
'Other': {
'shape': 'stringType'
}
}
},
'policyDocumentType': {
'type': 'string'
},
'stringType': {
'type': 'string'
},
}
}
model = ServiceModel(service_def)
op_model = model.operation_model('Foo')
handlers.json_decode_policies(parsed, op_model)
self.assertEqual(parsed['Document'], {'foo': 'foobarbaz'})
no_document = {'Other': 'bar'}
handlers.json_decode_policies(no_document, op_model)
self.assertEqual(no_document, {'Other': 'bar'})
def setUp(self):
self.service_description = {
'metadata': {
'serviceFullName': 'AWS MyService',
'apiVersion': '2014-01-01',
'endpointPrefix': 'myservice',
'signatureVersion': 'v4',
'protocol': 'query'
},
'operations': {},
'shapes': {},
}
self.service_model = ServiceModel(self.service_description,
'myservice')
class TestCLIArgument(unittest.TestCase):
def setUp(self):
self.service_name = 'baz'
self.service_model = ServiceModel(
{
'metadata': {
'endpointPrefix': 'bad',
},
'operations': {
'SampleOperation': {
'name': 'SampleOperation',
'input': {
'shape': 'Input'
}
}
},
'shapes': {
'StringShape': {
'type': 'string'
},
'Input': {
'type': 'structure',
'members': {
'Foo': {
'shape': 'StringShape'
}
}
}
}
}, self.service_name)
self.operation_model = self.service_model.operation_model(
'SampleOperation')
self.argument_model = self.operation_model.input_shape.members['Foo']
self.event_emitter = mock.Mock()
def create_argument(self):
return arguments.CLIArgument(self.argument_model.name,
self.argument_model, self.operation_model,
self.event_emitter)
def test_unpack_uses_service_name_in_event(self):
self.event_emitter.emit.return_value = ['value']
argument = self.create_argument()
params = {}
argument.add_to_params(params, 'value')
expected_event_name = 'process-cli-arg.%s.%s' % (self.service_name,
'sample-operation')
actual_event_name = self.event_emitter.emit.call_args[0][0]
self.assertEqual(actual_event_name, expected_event_name)
def setUp(self):
self.model = {
'metadata': {
'protocol': 'json',
'apiVersion': '2014-01-01',
'jsonVersion': '1.1',
'targetPrefix': 'foo'
},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {
'shape': 'InputShape'
},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {
'shape': 'TimestampType'
},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
def test_no_output(self):
service_model = ServiceModel({
'operations': {
'SampleOperation': {
'name': 'SampleOperation',
'input': {'shape': 'SampleOperationInputOutput'},
}
},
'shapes': {
'SampleOperationInput': {
'type': 'structure',
'members': {}
},
'String': {
'type': 'string'
}
}
})
operation_model = service_model.operation_model('SampleOperation')
parsed = {}
self.injector.inject_attribute_value_output(
parsed=parsed, model=operation_model)
self.assertEqual(parsed, {})
def test_no_output(self):
service_model = ServiceModel({
'operations': {
'SampleOperation': {
'name': 'SampleOperation',
'input': {'shape': 'SampleOperationInputOutput'},
}
},
'shapes': {
'SampleOperationInput': {
'type': 'structure',
'members': {}
},
'String': {
'type': 'string'
}
}
})
operation_model = service_model.operation_model('SampleOperation')
parsed = {}
self.injector.inject_attribute_value_output(
parsed=parsed, model=operation_model)
assert parsed == {}
def setUp(self):
self.service_name = "baz"
self.service_model = ServiceModel(
{
"metadata": {"endpointPrefix": "bad"},
"operations": {"SampleOperation": {"name": "SampleOperation", "input": {"shape": "Input"}}},
"shapes": {
"StringShape": {"type": "string"},
"Input": {"type": "structure", "members": {"Foo": {"shape": "StringShape"}}},
},
},
self.service_name,
)
self.operation_model = self.service_model.operation_model("SampleOperation")
self.argument_model = self.operation_model.input_shape.members["Foo"]
self.event_emitter = mock.Mock()
def get_service_model(self, service_name, api_version=None):
"""Get the service model object.
:type service_name: string
:param service_name: The service name
:type api_version: string
:param api_version: The API version of the service. If none is
provided, then the latest API version will be used.
:rtype: L{botocore.model.ServiceModel}
:return: The botocore service model for the service.
"""
service_description = self.get_service_data(service_name, api_version)
return ServiceModel(service_description, service_name=service_name)
class TestBinaryTypesJSON(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {
'protocol': 'json',
'apiVersion': '2014-01-01',
'jsonVersion': '1.1',
'targetPrefix': 'foo'
},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {
'shape': 'InputShape'
},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Blob': {
'shape': 'BlobType'
},
}
},
'BlobType': {
'type': 'blob',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_blob_accepts_bytes_type(self):
body = b'bytes body'
self.serialize_to_request(input_params={'Blob': body})
def test_can_generate_operation_with_describe_prefix(self):
model = deepcopy(BASIC_MODEL)
# Swap ListCertificates for DescribeCertificates.
# We should still be able to generate auto-completion data.
model['operations']['DescribeCertificates'] = model['operations'].pop(
'ListCertificates')
service_model = ServiceModel(model)
completion_data = self.heuristic.generate_completion_descriptions(
service_model)
# Ensure we're using the swapped 'DescribeCertificates' operation.
self.assertEqual(
completion_data['resources']['Certificate']['operation'],
'DescribeCertificates')
self.assertEqual(list(sorted(completion_data['operations'])), [
'DeleteCertificate', 'DescribeCertificate', 'ExportCertificate',
'GetCertificate', 'ImportCertificate'
])
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(
self.client, cache=cache, current_time=time
)
class TestBinaryTypesJSON(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01',
'jsonVersion': '1.1', 'targetPrefix': 'foo'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Blob': {'shape': 'BlobType'},
}
},
'BlobType': {
'type': 'blob',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_blob_accepts_bytes_type(self):
body = b'bytes body'
self.serialize_to_request(input_params={'Blob': body})
class TestTimestamps(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'TimestampType'},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_accepts_datetime_object(self):
request = self.serialize_to_request(
{'Timestamp': datetime.datetime(2014, 1, 1, 12, 12, 12,
tzinfo=dateutil.tz.tzutc())})
self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z')
def test_accepts_naive_datetime_object(self):
request = self.serialize_to_request(
{'Timestamp': datetime.datetime(2014, 1, 1, 12, 12, 12)})
self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z')
def test_accepts_iso_8601_format(self):
request = self.serialize_to_request({'Timestamp': '2014-01-01T12:12:12Z'})
self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z')
def test_accepts_timestamp_without_tz_info(self):
# If a timezone/utc is not specified, assume they meant
# UTC. This is also the previous behavior from older versions
# of botocore so we want to make sure we preserve this behavior.
request = self.serialize_to_request({'Timestamp': '2014-01-01T12:12:12'})
self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z')
def test_microsecond_timestamp_without_tz_info(self):
request = self.serialize_to_request(
{'Timestamp': '2014-01-01T12:12:12.123456'})
self.assertEqual(request['body']['Timestamp'],
'2014-01-01T12:12:12.123456Z')
class TestInstanceCreation(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'StringTestType'},
}
},
'StringTestType': {
'type': 'string',
'min': 15
}
}
}
self.service_model = ServiceModel(self.model)
def assert_serialize_valid_parameter(self, request_serializer):
valid_string = 'valid_string_with_min_15_chars'
request = request_serializer.serialize_to_request(
{'Timestamp': valid_string},
self.service_model.operation_model('TestOperation'))
self.assertEqual(request['body']['Timestamp'], valid_string)
def assert_serialize_invalid_parameter(self, request_serializer):
invalid_string = 'short string'
request = request_serializer.serialize_to_request(
{'Timestamp': invalid_string},
self.service_model.operation_model('TestOperation'))
self.assertEqual(request['body']['Timestamp'], invalid_string)
def test_instantiate_without_validation(self):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'], False)
try:
self.assert_serialize_valid_parameter(request_serializer)
except ParamValidationError as e:
self.fail("Shouldn't fail serializing valid parameter without validation")
try:
self.assert_serialize_invalid_parameter(request_serializer)
except ParamValidationError as e:
self.fail("Shouldn't fail serializing invalid parameter without validation")
def test_instantiate_with_validation(self):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'], True)
try:
self.assert_serialize_valid_parameter(request_serializer)
except ParamValidationError as e:
self.fail("Shouldn't fail serializing valid parameter with validation")
with self.assertRaises(ParamValidationError):
self.assert_serialize_invalid_parameter(request_serializer)
def serialize_to_request(self, input_params):
service_model = ServiceModel(self.model)
request_serializer = serialize.create_serializer(
service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, service_model.operation_model('TestOperation'))
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest):
def setUp(self):
super(TestEndpointDiscoveryManager, self).setUp()
self.construct_manager()
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(
self.client, cache=cache, current_time=time
)
def test_injects_api_version_if_endpoint_operation(self):
model = self.service_model.operation_model('DescribeEndpoints')
params = {'headers': {}}
inject_api_version_header_if_needed(model, params)
self.assertEqual(params['headers'].get('x-amz-api-version'),
'2018-08-31')
def test_no_inject_api_version_if_not_endpoint_operation(self):
model = self.service_model.operation_model('TestDiscoveryRequired')
params = {'headers': {}}
inject_api_version_header_if_needed(model, params)
self.assertNotIn('x-amz-api-version', params['headers'])
def test_gather_identifiers(self):
params = {
'Foo': 'value1',
'Nested': {'Bar': 'value2'}
}
operation = self.service_model.operation_model('TestDiscoveryRequired')
ids = self.manager.gather_identifiers(operation, params)
self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'})
def test_gather_identifiers_none(self):
operation = self.service_model.operation_model('TestDiscovery')
ids = self.manager.gather_identifiers(operation, {})
self.assertEqual(ids, {})
def test_describe_endpoint(self):
kwargs = {
'Operation': 'FooBar',
'Identifiers': {'Foo': 'value1', 'Bar': 'value2'},
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with(**kwargs)
def test_describe_endpoint_no_input(self):
describe = self.service_description['operations']['DescribeEndpoints']
del describe['input']
self.construct_manager()
self.manager.describe_endpoint(Operation='FooBar', Identifiers={})
self.client.describe_endpoints.assert_called_with()
def test_describe_endpoint_empty_input(self):
describe = self.service_description['operations']['DescribeEndpoints']
describe['input'] = {'shape': 'EmptyStruct'}
self.construct_manager()
self.manager.describe_endpoint(Operation='FooBar', Identifiers={})
self.client.describe_endpoints.assert_called_with()
def test_describe_endpoint_ids_and_operation(self):
cache = {}
self.construct_manager(cache=cache)
ids = {'Foo': 'value1', 'Bar': 'value2'}
kwargs = {
'Operation': 'TestDiscoveryRequired',
'Identifiers': ids,
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with(**kwargs)
key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired')
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(**kwargs)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_no_ids_or_operation(self):
cache = {}
describe = self.service_description['operations']['DescribeEndpoints']
describe['input'] = {'shape': 'EmptyStruct'}
self.construct_manager(cache=cache)
self.manager.describe_endpoint(
Operation='TestDiscoveryRequired', Identifiers={}
)
self.client.describe_endpoints.assert_called_with()
key = ()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(
Operation='TestDiscoveryRequired', Identifiers={}
)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_expired_entry(self):
current_time = time.time()
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': current_time - 10}]
}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(**kwargs)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_cache_expiration(self):
def _time():
return float(0)
cache = {}
self.construct_manager(cache=cache, time=_time)
self.manager.describe_endpoint(
Operation='TestDiscoveryRequired', Identifiers={}
)
key = ()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Expiration'], float(120))
def test_delete_endpoints_present(self):
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': 0}]
}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.delete_endpoints(**kwargs)
self.assertEqual(cache, {})
def test_delete_endpoints_absent(self):
cache = {}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.delete_endpoints(**kwargs)
self.assertEqual(cache, {})
def test_describe_endpoint_optional_fails_no_cache(self):
side_effect = [ConnectionError(error=None)]
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
# This second call should be blocked as we just failed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
self.client.describe_endpoints.call_args_list == [call()]
def test_describe_endpoint_optional_fails_stale_cache(self):
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': 0}]
}
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(cache=cache, side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
# This second call shouldn't go through as we just failed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
self.client.describe_endpoints.call_args_list == [call()]
def test_describe_endpoint_required_fails_no_cache(self):
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
# This second call should go through, as we have no cache
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
describe_count = self.client.describe_endpoints.call_count
self.assertEqual(describe_count, 2)
def test_describe_endpoint_required_fails_stale_cache(self):
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': 0}]
}
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(cache=cache, side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
# We have a stale endpoint, so this shouldn't fail or force a refresh
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
self.client.describe_endpoints.call_args_list == [call()]
def test_describe_endpoint_required_force_refresh_success(self):
side_effect = [
ConnectionError(error=None),
{'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]},
]
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
# First call will fail
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.call_args_list == [call()]
# Force a refresh if the cache is empty but discovery is required
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'new.com')
def test_describe_endpoint_retries_after_failing(self):
fake_time = Mock()
fake_time.side_effect = [0, 100, 200]
side_effect = [
ConnectionError(error=None),
{'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]},
]
self.construct_manager(side_effect=side_effect, time=fake_time)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
self.client.describe_endpoints.call_args_list == [call()]
# Second time should try again as enough time has elapsed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'new.com')
class TestEndpointDiscoveryHandler(BaseEndpointDiscoveryTest):
def setUp(self):
super(TestEndpointDiscoveryHandler, self).setUp()
self.manager = Mock(spec=EndpointDiscoveryManager)
self.handler = EndpointDiscoveryHandler(self.manager)
self.service_model = ServiceModel(self.service_description)
def test_register_handler(self):
events = Mock(spec=HierarchicalEmitter)
self.handler.register(events, 'foo-bar')
events.register.assert_any_call(
'before-parameter-build.foo-bar', self.handler.gather_identifiers
)
events.register.assert_any_call(
'needs-retry.foo-bar', self.handler.handle_retries
)
events.register_first.assert_called_with(
'request-created.foo-bar', self.handler.discover_endpoint
)
def test_discover_endpoint(self):
request = AWSRequest()
request.context = {
'discovery': {'identifiers': {}}
}
self.manager.describe_endpoint.return_value = 'https://new.foo'
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'https://new.foo')
self.manager.describe_endpoint.assert_called_with(
Operation='TestOperation', Identifiers={}
)
def test_discover_endpoint_fails(self):
request = AWSRequest()
request.context = {
'discovery': {'identifiers': {}}
}
request.url = 'old.com'
self.manager.describe_endpoint.return_value = None
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'old.com')
self.manager.describe_endpoint.assert_called_with(
Operation='TestOperation', Identifiers={}
)
def test_discover_endpoint_no_protocol(self):
request = AWSRequest()
request.context = {
'discovery': {'identifiers': {}}
}
self.manager.describe_endpoint.return_value = 'new.foo'
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'https://new.foo')
self.manager.describe_endpoint.assert_called_with(
Operation='TestOperation', Identifiers={}
)
def test_inject_no_context(self):
request = AWSRequest(url='https://original.foo')
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'https://original.foo')
self.manager.describe_endpoint.assert_not_called()
def test_gather_identifiers(self):
context = {}
params = {
'Foo': 'value1',
'Nested': {'Bar': 'value2'}
}
ids = {
'Foo': 'value1',
'Bar': 'value2'
}
model = self.service_model.operation_model('TestDiscoveryRequired')
self.manager.gather_identifiers.return_value = ids
self.handler.gather_identifiers(params, model, context)
self.assertEqual(context['discovery']['identifiers'], ids)
def test_gather_identifiers_not_discoverable(self):
context = {}
model = self.service_model.operation_model('DescribeEndpoints')
self.handler.gather_identifiers({}, model, context)
self.assertEqual(context, {})
def test_discovery_disabled_but_required(self):
model = self.service_model.operation_model('TestDiscoveryRequired')
with self.assertRaises(EndpointDiscoveryRequired):
block_endpoint_discovery_required_operations(model)
def test_discovery_disabled_but_optional(self):
context = {}
model = self.service_model.operation_model('TestDiscoveryOptional')
block_endpoint_discovery_required_operations(model, context=context)
self.assertEqual(context, {})
def test_does_not_retry_no_response(self):
retry = self.handler.handle_retries(None, None, None)
self.assertIsNone(retry)
def test_does_not_retry_other_errors(self):
parsed_response = {
'ResponseMetadata': {'HTTPStatusCode': 200}
}
response = (None, parsed_response)
retry = self.handler.handle_retries(None, response, None)
self.assertIsNone(retry)
def test_does_not_retry_if_no_context(self):
request_dict = {'context': {}}
parsed_response = {
'ResponseMetadata': {'HTTPStatusCode': 421}
}
response = (None, parsed_response)
retry = self.handler.handle_retries(request_dict, response, None)
self.assertIsNone(retry)
def _assert_retries(self, parsed_response):
request_dict = {
'context': {
'discovery': {'identifiers': {}}
}
}
response = (None, parsed_response)
model = self.service_model.operation_model('TestDiscoveryOptional')
retry = self.handler.handle_retries(request_dict, response, model)
self.assertEqual(retry, 0)
self.manager.delete_endpoints.assert_called_with(
Operation='TestDiscoveryOptional', Identifiers={}
)
def test_retries_421_status_code(self):
parsed_response = {
'ResponseMetadata': {'HTTPStatusCode': 421}
}
self._assert_retries(parsed_response)
def test_retries_invalid_endpoint_exception(self):
parsed_response = {'Error': {'Code': 'InvalidEndpointException'}}
self._assert_retries(parsed_response)
class TestTimestampHeadersWithRestXML(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'TimestampHeader': {
'shape': 'TimestampType',
'location': 'header',
'locationName': 'x-timestamp'
},
}
},
'TimestampType': {
'type': 'timestamp',
}
}
}
self.service_model = ServiceModel(self.model)
def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'])
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation'))
def test_accepts_datetime_object(self):
request = self.serialize_to_request(
{'TimestampHeader': datetime.datetime(2014, 1, 1, 12, 12, 12,
tzinfo=dateutil.tz.tzutc())})
self.assertEqual(request['headers']['x-timestamp'],
'Wed, 01 Jan 2014 12:12:12 GMT')
def test_accepts_iso_8601_format(self):
request = self.serialize_to_request(
{'TimestampHeader': '2014-01-01T12:12:12+00:00'})
self.assertEqual(request['headers']['x-timestamp'],
'Wed, 01 Jan 2014 12:12:12 GMT')
def test_accepts_iso_8601_format_non_utc(self):
request = self.serialize_to_request(
{'TimestampHeader': '2014-01-01T07:12:12-05:00'})
self.assertEqual(request['headers']['x-timestamp'],
'Wed, 01 Jan 2014 12:12:12 GMT')
def test_accepts_rfc_822_format(self):
request = self.serialize_to_request(
{'TimestampHeader': 'Wed, 01 Jan 2014 12:12:12 GMT'})
self.assertEqual(request['headers']['x-timestamp'],
'Wed, 01 Jan 2014 12:12:12 GMT')
def test_accepts_unix_timestamp_integer(self):
request = self.serialize_to_request(
{'TimestampHeader': 1388578332})
self.assertEqual(request['headers']['x-timestamp'],
'Wed, 01 Jan 2014 12:12:12 GMT')
def setUp(self):
super(TestEndpointDiscoveryHandler, self).setUp()
self.manager = Mock(spec=EndpointDiscoveryManager)
self.handler = EndpointDiscoveryHandler(self.manager)
self.service_model = ServiceModel(self.service_description)
