class TestCLIArgument(unittest.TestCase): def setUp(self): self.service_name = "baz" self.service_model = ServiceModel( { "metadata": {"endpointPrefix": "bad"}, "operations": {"SampleOperation": {"name": "SampleOperation", "input": {"shape": "Input"}}}, "shapes": { "StringShape": {"type": "string"}, "Input": {"type": "structure", "members": {"Foo": {"shape": "StringShape"}}}, }, }, self.service_name, ) self.operation_model = self.service_model.operation_model("SampleOperation") self.argument_model = self.operation_model.input_shape.members["Foo"] self.event_emitter = mock.Mock() def create_argument(self): return arguments.CLIArgument( self.argument_model.name, self.argument_model, self.operation_model, self.event_emitter ) def test_unpack_uses_service_name_in_event(self): self.event_emitter.emit.return_value = ["value"] argument = self.create_argument() params = {} argument.add_to_params(params, "value") expected_event_name = "process-cli-arg.%s.%s" % (self.service_name, "sample-operation") actual_event_name = self.event_emitter.emit.call_args[0][0] self.assertEqual(actual_event_name, expected_event_name)
class TestBinaryTypes(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Blob': {'shape': 'BlobType'}, } }, 'BlobType': { 'type': 'blob', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def assert_serialized_blob_equals(self, request, blob_bytes): # This method handles all the details of the base64 decoding. encoded = base64.b64encode(blob_bytes) # Now the serializers actually have the base64 encoded contents # as str types so we need to decode back. We know that this is # ascii so it's safe to use the ascii encoding. expected = encoded.decode('ascii') self.assertEqual(request['body']['Blob'], expected) def test_blob_accepts_bytes_type(self): body = b'bytes body' request = self.serialize_to_request(input_params={'Blob': body}) def test_blob_accepts_str_type(self): body = u'ascii text' request = self.serialize_to_request(input_params={'Blob': body}) self.assert_serialized_blob_equals( request, blob_bytes=body.encode('ascii')) def test_blob_handles_unicode_chars(self): body = u'\u2713' request = self.serialize_to_request(input_params={'Blob': body}) self.assert_serialized_blob_equals( request, blob_bytes=body.encode('utf-8'))
def test_no_output(self): service_model = ServiceModel({ 'operations': { 'SampleOperation': { 'name': 'SampleOperation', 'input': { 'shape': 'SampleOperationInputOutput' }, } }, 'shapes': { 'SampleOperationInput': { 'type': 'structure', 'members': {}, }, 'String': { 'type': 'string' }, }, }) operation_model = service_model.operation_model('SampleOperation') parsed = {} self.injector.inject_attribute_value_output(parsed=parsed, model=operation_model) assert parsed == {}
def test_validate_ignores_response_metadata(self): service_response = {'ResponseMetadata': {'foo': 'bar'}} service_model = ServiceModel({ 'documentation': '', 'operations': { 'foo': { 'name': 'foo', 'input': {'shape': 'StringShape'}, 'output': {'shape': 'StringShape'} } }, 'shapes': { 'StringShape': {'type': 'string'} } }) op_name = service_model.operation_names[0] output_shape = service_model.operation_model(op_name).output_shape self.client.meta.service_model = service_model self.stubber.add_response('TestOperation', service_response) self.validate_parameters_mock.assert_called_with( {}, output_shape) # Make sure service response hasn't been mutated self.assertEqual( service_response, {'ResponseMetadata': {'foo': 'bar'}})
class TestCLIArgument(unittest.TestCase): def setUp(self): self.service_name = 'baz' self.service_model = ServiceModel( { 'metadata': { 'endpointPrefix': 'bad', }, 'operations': { 'SampleOperation': { 'name': 'SampleOperation', 'input': { 'shape': 'Input' } } }, 'shapes': { 'StringShape': { 'type': 'string' }, 'Input': { 'type': 'structure', 'members': { 'Foo': { 'shape': 'StringShape' } } } } }, self.service_name) self.operation_model = self.service_model.operation_model( 'SampleOperation') self.argument_model = self.operation_model.input_shape.members['Foo'] self.event_emitter = mock.Mock() def create_argument(self): return arguments.CLIArgument(self.argument_model.name, self.argument_model, self.operation_model, self.event_emitter) def test_unpack_uses_service_name_in_event(self): self.event_emitter.emit.return_value = ['value'] argument = self.create_argument() params = {} argument.add_to_params(params, 'value') expected_event_name = 'process-cli-arg.%s.%s' % (self.service_name, 'sample-operation') actual_event_name = self.event_emitter.emit.call_args[0][0] self.assertEqual(actual_event_name, expected_event_name) def test_list_type_has_correct_nargs_value(self): # We don't actually care about the values, we just need a ListArgument # type. arg = arguments.ListArgument(argument_model=self.argument_model, event_emitter=self.event_emitter, is_required=True, name='test-nargs', operation_model=None, serialized_name='TestNargs') self.assertEqual(arg.nargs, '*')
def test_validate_ignores_response_metadata(self): service_response = {'ResponseMetadata': {'foo': 'bar'}} service_model = ServiceModel({ 'documentation': '', 'operations': { 'foo': { 'name': 'foo', 'input': { 'shape': 'StringShape' }, 'output': { 'shape': 'StringShape' } } }, 'shapes': { 'StringShape': { 'type': 'string' } } }) op_name = service_model.operation_names[0] output_shape = service_model.operation_model(op_name).output_shape self.client.meta.service_model = service_model self.stubber.add_response('TestOperation', service_response) self.validate_parameters_mock.assert_called_with({}, output_shape) # Make sure service response hasn't been mutated self.assertEqual(service_response, {'ResponseMetadata': { 'foo': 'bar' }})
def test_RdbSerializer(self): rdb_model = { "metadata": self.rdb_model_metadata, "operations": { "RdbOperation": { "http": { "method": "POST", "requestUri": "/" }, "input": { "shape": "RdbOperationRequest" }, "name": "rdbOperation", "output": { "shape": "RdbOperationResult" } } }, "shapes": { "RdbOperationRequest": { "members": { "Parameter": { "locationName": "Parameter", "shape": "String" } }, "name": "RdbOperationRequest", "type": "structure" }, "RdbOperationResult": { "members": { "Response": { "locationName": "Response", "shape": "String" } }, "name": "RdbOperationResult", "type": "structure" }, "String": { "name": "String", "type": "string" }, } } rdb_service_model = ServiceModel(rdb_model) params = { "Parameter": "test" } rdb_serializer = serialize.RdbSerializer() res = rdb_serializer.serialize_to_request( params, rdb_service_model.operation_model("RdbOperation")) assert res["body"] == {"Action": "RdbOperation", "Parameter": "test", "Version": "2013-05-15N2013-12-16"} assert res["headers"] == {"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"} assert res["method"] == "POST" assert res["query_string"] == "" assert res["url_path"] == "/"
def test_ComputingSerializer(self): computing_model = { "metadata": self.computing_model_metadata, "operations": { "ComputingOperation": { "http": { "method": "POST", "requestUri": "/api/" }, "input": { "shape": "ComputingOperationRequest" }, "name": "ComputingOperation", "output": { "shape": "ComputingOperationResult" } } }, "shapes": { "ComputingOperationRequest": { "members": { "Parameter": { "locationName": "Parameter", "shape": "String" } }, "name": "ComputingOperationRequest", "type": "structure" }, "ComputingOperationResult": { "members": { "Response": { "locationName": "Response", "shape": "String" } }, "name": "ComputingOperationResult", "type": "structure" }, "String": { "name": "String", "type": "string" }, } } computing_service_model = ServiceModel(computing_model) params = { "Parameter": "test" } computing_serializer = serialize.ComputingSerializer() res = computing_serializer.serialize_to_request( params, computing_service_model.operation_model("ComputingOperation")) assert res["body"] == {"Action": "ComputingOperation", "Parameter": "test", "Version": "3.0"} assert res["headers"] == {"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"} assert res["method"] == "POST" assert res["query_string"] == "" assert res["url_path"] == "/api/"
class TestRestXMLUnicodeSerialization(unittest.TestCase): def setUp(self): self.model = { 'metadata': { 'protocol': 'rest-xml', 'apiVersion': '2014-01-01' }, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': { 'shape': 'InputShape' }, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Foo': { 'shape': 'FooShape', 'locationName': 'Foo' }, }, 'payload': 'Foo' }, 'FooShape': { 'type': 'list', 'member': { 'shape': 'StringShape' } }, 'StringShape': { 'type': 'string', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_restxml_serializes_unicode(self): params = {'Foo': [u'\u65e5\u672c\u8a9e\u3067\u304a\uff4b']} try: self.serialize_to_request(params) except UnicodeEncodeError: self.fail("RestXML serializer failed to serialize unicode text.")
class TestJSONTimestampSerialization(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01', 'jsonVersion': '1.1', 'targetPrefix': 'foo'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Timestamp': {'shape': 'TimestampType'}, } }, 'TimestampType': { 'type': 'timestamp', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_accepts_iso_8601_format(self): body = json.loads(self.serialize_to_request( {'Timestamp': '1970-01-01T00:00:00'})['body'].decode('utf-8')) self.assertEqual(body['Timestamp'], 0) def test_accepts_epoch(self): body = json.loads(self.serialize_to_request( {'Timestamp': '0'})['body'].decode('utf-8')) self.assertEqual(body['Timestamp'], 0) # Can also be an integer 0. body = json.loads(self.serialize_to_request( {'Timestamp': 0})['body'].decode('utf-8')) self.assertEqual(body['Timestamp'], 0) def test_accepts_partial_iso_format(self): body = json.loads(self.serialize_to_request( {'Timestamp': '1970-01-01'})['body'].decode('utf-8')) self.assertEqual(body['Timestamp'], 0)
def handle_resource_action( client_name, class_name, action, method_path, fn_name, service_model: ServiceModel, shapes_path, resource_path ): operation_model = service_model.operation_model(action.request.operation) input_shape = operation_model.input_shape has_output_shape = action.resource and action.resource.model.shape in service_model.shape_names output_shape = service_model.shape_for(action.resource.model.shape) if has_output_shape else None output_name = action.resource.model.name if action.resource else None if output_name: new_path = get_resource_path_for(output_name, resource_path) append_return_type = ' -> ' + f'[{output_name}]({new_path})' else: append_return_type = '' sub_res_var_name = None is_sub_res = is_sub_resource(resource_path) parameters = input_shape.members if input_shape else {} if is_sub_res: sub_res_name = resource_path[resource_path.rindex('/') + 1 :] sub_res_var_name = get_variable_name_for(sub_res_name) request_params = list(map(lambda x: x.target, action.request.params)) if input_shape: include_params = {name: value for name, value in input_shape.members.items() if name not in request_params} else: include_params = {} param_str = get_param_str_params(input_shape, shapes_path, include_params) else: if input_shape: include_params = { name: value for name, value in input_shape.members.items() if name in input_shape.required_members } else: include_params = {} param_str = get_param_str(input_shape, shapes_path) signature = get_signature_string( client_name, class_name, input_shape, output_shape, fn_name, param_str, shapes_path, append_return_type, sub_res_var_name, parameters, include_params, ) documentation = get_operation_documentation(operation_model, service_model) headline = f'# {fn_name} action' list_item = f'- **[{fn_name}]({method_path})**({param_str}){append_return_type}' return list_item, signature, documentation, headline
def __init__(self, service: ServiceModel) -> None: super().__init__(service) # When parsing a request, we need to lookup the operation based on the HTTP method and URI. # Therefore we create a mapping when the parser is initialized. self.operation_lookup = defaultdict(lambda: defaultdict(OperationModel)) for operation in service.operation_names: operation_model = service.operation_model(operation) http = operation_model.http if len(http) > 0: method = http.get("method") request_uri = http.get("requestUri") self.operation_lookup[method][request_uri] = operation_model
class TestRestXMLUnicodeSerialization(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Foo': { 'shape': 'FooShape', 'locationName': 'Foo' }, }, 'payload': 'Foo' }, 'FooShape': { 'type': 'list', 'member': {'shape': 'StringShape'} }, 'StringShape': { 'type': 'string', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_restxml_serializes_unicode(self): params = { 'Foo': [u'\u65e5\u672c\u8a9e\u3067\u304a\uff4b'] } try: self.serialize_to_request(params) except UnicodeEncodeError: self.fail("RestXML serializer failed to serialize unicode text.")
class TestCLIArgument(unittest.TestCase): def setUp(self): self.service_name = 'baz' self.service_model = ServiceModel( { 'metadata': { 'endpointPrefix': 'bad', }, 'operations': { 'SampleOperation': { 'name': 'SampleOperation', 'input': { 'shape': 'Input' } } }, 'shapes': { 'StringShape': { 'type': 'string' }, 'Input': { 'type': 'structure', 'members': { 'Foo': { 'shape': 'StringShape' } } } } }, self.service_name) self.operation_model = self.service_model.operation_model( 'SampleOperation') self.argument_model = self.operation_model.input_shape.members['Foo'] self.event_emitter = mock.Mock() def create_argument(self): return arguments.CLIArgument(self.argument_model.name, self.argument_model, self.operation_model, self.event_emitter) def test_unpack_uses_service_name_in_event(self): self.event_emitter.emit.return_value = ['value'] argument = self.create_argument() params = {} argument.add_to_params(params, 'value') expected_event_name = 'process-cli-arg.%s.%s' % (self.service_name, 'sample-operation') actual_event_name = self.event_emitter.emit.call_args[0][0] self.assertEqual(actual_event_name, expected_event_name)
class TestBinaryTypesJSON(unittest.TestCase): def setUp(self): self.model = { 'metadata': { 'protocol': 'json', 'apiVersion': '2014-01-01', 'jsonVersion': '1.1', 'targetPrefix': 'foo' }, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': { 'shape': 'InputShape' }, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Blob': { 'shape': 'BlobType' }, } }, 'BlobType': { 'type': 'blob', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_blob_accepts_bytes_type(self): body = b'bytes body' self.serialize_to_request(input_params={'Blob': body})
def _create_service_map(service: ServiceModel) -> Map: """ Creates a Werkzeug Map object with all rules necessary for the specific service. :param service: botocore service model to create the rules for :return: a Map instance which is used to perform the in-service operation routing """ ops = [ service.operation_model(op_name) for op_name in service.operation_names ] rules = [] # group all operations by their path and method path_index: Dict[(str, str), List[_HttpOperation]] = defaultdict(list) for op in ops: http_op = _HttpOperation.from_operation(op) path_index[(http_op.path, http_op.method)].append(http_op) # create a matching rule for each (path, method) combination for (path, method), ops in path_index.items(): # translate the requestUri to a Werkzeug rule string rule_string = _path_param_regex.sub( _transform_path_params_to_rule_vars, path) if len(ops) == 1: # if there is only a single operation for a (path, method) combination, # the default Werkzeug rule can be used directly (this is the case for most rules) op = ops[0] rules.append( _StrictMethodRule(string=rule_string, method=method, endpoint=op.operation)) # type: ignore else: # if there is an ambiguity with only the (path, method) combination, # a custom rule - which can use additional request metadata - needs to be used rules.append( _RequestMatchingRule(string=rule_string, method=method, operations=ops)) return Map( rules=rules, strict_slashes=False, merge_slashes=False, converters={"path": GreedyPathConverter}, )
def test_decode_json_policy(self): parsed = { 'Document': '{"foo": "foobarbaz"}', 'Other': 'bar', } service_def = { 'operations': { 'Foo': { 'output': { 'shape': 'PolicyOutput' }, } }, 'shapes': { 'PolicyOutput': { 'type': 'structure', 'members': { 'Document': { 'shape': 'policyDocumentType' }, 'Other': { 'shape': 'stringType' } } }, 'policyDocumentType': { 'type': 'string' }, 'stringType': { 'type': 'string' }, } } model = ServiceModel(service_def) op_model = model.operation_model('Foo') handlers.json_decode_policies(parsed, op_model) self.assertEqual(parsed['Document'], {'foo': 'foobarbaz'}) no_document = {'Other': 'bar'} handlers.json_decode_policies(no_document, op_model) self.assertEqual(no_document, {'Other': 'bar'})
def test_decode_json_policy(self): parsed = {"Document": '{"foo": "foobarbaz"}', "Other": "bar"} service_def = { "operations": {"Foo": {"output": {"shape": "PolicyOutput"}}}, "shapes": { "PolicyOutput": { "type": "structure", "members": {"Document": {"shape": "policyDocumentType"}, "Other": {"shape": "stringType"}}, }, "policyDocumentType": {"type": "string"}, "stringType": {"type": "string"}, }, } model = ServiceModel(service_def) op_model = model.operation_model("Foo") handlers.json_decode_policies(parsed, op_model) self.assertEqual(parsed["Document"], {"foo": "foobarbaz"}) no_document = {"Other": "bar"} handlers.json_decode_policies(no_document, op_model) self.assertEqual(no_document, {"Other": "bar"})
def test_decode_json_policy(self): parsed = { 'Document': '{"foo": "foobarbaz"}', 'Other': 'bar', } service_def = { 'operations': { 'Foo': { 'output': {'shape': 'PolicyOutput'}, } }, 'shapes': { 'PolicyOutput': { 'type': 'structure', 'members': { 'Document': { 'shape': 'policyDocumentType' }, 'Other': { 'shape': 'stringType' } } }, 'policyDocumentType': { 'type': 'string' }, 'stringType': { 'type': 'string' }, } } model = ServiceModel(service_def) op_model = model.operation_model('Foo') handlers.json_decode_policies(parsed, op_model) self.assertEqual(parsed['Document'], {'foo': 'foobarbaz'}) no_document = {'Other': 'bar'} handlers.json_decode_policies(no_document, op_model) self.assertEqual(no_document, {'Other': 'bar'})
class TestBinaryTypesJSON(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01', 'jsonVersion': '1.1', 'targetPrefix': 'foo'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Blob': {'shape': 'BlobType'}, } }, 'BlobType': { 'type': 'blob', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_blob_accepts_bytes_type(self): body = b'bytes body' self.serialize_to_request(input_params={'Blob': body})
def __init__(self, service: ServiceModel) -> None: super().__init__(service) # When parsing a request, we need to lookup the operation based on the HTTP method and URI. # We create a mapping when the parser is initialized. # Since the path can contain URI path parameters, the key of the dict is a regex. self.operation_lookup: DefaultDict[str, OrderedDict[ Pattern[str], OperationModel]] = defaultdict(lambda: OrderedDict()) # Extract all operation models from the service spec operation_models = [ service.operation_model(operation) for operation in service.operation_names ] # Sort the operation models descending based on their normalized request URIs. # This is necessary, to ensure that greedy regex matches do not cause wrong method lookups. # f.e. /fuu/{bar}/baz should have precedence over /fuu/{bar}. sorted_operation_models = sorted( operation_models, key=self._get_normalized_request_uri_length, reverse=True) for operation_model in sorted_operation_models: http = operation_model.http method = http.get("method") request_uri_regex = self._get_request_uri_regex(operation_model) self.operation_lookup[method][request_uri_regex] = operation_model
def test_no_output(self): service_model = ServiceModel({ 'operations': { 'SampleOperation': { 'name': 'SampleOperation', 'input': {'shape': 'SampleOperationInputOutput'}, } }, 'shapes': { 'SampleOperationInput': { 'type': 'structure', 'members': {} }, 'String': { 'type': 'string' } } }) operation_model = service_model.operation_model('SampleOperation') parsed = {} self.injector.inject_attribute_value_output( parsed=parsed, model=operation_model) self.assertEqual(parsed, {})
class TestEndpointDiscoveryHandler(BaseEndpointDiscoveryTest): def setUp(self): super(TestEndpointDiscoveryHandler, self).setUp() self.manager = mock.Mock(spec=EndpointDiscoveryManager) self.handler = EndpointDiscoveryHandler(self.manager) self.service_model = ServiceModel(self.service_description) def test_register_handler(self): events = mock.Mock(spec=HierarchicalEmitter) self.handler.register(events, 'foo-bar') events.register.assert_any_call('before-parameter-build.foo-bar', self.handler.gather_identifiers) events.register.assert_any_call('needs-retry.foo-bar', self.handler.handle_retries) events.register_first.assert_called_with( 'request-created.foo-bar', self.handler.discover_endpoint) def test_discover_endpoint(self): request = AWSRequest() request.context = {'discovery': {'identifiers': {}}} self.manager.describe_endpoint.return_value = 'https://new.foo' self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'https://new.foo') self.manager.describe_endpoint.assert_called_with( Operation='TestOperation', Identifiers={}) def test_discover_endpoint_fails(self): request = AWSRequest() request.context = {'discovery': {'identifiers': {}}} request.url = 'old.com' self.manager.describe_endpoint.return_value = None self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'old.com') self.manager.describe_endpoint.assert_called_with( Operation='TestOperation', Identifiers={}) def test_discover_endpoint_no_protocol(self): request = AWSRequest() request.context = {'discovery': {'identifiers': {}}} self.manager.describe_endpoint.return_value = 'new.foo' self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'https://new.foo') self.manager.describe_endpoint.assert_called_with( Operation='TestOperation', Identifiers={}) def test_inject_no_context(self): request = AWSRequest(url='https://original.foo') self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'https://original.foo') self.manager.describe_endpoint.assert_not_called() def test_gather_identifiers(self): context = {} params = {'Foo': 'value1', 'Nested': {'Bar': 'value2'}} ids = {'Foo': 'value1', 'Bar': 'value2'} model = self.service_model.operation_model('TestDiscoveryRequired') self.manager.gather_identifiers.return_value = ids self.handler.gather_identifiers(params, model, context) self.assertEqual(context['discovery']['identifiers'], ids) def test_gather_identifiers_not_discoverable(self): context = {} model = self.service_model.operation_model('DescribeEndpoints') self.handler.gather_identifiers({}, model, context) self.assertEqual(context, {}) def test_discovery_disabled_but_required(self): model = self.service_model.operation_model('TestDiscoveryRequired') with self.assertRaises(EndpointDiscoveryRequired): block_endpoint_discovery_required_operations(model) def test_discovery_disabled_but_optional(self): context = {} model = self.service_model.operation_model('TestDiscoveryOptional') block_endpoint_discovery_required_operations(model, context=context) self.assertEqual(context, {}) def test_does_not_retry_no_response(self): retry = self.handler.handle_retries(None, None, None) self.assertIsNone(retry) def test_does_not_retry_other_errors(self): parsed_response = {'ResponseMetadata': {'HTTPStatusCode': 200}} response = (None, parsed_response) retry = self.handler.handle_retries(None, response, None) self.assertIsNone(retry) def test_does_not_retry_if_no_context(self): request_dict = {'context': {}} parsed_response = {'ResponseMetadata': {'HTTPStatusCode': 421}} response = (None, parsed_response) retry = self.handler.handle_retries(request_dict, response, None) self.assertIsNone(retry) def _assert_retries(self, parsed_response): request_dict = {'context': {'discovery': {'identifiers': {}}}} response = (None, parsed_response) model = self.service_model.operation_model('TestDiscoveryOptional') retry = self.handler.handle_retries(request_dict, response, model) self.assertEqual(retry, 0) self.manager.delete_endpoints.assert_called_with( Operation='TestDiscoveryOptional', Identifiers={}) def test_retries_421_status_code(self): parsed_response = {'ResponseMetadata': {'HTTPStatusCode': 421}} self._assert_retries(parsed_response) def test_retries_invalid_endpoint_exception(self): parsed_response = {'Error': {'Code': 'InvalidEndpointException'}} self._assert_retries(parsed_response)
def serialize_to_request(self, input_params): service_model = ServiceModel(self.model) request_serializer = serialize.create_serializer( service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, service_model.operation_model('TestOperation'))
def generate_service_api(output, service: ServiceModel, doc=True): service_name = service.service_name.replace("-", "_") class_name = service_name + "_api" class_name = snake_to_camel_case(class_name) output.write(f"class {class_name}:\n") output.write("\n") output.write(f' service = "{service.service_name}"\n') output.write(f' version = "{service.api_version}"\n') for op_name in service.operation_names: operation: OperationModel = service.operation_model(op_name) fn_name = camel_to_snake_case(op_name) if operation.output_shape: output_shape = operation.output_shape.name else: output_shape = "None" output.write("\n") parameters = OrderedDict() param_shapes = OrderedDict() input_shape = operation.input_shape if input_shape is not None: members = list(input_shape.members) for m in input_shape.required_members: members.remove(m) m_shape = input_shape.members[m] parameters[xform_name(m)] = m_shape.name param_shapes[xform_name(m)] = m_shape for m in members: m_shape = input_shape.members[m] param_shapes[xform_name(m)] = m_shape parameters[xform_name(m)] = f"{m_shape.name} = None" if any(map(is_bad_param_name, parameters.keys())): # if we cannot render the parameter name, don't expand the parameters in the handler param_list = f"request: {input_shape.name}" if input_shape else "" output.write(f' @handler("{operation.name}", expand=False)\n') else: param_list = ", ".join([f"{k}: {v}" for k, v in parameters.items()]) output.write(f' @handler("{operation.name}")\n') output.write( f" def {fn_name}(self, context: RequestContext, {param_list}) -> {output_shape}:\n" ) # convert html documentation to rst and print it into to the signature if doc: html = operation.documentation import pypandoc doc = pypandoc.convert_text(html, "rst", format="html") output.write(' """') output.write(f"{doc.strip()}\n") output.write("\n") # parameters for param_name, shape in param_shapes.items(): # FIXME: this doesn't work properly pdoc = pypandoc.convert_text(shape.documentation, "rst", format="html") pdoc = pdoc.strip().split(".")[0] + "." output.write(f":param {param_name}: {pdoc}\n") # return value if operation.output_shape: output.write(f":returns: {operation.output_shape.name}\n") # errors for error in operation.error_shapes: output.write(f":raises {error.name}:\n") output.write(' """\n') output.write(" raise NotImplementedError\n")
class TestTimestamps(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Timestamp': {'shape': 'TimestampType'}, } }, 'TimestampType': { 'type': 'timestamp', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_accepts_datetime_object(self): request = self.serialize_to_request( {'Timestamp': datetime.datetime(2014, 1, 1, 12, 12, 12, tzinfo=dateutil.tz.tzutc())}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_accepts_naive_datetime_object(self): request = self.serialize_to_request( {'Timestamp': datetime.datetime(2014, 1, 1, 12, 12, 12)}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_accepts_iso_8601_format(self): request = self.serialize_to_request({'Timestamp': '2014-01-01T12:12:12Z'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_accepts_timestamp_without_tz_info(self): # If a timezone/utc is not specified, assume they meant # UTC. This is also the previous behavior from older versions # of botocore so we want to make sure we preserve this behavior. request = self.serialize_to_request({'Timestamp': '2014-01-01T12:12:12'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_microsecond_timestamp_without_tz_info(self): request = self.serialize_to_request( {'Timestamp': '2014-01-01T12:12:12.123456'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12.123456Z')
class TestInstanceCreation(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Timestamp': {'shape': 'StringTestType'}, } }, 'StringTestType': { 'type': 'string', 'min': 15 } } } self.service_model = ServiceModel(self.model) def assert_serialize_valid_parameter(self, request_serializer): valid_string = 'valid_string_with_min_15_chars' request = request_serializer.serialize_to_request( {'Timestamp': valid_string}, self.service_model.operation_model('TestOperation')) self.assertEqual(request['body']['Timestamp'], valid_string) def assert_serialize_invalid_parameter(self, request_serializer): invalid_string = 'short string' request = request_serializer.serialize_to_request( {'Timestamp': invalid_string}, self.service_model.operation_model('TestOperation')) self.assertEqual(request['body']['Timestamp'], invalid_string) def test_instantiate_without_validation(self): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol'], False) try: self.assert_serialize_valid_parameter(request_serializer) except ParamValidationError as e: self.fail("Shouldn't fail serializing valid parameter without validation") try: self.assert_serialize_invalid_parameter(request_serializer) except ParamValidationError as e: self.fail("Shouldn't fail serializing invalid parameter without validation") def test_instantiate_with_validation(self): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol'], True) try: self.assert_serialize_valid_parameter(request_serializer) except ParamValidationError as e: self.fail("Shouldn't fail serializing valid parameter with validation") with self.assertRaises(ParamValidationError): self.assert_serialize_invalid_parameter(request_serializer)
def test_ComputingSerializer_fix_describe_load_balancers_params_with_loadbalancer_name(self): computing_model = { "metadata": self.computing_model_metadata, "operations": { "DescribeLoadBalancers": { "http": { "method": "POST", "requestUri": "/api/" }, "input": { "shape": "DescribeLoadBalancersRequest" }, "name": "DescribeLoadBalancers", "output": { "shape": "ComputingOperationResult" } } }, "shapes": { "DescribeLoadBalancersRequest": { "members": { "LoadBalancerNames": { "locationName": "LoadBalancerNames", "shape": "ListOfRequestLoadBalancerNames" }, "Patamator": { "locationName": "Patamator", "shape": "String" } }, "name": "DescribeLoadBalancersRequest", "type": "structure" }, "ListOfRequestLoadBalancerNames": { "member": { "locationName": "member", "shape": "RequestLoadBalancerNames" }, "name": "ListOfRequestLoadBalancerNames", "type": "list" }, "RequestLoadBalancerNames": { "members": { "InstancePort": { "locationName": "InstancePort", "shape": "Integer" }, "LoadBalancerName": { "locationName": "LoadBalancerName", "shape": "String" }, "LoadBalancerPort": { "locationName": "LoadBalancerPort", "shape": "Integer" } }, "name": "RequestLoadBalancerNames", "type": "structure" }, "ComputingOperationResult": { "members": { "Response": { "locationName": "Response", "shape": "String" } }, "name": "ComputingOperationResult", "type": "structure" }, "String": { "name": "String", "type": "string" }, "Integer": { "name": "Integer", "type": "integer" }, } } computing_service_model = ServiceModel(computing_model) params = { "LoadBalancerNames": [ { "LoadBalancerName": "test_load_balancer_name", "LoadBalancerPort": "test_load_balancer_port", "InstancePort": "test_instance_port" } ] } computing_serializer = serialize.ComputingSerializer() res = computing_serializer.serialize_to_request( params, computing_service_model.operation_model("DescribeLoadBalancers")) assert res["body"] == { "Action": "DescribeLoadBalancers", "Version": "3.0", "LoadBalancerNames.member.1": "test_load_balancer_name", "LoadBalancerNames.LoadBalancerPort.1": "test_load_balancer_port", "LoadBalancerNames.InstancePort.1": "test_instance_port" } assert res["headers"] == {"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"} assert res["method"] == "POST" assert res["query_string"] == "" assert res["url_path"] == "/api/"
class TestTimestamps(unittest.TestCase): def setUp(self): self.model = { 'metadata': { 'protocol': 'query', 'apiVersion': '2014-01-01' }, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': { 'shape': 'InputShape' }, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Timestamp': { 'shape': 'TimestampType' }, } }, 'TimestampType': { 'type': 'timestamp', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_accepts_datetime_object(self): request = self.serialize_to_request({ 'Timestamp': datetime.datetime(2014, 1, 1, 12, 12, 12, tzinfo=dateutil.tz.tzutc()) }) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_accepts_naive_datetime_object(self): request = self.serialize_to_request( {'Timestamp': datetime.datetime(2014, 1, 1, 12, 12, 12)}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_accepts_iso_8601_format(self): request = self.serialize_to_request( {'Timestamp': '2014-01-01T12:12:12Z'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_accepts_timestamp_without_tz_info(self): # If a timezone/utc is not specified, assume they meant # UTC. This is also the previous behavior from older versions # of botocore so we want to make sure we preserve this behavior. request = self.serialize_to_request( {'Timestamp': '2014-01-01T12:12:12'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12Z') def test_microsecond_timestamp_without_tz_info(self): request = self.serialize_to_request( {'Timestamp': '2014-01-01T12:12:12.123456'}) self.assertEqual(request['body']['Timestamp'], '2014-01-01T12:12:12.123456Z')
class ShapeParser: """ Parser for botocore shape files. Arguments: session -- Boto3 session. service_name -- ServiceName. """ # Type map for shape types. SHAPE_TYPE_MAP: Mapping[str, FakeAnnotation] = { "integer": Type.int, "long": Type.int, "boolean": Type.bool, "double": Type.float, "float": Type.float, "timestamp": TypeSubscript(Type.Union, [Type.datetime, Type.str]), "blob": TypeSubscript(Type.Union, [Type.bytes, Type.IOBytes, TypeClass(StreamingBody)]), "blob_streaming": TypeSubscript(Type.Union, [Type.bytes, Type.IOBytes, TypeClass(StreamingBody)]), } OUTPUT_SHAPE_TYPE_MAP: Mapping[str, FakeAnnotation] = { "timestamp": Type.datetime, "blob": Type.bytes, "blob_streaming": TypeClass(StreamingBody), } # Alias map fixes added by botocore for documentation build. # https://github.com/boto/botocore/blob/develop/botocore/handlers.py#L773 # https://github.com/boto/botocore/blob/develop/botocore/handlers.py#L1055 ARGUMENT_ALIASES: dict[str, dict[str, dict[str, str]]] = { ServiceNameCatalog.cloudsearchdomain.boto3_name: { "Search": { "return": "returnFields" } }, ServiceNameCatalog.logs.boto3_name: { "CreateExportTask": { "from": "fromTime" } }, ServiceNameCatalog.ec2.boto3_name: { "*": { "Filter": "Filters" } }, ServiceNameCatalog.s3.boto3_name: { "PutBucketAcl": { "ContentMD5": "None" }, "PutBucketCors": { "ContentMD5": "None" }, "PutBucketLifecycle": { "ContentMD5": "None" }, "PutBucketLogging": { "ContentMD5": "None" }, "PutBucketNotification": { "ContentMD5": "None" }, "PutBucketPolicy": { "ContentMD5": "None" }, "PutBucketReplication": { "ContentMD5": "None" }, "PutBucketRequestPayment": { "ContentMD5": "None" }, "PutBucketTagging": { "ContentMD5": "None" }, "PutBucketVersioning": { "ContentMD5": "None" }, "PutBucketWebsite": { "ContentMD5": "None" }, "PutObjectAcl": { "ContentMD5": "None" }, }, } def __init__(self, session: Session, service_name: ServiceName): loader = session._loader botocore_session: BotocoreSession = session._session service_data = botocore_session.get_service_data( service_name.boto3_name) self.service_name = service_name self.service_model = ServiceModel(service_data, service_name.boto3_name) self._typed_dict_map: dict[str, TypeTypedDict] = {} self._waiters_shape: Mapping[str, Any] | None = None try: self._waiters_shape = loader.load_service_model( service_name.boto3_name, "waiters-2") except UnknownServiceError: pass self._paginators_shape: Mapping[str, Any] | None = None try: self._paginators_shape = loader.load_service_model( service_name.boto3_name, "paginators-1") except UnknownServiceError: pass self._resources_shape: Mapping[str, Any] | None = None try: self._resources_shape = loader.load_service_model( service_name.boto3_name, "resources-1") except UnknownServiceError: pass self.logger = get_logger() self.response_metadata_typed_dict = TypeTypedDict( "ResponseMetadataTypeDef", [ TypedDictAttribute("RequestId", Type.str, True), TypedDictAttribute("HostId", Type.str, True), TypedDictAttribute("HTTPStatusCode", Type.int, True), TypedDictAttribute("HTTPHeaders", Type.DictStrStr, True), TypedDictAttribute("RetryAttempts", Type.int, True), ], ) self.proxy_operation_model = OperationModel({}, self.service_model) def _get_operation(self, name: str) -> OperationModel: return self.service_model.operation_model(name) def _get_operation_names(self) -> list[str]: return list(self.service_model.operation_names) def _get_paginator(self, name: str) -> dict[str, Any]: if not self._paginators_shape: raise ShapeParserError(f"Unknown paginator: {name}") try: return self._paginators_shape["pagination"][name] except KeyError as e: raise ShapeParserError(f"Unknown paginator: {name}") from e def _get_service_resource(self) -> dict[str, Any]: if not self._resources_shape: raise ShapeParserError("Resource shape not found") return self._resources_shape["service"] def _get_resource_shape(self, name: str) -> dict[str, Any]: if not self._resources_shape: raise ShapeParserError("Resource shape not found") try: return self._resources_shape["resources"][name] except KeyError as e: raise ShapeParserError(f"Unknown resource: {name}") from e def get_paginator_names(self) -> list[str]: """ Get available paginator names. Returns: A list of paginator names. """ result: list[str] = [] if self._paginators_shape: for name in self._paginators_shape.get("pagination", []): result.append(name) result.sort() return result def _get_argument_alias(self, operation_name: str, argument_name: str) -> str: service_map = self.ARGUMENT_ALIASES.get(self.service_name.boto3_name) if not service_map: return argument_name operation_map: dict[str, str] = {} if "*" in service_map: operation_map = service_map["*"] if operation_name in service_map: operation_map = service_map[operation_name] if not operation_map: return argument_name if argument_name not in operation_map: return argument_name return operation_map[argument_name] def _parse_arguments( self, class_name: str, method_name: str, operation_name: str, shape: StructureShape, exclude_names: Iterable[str] = tuple(), optional_only: bool = False, ) -> list[Argument]: result: list[Argument] = [] required = shape.required_members for argument_name, argument_shape in shape.members.items(): if argument_name in exclude_names: continue argument_alias = self._get_argument_alias(operation_name, argument_name) if argument_alias == "None": continue argument_type_stub = get_method_type_stub(self.service_name, class_name, method_name, argument_name) if argument_type_stub is Type.RemoveArgument: continue if argument_type_stub is not None: argument_type = argument_type_stub else: argument_type = self.parse_shape(argument_shape) argument = Argument(argument_alias, argument_type) if argument_name not in required: argument.default = Type.Ellipsis if optional_only and argument.required: continue # FIXME: https://github.com/boto/boto3/issues/2813 # if not argument.required and argument.type_annotation: # argument.type_annotation = Type.get_optional(argument.type_annotation) result.append(argument) result.sort(key=lambda x: not x.required) return result def _parse_return_type(self, class_name: str, method_name: str, shape: Shape | None) -> FakeAnnotation: argument_type_stub = get_method_type_stub(self.service_name, class_name, method_name, "return") if argument_type_stub is not None: return argument_type_stub if shape: return self.parse_shape(shape, output=True) return Type.none @staticmethod def _get_kw_flags(method_name: str, arguments: Sequence[Argument]) -> list[Argument]: if len(arguments) and not method_name[0].isupper(): return [Argument.kwflag()] return [] def get_client_method_map(self) -> dict[str, Method]: """ Get client methods from shape. Returns: A map of method name to Method. """ result: dict[str, Method] = { "can_paginate": Method( "can_paginate", [Argument("self", None), Argument("operation_name", Type.str)], Type.bool, ), "generate_presigned_url": Method( "generate_presigned_url", [ Argument("self", None), Argument("ClientMethod", Type.str), Argument("Params", Type.MappingStrAny, Type.Ellipsis), Argument("ExpiresIn", Type.int, TypeConstant(3600)), Argument("HttpMethod", Type.str, Type.Ellipsis), ], Type.str, ), } for operation_name in self._get_operation_names(): operation_model = self._get_operation(operation_name) arguments: list[Argument] = [Argument("self", None)] method_name = xform_name(operation_name) if operation_model.input_shape is not None: shape_arguments = self._parse_arguments( "Client", method_name, operation_name, operation_model.input_shape, ) arguments.extend( self._get_kw_flags(method_name, shape_arguments)) arguments.extend(shape_arguments) return_type = self._parse_return_type("Client", method_name, operation_model.output_shape) method = Method(name=method_name, arguments=arguments, return_type=return_type) if operation_model.input_shape: method.request_type_annotation = method.get_request_type_annotation( self._get_typed_dict_name(operation_model.input_shape, postfix="Request")) result[method.name] = method return result @staticmethod def _get_typed_dict_name(shape: Shape, postfix: str = "") -> str: return f"{shape.name}{postfix}TypeDef" def _parse_shape_string(self, shape: StringShape) -> FakeAnnotation: if not shape.enum: return Type.str literal_name = f"{shape.name}Type" literal_type_stub = get_literal_type_stub(self.service_name, literal_name) if literal_type_stub: return literal_type_stub return TypeLiteral(f"{shape.name}Type", [option for option in shape.enum]) def _parse_shape_map( self, shape: MapShape, output_child: bool = False, is_streaming: bool = False, ) -> FakeAnnotation: type_subscript = TypeSubscript( Type.Dict) if output_child else TypeSubscript(Type.Mapping) if shape.key: type_subscript.add_child( self.parse_shape(shape.key, output_child=output_child, is_streaming=is_streaming)) else: type_subscript.add_child(Type.str) if shape.value: type_subscript.add_child( self.parse_shape(shape.value, output_child=output_child, is_streaming=is_streaming)) else: type_subscript.add_child(Type.Any) return type_subscript def _parse_shape_structure( self, shape: StructureShape, output: bool = False, output_child: bool = False, is_streaming: bool = False, ) -> FakeAnnotation: if not shape.members.items(): return Type.DictStrAny if output_child else Type.MappingStrAny required = shape.required_members typed_dict_name = self._get_typed_dict_name(shape) shape_type_stub = get_shape_type_stub(self.service_name, typed_dict_name) if shape_type_stub: return shape_type_stub typed_dict = TypeTypedDict(typed_dict_name) if typed_dict.name in self._typed_dict_map: old_typed_dict = self._typed_dict_map[typed_dict.name] child_names = {i.name for i in old_typed_dict.children} if output and "ResponseMetadata" in child_names: return self._typed_dict_map[typed_dict.name] if not output and "ResponseMetadata" not in child_names: return self._typed_dict_map[typed_dict.name] if output: typed_dict.name = self._get_typed_dict_name( shape, postfix="ResponseMetadata") self.logger.debug( f"Marking {typed_dict.name} as ResponseMetadataTypeDef") else: old_typed_dict.name = self._get_typed_dict_name( shape, postfix="ResponseMetadata") self._typed_dict_map[old_typed_dict.name] = old_typed_dict self.logger.debug( f"Marking {old_typed_dict.name} as ResponseMetadataTypeDef" ) self._typed_dict_map[typed_dict.name] = typed_dict for attr_name, attr_shape in shape.members.items(): typed_dict.add_attribute( attr_name, self.parse_shape( attr_shape, output_child=output or output_child, is_streaming=is_streaming, ), attr_name in required, ) if output: self._make_output_typed_dict(typed_dict) return typed_dict def _make_output_typed_dict(self, typed_dict: TypeTypedDict) -> None: for attribute in typed_dict.children: attribute.required = True child_names = {i.name for i in typed_dict.children} if "ResponseMetadata" not in child_names: typed_dict.add_attribute( "ResponseMetadata", self.response_metadata_typed_dict, True, ) def _parse_shape_list(self, shape: ListShape, output_child: bool = False) -> FakeAnnotation: type_subscript = TypeSubscript( Type.List) if output_child else TypeSubscript(Type.Sequence) if shape.member: type_subscript.add_child( self.parse_shape(shape.member, output_child=output_child)) else: type_subscript.add_child(Type.Any) return type_subscript def parse_shape( self, shape: Shape, output: bool = False, output_child: bool = False, is_streaming: bool = False, ) -> FakeAnnotation: """ Parse any botocore shape to TypeAnnotation. Arguments: shape -- Botocore shape. output -- Whether shape should use strict output types. output_child -- Whether shape parent is marked as output. is_streaming -- Whether shape should be streaming. Returns: TypeAnnotation or similar class. """ if not is_streaming: is_streaming = "streaming" in shape.serialization and shape.serialization[ "streaming"] if output or output_child: is_streaming = self.proxy_operation_model._get_streaming_body( shape) is not None # type: ignore type_name = shape.type_name if is_streaming and type_name == "blob": type_name = "blob_streaming" if output or output_child: if type_name in self.OUTPUT_SHAPE_TYPE_MAP: return self.OUTPUT_SHAPE_TYPE_MAP[type_name] if type_name in self.SHAPE_TYPE_MAP: return self.SHAPE_TYPE_MAP[type_name] if isinstance(shape, StringShape): return self._parse_shape_string(shape) if isinstance(shape, MapShape): return self._parse_shape_map( shape, output_child=output or output_child, is_streaming=is_streaming, ) if isinstance(shape, StructureShape): return self._parse_shape_structure( shape, output=output, output_child=output or output_child, is_streaming=is_streaming, ) if isinstance(shape, ListShape): return self._parse_shape_list(shape, output_child=output or output_child) if self._resources_shape and shape.type_name in self._resources_shape[ "resources"]: return AliasInternalImport(shape.type_name) self.logger.warning(f"Unknown shape: {shape} {type_name}") return Type.Any def get_paginate_method(self, paginator_name: str) -> Method: """ Get Paginator `paginate` method. Arguments: paginator_name -- Paginator name. Returns: Method. """ operation_name = paginator_name paginator_shape = self._get_paginator(paginator_name) operation_shape = self._get_operation(operation_name) skip_argument_names: list[str] = [] input_token = paginator_shape["input_token"] if isinstance(input_token, list): skip_argument_names.extend(input_token) else: skip_argument_names.append(input_token) if "limit_key" in paginator_shape: skip_argument_names.append(paginator_shape["limit_key"]) arguments: list[Argument] = [Argument("self", None)] if operation_shape.input_shape is not None: shape_arguments = self._parse_arguments( "Paginator", "paginate", operation_name, operation_shape.input_shape, exclude_names=skip_argument_names, ) shape_arguments.append( Argument("PaginationConfig", paginator_config_type, Type.Ellipsis)) arguments.extend(self._get_kw_flags("paginate", shape_arguments)) arguments.extend(shape_arguments) return_type: FakeAnnotation = Type.none if operation_shape.output_shape is not None: page_iterator_import = InternalImport("_PageIterator", stringify=False) return_item = self._parse_return_type("Paginator", "paginate", operation_shape.output_shape) return_type = TypeSubscript(page_iterator_import, [return_item]) return Method("paginate", arguments, return_type) def get_wait_method(self, waiter_name: str) -> Method: """ Get Waiter `wait` method. Arguments: waiter_name -- Waiter name. Returns: Method. """ if not self._waiters_shape: raise ShapeParserError("Waiter not found") operation_name = self._waiters_shape["waiters"][waiter_name][ "operation"] operation_shape = self._get_operation(operation_name) arguments: list[Argument] = [Argument("self", None)] if operation_shape.input_shape is not None: shape_arguments = self._parse_arguments( "Waiter", "wait", operation_name, operation_shape.input_shape) shape_arguments.append( Argument("WaiterConfig", waiter_config_type, Type.Ellipsis)) arguments.extend(self._get_kw_flags("wait", shape_arguments)) arguments.extend(shape_arguments) return Method(name="wait", arguments=arguments, return_type=Type.none) def get_service_resource_method_map(self) -> dict[str, Method]: """ Get methods for ServiceResource. Returns: A map of method name to Method. """ result: dict[str, Method] = { "get_available_subresources": Method( "get_available_subresources", [Argument("self", None)], TypeSubscript(Type.Sequence, [Type.str]), ), } service_resource_shape = self._get_service_resource() for action_name, action_shape in service_resource_shape.get( "actions", {}).items(): method = self._get_resource_method("ServiceResource", action_name, action_shape) result[method.name] = method return result def get_resource_method_map(self, resource_name: str) -> dict[str, Method]: """ Get methods for Resource. Arguments: resource_name -- Resource name. Returns: A map of method name to Method. """ resource_shape = self._get_resource_shape(resource_name) result: dict[str, Method] = { "get_available_subresources": Method( "get_available_subresources", [Argument("self", None)], TypeSubscript(Type.Sequence, [Type.str]), ), "load": Method("load", [Argument("self", None)], Type.none), "reload": Method("reload", [Argument("self", None)], Type.none), } for action_name, action_shape in resource_shape.get("actions", {}).items(): method = self._get_resource_method(resource_name, action_name, action_shape) result[method.name] = method for waiter_name in resource_shape.get("waiters", {}): method = Method( f"wait_until_{xform_name(waiter_name)}", [Argument("self", None)], Type.none, ) result[method.name] = method return result @staticmethod def _get_arg_from_target(target: str) -> str: if "[" not in target: return target return target.split("[")[0] def _get_resource_method(self, resource_name: str, action_name: str, action_shape: dict[str, Any]) -> Method: return_type: FakeAnnotation = Type.none method_name = xform_name(action_name) arguments: list[Argument] = [Argument("self", None)] if "resource" in action_shape: return_type = self._parse_return_type( resource_name, method_name, Shape("resource", action_shape["resource"])) path = action_shape["resource"].get("path", "") if path.endswith("[]"): return_type = TypeSubscript(Type.List, [return_type]) operation_shape = None if "request" in action_shape: operation_name = action_shape["request"]["operation"] operation_shape = self._get_operation(operation_name) skip_argument_names = { self._get_arg_from_target(i["target"]) for i in action_shape["request"].get("params", {}) if i["source"] == "identifier" } if operation_shape.input_shape is not None: shape_arguments = self._parse_arguments( resource_name, method_name, operation_name, operation_shape.input_shape, exclude_names=skip_argument_names, ) arguments.extend( self._get_kw_flags(method_name, shape_arguments)) arguments.extend(shape_arguments) if operation_shape.output_shape is not None and return_type is Type.none: operation_return_type = self.parse_shape( operation_shape.output_shape, output=True) return_type = operation_return_type method = Method(name=method_name, arguments=arguments, return_type=return_type) if operation_shape and operation_shape.input_shape is not None: method.request_type_annotation = method.get_request_type_annotation( self._get_typed_dict_name(operation_shape.input_shape, postfix=resource_name)) return method def get_collection_filter_method(self, name: str, collection: Collection, self_type: FakeAnnotation) -> Method: """ Get `filter` classmethod for Resource collection. Arguments: name -- Collection record name. collection -- Boto3 Collection. class_type -- Collection class type annotation. Returns: Filter Method record. """ result = Method( name="filter", arguments=[Argument("self", None)], return_type=self_type, ) if not collection.request: return result operation_name = collection.request.operation operation_model = self._get_operation(operation_name) if operation_model.input_shape is not None: shape_arguments = self._parse_arguments( name, result.name, operation_name, operation_model.input_shape, optional_only=True, ) result.arguments.extend( self._get_kw_flags(result.name, shape_arguments)) result.arguments.extend(shape_arguments) return result def get_collection_batch_methods(self, name: str, collection: Collection) -> list[Method]: """ Get batch operations for Resource collection. Arguments: name -- Collection record name. collection -- Boto3 Collection. class_type -- Collection self type annotation. Returns: List of Method records. """ result = [] for batch_action in collection.batch_actions: method = Method( name=batch_action.name, arguments=[Argument("self", None)], return_type=Type.none, ) result.append(method) if batch_action.request: operation_name = batch_action.request.operation operation_model = self._get_operation(operation_name) if operation_model.input_shape is not None: shape_arguments = self._parse_arguments( name, batch_action.name, operation_name, operation_model.input_shape, optional_only=True, ) method.arguments.extend( self._get_kw_flags(batch_action.name, shape_arguments)) method.arguments.extend(shape_arguments) if operation_model.output_shape is not None: item_return_type = self.parse_shape( operation_model.output_shape, output=True) return_type = TypeSubscript(Type.List, [item_return_type]) method.return_type = return_type return result
def handle_method(fragment): if fragment["Type"] != "AWS::ApiGateway::Method": response_string = "Macro only supports \"AWS::ApiGateway::Method\", user supplied {}" raise InvalidTypeException(response_string.format(fragment["Type"])) service_name = fragment["Properties"]["Integration"].pop("Service").lower() action = fragment["Properties"]["Integration"].pop("Action") response_maps = fragment["Properties"]["Integration"].pop("ResponseMaps") try: fragment.pop("Fn::Transform") except: pass loader = Loader() service_description = loader.load_service_model(service_name=service_name, type_name='service-2') service_model = ServiceModel(service_description) protocol = service_model.protocol op_model = service_model.operation_model(action["Name"]) request_parameters = action.get("Parameters", {}) params = dict(ChainMap(*request_parameters)) print("params: {}".format(params)) serializer = create_serializer(protocol) response_parser = create_parser(protocol) print(service_model.protocol) request = serializer.serialize_to_request(params, op_model) request_object = AWSRequest( method=request['method'], url=get_endpoint(service_model.service_name), data=request['body'], headers=request['headers']) X = request_object.prepare() print("Raw request: {}".format(request)) print("Prepared request: {}".format(X)) integration = fragment["Properties"]["Integration"] new_integration = integration_template() # Copy the existing values to the new template for entry in integration.keys(): new_integration[entry] = integration[entry] # Add headers to cfn template if X.headers is not None and callable(getattr(X.headers, "keys", None)): for header in X.headers.keys(): if header.lower() != 'Content-Length'.lower(): new_integration["RequestParameters"].update({"integration.request.header.{}".format(header): "'{}'".format(X.headers[header])}) # Add Query Strings to cfn template if 'query_string' in request and callable(getattr(request['query_string'], "keys", None)): for query in request['query_string'].keys(): new_integration["RequestParameters"].update({"integration.request.querystring.{}".format(query): "'{}'".format(request['query_string'][query])}) # Set the body if isinstance(X.body, str): new_integration["RequestTemplates"]["application/json"] = X.body else: new_integration["RequestTemplates"]["application/json"] = str(X.body, "utf-8") if X.body else '' new_integration["Uri"] = ":".join([ "arn", "aws", "apigateway", REGION, service_model.endpoint_prefix, "path/" + request["url_path"] ]) new_integration["IntegrationHttpMethod"] = X.method fragment["Properties"]["Integration"] = new_integration print(fragment) return fragment
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest): def setUp(self): super(TestEndpointDiscoveryManager, self).setUp() self.construct_manager() def construct_manager(self, cache=None, time=None, side_effect=None): self.service_model = ServiceModel(self.service_description) self.meta = Mock(spec=ClientMeta) self.meta.service_model = self.service_model self.client = Mock() if side_effect is None: side_effect = [{ 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }] self.client.describe_endpoints.side_effect = side_effect self.client.meta = self.meta self.manager = EndpointDiscoveryManager( self.client, cache=cache, current_time=time ) def test_injects_api_version_if_endpoint_operation(self): model = self.service_model.operation_model('DescribeEndpoints') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertEqual(params['headers'].get('x-amz-api-version'), '2018-08-31') def test_no_inject_api_version_if_not_endpoint_operation(self): model = self.service_model.operation_model('TestDiscoveryRequired') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertNotIn('x-amz-api-version', params['headers']) def test_gather_identifiers(self): params = { 'Foo': 'value1', 'Nested': {'Bar': 'value2'} } operation = self.service_model.operation_model('TestDiscoveryRequired') ids = self.manager.gather_identifiers(operation, params) self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'}) def test_gather_identifiers_none(self): operation = self.service_model.operation_model('TestDiscovery') ids = self.manager.gather_identifiers(operation, {}) self.assertEqual(ids, {}) def test_describe_endpoint(self): kwargs = { 'Operation': 'FooBar', 'Identifiers': {'Foo': 'value1', 'Bar': 'value2'}, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) def test_describe_endpoint_no_input(self): describe = self.service_description['operations']['DescribeEndpoints'] del describe['input'] self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_empty_input(self): describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_ids_and_operation(self): cache = {} self.construct_manager(cache=cache) ids = {'Foo': 'value1', 'Bar': 'value2'} kwargs = { 'Operation': 'TestDiscoveryRequired', 'Identifiers': ids, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired') self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_no_ids_or_operation(self): cache = {} describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager(cache=cache) self.manager.describe_endpoint( Operation='TestDiscoveryRequired', Identifiers={} ) self.client.describe_endpoints.assert_called_with() key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint( Operation='TestDiscoveryRequired', Identifiers={} ) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_expired_entry(self): current_time = time.time() key = () cache = { key: [{'Address': 'old.com', 'Expiration': current_time - 10}] } self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with() self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_cache_expiration(self): def _time(): return float(0) cache = {} self.construct_manager(cache=cache, time=_time) self.manager.describe_endpoint( Operation='TestDiscoveryRequired', Identifiers={} ) key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Expiration'], float(120)) def test_delete_endpoints_present(self): key = () cache = { key: [{'Address': 'old.com', 'Expiration': 0}] } self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_delete_endpoints_absent(self): cache = {} self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_describe_endpoint_optional_fails_no_cache(self): side_effect = [ConnectionError(error=None)] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) # This second call should be blocked as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [call()] def test_describe_endpoint_optional_fails_stale_cache(self): key = () cache = { key: [{'Address': 'old.com', 'Expiration': 0}] } side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # This second call shouldn't go through as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [call()] def test_describe_endpoint_required_fails_no_cache(self): side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) # This second call should go through, as we have no cache with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) describe_count = self.client.describe_endpoints.call_count self.assertEqual(describe_count, 2) def test_describe_endpoint_required_fails_stale_cache(self): key = () cache = { key: [{'Address': 'old.com', 'Expiration': 0}] } side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # We have a stale endpoint, so this shouldn't fail or force a refresh endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [call()] def test_describe_endpoint_required_force_refresh_success(self): side_effect = [ ConnectionError(error=None), {'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }]}, ] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} # First call will fail with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.call_args_list == [call()] # Force a refresh if the cache is empty but discovery is required endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com') def test_describe_endpoint_retries_after_failing(self): fake_time = Mock() fake_time.side_effect = [0, 100, 200] side_effect = [ ConnectionError(error=None), {'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }]}, ] self.construct_manager(side_effect=side_effect, time=fake_time) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [call()] # Second time should try again as enough time has elapsed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com')
def test_ScriptSerializer(self): script_model = { "metadata": self.script_model_metadata, "operations": { "ExecuteScript": { "http": { "method": "POST", "requestUri": "/2015-09-01/" }, "input": { "shape": "ExecuteScriptRequest" }, "name": "ExecuteScript", "output": { "resultWrapper": "ExecuteScriptResult", "shape": "ExecuteScriptResult" } } }, "shapes": { "ExecuteScriptRequest": { "members": { "Body": { "locationName": "Body", "shape": "String" }, "Header": { "locationName": "Header", "shape": "String" }, "Method": { "locationName": "Method", "shape": "String" }, "Query": { "locationName": "Query", "shape": "String" }, "ScriptIdentifier": { "locationName": "ScriptIdentifier", "shape": "String" } }, "name": "ExecuteScriptRequest", "required": ["Method", "ScriptIdentifier"], "type": "structure" }, "ExecuteScriptResult": { "members": { "Response": { "locationName": "Response", "shape": "String" } }, "name": "ExecuteScriptResult", "type": "structure" }, "String": { "name": "String", "type": "string" }, } } script_service_model = ServiceModel(script_model) params = { "Body": "test_body", "Header": "test_header", "Method": "test_method", "Query": "test_query", "ScriptIdentifier": "test_script_identifier" } script_serializer = serialize.ScriptSerializer() res = script_serializer.serialize_to_request( params, script_service_model.operation_model("ExecuteScript")) assert res["body"] == { "Body": "test_body", "Header": "test_header", "Method": "test_method", "Query": "test_query", "ScriptIdentifier": "test_script_identifier" } assert res["headers"] == { "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", "X-Amz-Target": "2015-09-01.ExecuteScript" } assert res["method"] == "POST" assert res["query_string"] == "" assert res["url_path"] == "/2015-09-01/"
class TestTimestampHeadersWithRestXML(unittest.TestCase): def setUp(self): self.model = { 'metadata': { 'protocol': 'rest-xml', 'apiVersion': '2014-01-01' }, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': { 'shape': 'InputShape' }, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'TimestampHeader': { 'shape': 'TimestampType', 'location': 'header', 'locationName': 'x-timestamp' }, } }, 'TimestampType': { 'type': 'timestamp', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_accepts_datetime_object(self): request = self.serialize_to_request({ 'TimestampHeader': datetime.datetime(2014, 1, 1, 12, 12, 12, tzinfo=dateutil.tz.tzutc()) }) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_iso_8601_format(self): request = self.serialize_to_request( {'TimestampHeader': '2014-01-01T12:12:12+00:00'}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_iso_8601_format_non_utc(self): request = self.serialize_to_request( {'TimestampHeader': '2014-01-01T07:12:12-05:00'}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_rfc_822_format(self): request = self.serialize_to_request( {'TimestampHeader': 'Wed, 01 Jan 2014 12:12:12 GMT'}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_unix_timestamp_integer(self): request = self.serialize_to_request({'TimestampHeader': 1388578332}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT')
class TestBinaryTypes(unittest.TestCase): def setUp(self): self.model = { 'metadata': { 'protocol': 'query', 'apiVersion': '2014-01-01' }, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': { 'shape': 'InputShape' }, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Blob': { 'shape': 'BlobType' }, } }, 'BlobType': { 'type': 'blob', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def assert_serialized_blob_equals(self, request, blob_bytes): # This method handles all the details of the base64 decoding. encoded = base64.b64encode(blob_bytes) # Now the serializers actually have the base64 encoded contents # as str types so we need to decode back. We know that this is # ascii so it's safe to use the ascii encoding. expected = encoded.decode('ascii') self.assertEqual(request['body']['Blob'], expected) def test_blob_accepts_bytes_type(self): body = b'bytes body' request = self.serialize_to_request(input_params={'Blob': body}) def test_blob_accepts_str_type(self): body = u'ascii text' request = self.serialize_to_request(input_params={'Blob': body}) self.assert_serialized_blob_equals(request, blob_bytes=body.encode('ascii')) def test_blob_handles_unicode_chars(self): body = u'\u2713' request = self.serialize_to_request(input_params={'Blob': body}) self.assert_serialized_blob_equals(request, blob_bytes=body.encode('utf-8'))
class TestInstanceCreation(unittest.TestCase): def setUp(self): self.model = { 'metadata': { 'protocol': 'query', 'apiVersion': '2014-01-01' }, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': { 'shape': 'InputShape' }, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'Timestamp': { 'shape': 'StringTestType' }, } }, 'StringTestType': { 'type': 'string', 'min': 15 } } } self.service_model = ServiceModel(self.model) def assert_serialize_valid_parameter(self, request_serializer): valid_string = 'valid_string_with_min_15_chars' request = request_serializer.serialize_to_request( {'Timestamp': valid_string}, self.service_model.operation_model('TestOperation')) self.assertEqual(request['body']['Timestamp'], valid_string) def assert_serialize_invalid_parameter(self, request_serializer): invalid_string = 'short string' request = request_serializer.serialize_to_request( {'Timestamp': invalid_string}, self.service_model.operation_model('TestOperation')) self.assertEqual(request['body']['Timestamp'], invalid_string) def test_instantiate_without_validation(self): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol'], False) try: self.assert_serialize_valid_parameter(request_serializer) except ParamValidationError as e: self.fail( "Shouldn't fail serializing valid parameter without validation" ) try: self.assert_serialize_invalid_parameter(request_serializer) except ParamValidationError as e: self.fail( "Shouldn't fail serializing invalid parameter without validation" ) def test_instantiate_with_validation(self): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol'], True) try: self.assert_serialize_valid_parameter(request_serializer) except ParamValidationError as e: self.fail( "Shouldn't fail serializing valid parameter with validation") with self.assertRaises(ParamValidationError): self.assert_serialize_invalid_parameter(request_serializer)
def test_EssSerializer_GetDeliveryLog(self): ess_model = { "metadata": self.ess_model_metadata, "operations": { "GetDeliveryLog": { "http": { "method": "POST", "requestUri": "/" }, "input": { "shape": "GetDeliveryLogRequest" }, "name": "essOperation", "output": { "shape": "EssOperationResult" } } }, "shapes": { "GetDeliveryLogRequest": { "members": { "EndDate": { "locationName": "EndDate", "shape": "TStamp" }, "MaxItems": { "locationName": "MaxItems", "shape": "Integer" }, "NextToken": { "locationName": "NextToken", "shape": "String" }, "StartDate": { "locationName": "StartDate", "shape": "TStamp" }, "Status": { "locationName": "Status", "shape": "Integer" } }, "name": "GetDeliveryLogRequest", "required": [ "EndDate", "StartDate" ], "type": "structure" }, "EssOperationResult": { "members": { "Response": { "locationName": "Response", "shape": "String" } }, "name": "EssOperationResult", "type": "structure" }, "Integer": { "name": "Integer", "type": "integer" }, "TStamp": { "name": "TStamp", "type": "timestamp" }, "String": { "name": "String", "type": "string" } } } ess_service_model = ServiceModel(ess_model) params = {} ess_serializer = serialize.EssSerializer() res = ess_serializer.serialize_to_request( params, ess_service_model.operation_model("GetDeliveryLog")) assert res["body"] == {"Action": "GetDeliveryLog", "Version": "2010-12-01N2014-05-28"} assert res["headers"] == {"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"} assert res["method"] == "POST" assert res["query_string"] == "" assert res["url_path"] == "/"
class TestTimestampHeadersWithRestXML(unittest.TestCase): def setUp(self): self.model = { 'metadata': {'protocol': 'rest-xml', 'apiVersion': '2014-01-01'}, 'documentation': '', 'operations': { 'TestOperation': { 'name': 'TestOperation', 'http': { 'method': 'POST', 'requestUri': '/', }, 'input': {'shape': 'InputShape'}, } }, 'shapes': { 'InputShape': { 'type': 'structure', 'members': { 'TimestampHeader': { 'shape': 'TimestampType', 'location': 'header', 'locationName': 'x-timestamp' }, } }, 'TimestampType': { 'type': 'timestamp', } } } self.service_model = ServiceModel(self.model) def serialize_to_request(self, input_params): request_serializer = serialize.create_serializer( self.service_model.metadata['protocol']) return request_serializer.serialize_to_request( input_params, self.service_model.operation_model('TestOperation')) def test_accepts_datetime_object(self): request = self.serialize_to_request( {'TimestampHeader': datetime.datetime(2014, 1, 1, 12, 12, 12, tzinfo=dateutil.tz.tzutc())}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_iso_8601_format(self): request = self.serialize_to_request( {'TimestampHeader': '2014-01-01T12:12:12+00:00'}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_iso_8601_format_non_utc(self): request = self.serialize_to_request( {'TimestampHeader': '2014-01-01T07:12:12-05:00'}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_rfc_822_format(self): request = self.serialize_to_request( {'TimestampHeader': 'Wed, 01 Jan 2014 12:12:12 GMT'}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT') def test_accepts_unix_timestamp_integer(self): request = self.serialize_to_request( {'TimestampHeader': 1388578332}) self.assertEqual(request['headers']['x-timestamp'], 'Wed, 01 Jan 2014 12:12:12 GMT')
def test_DnsSerializer(self): dns_model = { "metadata": self.dns_model_metadata, "operations": { "DnsOperation": { "http": { "method": "POST", "requestUri": "/2012-12-12N2013-12-16/operation" }, "input": { "locationName": "DnsOperationRequest", "shape": "DnsOperationRequest", "xmlNamespace": { "uri": "https://route53.amazonaws.com/doc/2012-12-12/" } }, "name": "DnsOperation", "output": { "shape": "DnsOperationResult" } }, }, "shapes": { "DnsOperationRequest": { "members": { "Parameter": { "locationName": "Parameter", "shape": "String" }, }, "name": "DnsOperationRequest", "type": "structure" }, "DnsOperationResult": { "members": { "Parameter": { "locationName": "Parameter", "shape": "String" } }, "name": "DnsOperationResult", "type": "structure" }, "String": { "name": "String", "type": "string" }, } } dns_service_model = ServiceModel(dns_model) params = {"Parameter": "test"} dns_serializer = serialize.DnsSerializer() res = dns_serializer.serialize_to_request( params, dns_service_model.operation_model("DnsOperation")) assert res[ "body"] == b'<DnsOperationRequest xmlns="https://route53.amazonaws.com/doc/2012-12-12/"><Parameter>test</Parameter></DnsOperationRequest>' # noqa: E501 assert res["headers"] == {} assert res["method"] == "POST" assert res["query_string"] == {} assert res["url_path"] == "/2012-12-12N2013-12-16/operation"
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest): def setUp(self): super(TestEndpointDiscoveryManager, self).setUp() self.construct_manager() def construct_manager(self, cache=None, time=None, side_effect=None): self.service_model = ServiceModel(self.service_description) self.meta = mock.Mock(spec=ClientMeta) self.meta.service_model = self.service_model self.client = mock.Mock() if side_effect is None: side_effect = [{ 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }] self.client.describe_endpoints.side_effect = side_effect self.client.meta = self.meta self.manager = EndpointDiscoveryManager(self.client, cache=cache, current_time=time) def test_injects_api_version_if_endpoint_operation(self): model = self.service_model.operation_model('DescribeEndpoints') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertEqual(params['headers'].get('x-amz-api-version'), '2018-08-31') def test_no_inject_api_version_if_not_endpoint_operation(self): model = self.service_model.operation_model('TestDiscoveryRequired') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertNotIn('x-amz-api-version', params['headers']) def test_gather_identifiers(self): params = {'Foo': 'value1', 'Nested': {'Bar': 'value2'}} operation = self.service_model.operation_model('TestDiscoveryRequired') ids = self.manager.gather_identifiers(operation, params) self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'}) def test_gather_identifiers_none(self): operation = self.service_model.operation_model('TestDiscovery') ids = self.manager.gather_identifiers(operation, {}) self.assertEqual(ids, {}) def test_describe_endpoint(self): kwargs = { 'Operation': 'FooBar', 'Identifiers': { 'Foo': 'value1', 'Bar': 'value2' }, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) def test_describe_endpoint_no_input(self): describe = self.service_description['operations']['DescribeEndpoints'] del describe['input'] self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_empty_input(self): describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_ids_and_operation(self): cache = {} self.construct_manager(cache=cache) ids = {'Foo': 'value1', 'Bar': 'value2'} kwargs = { 'Operation': 'TestDiscoveryRequired', 'Identifiers': ids, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired') self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_no_ids_or_operation(self): cache = {} describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager(cache=cache) self.manager.describe_endpoint(Operation='TestDiscoveryRequired', Identifiers={}) self.client.describe_endpoints.assert_called_with() key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(Operation='TestDiscoveryRequired', Identifiers={}) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_expired_entry(self): current_time = time.time() key = () cache = { key: [{ 'Address': 'old.com', 'Expiration': current_time - 10 }] } self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with() self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_cache_expiration(self): def _time(): return float(0) cache = {} self.construct_manager(cache=cache, time=_time) self.manager.describe_endpoint(Operation='TestDiscoveryRequired', Identifiers={}) key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Expiration'], float(120)) def test_delete_endpoints_present(self): key = () cache = {key: [{'Address': 'old.com', 'Expiration': 0}]} self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_delete_endpoints_absent(self): cache = {} self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_describe_endpoint_optional_fails_no_cache(self): side_effect = [ConnectionError(error=None)] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) # This second call should be blocked as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [mock.call()] def test_describe_endpoint_optional_fails_stale_cache(self): key = () cache = {key: [{'Address': 'old.com', 'Expiration': 0}]} side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # This second call shouldn't go through as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [mock.call()] def test_describe_endpoint_required_fails_no_cache(self): side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) # This second call should go through, as we have no cache with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) describe_count = self.client.describe_endpoints.call_count self.assertEqual(describe_count, 2) def test_describe_endpoint_required_fails_stale_cache(self): key = () cache = {key: [{'Address': 'old.com', 'Expiration': 0}]} side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # We have a stale endpoint, so this shouldn't fail or force a refresh endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [mock.call()] def test_describe_endpoint_required_force_refresh_success(self): side_effect = [ ConnectionError(error=None), { 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }, ] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} # First call will fail with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.call_args_list == [mock.call()] # Force a refresh if the cache is empty but discovery is required endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com') def test_describe_endpoint_retries_after_failing(self): fake_time = mock.Mock() fake_time.side_effect = [0, 100, 200] side_effect = [ ConnectionError(error=None), { 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }, ] self.construct_manager(side_effect=side_effect, time=fake_time) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [mock.call()] # Second time should try again as enough time has elapsed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com')
class TestEndpointDiscoveryHandler(BaseEndpointDiscoveryTest): def setUp(self): super(TestEndpointDiscoveryHandler, self).setUp() self.manager = Mock(spec=EndpointDiscoveryManager) self.handler = EndpointDiscoveryHandler(self.manager) self.service_model = ServiceModel(self.service_description) def test_register_handler(self): events = Mock(spec=HierarchicalEmitter) self.handler.register(events, 'foo-bar') events.register.assert_any_call( 'before-parameter-build.foo-bar', self.handler.gather_identifiers ) events.register.assert_any_call( 'needs-retry.foo-bar', self.handler.handle_retries ) events.register_first.assert_called_with( 'request-created.foo-bar', self.handler.discover_endpoint ) def test_discover_endpoint(self): request = AWSRequest() request.context = { 'discovery': {'identifiers': {}} } self.manager.describe_endpoint.return_value = 'https://new.foo' self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'https://new.foo') self.manager.describe_endpoint.assert_called_with( Operation='TestOperation', Identifiers={} ) def test_discover_endpoint_fails(self): request = AWSRequest() request.context = { 'discovery': {'identifiers': {}} } request.url = 'old.com' self.manager.describe_endpoint.return_value = None self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'old.com') self.manager.describe_endpoint.assert_called_with( Operation='TestOperation', Identifiers={} ) def test_discover_endpoint_no_protocol(self): request = AWSRequest() request.context = { 'discovery': {'identifiers': {}} } self.manager.describe_endpoint.return_value = 'new.foo' self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'https://new.foo') self.manager.describe_endpoint.assert_called_with( Operation='TestOperation', Identifiers={} ) def test_inject_no_context(self): request = AWSRequest(url='https://original.foo') self.handler.discover_endpoint(request, 'TestOperation') self.assertEqual(request.url, 'https://original.foo') self.manager.describe_endpoint.assert_not_called() def test_gather_identifiers(self): context = {} params = { 'Foo': 'value1', 'Nested': {'Bar': 'value2'} } ids = { 'Foo': 'value1', 'Bar': 'value2' } model = self.service_model.operation_model('TestDiscoveryRequired') self.manager.gather_identifiers.return_value = ids self.handler.gather_identifiers(params, model, context) self.assertEqual(context['discovery']['identifiers'], ids) def test_gather_identifiers_not_discoverable(self): context = {} model = self.service_model.operation_model('DescribeEndpoints') self.handler.gather_identifiers({}, model, context) self.assertEqual(context, {}) def test_discovery_disabled_but_required(self): model = self.service_model.operation_model('TestDiscoveryRequired') with self.assertRaises(EndpointDiscoveryRequired): block_endpoint_discovery_required_operations(model) def test_discovery_disabled_but_optional(self): context = {} model = self.service_model.operation_model('TestDiscoveryOptional') block_endpoint_discovery_required_operations(model, context=context) self.assertEqual(context, {}) def test_does_not_retry_no_response(self): retry = self.handler.handle_retries(None, None, None) self.assertIsNone(retry) def test_does_not_retry_other_errors(self): parsed_response = { 'ResponseMetadata': {'HTTPStatusCode': 200} } response = (None, parsed_response) retry = self.handler.handle_retries(None, response, None) self.assertIsNone(retry) def test_does_not_retry_if_no_context(self): request_dict = {'context': {}} parsed_response = { 'ResponseMetadata': {'HTTPStatusCode': 421} } response = (None, parsed_response) retry = self.handler.handle_retries(request_dict, response, None) self.assertIsNone(retry) def _assert_retries(self, parsed_response): request_dict = { 'context': { 'discovery': {'identifiers': {}} } } response = (None, parsed_response) model = self.service_model.operation_model('TestDiscoveryOptional') retry = self.handler.handle_retries(request_dict, response, model) self.assertEqual(retry, 0) self.manager.delete_endpoints.assert_called_with( Operation='TestDiscoveryOptional', Identifiers={} ) def test_retries_421_status_code(self): parsed_response = { 'ResponseMetadata': {'HTTPStatusCode': 421} } self._assert_retries(parsed_response) def test_retries_invalid_endpoint_exception(self): parsed_response = {'Error': {'Code': 'InvalidEndpointException'}} self._assert_retries(parsed_response)
class ShapeParser: """ Parser for botocore shape files. Arguments: session -- Boto3 session. service_name -- ServiceName. """ # Type map for shape types. SHAPE_TYPE_MAP = { "integer": Type.int, "long": Type.int, "boolean": Type.bool, "double": Type.float, "float": Type.float, "timestamp": ExternalImport(ImportString("datetime"), "datetime"), "blob": TypeSubscript(Type.Union, [Type.bytes, Type.IO]), } # Alias map fixes added by botocore for documentation build. # https://github.com/boto/botocore/blob/develop/botocore/handlers.py#L773 # https://github.com/boto/botocore/blob/develop/botocore/handlers.py#L1055 ARGUMENT_ALIASES: Dict[str, Dict[str, Dict[str, str]]] = { ServiceNameCatalog.cloudsearchdomain.boto3_name: { "Search": {"return": "returnFields"} }, ServiceNameCatalog.logs.boto3_name: {"CreateExportTask": {"from": "fromTime"}}, ServiceNameCatalog.ec2.boto3_name: {"*": {"Filter": "Filters"}}, ServiceNameCatalog.s3.boto3_name: { "PutBucketAcl": {"ContentMD5": "None"}, "PutBucketCors": {"ContentMD5": "None"}, "PutBucketLifecycle": {"ContentMD5": "None"}, "PutBucketLogging": {"ContentMD5": "None"}, "PutBucketNotification": {"ContentMD5": "None"}, "PutBucketPolicy": {"ContentMD5": "None"}, "PutBucketReplication": {"ContentMD5": "None"}, "PutBucketRequestPayment": {"ContentMD5": "None"}, "PutBucketTagging": {"ContentMD5": "None"}, "PutBucketVersioning": {"ContentMD5": "None"}, "PutBucketWebsite": {"ContentMD5": "None"}, "PutObjectAcl": {"ContentMD5": "None"}, }, } def __init__(self, session: Session, service_name: ServiceName): loader = session._loader # pylint: disable=protected-access botocore_session: BotocoreSession = session._session # pylint: disable=protected-access service_data = botocore_session.get_service_data(service_name.boto3_name) self.service_name = service_name self.service_model = ServiceModel(service_data, service_name.boto3_name) self._typed_dict_map: Dict[str, TypeTypedDict] = {} self._waiters_shape: Shape = {} try: self._waiters_shape = loader.load_service_model( service_name.boto3_name, "waiters-2" ) except UnknownServiceError: pass self._paginators_shape: Shape = {} try: self._paginators_shape = loader.load_service_model( service_name.boto3_name, "paginators-1" ) except UnknownServiceError: pass self._resources_shape: Shape = {} try: self._resources_shape = loader.load_service_model( service_name.boto3_name, "resources-1" ) except UnknownServiceError: pass self.logger = get_logger() def _get_operation(self, name: str) -> OperationModel: return self.service_model.operation_model(name) def _get_operation_names(self) -> List[str]: return list( self.service_model.operation_names ) # pylint: disable=not-an-iterable def _get_paginator(self, name: str) -> Shape: try: return self._paginators_shape["pagination"][name] except KeyError: raise ShapeParserError(f"Unknown paginator: {name}") def _get_service_resource(self) -> Shape: return self._resources_shape["service"] def _get_resource_shape(self, name: str) -> Shape: try: return self._resources_shape["resources"][name] except KeyError: raise ShapeParserError(f"Unknown resource: {name}") def get_paginator_names(self) -> List[str]: """ Get available paginator names. Returns: A list of paginator names. """ result: List[str] = [] for name in self._paginators_shape.get("pagination", []): result.append(name) result.sort() return result def _get_argument_alias(self, operation_name: str, argument_name: str) -> str: service_map = self.ARGUMENT_ALIASES.get(self.service_name.boto3_name) if not service_map: return argument_name operation_map: Dict[str, str] = {} if "*" in service_map: operation_map = service_map["*"] if operation_name in service_map: operation_map = service_map[operation_name] if not operation_map: return argument_name if argument_name not in operation_map: return argument_name return operation_map[argument_name] def _parse_arguments( self, class_name: str, method_name: str, operation_name: str, shape: StructureShape, ) -> List[Argument]: result: List[Argument] = [] required = shape.required_members for argument_name, argument_shape in shape.members.items(): argument_alias = self._get_argument_alias(operation_name, argument_name) if argument_alias == "None": continue argument_type_stub = get_method_type_stub( self.service_name, class_name, method_name, argument_name ) if argument_type_stub is not None: argument_type = argument_type_stub else: argument_type = self._parse_shape(argument_shape) argument = Argument(argument_alias, argument_type) if argument_name not in required: argument.default = Type.none result.append(argument) result.sort(key=lambda x: not x.required) return result def _parse_return_type( self, class_name: str, method_name: str, shape: Optional[Shape] ) -> FakeAnnotation: argument_type_stub = get_method_type_stub( self.service_name, class_name, method_name, "return" ) if argument_type_stub is not None: return argument_type_stub if shape: return self._parse_shape(shape) return Type.none def get_client_method_map(self) -> Dict[str, Method]: """ Get client methods from shape. Returns: A map of method name to Method. """ result: Dict[str, Method] = { "can_paginate": Method( "can_paginate", [Argument("self", None), Argument("operation_name", Type.str)], Type.bool, ), "generate_presigned_url": Method( "generate_presigned_url", [ Argument("self", None), Argument("ClientMethod", Type.str), Argument("Params", Type.DictStrAny, Type.none), Argument("ExpiresIn", Type.int, TypeConstant(3600)), Argument("HttpMethod", Type.str, Type.none), ], Type.str, ), } for operation_name in self._get_operation_names(): operation_model = self._get_operation(operation_name) arguments: List[Argument] = [Argument("self", None)] method_name = xform_name(operation_name) if operation_model.input_shape is not None: arguments.extend( self._parse_arguments( "Client", method_name, operation_name, operation_model.input_shape, ) ) return_type = self._parse_return_type( "Client", method_name, operation_model.output_shape ) method = Method( name=method_name, arguments=arguments, return_type=return_type ) result[method.name] = method return result @staticmethod def _parse_shape_string(shape: StringShape) -> FakeAnnotation: if not shape.enum: return Type.str type_literal = TypeLiteral() for option in shape.enum: type_literal.add_literal_child(option) return type_literal def _parse_shape_map(self, shape: MapShape) -> FakeAnnotation: type_subscript = TypeSubscript(Type.Dict) if shape.key: type_subscript.add_child(self._parse_shape(shape.key)) else: type_subscript.add_child(Type.str) if shape.value: type_subscript.add_child(self._parse_shape(shape.value)) else: type_subscript.add_child(Type.Any) return type_subscript def _parse_shape_structure(self, shape: StructureShape) -> FakeAnnotation: if not shape.members.items(): return Type.DictStrAny required = shape.required_members typed_dict_name = f"{shape.name}TypeDef" shape_type_stub = get_shape_type_stub(self.service_name, typed_dict_name) if shape_type_stub: return shape_type_stub if typed_dict_name in self._typed_dict_map: return self._typed_dict_map[typed_dict_name] typed_dict = TypeTypedDict(typed_dict_name) self._typed_dict_map[typed_dict_name] = typed_dict for attr_name, attr_shape in shape.members.items(): typed_dict.add_attribute( attr_name, self._parse_shape(attr_shape), attr_name in required, ) return typed_dict def _parse_shape_list(self, shape: ListShape) -> FakeAnnotation: type_subscript = TypeSubscript(Type.List) if shape.member: type_subscript.add_child(self._parse_shape(shape.member)) else: type_subscript.add_child(Type.Any) return type_subscript def _parse_shape(self, shape: Shape) -> FakeAnnotation: if shape.type_name in self.SHAPE_TYPE_MAP: return self.SHAPE_TYPE_MAP[shape.type_name] if isinstance(shape, StringShape): return self._parse_shape_string(shape) if isinstance(shape, MapShape): return self._parse_shape_map(shape) if isinstance(shape, StructureShape): return self._parse_shape_structure(shape) if isinstance(shape, ListShape): return self._parse_shape_list(shape) if shape.type_name in self._resources_shape["resources"]: return AliasInternalImport(shape.type_name) self.logger.warning(f"Unknown shape: {shape}") return Type.Any def get_paginate_method(self, paginator_name: str) -> Method: """ Get Paginator `paginate` method. Arguments: paginator_name -- Paginator name. Returns: Method. """ operation_name = paginator_name paginator_shape = self._get_paginator(paginator_name) operation_shape = self._get_operation(operation_name) skip_argument_names: List[str] = [] input_token = paginator_shape["input_token"] if isinstance(input_token, list): skip_argument_names.extend(input_token) else: skip_argument_names.append(input_token) if "limit_key" in paginator_shape: skip_argument_names.append(paginator_shape["limit_key"]) arguments: List[Argument] = [Argument("self", None)] if operation_shape.input_shape is not None: for argument in self._parse_arguments( "Paginator", "paginate", operation_name, operation_shape.input_shape ): if argument.name in skip_argument_names: continue arguments.append(argument) arguments.append(Argument("PaginationConfig", paginator_config_type, Type.none)) return_type: FakeAnnotation = Type.none if operation_shape.output_shape is not None: return_type = TypeSubscript( Type.Iterator, [ self._parse_return_type( "Paginator", "paginate", operation_shape.output_shape ), ], ) return Method("paginate", arguments, return_type) def get_wait_method(self, waiter_name: str) -> Method: """ Get Waiter `wait` method. Arguments: waiter_name -- Waiter name. Returns: Method. """ operation_name = self._waiters_shape["waiters"][waiter_name]["operation"] operation_shape = self._get_operation(operation_name) arguments: List[Argument] = [Argument("self", None)] if operation_shape.input_shape is not None: arguments.extend( self._parse_arguments( "Waiter", "wait", operation_name, operation_shape.input_shape ) ) arguments.append(Argument("WaiterConfig", waiter_config_type, Type.none)) return Method(name="wait", arguments=arguments, return_type=Type.none) def get_service_resource_method_map(self) -> Dict[str, Method]: """ Get methods for ServiceResource. Returns: A map of method name to Method. """ result: Dict[str, Method] = { "get_available_subresources": Method( "get_available_subresources", [Argument("self", None)], TypeSubscript(Type.List, [Type.str]), ), } service_resource_shape = self._get_service_resource() for action_name, action_shape in service_resource_shape.get( "actions", {} ).items(): method = self._get_resource_method( "ServiceResource", action_name, action_shape ) result[method.name] = method return result def get_resource_method_map(self, resource_name: str) -> Dict[str, Method]: """ Get methods for Resource. Arguments: resource_name -- Resource name. Returns: A map of method name to Method. """ resource_shape = self._get_resource_shape(resource_name) result: Dict[str, Method] = { "get_available_subresources": Method( "get_available_subresources", [Argument("self", None)], TypeSubscript(Type.List, [Type.str]), ), "load": Method("load", [Argument("self", None)], Type.none), "reload": Method("reload", [Argument("self", None)], Type.none), } for action_name, action_shape in resource_shape.get("actions", {}).items(): method = self._get_resource_method(resource_name, action_name, action_shape) result[method.name] = method for waiter_name in resource_shape.get("waiters", {}): method = Method( f"wait_until_{xform_name(waiter_name)}", [Argument("self", None)], Type.none, ) result[method.name] = method return result def _get_resource_method( self, resource_name: str, action_name: str, action_shape: Dict[str, Any] ) -> Method: return_type: FakeAnnotation = Type.none method_name = xform_name(action_name) arguments: List[Argument] = [Argument("self", None)] if "resource" in action_shape: return_type = self._parse_return_type( resource_name, method_name, Shape("resource", action_shape["resource"]) ) path = action_shape["resource"].get("path", "") if path.endswith("[]"): return_type = TypeSubscript(Type.List, [return_type]) if "request" in action_shape: operation_name = action_shape["request"]["operation"] operation_shape = self._get_operation(operation_name) skip_argument_names: List[str] = [ i["target"] for i in action_shape["request"].get("params", {}) if i["source"] == "identifier" ] if operation_shape.input_shape is not None: for argument in self._parse_arguments( resource_name, method_name, operation_name, operation_shape.input_shape, ): if argument.name not in skip_argument_names: arguments.append(argument) if operation_shape.output_shape is not None and return_type is Type.none: operation_return_type = self._parse_shape(operation_shape.output_shape) return_type = operation_return_type return Method(name=method_name, arguments=arguments, return_type=return_type) def get_collection_filter_method( self, name: str, collection: Collection, self_type: FakeAnnotation ) -> Method: """ Get `filter` classmethod for Resource collection. Arguments: name -- Collection record name. collection -- Boto3 Collection. class_type -- Collection class type annotation. Returns: Filter Method record. """ arguments: List[Argument] = [Argument("self", None)] result = Method("filter", arguments, self_type) if not collection.request: return result operation_name = collection.request.operation operation_model = self._get_operation(operation_name) if operation_model.input_shape is not None: for argument in self._parse_arguments( name, result.name, operation_name, operation_model.input_shape, ): if argument.required: continue arguments.append(argument) return result def get_collection_batch_methods( self, name: str, collection: Collection ) -> List[Method]: """ Get batch operations for Resource collection. Arguments: name -- Collection record name. collection -- Boto3 Collection. class_type -- Collection self type annotation. Returns: List of Method records. """ result = [] for batch_action in collection.batch_actions: method = Method(batch_action.name, [Argument("self", None)], Type.none) result.append(method) if batch_action.request: operation_name = batch_action.request.operation operation_model = self._get_operation(operation_name) if operation_model.input_shape is not None: for argument in self._parse_arguments( name, batch_action.name, operation_name, operation_model.input_shape, ): if argument.required: continue method.arguments.append(argument) if operation_model.output_shape is not None: return_type = self._parse_shape(operation_model.output_shape) method.return_type = return_type return result
def test_RdbSerializer_fix_get_metrics_statistics_params(self): rdb_model = { "metadata": self.rdb_model_metadata, "operations": { "NiftyGetMetricStatistics": { "http": { "method": "POST", "requestUri": "/" }, "input": { "shape": "NiftyGetMetricStatisticsRequest" }, "name": "NiftyGetMetricStatistics", "output": { "resultWrapper": "RdbOperationResult", "shape": "RdbOperationResult" } }, }, "shapes": { "NiftyGetMetricStatisticsRequest": { "members": { "Dimensions": { "locationName": "Dimensions", "shape": "ListOfRequestDimensions" }, "EndTime": { "locationName": "EndTime", "shape": "TStamp" }, "MetricName": { "locationName": "MetricName", "shape": "String" }, "StartTime": { "locationName": "StartTime", "shape": "TStamp" } }, "name": "NiftyGetMetricStatisticsRequest", "type": "structure" }, "ListOfRequestDimensions": { "member": { "locationName": "member", "shape": "RequestDimensions" }, "name": "ListOfRequestDimensions", "type": "list" }, "RequestDimensions": { "members": { "Name": { "locationName": "Name", "shape": "String" }, "Value": { "locationName": "Value", "shape": "String" } }, "name": "RequestDimensions", "required": [ "Name", "Value" ], "type": "structure" }, "RdbOperationResult": { "members": { "Response": { "locationName": "Response", "shape": "String" } }, "name": "RdbOperationResult", "type": "structure" }, "String": { "name": "String", "type": "string" }, "TStamp": { "name": "TStamp", "type": "timestamp" } } } rdb_service_model = ServiceModel(rdb_model) params = {} rdb_serializer = serialize.RdbSerializer() res = rdb_serializer.serialize_to_request( params, rdb_service_model.operation_model("NiftyGetMetricStatistics")) assert res["body"] == { "Action": "NiftyGetMetricStatistics", "Version": "2013-05-15N2013-12-16" } assert res["headers"] == {"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"} assert res["method"] == "POST" assert res["query_string"] == "" assert res["url_path"] == "/"