def test_result_key_validation(self):
# result_key must exist.
extra = {
"pagination": {
"AssumeRole": {"input_token": ["Token", "TokenToken"], "output_token": ["NextToken", "NextTokenToken"]}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_result_key_validation(self):
# result_key must exist.
extra = {
'pagination': {
'AssumeRole': {
'input_token': ['Token', 'TokenToken'],
'output_token': ['NextToken', 'NextTokenToken']
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_result_key_exists_in_output(self):
extra = {
'pagination': {
'AssumeRole': {
'input_token': ['Token', 'TokenToken'],
'output_token': ['NextToken', 'NextTokenToken'],
'result_key': 'DoesNotExist',
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_result_key_exists_in_output(self):
extra = {
"pagination": {
"AssumeRole": {
"input_token": ["Token", "TokenToken"],
"output_token": ["NextToken", "NextTokenToken"],
"result_key": "DoesNotExist",
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_expected_schema_exists(self):
# In this case, the key 'output_tokens' is suppose to be 'output_token'
# so we should get an error when this happens.
extra = {
'pagination': {
'AssumeRole': {
'input_token': ['Token', 'TokenToken'],
'output_tokens': ['NextToken', 'NextTokenToken'],
'result_key': ['Credentials', 'AssumedRoleUser'],
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_input_tokens_exist_in_model(self):
extra = {
'pagination': {
'AssumeRole': {
# In this case, "DoesNotExist" token is not in the input
# model, so we get an exception complaining about this.
'input_token': ['NextToken', 'DoesNotExist'],
'output_token': ['NextToken', 'NextTokenToken'],
'result_key': ['Credentials', 'AssumedRoleUser'],
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_input_tokens_exist_in_model(self):
extra = {
"pagination": {
"AssumeRole": {
# In this case, "DoesNotExist" token is not in the input
# model, so we get an exception complaining about this.
"input_token": ["NextToken", "DoesNotExist"],
"output_token": ["NextToken", "NextTokenToken"],
"result_key": ["Credentials", "AssumedRoleUser"],
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_expected_schema_exists(self):
# In this case, the key 'output_tokens' is suppose to be 'output_token'
# so we should get an error when this happens.
extra = {
"pagination": {
"AssumeRole": {
"input_token": ["Token", "TokenToken"],
"output_tokens": ["NextToken", "NextTokenToken"],
"result_key": ["Credentials", "AssumedRoleUser"],
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
translate(self.model)
def test_merging_occurs_after_transformation(self):
enhancements = {
'transformations': {
'operation-name': {
'remove': r'\d{4}_\d{2}_\d{2}'
}
},
'operations': {
'RealOperation': {
'input': {
'checksum': 'md5',
}
},
}
}
model = ModelFiles(SERVICES,
regions={},
retry={},
enhancements=enhancements)
model.enhancements = enhancements
new_model = translate(model)
self.assertIn('RealOperation', new_model['operations'])
self.assertEqual(
new_model['operations']['RealOperation']['input']['checksum'],
'md5')
def test_cant_add_pagination_to_nonexistent_operation(self):
extra = {
'pagination': {
'ThisOperationDoesNotExist': {
'input_token': 'NextToken',
'output_token': ['NextToken', 'NextTokenToken'],
'result_key': ['Credentials', 'AssumedRoleUser'],
'limit_key': 'Foo',
}
}
}
self.model.enhancements = extra
with self.assertRaisesRegexp(
ValueError, "Trying to add pagination config for non "
"existent operation: ThisOperationDoesNotExist"):
translate(self.model)
def test_iterators_are_merged_into_operations(self):
# This may or may not pan out, but if a pagination config is
# specified, that info is merged into the specific operations.
extra = {
'pagination': {
'AssumeRole': {
'input_token': 'NextToken',
'output_token': 'NextToken',
'limit_key': 'MaxResults',
'result_key': 'Credentials',
'non_aggregate_keys': ['PackedPolicySize',
'AssumedRoleUser'],
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
op = new_model['operations']['AssumeRole']
self.assertDictEqual(
op['pagination'], {
'input_token': 'NextToken',
'py_input_token': 'next_token',
'output_token': 'NextToken',
'limit_key': 'MaxResults',
'result_key': 'Credentials',
'non_aggregate_keys': ['PackedPolicySize',
'AssumedRoleUser'],
})
def test_validate_valid_operation(self):
extra = {
'waiters': {
'__default__': {
'interval': 20,
'max_attempts': 25,
},
'__RoleResource': {
'operation': 'THISOPERATIONDOESNOTEXIST',
'max_attempts': 50,
},
'RoleExists': {
'extends': '__RoleResource',
'ignore_errors': ['Error1'],
'success_type': 'output',
'success_path': 'Table.TableStatus',
'success_value': 'ACTIVE',
'failure_type': 'output',
'failure_path': 'Table.TableStatus',
'failure_value': 'ACTIVE',
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
new_model = translate(self.model)
def test_only_acceptors_provided(self):
extra = {
'waiters': {
'RoleExists': {
'operation': 'AssumeRole',
'acceptor_type': 'output',
'acceptor_path': 'acceptor_path',
'acceptor_value': 'acceptor_value',
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
denormalized = {
'RoleExists': {
'operation': 'AssumeRole',
'success': {
'type': 'output',
'path': 'acceptor_path',
'value': ['acceptor_value'],
},
'failure': {
'type': 'output',
'path': 'acceptor_path',
'value': ['acceptor_value'],
}
}
}
self.assertEqual(new_model['waiters'], denormalized)
def test_can_add_fields_to_op_params(self):
# A use case would be if we want to annotate that a
# string type might also come from a file (keypairs, s3 uploads, etc).
new_keys = {
'operations': {
'AssumeRole': {
'input': {
'members': {
'Policy': {
'alsofrom': 'filename',
},
}
}
},
}
}
self.model.enhancements = new_keys
new_model = translate(self.model)
self.assertEqual(
new_model['operations']['AssumeRole']['input']['members']\
['Policy']['alsofrom'],
'filename')
self.assertEqual(
new_model['operations']['AssumeRole']['input']['members']\
['Policy']['type'],
'string')
self.assertEqual(
new_model['operations']['AssumeRole']['input']['members']\
['RoleArn']['shape_name'],
'arnType')
def test_extra_key(self):
# Anything in "extra" is merged as a top level key.
extra = {"extra": {"signature_version": "v2"}}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model["signature_version"], "v2")
self.assertEqual(new_model["documentation"], "docs")
def test_operations_is_a_dict_not_list(self):
# In order to make overriding easier, we want the list of
# operations to be a dict, not a list. The way we don't have
# to search through the list to find the operation we want to
# change. It also makes it easier to annontate operations.
new_model = translate(self.model)
self.assertIn('AssumeRole', new_model['operations'])
def test_py_input_name_is_not_added_if_it_exists(self):
# This may or may not pan out, but if a pagination config is
# specified, that info is merged into the specific operations.
extra = {
'pagination': {
'AssumeRole': {
'input_token': 'NextToken',
'output_token': 'NextToken',
'py_input_token': 'other_value',
'limit_key': 'MaxResults',
'result_key': 'Credentials',
'non_aggregate_keys': ['foo'],
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
op = new_model['operations']['AssumeRole']
# Note how 'py_input_token' is left untouched. This allows us
# to override this naming if we need to.
self.assertDictEqual(
op['pagination'], {
'input_token': 'NextToken',
'py_input_token': 'other_value',
'output_token': 'NextToken',
'limit_key': 'MaxResults',
'result_key': 'Credentials',
'non_aggregate_keys': ['foo'],
})
def test_py_input_name_is_not_added_if_it_exists(self):
# This may or may not pan out, but if a pagination config is
# specified, that info is merged into the specific operations.
extra = {
"pagination": {
"AssumeRole": {
"input_token": "NextToken",
"output_token": "NextToken",
"py_input_token": "other_value",
"limit_key": "MaxResults",
"result_key": "Credentials",
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
op = new_model["operations"]["AssumeRole"]
# Note how 'py_input_token' is left untouched. This allows us
# to override this naming if we need to.
self.assertDictEqual(
op["pagination"],
{
"input_token": "NextToken",
"py_input_token": "other_value",
"output_token": "NextToken",
"limit_key": "MaxResults",
"result_key": "Credentials",
},
)
def test_iterators_are_merged_into_operations(self):
# This may or may not pan out, but if a pagination config is
# specified, that info is merged into the specific operations.
extra = {
"pagination": {
"AssumeRole": {
"input_token": "NextToken",
"output_token": "NextToken",
"limit_key": "MaxResults",
"result_key": "Credentials",
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
op = new_model["operations"]["AssumeRole"]
self.assertDictEqual(
op["pagination"],
{
"input_token": "NextToken",
"py_input_token": "next_token",
"output_token": "NextToken",
"limit_key": "MaxResults",
"result_key": "Credentials",
},
)
def test_iterators_are_merged_into_operations(self):
# This may or may not pan out, but if a pagination config is
# specified, that info is merged into the specific operations.
extra = {
'pagination': {
'AssumeRole': {
'input_token': 'NextToken',
'output_token': 'NextToken',
'limit_key': 'MaxResults',
'result_key': 'Credentials',
'non_aggregate_keys': ['PackedPolicySize',
'AssumedRoleUser'],
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
op = new_model['operations']['AssumeRole']
self.assertDictEqual(
op['pagination'], {
'input_token': 'NextToken',
'py_input_token': 'next_token',
'output_token': 'NextToken',
'limit_key': 'MaxResults',
'result_key': 'Credentials',
'non_aggregate_keys': ['PackedPolicySize',
'AssumedRoleUser'],
})
def test_resolve_reference(self):
model = ModelFiles(SERVICES, {}, self.retry, {})
new_model = translate(model)
operation_config = new_model['retry']['AssumeRole']
# And we should resolve references.
self.assertEqual(operation_config['policies']['other'],
{"from": {"definition": "file"}})
def test_can_add_extras_top_level_keys(self):
# A use case here would be adding the iterator/waiter configs.
new_keys = {"extra": {"paginators": "paginator_config"}}
self.model.enhancements = new_keys
new_model = translate(self.model)
# There should be a new top level key 'iterators' that was merged in.
self.assertEqual(new_model["paginators"], "paginator_config")
self.assertIn("operations", new_model)
def test_remove_deprecated_params(self):
enhancements = {"transformations": {"remove-deprecated-params": {"deprecated_keyword": "deprecated"}}}
model = ModelFiles(SERVICES, regions={}, retry={}, enhancements=enhancements)
new_model = translate(model)
operation = new_model["operations"]["DeprecatedOperation"]
# The deprecated param should be gone, the other should remain
self.assertIn("FooBar", operation["input"]["members"])
self.assertNotIn("FieBaz", operation["input"]["members"])
def test_can_add_extras_top_level_keys(self):
# A use case here would be adding the iterator/waiter configs.
new_keys = {'extra': {'paginators': 'paginator_config'}}
self.model.enhancements = new_keys
new_model = translate(self.model)
# There should be a new top level key 'iterators' that was merged in.
self.assertEqual(new_model['paginators'], 'paginator_config')
self.assertIn('operations', new_model)
def test_can_add_fields_to_operation(self):
# A use case would be to add checksum info for a param.
# We could go for a more streamlined syntax, but this way, it's
# clear how this maps onto the existing json model.
new_keys = {"operations": {"AssumeRole": {"input": {"checksum": "md5"}}}}
self.model.enhancements = new_keys
new_model = translate(self.model)
self.assertEqual(new_model["operations"]["AssumeRole"]["input"]["checksum"], "md5")
def test_paginators_are_placed_into_top_level_key(self):
extra = {
"pagination": {
"AssumeRole": {"input_token": "NextToken", "output_token": "NextToken", "result_key": "Credentials"}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model["pagination"], extra["pagination"])
def test_can_add_fields_to_op_params(self):
# A use case would be if we want to annotate that a
# string type might also come from a file (keypairs, s3 uploads, etc).
new_keys = {"operations": {"AssumeRole": {"input": {"members": {"Policy": {"alsofrom": "filename"}}}}}}
self.model.enhancements = new_keys
new_model = translate(self.model)
self.assertEqual(new_model["operations"]["AssumeRole"]["input"]["members"]["Policy"]["alsofrom"], "filename")
self.assertEqual(new_model["operations"]["AssumeRole"]["input"]["members"]["Policy"]["type"], "string")
self.assertEqual(new_model["operations"]["AssumeRole"]["input"]["members"]["RoleArn"]["shape_name"], "arnType")
def test_result_key_validation_with_no_output(self):
extra = {
'pagination': {
# RealOperation does not have any output members so
# we should get an error message telling us this.
'NoOutputOperation': {
'input_token': 'NextToken',
'output_token': ['NextToken', 'NextTokenToken'],
'result_key': ['Credentials', 'AssumedRoleUser'],
'limit_key': 'Foo',
}
}
}
self.model.enhancements = extra
with self.assertRaisesRegexp(
ValueError, "Trying to add pagination config for an "
"operation with no output members: "
"NoOutputOperation"):
translate(self.model)
def test_remove_deprecated_ops(self):
enhancements = {
'transformations': {
'remove-deprecated-operations':
{'deprecated_keyword': 'deprecated'}
}
}
model = ModelFiles(SERVICES, retry={}, enhancements=enhancements)
new_model = translate(model)
# The deprecated operation should be gone
self.assertNotIn('DeprecatedOperation2', new_model['operations'])
def test_extra_key(self):
# Anything in "extra" is merged as a top level key.
extra = {
"extra": {
"signature_version": "v2",
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model['signature_version'], 'v2')
self.assertEqual(new_model['documentation'], 'docs')
def test_remove_deprecated_ops(self):
enhancements = {
'transformations': {
'remove-deprecated-operations':
{'deprecated_keyword': 'deprecated'}
}
}
model = ModelFiles(SERVICES, retry={}, enhancements=enhancements)
new_model = translate(model)
# The deprecated operation should be gone
self.assertNotIn('DeprecatedOperation2', new_model['operations'])
def test_remove_deprecated_params(self):
enhancements = {
'transformations': {
'remove-deprecated-params': {'deprecated_keyword': 'deprecated'}
}
}
model = ModelFiles(SERVICES, retry={}, enhancements=enhancements)
new_model = translate(model)
operation = new_model['operations']['DeprecatedOperation']
# The deprecated param should be gone, the other should remain
self.assertIn('FooBar', operation['input']['members'])
self.assertNotIn('FieBaz', operation['input']['members'])
def test_replace_operation_key_name(self):
enhancements = {"transformations": {"operation-name": {"remove": r"\d{4}_\d{2}_\d{2}"}}}
model = ModelFiles(SERVICES, regions={}, retry={}, enhancements=enhancements)
new_model = translate(model)
# But the key into the operation dict is stripped of the
# matched regex.
self.assertEqual(
list(sorted(new_model["operations"].keys())),
["AssumeRole", "DeprecatedOperation", "DeprecatedOperation2", "RealOperation"],
)
# But the name key attribute is left unchanged.
self.assertEqual(new_model["operations"]["RealOperation"]["name"], "RealOperation2013_02_04")
def test_remove_deprecated_params(self):
enhancements = {
'transformations': {
'remove-deprecated-params': {'deprecated_keyword': 'deprecated'}
}
}
model = ModelFiles(SERVICES, retry={}, enhancements=enhancements)
new_model = translate(model)
operation = new_model['operations']['DeprecatedOperation']
# The deprecated param should be gone, the other should remain
self.assertIn('FooBar', operation['input']['members'])
self.assertNotIn('FieBaz', operation['input']['members'])
def test_paginators_are_placed_into_top_level_key(self):
extra = {
'pagination': {
'AssumeRole': {
'input_token': 'NextToken',
'output_token': 'NextToken',
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model['pagination'], extra['pagination'])
def test_rename_param(self):
enhancements = {
'transformations': {
'renames': {
'RenameOperation.input.members.RenameMe': 'BeenRenamed',
}
}
}
model = ModelFiles(SERVICES, retry={}, enhancements=enhancements)
new_model = translate(model)
arguments = new_model['operations']['RenameOperation']['input']['members']
self.assertNotIn('RenameMe', arguments)
self.assertIn('BeenRenamed', arguments)
def test_echoed_input_params_ignored(self):
extra = {
'pagination': {
'EchoedOutputParams': {
'input_token': ['Marker'],
'output_token': ['NextMarker'],
'result_key': 'ResultKey',
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model['pagination'], extra['pagination'])
def test_result_key_can_be_a_list(self):
extra = {
'pagination': {
'AssumeRole': {
'input_token': ['NextToken'],
'output_token': ['NextToken', 'NextTokenToken'],
'result_key': ['Credentials', 'AssumedRoleUser'],
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model['pagination'], extra['pagination'])
def test_paginators_are_placed_into_top_level_key(self):
extra = {
'pagination': {
'AssumeRole': {
'input_token': 'NextToken',
'output_token': 'NextToken',
'result_key': 'Credentials',
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model['pagination'], extra['pagination'])
def test_skip_jmespath_validation(self):
# This would fail previously.
extra = {
'pagination': {
'AssumeRole': {
'input_token': ['NextToken'],
'output_token': ['NextToken', 'NextTokenToken'],
'result_key': 'Credentials.AssumedRoleUser',
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
self.assertEqual(new_model['pagination'], extra['pagination'])
def test_paginators_are_validated(self):
# Can't create a paginator config that refers to a non existent
# operation.
extra = {
'pagination': {
'UnknownOperation': {
'input_token': 'NextToken',
'output_token': 'NextToken',
'max_results': 'MaxResults',
}
}
}
self.model.enhancements = extra
with self.assertRaises(ValueError):
new_model = translate(self.model)
def test_default_and_extends(self):
extra = {
'waiters': {
'__default__': {
'interval': 20,
'max_attempts': 25,
},
'__RoleResource': {
'operation': 'AssumeRole',
'max_attempts': 50,
},
'RoleExists': {
'extends': '__RoleResource',
'ignore_errors': ['Error1'],
'success_type': 'output',
'success_path': 'Table.TableStatus',
'success_value': 'ACTIVE',
'failure_type': 'output',
'failure_path': 'Table.TableStatus',
'failure_value': 'ACTIVE',
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
denormalized = {
'RoleExists': {
# From __default__
'interval': 20,
# Overriden from __RoleResource
'max_attempts': 50,
# Defined in __RoleResource
'operation': 'AssumeRole',
# Defined in RoleExists
'ignore_errors': ['Error1'],
'success': {
'type': 'output',
'path': 'Table.TableStatus',
'value': ['ACTIVE'],
},
'failure': {
'type': 'output',
'path': 'Table.TableStatus',
'value': ['ACTIVE'],
}
}
}
self.assertEqual(new_model['waiters'], denormalized)
def test_supported_regions_are_merged_into_service(self):
extra = {
"extra": {
"metadata": {
"regions": {
"us-east-1": "https://sts.amazonaws.com/",
"us-gov-west-1": None,
},
"protocols": ["https"]
}
}
}
self.model.enhancements = extra
self.model.name = 'sts'
new_model = translate(self.model)
self.maxDiff = None
self.assertDictEqual(new_model['metadata'], extra['extra']['metadata'])
def test_replace_operation_key_name(self):
enhancements = {
'transformations': {
'operation-name': {'remove': r'\d{4}_\d{2}_\d{2}'}
}
}
model = ModelFiles(SERVICES, regions={}, retry={},
enhancements=enhancements)
new_model = translate(model)
# But the key into the operation dict is stripped of the
# matched regex.
self.assertEqual(list(sorted(new_model['operations'].keys())),
['AssumeRole', 'DeprecatedOperation',
'DeprecatedOperation2', 'NoOutputOperation',
'RealOperation', 'RenameOperation'])
# But the name key attribute is left unchanged.
self.assertEqual(new_model['operations']['RealOperation']['name'],
'RealOperation2013_02_04')
def test_remove_deprecated_params(self):
enhancements = {
"transformations": {
"filter-documentation": {
"filter": {
"regex": "<!\\[CDATA\\[.*\\]\\]>",
"replacement": ""
}
}
}
}
model = ModelFiles(SERVICES, retry={}, enhancements=enhancements)
new_model = translate(model)
operation = new_model['operations']['DeprecatedOperation']
# The deprecated param should be gone, the other should remain
self.assertEqual(operation['documentation'], 'This is my stuff')
param = operation['input']['members']['FooBar']
self.assertEqual(param['documentation'], 'blah blah blah blah')
def test_can_add_fields_to_operation(self):
# A use case would be to add checksum info for a param.
# We could go for a more streamlined syntax, but this way, it's
# clear how this maps onto the existing json model.
new_keys = {
'operations': {
'AssumeRole': {
'input': {
'checksum': 'md5',
}
},
}
}
self.model.enhancements = new_keys
new_model = translate(self.model)
self.assertEqual(
new_model['operations']['AssumeRole']['input']['checksum'], 'md5')
def test_waiter_default_resolved(self):
extra = {
'waiters': {
'__default__': {
'interval': 20,
'operation': 'AssumeRole',
'max_attempts': 25,
'acceptor_type': 'output',
'acceptor_path': 'path',
'acceptor_value': 'value',
},
# Note that this config doesn't make any actual sense,
# this is just testing we denormalize fields properly.
'RoleExists': {
'operation': 'AssumeRole',
'ignore_errors': ['Error1'],
'success_type': 'output',
'success_path': 'Table.TableStatus',
'success_value': ['ACTIVE'],
}
}
}
self.model.enhancements = extra
new_model = translate(self.model)
denormalized = {
'RoleExists': {
'interval': 20,
'max_attempts': 25,
'operation': 'AssumeRole',
'ignore_errors': ['Error1'],
'success': {
'type': 'output',
'path': 'Table.TableStatus',
'value': ['ACTIVE'],
},
'failure': {
'type': 'output',
'path': 'path',
'value': ['value'],
}
}
}
self.assertEqual(new_model['waiters'], denormalized)
