def test_validate_invalid_int_value_name(self):
self.test_input['name'] = 30
with self.assertRaisesRegex(Invalid, 'key: "name" contains invalid item "30" with type "int": not of type string'):
_ = maat_scale(self.test_input, self.test_validation)
# test validator of name to make the previous invalid value valid.
self.test_validation['name'] = {'validator': 'int'}
validated_items = maat_scale(self.test_input, self.test_validation)
difference = ddiff(validated_items, self.test_input)
self.assertEqual(difference, {})
def test_non_dictionary_function(self):
"""a helpfull message is shown when trying to use post transformation that is not registered"""
test_input_list = ['id', float(23)]
test_input_tuple = ('id', float(23))
test_validation = {'id': {'validator': 'int', 'transform': 'int'}}
with self.assertRaisesRegex(Invalid, "\['id', 23.0\] not a dictionary but is of type list"):
_ = maat_scale(test_input_list, test_validation)
with self.assertRaisesRegex(Invalid, "\('id', 23.0\) not a dictionary but is of type tuple"):
_ = maat_scale(test_input_tuple, test_validation)
def test_item_nullable(self):
"""a helpfull message is shown when trying to use post transformation that is not registered"""
test_input = {'id': None}
test_validation = {'id': {'validator': 'float'}}
with self.assertRaisesRegex(Invalid, 'key: \"id\" contains invalid item \"None\" with type \"NoneType\": not of type float'):
_ = maat_scale(test_input, test_validation)
test_validation = {'id': {'validator': 'float', 'null_able': True}}
validated_items = maat_scale(test_input, test_validation)
difference = ddiff(validated_items, test_input)
self.assertEqual(difference, {})
def test_validation_test_invalid_keys_exception(self):
"""Test invalid keys exception"""
del self.test_input['type']
with self.assertRaisesRegex(Invalid, 'key:"type" is not set'):
_ = maat_scale(self.test_input, self.test_validation)
def test_validate_fail_within_nested_list_dicts_with_custom_validation(self):
from maat import registered_functions
def blacklist_address(key, val):
"""Since this is a example blacklisted is hardcoded.
It could come from config or an key value store at runtime
Either within this function or given as argument.
"""
blacklist = ['blacklisted', 'black_listed', 'BLACKLISTED']
if val in blacklist:
raise Invalid('{0} is in blacklist of {1}'.format(key, val))
return val
registered_functions['valid_address'] = blacklist_address
test_input = {
'id': 23,
'addresses': [
{'street': 'valid adress'},
{'street': 'blacklisted'},
]
}
test_validation = {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'list', 'list_dicts': True, 'nested': {
'street': {'validator': 'valid_address'}}
}
}
with self.assertRaisesRegex(Invalid, "street is in blacklist of blacklisted"):
_ = maat_scale(test_input, test_validation)
def test_validation(self):
"""Happy path test"""
validated_items = maat_scale(self.test_input, self.test_validation)
difference = ddiff(validated_items, self.test_input)
# if the differ finds no difference a empty dictionary is returned
self.assertEqual(difference, {})
def test_pre_post_transformation(self):
"""Input dict has item with type float, transform to int, validate as int, tranform back to int"""
test_input = {'id': float(23)}
test_validation = {'id': {'validator': 'int', 'transform': 'float', 'pre_transform': 'int'}}
validated_items = maat_scale(test_input, test_validation)
difference = ddiff(validated_items, test_input)
self.assertEqual(difference, {})
def test_not_set_validator(self):
"""Approriate message shown when trying to use a validator that is not registered"""
test_input = {'id': 23}
test_validation = {'id': {'validator': 'integer'}}
with self.assertRaisesRegex(Invalid, 'integer is not registered as validator'):
_ = maat_scale(test_input, test_validation)
def test_validate_programming_error_trying_to_validate_non_dict(self):
test_input = [1, 2, 4]
with self.assertRaisesRegex(Invalid, "\[1, 2, 4\] not a dictionary but is of type list"):
_ = maat_scale(test_input, self.test_validation)
test_input = set([1, 2, 3])
with self.assertRaisesRegex(Invalid, "\{1, 2, 3\} not a dictionary but is of type set"):
_ = maat_scale(test_input, self.test_validation)
test_input = 1
with self.assertRaisesRegex(Invalid, "1 not a dictionary but is of type int"):
_ = maat_scale(test_input, self.test_validation)
test_input = None
with self.assertRaisesRegex(Invalid, "None not a dictionary but is of type None"):
_ = maat_scale(test_input, self.test_validation)
def test_validation_test_remove_key_and_set_optional(self):
"""Test remove key and set optional"""
del self.test_input['type']
self.test_validation['type']['optional'] = True
validated_items = maat_scale(self.test_input, self.test_validation)
difference = ddiff(validated_items, self.test_input)
self.assertEqual(difference, {})
def test_missing_key_validation(self):
"""Items that will raise an exception"""
del(self.test_validation['name'])
expected_expection_msg = 'invalid keys: name :expected keys: id, type'
if sys.version_info[1] < 6:
expected_expection_msg = 'invalid keys: name'
with self.assertRaisesRegex(Invalid, expected_expection_msg):
_ = maat_scale(self.test_input, self.test_validation)
def test_validation_test_remove_key_and_set_default(self):
"""Test remove key and set optional"""
del self.test_input['type']
excepted_value = 'banana'
self.test_validation['type']['default_value'] = excepted_value
validated_items = maat_scale(self.test_input, self.test_validation)
difference = ddiff(validated_items, self.test_input)
self.assertEqual(difference, {'dictionary_item_removed': set(["root['type']"])})
self.assertEqual(validated_items['type'], excepted_value)
def test_validate_nested_dict(self):
self.test_input = {
'id': 23,
'addresses': {
'street': 'John Doe Street',
'number': 123,
}
}
self.test_validation = {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'dict', 'key_regex': r'(\w+ )', 'nested': {
'street': {'validator': 'str', 'min_length': 5, 'max_length': 99},
'number': {'validator': 'int', 'min_amount': 1},
}
}
}
validated_items = maat_scale(self.test_input, self.test_validation)
difference = ddiff(validated_items, self.test_input)
self.assertEqual(difference, {})
def test_validate_nested_list(self):
self.test_input = {
'id': 23,
'addresses': [
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
]
}
self.test_validation = {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'list',
'list_dicts': {
'street': {'validator': 'str', 'min_length': 5, 'max_length': 99},
}
}
}
validated_items = maat_scale(self.test_input, self.test_validation)
difference = ddiff(validated_items, self.test_input)
self.assertEqual(difference, {})
def test_validate_skip_instead_of_fail_within_nested_list_with_custom_validation(self):
from maat import registered_functions
def blacklist_address(key, val):
"""Since this is a example blacklisted is hardcoded.
It could come from config or an key value store at runtime
Either within this function or given as argument.
"""
blacklist = ['blacklisted', 'black_listed', 'BLACKLISTED']
if val in blacklist:
raise Invalid('{0} is in blacklist of {1}'.format(key, val))
return val
registered_functions['valid_address'] = blacklist_address
test_input = {
'id': 23,
'addresses': [
'blacklisted',
'valid adress',
'also valid address',
'black_listed',
'valid again',
'BLACKLISTED',
]
}
test_validation = {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'valid_address', 'skip_failed': True, 'list': True, 'nested': True}
}
expected_result = {
'id': 23,
'addresses': [
'valid adress',
'also valid address',
'valid again',
]
}
validated_items = maat_scale(test_input, test_validation)
difference = ddiff(validated_items, expected_result)
self.assertEqual(difference, {})
def test_validate_item_800_deep_invalid_item(self):
input_dict = current = {}
counter_dict = counter_current = {}
times = 800
for _ in range(times):
current['nested_dic'] = {}
current = current['nested_dic']
# set last item
current['last'] = 4
counter_current['nested_dic'] = {'validator': 'dict', 'min_amount': 0, 'max_amount': 10, 'nested': {}}
for _ in range(times - 1):
counter_current['nested_dic']['nested'] = {'nested_dic': {'validator': 'dict', 'min_amount': 0, 'max_amount': 10, 'nested': {}}}
counter_current = counter_current['nested_dic']['nested']
# increase minimal amount to 5
counter_current['nested_dic']['nested'] = {'last': {'validator': 'int', 'min_amount': 5, 'max_amount': 10}}
with self.assertRaisesRegex(Invalid, 'key: "last" contains invalid item "4": integer is less then 5'):
_ = maat_scale(input_dict, counter_dict)
def test_validate_invalid_depth(self):
input_dict = current = {}
counter_dict = counter_current = {}
times = sys.getrecursionlimit()
for _ in range(times):
current['nested_dic'] = {}
current = current['nested_dic']
# set last item
current['last'] = 4
counter_current['nested_dic'] = {'validator': 'dict', 'min_amount': 0, 'max_amount': 10, 'nested': {}}
for _ in range(times - 1):
counter_current['nested_dic']['nested'] = {'nested_dic': {'validator': 'dict', 'min_amount': 0, 'max_amount': 10, 'nested': {}}}
counter_current = counter_current['nested_dic']['nested']
counter_current['nested_dic']['nested'] = {'last': {'validator': 'int', 'min_amount': 1, 'max_amount': 10}}
error_msg = sys.getrecursionlimit() - 49
with self.assertRaisesRegex(Invalid, '{}: invalid depth of dict'.format(error_msg)):
_ = maat_scale(input_dict, counter_dict)
def test_validate_item_200_deep(self):
"""Lower depth for deepdiff limits, then later tests"""
input_dict = current = {}
counter_dict = counter_current = {}
times = 200
for _ in range(times):
current['nested_dic'] = {}
current = current['nested_dic']
# set last item
current['last'] = 4
counter_current['nested_dic'] = {'validator': 'dict', 'min_amount': 0, 'max_amount': 10, 'nested': {}}
for _ in range(times - 1):
counter_current['nested_dic']['nested'] = {'nested_dic': {'validator': 'dict', 'min_amount': 0, 'max_amount': 10, 'nested': {}}}
counter_current = counter_current['nested_dic']['nested']
counter_current['nested_dic']['nested'] = {'last': {'validator': 'int', 'min_amount': 1, 'max_amount': 10}}
validated_items = maat_scale(input_dict, counter_dict)
difference = ddiff(validated_items, input_dict)
self.assertEqual(difference, {})
def test_validation_new_syntax_str_cast(self):
test_input = {'id': 23}
expected = {'id': '23'}
counter_dict = {'id': {'validator': 'str', 'cast': True}}
result = maat.maat_scale(test_input, counter_dict)
self.assertEqual(expected, result)
def test_validate_very_nested_dict(self):
nested_dict = {
'data': {
'people': {
'7': {
'id': 7,
'name': 'John Doe',
'type': 'mimic',
'x': 823.6228647149701,
'y': 157.57736006592654,
'address': {
'id': 23,
'addresses': {
'street': {
'two': 'deep',
'222': 'deep',
}
}
}
},
'208': {
'id': 208,
'name': 'John Doe Too',
'type': 'person',
'x': 434.9446032612515,
'y': 580.0,
'address': {
'id': 23,
'addresses': {
'street': {
'two': 'deep',
'222': 'deep',
}
}
}
}
},
'streets': {
'id': 23,
'addresses': [
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
]
}
}
}
addresses_item = {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'list', 'nested': {
'street': {'validator': 'dict', 'min_amount': 5, 'max_length': 99, 'nested': {
'two': {'validator': 'str', 'min_length': 3, 'max_length': 99},
'222': {'validator': 'str', 'min_length': 3, 'max_length': 99},
}
}
}
}
}
geo_item = {
'id': {'validator': 'int', 'min_amount': 1},
'name': {'validator': 'str', 'min_length': 1, 'max_length': 35, 'regex': '([^\s]+)'},
'type': {'validator': 'str', 'min_length': 1, 'max_length': 25, 'regex': r'([^\s]+)'},
'x': {'validator': 'float'},
'y': {'validator': 'float'},
'address': {'validator': 'dict',
'nested': addresses_item}
}
nested_dict_validation = {
'data': {'validator': 'dict', 'nested': {
'people': {'validator': 'dict', 'min_amount': 1, 'max_amount': 99, 'aso_array': True,
'nested': geo_item},
'streets': {'validator': 'dict', 'nested': {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'list', 'list_dicts': True, 'nested': {
'street': {'validator': 'str', 'min_length': 1, 'max_length': 99}
}
}
}
}
}
}
}
validated_items = maat_scale(nested_dict, nested_dict_validation)
difference = ddiff(validated_items, nested_dict)
self.assertEqual(difference, {})
def test_validate_invalid_id(self):
"""Set id to invalid value, expect Exception"""
self.test_input['id'] = -1
with self.assertRaisesRegex(Invalid, 'key: "id" contains invalid item "-1": integer is less then 1'):
_ = maat_scale(self.test_input, self.test_validation)
def test_validate_very_nested_dict_fail_lowest_item(self):
nested_dict = {
'data': {
'people': {
'7': {
'id': 7,
'name': 'John Doe',
'type': 'mimic',
'x': 823.6228647149701,
'y': 157.57736006592654,
'address': {
'id': 23,
'addresses': {
'street': {
'two': 'deep',
'222': 'deep',
}
}
}
},
'208': {
'id': 208,
'name': 'John Doe Too',
'type': 'person',
'x': 434.9446032612515,
'y': 580.0,
'address': {
'id': 23,
'addresses': {
'street': {
'two': 'deep',
'222': 'deep',
}
}
}
}
},
'streets': {
'id': 23,
'addresses': [
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
{'street': 'John Doe Street'},
]
}
}
}
addresses_item = {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'list', 'nested': {
'street': {'validator': 'dict', 'min_amount': 5, 'max_length': 99, 'nested': {
'two': {'validator': 'str', 'min_length': 3, 'max_length': 99},
'222': {'validator': 'str', 'min_length': 3, 'max_length': 99, 'choices': [
'not_part_of_choices',
'fail_here']},
}
}
}
}
}
geo_item = {
'id': {'validator': 'int', 'min_amount': 1},
'name': {'validator': 'str', 'min_length': 1, 'max_length': 35, 'regex': '([^\s]+)'},
'type': {'validator': 'str', 'min_length': 1, 'max_length': 25, 'regex': r'([^\s]+)'},
'x': {'validator': 'float'},
'y': {'validator': 'float'},
'address': {'validator': 'dict',
'nested': addresses_item}
}
nested_dict_validation = {
'data': {'validator': 'dict', 'nested': {
'people': {'validator': 'dict', 'min_amount': 1, 'max_amount': 99, 'aso_array': True,
'nested': geo_item},
'streets': {'validator': 'dict', 'nested': {
'id': {'validator': 'int', 'min_amount': 1},
'addresses': {'validator': 'list', 'list_dicts': True, 'nested': {
'street': {'validator': 'str', 'min_length': 1, 'max_length': 99}
}
}
}
}
}
}
}
exp_exc_msg = "key: \"222\" contains invalid item \"deep\": not in valid choices \['not_part_of_choices', 'fail_here'\]"
with self.assertRaisesRegex(Invalid, exp_exc_msg):
_ = maat_scale(nested_dict, nested_dict_validation)
def test_not_set_post_transformation_function(self):
"""a helpfull message is shown when trying to use post transformation that is not registered"""
test_input = {'id': 23}
test_validation = {'id': {'validator': 'float', 'transform': 'integer'}}
with self.assertRaisesRegex(Invalid, 'integer is not registered as transformation'):
_ = maat_scale(test_input, test_validation)
def test_not_set_pre_transformation_function(self):
"""Approriate message shown when trying to use pre transformation that is not registered"""
test_input = {'id': float(23)}
test_validation = {'id': {'validator': 'int', 'pre_transform': 'integer'}}
with self.assertRaisesRegex(Invalid, 'integer is not registered as transformation'):
_ = maat_scale(test_input, test_validation)
def test_missing_input_key_validation(self):
"""Items that will raise an exception"""
del(self.test_input['id'])
with self.assertRaisesRegex(Invalid, 'key:"id" is not set'):
_ = maat_scale(self.test_input, self.test_validation)
