class dynamodb_client_UnitTestCase(unittest.TestCase):
TEST_CONFIG = {
'row_mapper': {
'lambda_name': 'S',
'invocation_id': 'S',
'en_time': 'N',
'hash_col': 'S',
'range_col': 'N',
'other_col': 'S',
'new_col': 'S',
'some_col': 'S',
'some_counter': 'N',
'some_bool': 'BOOL',
'some_bool2': 'BOOL',
'some_map': 'M',
'some_list': 'L'
},
'required_fields': ['lambda_name'],
'table_name': 'autotest_dynamo_db'
}
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.patcher = patch("boto3.client")
self.paginator_mock = MagicMock()
self.dynamo_mock = MagicMock()
self.dynamo_mock.get_paginator.return_value = self.paginator_mock
self.boto3_client_patch = self.patcher.start()
self.boto3_client_patch.return_value = self.dynamo_mock
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def tearDown(self):
self.patcher.stop()
def test_dict_to_dynamo_strict(self):
dict_row = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': True,
'some_bool2': 'True',
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row)
expected = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'some_bool': {
'BOOL': True
},
'some_bool2': {
'BOOL': True
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_not_strict(self):
dict_row = {
'name': 'cat',
'age': 3,
'other_bool': False,
'other_bool2': 'False',
'other_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {
'name': {
'S': 'cat'
},
'age': {
'N': '3'
},
'other_bool': {
'BOOL': False
},
'other_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo__not_strict__map_type(self):
dict_row = {
'accept_mimetypes': {
'image/webp': 1,
'image/apng': 1,
'image/*': 1,
'*/*': 0.8
},
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {}
logging.info(f"dynamo_row: {dynamo_row}")
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_prefix(self):
dict_row = {'hash_col': 'cat', 'range_col': '123', 'some_col': 'no'}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row,
add_prefix="#")
expected = {
'#hash_col': {
'S': 'cat'
},
'#range_col': {
'N': '123'
},
'#some_col': {
'S': 'no'
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dynamo_to_dict(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key': {
'N': '42'
},
'some_bool': {
'BOOL': False
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': False,
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
self.assertDictEqual(expected, dict_row)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
for k, v in dict_row['some_map'].items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict_no_strict_row_mapper(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key_n': {
'N': '42'
},
'extra_key_s': {
'S': 'wowie'
},
'other_bool': {
'BOOL': True
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'extra_key_n': 42,
'extra_key_s': 'wowie',
'other_bool': True
}
self.assertDictEqual(dict_row, expected)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict__dont_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = True
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}',
'duck_quack': '{"quack": "duck"}'
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}'
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__do_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = False
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
},
'duck_quack': {
"quack": "duck"
}
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
}
}
self.assertDictEqual(res, expected)
def test_get_by_query__validates_comparison(self):
self.assertRaises(AssertionError,
self.dynamo_client.get_by_query,
keys={'k': '1'},
comparisons={'k': 'unsupported'})
def test_get_by_query__between(self):
keys = {
'hash_col': 'cat',
'st_between_range_col': '3',
'en_between_range_col': '6'
}
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys)
# print(f"Call_args for paginate: {self.paginator_mock.paginate.call_args}")
args, kwargs = self.paginator_mock.paginate.call_args
# print(kwargs)
self.assertEqual(len(kwargs['ExpressionAttributeValues']), 3)
self.assertIn(
'range_col between :st_between_range_col and :en_between_range_col',
kwargs['KeyConditionExpression'])
def test_get_by_query__return_count(self):
# Make sure dynamo paginator is mocked.
self.paginator_mock.paginate.return_value = [{
'Count': 24,
'LastEvaluatedKey': 'bzz'
}, {
'Count': 12
}]
self.dynamo_client.dynamo_client.get_paginator.return_value = self.paginator_mock
# Call the manager
result = self.dynamo_client.get_by_query(keys={'a': 'b'},
return_count=True)
# Validate result
self.assertEqual(
result, 36,
f"Result from 2 pages should be 24 + 12, but we received: {result}"
)
# Make sure the paginator was called
self.dynamo_client.dynamo_client.get_paginator.assert_called()
def test__parse_filter_expression(self):
TESTS = {
'key = 42': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
' key = 42 ': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
'cat = meaw': ("cat = :filter_cat", {
":filter_cat": {
'S': 'meaw'
}
}),
'magic between 41 and 42':
("magic between :st_between_magic and :en_between_magic", {
":st_between_magic": {
'N': '41'
},
":en_between_magic": {
'N': '42'
}
}),
'attribute_not_exists boo': ("attribute_not_exists (boo)", {})
}
for data, expected in TESTS.items():
self.assertEqual(self.dynamo_client._parse_filter_expression(data),
expected)
def test__parse_filter_expression__raises(self):
TESTS = [
{
'k': 1
},
[1, 2],
None, # Invalid input types
'key == 42',
'foo ~ 1',
'foo3 <=> 0',
'key between 42', # Invalid operators
'key between 23, 25',
'key between [23, 25]',
'key 23 between 21', # Invalid between formats.
]
for data in TESTS:
self.assertRaises((AssertionError, ValueError),
self.dynamo_client._parse_filter_expression,
data)
def test_create__calls_put(self):
row = {'hash_col': 'cat', 'range_key': 'test', 'another_col': 'wow'}
self.dynamo_client.put = MagicMock(return_value=None)
self.dynamo_client.create(row)
self.dynamo_client.put.assert_called_once_with(
row, None, overwrite_existing=False)
def test_batch_get_items_one_table__strict(self):
# Strict - returns only fields that are in the row mapper
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=False)
self.assertEqual(result, [{'hash_col': 'b', 'range_col': 10}])
def test_batch_get_items_one_table__not_strict(self):
# Not strict - returns all fields
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=True)
self.assertEqual(result, [{
'hash_col': 'b',
'range_col': 10,
'unknown_col': 'not_strict'
}])
def test_get_by_query__max_items_and_count__raises(self):
with self.assertRaises(Exception) as e:
self.dynamo_client.get_by_query({'hash_col': 'key'},
table_name=self.table_name,
max_items=3,
return_count=True)
expected_msg = "DynamoDbCLient.get_by_query does not support `max_items` and `return_count` together"
self.assertEqual(e.exception.args[0], expected_msg)
class dynamodb_client_UnitTestCase(unittest.TestCase):
TEST_CONFIG = {
'row_mapper': {
'lambda_name': 'S',
'invocation_id': 'S',
'en_time': 'N',
'hash_col': 'S',
'range_col': 'N',
'other_col': 'S',
'new_col': 'S',
'some_col': 'S',
'some_counter': 'N',
'some_bool': 'BOOL',
'some_bool2': 'BOOL',
'some_map': 'M',
'some_list': 'L'
},
'required_fields': ['lambda_name'],
'table_name': 'autotest_dynamo_db',
'hash_key': 'hash_col',
}
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.patcher = patch("boto3.client")
self.paginator_mock = MagicMock()
self.dynamo_mock = MagicMock()
self.dynamo_mock.get_paginator.return_value = self.paginator_mock
self.boto3_client_patch = self.patcher.start()
self.boto3_client_patch.return_value = self.dynamo_mock
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def tearDown(self):
self.patcher.stop()
def test_create__raises__if_no_hash_col_configured(self):
bad_config = deepcopy(self.TEST_CONFIG)
del bad_config['hash_key']
dynamo_client = DynamoDbClient(config=bad_config)
row = {self.HASH_KEY: 'cat', self.RANGE_KEY: '123'}
self.assertRaises(AssertionError, dynamo_client.create, row,
self.table_name)
def test_create__calls_boto_client(self):
self.dynamo_mock.put_item.assert_not_called()
self.dynamo_client.put({
self.HASH_KEY: 'cat',
self.RANGE_KEY: '123'
}, self.table_name)
self.dynamo_mock.put_item.assert_called_once()
def test_dict_to_dynamo_strict(self):
dict_row = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': True,
'some_bool2': 'True',
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row)
expected = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'some_bool': {
'BOOL': True
},
'some_bool2': {
'BOOL': True
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_not_strict(self):
dict_row = {
'name': 'cat',
'age': 3,
'other_bool': False,
'other_bool2': 'False',
'other_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {
'name': {
'S': 'cat'
},
'age': {
'N': '3'
},
'other_bool': {
'BOOL': False
},
'other_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo__not_strict__map_type(self):
dict_row = {
'accept_mimetypes': {
'image/webp': 1,
'image/apng': 1,
'image/*': 1,
'*/*': 0.8
},
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {}
logging.info(f"dynamo_row: {dynamo_row}")
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_prefix(self):
dict_row = {'hash_col': 'cat', 'range_col': '123', 'some_col': 'no'}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row,
add_prefix="#")
expected = {
'#hash_col': {
'S': 'cat'
},
'#range_col': {
'N': '123'
},
'#some_col': {
'S': 'no'
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dynamo_to_dict(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key': {
'N': '42'
},
'some_bool': {
'BOOL': False
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': False,
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
self.assertDictEqual(expected, dict_row)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
for k, v in dict_row['some_map'].items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict_no_strict_row_mapper(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key_n': {
'N': '42'
},
'extra_key_s': {
'S': 'wowie'
},
'other_bool': {
'BOOL': True
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'extra_key_n': 42,
'extra_key_s': 'wowie',
'other_bool': True
}
self.assertDictEqual(dict_row, expected)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict__dont_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = True
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}',
'duck_quack': '{"quack": "duck"}'
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}'
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__do_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = False
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
},
'duck_quack': {
"quack": "duck"
}
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
}
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__mapping_doesnt_match__raises(self):
# If the value type in the DB doesn't match the expected type in row_mapper - raise ValueError
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'N': '111'
} # In the row_mapper, other_col is of type 'S'
}
with self.assertRaises(ValueError) as e:
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
self.assertEqual(
"'other_col' is expected to be of type 'S' in row_mapper, but real value is of type 'N'",
str(e.exception))
def test_get_by_query__validates_comparison(self):
self.assertRaises(AssertionError,
self.dynamo_client.get_by_query,
keys={'k': '1'},
comparisons={'k': 'unsupported'})
def test_get_by_query__between(self):
keys = {
'hash_col': 'cat',
'st_between_range_col': '3',
'en_between_range_col': '6'
}
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys)
# print(f"Call_args for paginate: {self.paginator_mock.paginate.call_args}")
args, kwargs = self.paginator_mock.paginate.call_args
# print(kwargs)
self.assertEqual(len(kwargs['ExpressionAttributeValues']), 3)
self.assertIn(
'range_col between :st_between_range_col and :en_between_range_col',
kwargs['KeyConditionExpression'])
def test_get_by_query__return_count(self):
# Make sure dynamo paginator is mocked.
self.paginator_mock.paginate.return_value = [{
'Count': 24,
'LastEvaluatedKey': 'bzz'
}, {
'Count': 12
}]
self.dynamo_client.dynamo_client.get_paginator.return_value = self.paginator_mock
# Call the manager
result = self.dynamo_client.get_by_query(keys={'a': 'b'},
return_count=True)
# Validate result
self.assertEqual(
result, 36,
f"Result from 2 pages should be 24 + 12, but we received: {result}"
)
# Make sure the paginator was called
self.dynamo_client.dynamo_client.get_paginator.assert_called()
def test_get_by_query__expr_attr(self):
keys = {
'st_between_range_col': '3',
'en_between_range_col': '6',
'session': 'ses1'
}
expr_attrs_names = ['range_col', 'session']
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys,
expr_attrs_names=expr_attrs_names)
args, kwargs = self.paginator_mock.paginate.call_args
self.assertIn('#range_col', kwargs['ExpressionAttributeNames'])
self.assertIn('#session', kwargs['ExpressionAttributeNames'])
self.assertIn(
'#range_col between :st_between_range_col and :en_between_range_col AND #session = :session',
kwargs['KeyConditionExpression'])
def test__parse_filter_expression(self):
TESTS = {
'key = 42': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
' key = 42 ': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
'cat = meaw': ("cat = :filter_cat", {
":filter_cat": {
'S': 'meaw'
}
}),
'magic between 41 and 42':
("magic between :st_between_magic and :en_between_magic", {
":st_between_magic": {
'N': '41'
},
":en_between_magic": {
'N': '42'
}
}),
'attribute_not_exists boo': ("attribute_not_exists (boo)", {})
}
for data, expected in TESTS.items():
self.assertEqual(self.dynamo_client._parse_filter_expression(data),
expected)
def test__parse_filter_expression__raises(self):
TESTS = [
{
'k': 1
},
[1, 2],
None, # Invalid input types
'key == 42',
'foo ~ 1',
'foo3 <=> 0',
'key between 42', # Invalid operators
'key between 23, 25',
'key between [23, 25]',
'key 23 between 21', # Invalid between formats.
]
for data in TESTS:
self.assertRaises((AssertionError, ValueError),
self.dynamo_client._parse_filter_expression,
data)
def test_create__calls_put(self):
row = {'hash_col': 'cat', 'range_key': 'test', 'another_col': 'wow'}
self.dynamo_client.put = MagicMock(return_value=None)
self.dynamo_client.create(row)
self.dynamo_client.put.assert_called_once_with(
row, None, overwrite_existing=False)
def test_batch_get_items_one_table__strict(self):
# Strict - returns only fields that are in the row mapper
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=False)
self.assertEqual(result, [{'hash_col': 'b', 'range_col': 10}])
def test_batch_get_items_one_table__not_strict(self):
# Not strict - returns all fields
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=True)
self.assertEqual(result, [{
'hash_col': 'b',
'range_col': 10,
'unknown_col': 'not_strict'
}])
def test_get_by_query__max_items_and_count__raises(self):
with self.assertRaises(Exception) as e:
self.dynamo_client.get_by_query({'hash_col': 'key'},
table_name=self.table_name,
max_items=3,
return_count=True)
expected_msg = "DynamoDbCLient.get_by_query does not support `max_items` and `return_count` together"
self.assertEqual(e.exception.args[0], expected_msg)
def test_patch__transfers_attrs_to_remove(self):
keys = {'hash_col': 'a'}
attributes_to_update = {'some_col': 'b'}
attributes_to_increment = {'some_counter': 3}
table_name = 'the_table'
attributes_to_remove = ['remove_me']
# using kwargs
self.dynamo_client.update = Mock()
self.dynamo_client.patch(
keys=keys,
attributes_to_update=attributes_to_update,
attributes_to_increment=attributes_to_increment,
table_name=table_name,
attributes_to_remove=attributes_to_remove)
self.dynamo_client.update.assert_called_once_with(
keys=keys,
attributes_to_update=attributes_to_update,
attributes_to_increment=attributes_to_increment,
table_name=table_name,
attributes_to_remove=attributes_to_remove,
condition_expression='attribute_exists hash_col')
# not kwargs
self.dynamo_client.update = Mock()
self.dynamo_client.patch(keys, attributes_to_update,
attributes_to_increment, table_name,
attributes_to_remove)
self.dynamo_client.update.assert_called_once_with(
keys=keys,
attributes_to_update=attributes_to_update,
attributes_to_increment=attributes_to_increment,
table_name=table_name,
attributes_to_remove=attributes_to_remove,
condition_expression='attribute_exists hash_col')
def test_sleep_db__get_capacity_called(self):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.sleep_db(last_action_time=datetime.datetime.now(),
action='write',
table_name='autotest_new')
self.dynamo_client.dynamo_client.describe_table.assert_called_once()
def test_sleep_db__wrong_action(self):
self.assertRaises(KeyError,
self.dynamo_client.sleep_db,
last_action_time=datetime.datetime.now(),
action='call')
@patch.object(time, 'sleep')
def test_sleep_db__fell_asleep(self, mock_sleep):
""" Test for table if BillingMode is PROVISIONED """
self.dynamo_client.get_capacity = MagicMock(return_value={
'read': 10,
'write': 5
})
# Check that went to sleep
time_between_ms = 100
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=time_between_ms)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write')
self.assertEqual(mock_sleep.call_count, 1)
args, kwargs = mock_sleep.call_args
# Should sleep around 1 / capacity second minus "time_between_ms" minus code execution time
self.assertGreater(
args[0], 1 / self.dynamo_client.get_capacity()['write'] -
time_between_ms - 0.02)
self.assertLess(args[0],
1 / self.dynamo_client.get_capacity()['write'])
@patch.object(time, 'sleep')
def test_sleep_db__fell_asleep(self, mock_sleep):
""" Test for table if BillingMode is PAY_PER_REQUEST """
self.dynamo_client.get_capacity = MagicMock(return_value={
'read': 0,
'write': 0
})
self.dynamo_client.sleep_db(last_action_time=datetime.datetime.now(),
action='write')
# Check that didn't go to sleep
time_between_ms = 100
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=time_between_ms)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write')
self.assertEqual(mock_sleep.call_count, 0)
@patch.object(time, 'sleep')
def test_sleep_db__(self, mock_sleep):
self.dynamo_client.get_capacity = MagicMock(return_value={
'read': 10,
'write': 5
})
# Shouldn't go to sleep
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=900)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write')
# Sleep function should not be called
self.assertEqual(mock_sleep.call_count, 0)
@patch.object(time, 'sleep')
def test_sleep_db__returns_none_for_on_demand(self, mock_sleep):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.dynamo_client.describe_table.return_value = {
'TableName': 'autotest_OnDemand'
}
# Check that went to sleep
time_between_ms = 10
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=time_between_ms)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write',
table_name='autotest_OnDemand')
self.assertEqual(mock_sleep.call_count, 0,
"Should not have called time.sleep")
def test_on_demand_provisioned_throughput__get_capacity(self):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.dynamo_client.describe_table.return_value = {
'TableName': 'autotest_OnDemand'
}
result = self.dynamo_client.get_capacity(
table_name='autotest_OnDemand')
self.assertIsNone(result)
def test_on_demand_provisioned_throughput__get_table_indexes(self):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.dynamo_client.describe_table.return_value = {
'Table': {
'TableName':
'autotest_OnDemandTable',
'LocalSecondaryIndexes': [],
'GlobalSecondaryIndexes': [{
'IndexName':
'IndexA',
'KeySchema': [
{
'AttributeName': 'SomeAttr',
'KeyType': 'HASH',
},
],
'Projection': {
'ProjectionType': 'ALL',
}
}]
}
}
result = self.dynamo_client.get_table_indexes(
table_name='autotest_OnDemandTable')
self.assertIsNone(result['IndexA'].get('ProvisionedThroughput'))
def test_get_table_indexes__ppr(self):
""" Check return value of get_table_indexes function in case table BillingMode is PAY_PER_REQUEST """
self.dynamo_client._describe_table = Mock(
return_value=PPR_DESCRIBE_TABLE)
expected_indexes = {
'session': {
'projection_type': 'ALL',
'hash_key': 'session',
'provisioned_throughput': {
'write_capacity': 0,
'read_capacity': 0
}
},
'session_id': {
'projection_type': 'ALL',
'hash_key': 'session_id',
'provisioned_throughput': {
'write_capacity': 0,
'read_capacity': 0
}
},
}
self.assertEqual(expected_indexes,
self.dynamo_client.get_table_indexes('actions'))
def test_get_table_indexes__pt(self):
""" Check return value of get_table_indexes function in case table BillingMode is PROVISIONED """
self.dynamo_client._describe_table = Mock(
return_value=PT_DESCRIBE_TABLE)
expected_indexes = {
'name': {
'projection_type': 'ALL',
'hash_key': 'name',
'provisioned_throughput': {
'write_capacity': 10,
'read_capacity': 100
}
},
'city': {
'projection_type': 'ALL',
'hash_key': 'city',
'provisioned_throughput': {
'write_capacity': 10,
'read_capacity': 100
}
},
}
self.assertEqual(expected_indexes,
self.dynamo_client.get_table_indexes('partners'))
class dynamodb_client_UnitTestCase(unittest.TestCase):
TEST_CONFIG = {
'row_mapper': {
'lambda_name': 'S',
'invocation_id': 'S',
'en_time': 'N',
'hash_col': 'S',
'range_col': 'N',
'other_col': 'S',
'new_col': 'S',
'some_col': 'S',
'some_counter': 'N'
},
'required_fields': ['lambda_name'],
'table_name': 'autotest_dynamo_db'
}
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.patcher = patch("boto3.client")
self.dynamo_mock = MagicMock()
self.paginator_mock = MagicMock()
self.boto3_client_patch = self.patcher.start()
self.boto3_client_patch.return_value = self.dynamo_mock
self.dynamo_mock.get_paginator.return_value = self.paginator_mock
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def tearDown(self):
self.patcher.stop()
def test_dict_to_dynamo_strict(self):
dict_row = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row)
expected = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_not_strict(self):
dict_row = {'name': 'cat', 'age': 3}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {'name': {'S': 'cat'}, 'age': {'N': '3'}}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_prefix(self):
dict_row = {'hash_col': 'cat', 'range_col': '123', 'some_col': 'no'}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row,
add_prefix="#")
expected = {
'#hash_col': {
'S': 'cat'
},
'#range_col': {
'N': '123'
},
'#some_col': {
'S': 'no'
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dynamo_to_dict(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key': {
'N': '42'
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456
}
self.assertDictEqual(dict_row, expected)
def test_dynamo_to_dict_no_strict_row_mapper(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key_n': {
'N': '42'
},
'extra_key_s': {
'S': 'wowie'
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row, strict=False)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'extra_key_n': 42,
'extra_key_s': 'wowie'
}
self.assertDictEqual(dict_row, expected)
def test_dynamo_to_dict__dont_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = True
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row, strict=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}',
'duck_quack': '{"quack": "duck"}'
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row, strict=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}'
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__do_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = False
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row, strict=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
},
'duck_quack': {
"quack": "duck"
}
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row, strict=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
}
}
self.assertDictEqual(res, expected)
def test_get_by_query__validates_comparison(self):
self.assertRaises(AssertionError,
self.dynamo_client.get_by_query,
keys={'k': '1'},
comparisons={'k': 'unsupported'})
def test_get_by_query__between(self):
keys = {
'hash_col': 'cat',
'st_between_range_col': '3',
'en_between_range_col': '6'
}
# paginator = MagicMock()
# self.dynamo_mock.get_paginator.return_value = p
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys)
# print(f"Call_args for paginate: {self.paginator_mock.paginate.call_args}")
args, kwargs = self.paginator_mock.paginate.call_args
# print(kwargs)
self.assertEqual(len(kwargs['ExpressionAttributeValues']), 3)
self.assertIn(
'range_col between :st_between_range_col and :en_between_range_col',
kwargs['KeyConditionExpression'])
def test__parse_filter_expression(self):
TESTS = {
'key = 42': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
' key = 42 ': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
'cat = meaw': ("cat = :filter_cat", {
":filter_cat": {
'S': 'meaw'
}
}),
'magic between 41 and 42':
("magic between :st_between_magic and :en_between_magic", {
":st_between_magic": {
'N': '41'
},
":en_between_magic": {
'N': '42'
}
}),
'attribute_not_exists boo': ("attribute_not_exists (boo)", {})
}
for data, expected in TESTS.items():
self.assertEqual(self.dynamo_client._parse_filter_expression(data),
expected)
def test__parse_filter_expression__raises(self):
TESTS = [
{
'k': 1
},
[1, 2],
None, # Invalid input types
'key == 42',
'foo ~ 1',
'foo3 <=> 0',
'key between 42', # Invalid operators
'key between 23, 25',
'key between [23, 25]',
'key 23 between 21', # Invalid between formats.
]
for data in TESTS:
self.assertRaises((AssertionError, ValueError),
self.dynamo_client._parse_filter_expression,
data)