class dynamodb_client_UnitTestCase(unittest.TestCase):
TEST_CONFIG = {
'row_mapper': {
'lambda_name': 'S',
'invocation_id': 'S',
'en_time': 'N',
'hash_col': 'S',
'range_col': 'N',
'other_col': 'S',
'new_col': 'S',
'some_col': 'S',
'some_counter': 'N',
'some_bool': 'BOOL',
'some_bool2': 'BOOL',
'some_map': 'M',
'some_list': 'L'
},
'required_fields': ['lambda_name'],
'table_name': 'autotest_dynamo_db',
'hash_key': 'hash_col',
}
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.patcher = patch("boto3.client")
self.paginator_mock = MagicMock()
self.dynamo_mock = MagicMock()
self.dynamo_mock.get_paginator.return_value = self.paginator_mock
self.boto3_client_patch = self.patcher.start()
self.boto3_client_patch.return_value = self.dynamo_mock
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def tearDown(self):
self.patcher.stop()
def test_create__raises__if_no_hash_col_configured(self):
bad_config = deepcopy(self.TEST_CONFIG)
del bad_config['hash_key']
dynamo_client = DynamoDbClient(config=bad_config)
row = {self.HASH_KEY: 'cat', self.RANGE_KEY: '123'}
self.assertRaises(AssertionError, dynamo_client.create, row,
self.table_name)
def test_create__calls_boto_client(self):
self.dynamo_mock.put_item.assert_not_called()
self.dynamo_client.put({
self.HASH_KEY: 'cat',
self.RANGE_KEY: '123'
}, self.table_name)
self.dynamo_mock.put_item.assert_called_once()
def test_dict_to_dynamo_strict(self):
dict_row = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': True,
'some_bool2': 'True',
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row)
expected = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'some_bool': {
'BOOL': True
},
'some_bool2': {
'BOOL': True
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_not_strict(self):
dict_row = {
'name': 'cat',
'age': 3,
'other_bool': False,
'other_bool2': 'False',
'other_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {
'name': {
'S': 'cat'
},
'age': {
'N': '3'
},
'other_bool': {
'BOOL': False
},
'other_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo__not_strict__map_type(self):
dict_row = {
'accept_mimetypes': {
'image/webp': 1,
'image/apng': 1,
'image/*': 1,
'*/*': 0.8
},
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {}
logging.info(f"dynamo_row: {dynamo_row}")
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_prefix(self):
dict_row = {'hash_col': 'cat', 'range_col': '123', 'some_col': 'no'}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row,
add_prefix="#")
expected = {
'#hash_col': {
'S': 'cat'
},
'#range_col': {
'N': '123'
},
'#some_col': {
'S': 'no'
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dynamo_to_dict(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key': {
'N': '42'
},
'some_bool': {
'BOOL': False
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': False,
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
self.assertDictEqual(expected, dict_row)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
for k, v in dict_row['some_map'].items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict_no_strict_row_mapper(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key_n': {
'N': '42'
},
'extra_key_s': {
'S': 'wowie'
},
'other_bool': {
'BOOL': True
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'extra_key_n': 42,
'extra_key_s': 'wowie',
'other_bool': True
}
self.assertDictEqual(dict_row, expected)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict__dont_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = True
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}',
'duck_quack': '{"quack": "duck"}'
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}'
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__do_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = False
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
},
'duck_quack': {
"quack": "duck"
}
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
}
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__mapping_doesnt_match__raises(self):
# If the value type in the DB doesn't match the expected type in row_mapper - raise ValueError
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'N': '111'
} # In the row_mapper, other_col is of type 'S'
}
with self.assertRaises(ValueError) as e:
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
self.assertEqual(
"'other_col' is expected to be of type 'S' in row_mapper, but real value is of type 'N'",
str(e.exception))
def test_get_by_query__validates_comparison(self):
self.assertRaises(AssertionError,
self.dynamo_client.get_by_query,
keys={'k': '1'},
comparisons={'k': 'unsupported'})
def test_get_by_query__between(self):
keys = {
'hash_col': 'cat',
'st_between_range_col': '3',
'en_between_range_col': '6'
}
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys)
# print(f"Call_args for paginate: {self.paginator_mock.paginate.call_args}")
args, kwargs = self.paginator_mock.paginate.call_args
# print(kwargs)
self.assertEqual(len(kwargs['ExpressionAttributeValues']), 3)
self.assertIn(
'range_col between :st_between_range_col and :en_between_range_col',
kwargs['KeyConditionExpression'])
def test_get_by_query__return_count(self):
# Make sure dynamo paginator is mocked.
self.paginator_mock.paginate.return_value = [{
'Count': 24,
'LastEvaluatedKey': 'bzz'
}, {
'Count': 12
}]
self.dynamo_client.dynamo_client.get_paginator.return_value = self.paginator_mock
# Call the manager
result = self.dynamo_client.get_by_query(keys={'a': 'b'},
return_count=True)
# Validate result
self.assertEqual(
result, 36,
f"Result from 2 pages should be 24 + 12, but we received: {result}"
)
# Make sure the paginator was called
self.dynamo_client.dynamo_client.get_paginator.assert_called()
def test_get_by_query__expr_attr(self):
keys = {
'st_between_range_col': '3',
'en_between_range_col': '6',
'session': 'ses1'
}
expr_attrs_names = ['range_col', 'session']
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys,
expr_attrs_names=expr_attrs_names)
args, kwargs = self.paginator_mock.paginate.call_args
self.assertIn('#range_col', kwargs['ExpressionAttributeNames'])
self.assertIn('#session', kwargs['ExpressionAttributeNames'])
self.assertIn(
'#range_col between :st_between_range_col and :en_between_range_col AND #session = :session',
kwargs['KeyConditionExpression'])
def test__parse_filter_expression(self):
TESTS = {
'key = 42': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
' key = 42 ': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
'cat = meaw': ("cat = :filter_cat", {
":filter_cat": {
'S': 'meaw'
}
}),
'magic between 41 and 42':
("magic between :st_between_magic and :en_between_magic", {
":st_between_magic": {
'N': '41'
},
":en_between_magic": {
'N': '42'
}
}),
'attribute_not_exists boo': ("attribute_not_exists (boo)", {})
}
for data, expected in TESTS.items():
self.assertEqual(self.dynamo_client._parse_filter_expression(data),
expected)
def test__parse_filter_expression__raises(self):
TESTS = [
{
'k': 1
},
[1, 2],
None, # Invalid input types
'key == 42',
'foo ~ 1',
'foo3 <=> 0',
'key between 42', # Invalid operators
'key between 23, 25',
'key between [23, 25]',
'key 23 between 21', # Invalid between formats.
]
for data in TESTS:
self.assertRaises((AssertionError, ValueError),
self.dynamo_client._parse_filter_expression,
data)
def test_create__calls_put(self):
row = {'hash_col': 'cat', 'range_key': 'test', 'another_col': 'wow'}
self.dynamo_client.put = MagicMock(return_value=None)
self.dynamo_client.create(row)
self.dynamo_client.put.assert_called_once_with(
row, None, overwrite_existing=False)
def test_batch_get_items_one_table__strict(self):
# Strict - returns only fields that are in the row mapper
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=False)
self.assertEqual(result, [{'hash_col': 'b', 'range_col': 10}])
def test_batch_get_items_one_table__not_strict(self):
# Not strict - returns all fields
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=True)
self.assertEqual(result, [{
'hash_col': 'b',
'range_col': 10,
'unknown_col': 'not_strict'
}])
def test_get_by_query__max_items_and_count__raises(self):
with self.assertRaises(Exception) as e:
self.dynamo_client.get_by_query({'hash_col': 'key'},
table_name=self.table_name,
max_items=3,
return_count=True)
expected_msg = "DynamoDbCLient.get_by_query does not support `max_items` and `return_count` together"
self.assertEqual(e.exception.args[0], expected_msg)
def test_patch__transfers_attrs_to_remove(self):
keys = {'hash_col': 'a'}
attributes_to_update = {'some_col': 'b'}
attributes_to_increment = {'some_counter': 3}
table_name = 'the_table'
attributes_to_remove = ['remove_me']
# using kwargs
self.dynamo_client.update = Mock()
self.dynamo_client.patch(
keys=keys,
attributes_to_update=attributes_to_update,
attributes_to_increment=attributes_to_increment,
table_name=table_name,
attributes_to_remove=attributes_to_remove)
self.dynamo_client.update.assert_called_once_with(
keys=keys,
attributes_to_update=attributes_to_update,
attributes_to_increment=attributes_to_increment,
table_name=table_name,
attributes_to_remove=attributes_to_remove,
condition_expression='attribute_exists hash_col')
# not kwargs
self.dynamo_client.update = Mock()
self.dynamo_client.patch(keys, attributes_to_update,
attributes_to_increment, table_name,
attributes_to_remove)
self.dynamo_client.update.assert_called_once_with(
keys=keys,
attributes_to_update=attributes_to_update,
attributes_to_increment=attributes_to_increment,
table_name=table_name,
attributes_to_remove=attributes_to_remove,
condition_expression='attribute_exists hash_col')
def test_sleep_db__get_capacity_called(self):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.sleep_db(last_action_time=datetime.datetime.now(),
action='write',
table_name='autotest_new')
self.dynamo_client.dynamo_client.describe_table.assert_called_once()
def test_sleep_db__wrong_action(self):
self.assertRaises(KeyError,
self.dynamo_client.sleep_db,
last_action_time=datetime.datetime.now(),
action='call')
@patch.object(time, 'sleep')
def test_sleep_db__fell_asleep(self, mock_sleep):
""" Test for table if BillingMode is PROVISIONED """
self.dynamo_client.get_capacity = MagicMock(return_value={
'read': 10,
'write': 5
})
# Check that went to sleep
time_between_ms = 100
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=time_between_ms)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write')
self.assertEqual(mock_sleep.call_count, 1)
args, kwargs = mock_sleep.call_args
# Should sleep around 1 / capacity second minus "time_between_ms" minus code execution time
self.assertGreater(
args[0], 1 / self.dynamo_client.get_capacity()['write'] -
time_between_ms - 0.02)
self.assertLess(args[0],
1 / self.dynamo_client.get_capacity()['write'])
@patch.object(time, 'sleep')
def test_sleep_db__fell_asleep(self, mock_sleep):
""" Test for table if BillingMode is PAY_PER_REQUEST """
self.dynamo_client.get_capacity = MagicMock(return_value={
'read': 0,
'write': 0
})
self.dynamo_client.sleep_db(last_action_time=datetime.datetime.now(),
action='write')
# Check that didn't go to sleep
time_between_ms = 100
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=time_between_ms)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write')
self.assertEqual(mock_sleep.call_count, 0)
@patch.object(time, 'sleep')
def test_sleep_db__(self, mock_sleep):
self.dynamo_client.get_capacity = MagicMock(return_value={
'read': 10,
'write': 5
})
# Shouldn't go to sleep
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=900)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write')
# Sleep function should not be called
self.assertEqual(mock_sleep.call_count, 0)
@patch.object(time, 'sleep')
def test_sleep_db__returns_none_for_on_demand(self, mock_sleep):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.dynamo_client.describe_table.return_value = {
'TableName': 'autotest_OnDemand'
}
# Check that went to sleep
time_between_ms = 10
last_action_time = datetime.datetime.now() - datetime.timedelta(
milliseconds=time_between_ms)
self.dynamo_client.sleep_db(last_action_time=last_action_time,
action='write',
table_name='autotest_OnDemand')
self.assertEqual(mock_sleep.call_count, 0,
"Should not have called time.sleep")
def test_on_demand_provisioned_throughput__get_capacity(self):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.dynamo_client.describe_table.return_value = {
'TableName': 'autotest_OnDemand'
}
result = self.dynamo_client.get_capacity(
table_name='autotest_OnDemand')
self.assertIsNone(result)
def test_on_demand_provisioned_throughput__get_table_indexes(self):
self.dynamo_client.dynamo_client = MagicMock()
self.dynamo_client.dynamo_client.describe_table.return_value = {
'Table': {
'TableName':
'autotest_OnDemandTable',
'LocalSecondaryIndexes': [],
'GlobalSecondaryIndexes': [{
'IndexName':
'IndexA',
'KeySchema': [
{
'AttributeName': 'SomeAttr',
'KeyType': 'HASH',
},
],
'Projection': {
'ProjectionType': 'ALL',
}
}]
}
}
result = self.dynamo_client.get_table_indexes(
table_name='autotest_OnDemandTable')
self.assertIsNone(result['IndexA'].get('ProvisionedThroughput'))
def test_get_table_indexes__ppr(self):
""" Check return value of get_table_indexes function in case table BillingMode is PAY_PER_REQUEST """
self.dynamo_client._describe_table = Mock(
return_value=PPR_DESCRIBE_TABLE)
expected_indexes = {
'session': {
'projection_type': 'ALL',
'hash_key': 'session',
'provisioned_throughput': {
'write_capacity': 0,
'read_capacity': 0
}
},
'session_id': {
'projection_type': 'ALL',
'hash_key': 'session_id',
'provisioned_throughput': {
'write_capacity': 0,
'read_capacity': 0
}
},
}
self.assertEqual(expected_indexes,
self.dynamo_client.get_table_indexes('actions'))
def test_get_table_indexes__pt(self):
""" Check return value of get_table_indexes function in case table BillingMode is PROVISIONED """
self.dynamo_client._describe_table = Mock(
return_value=PT_DESCRIBE_TABLE)
expected_indexes = {
'name': {
'projection_type': 'ALL',
'hash_key': 'name',
'provisioned_throughput': {
'write_capacity': 10,
'read_capacity': 100
}
},
'city': {
'projection_type': 'ALL',
'hash_key': 'city',
'provisioned_throughput': {
'write_capacity': 10,
'read_capacity': 100
}
},
}
self.assertEqual(expected_indexes,
self.dynamo_client.get_table_indexes('partners'))
class MetaHandler:
"""
MetaHandler is helper class for Essential classes.
It works with DynamoDB table to store the meta data of operations on Tasks.
"""
DEFAULT_CONFIG = {
'write_meta_to_ddb': True,
'dynamo_db_config': {
'table_name':
'sosw_tasks_meta',
'hash_key':
'task_id',
'row_mapper': {
'task_id': 'S',
'created_at': 'N',
'author': 'S',
'invocation_id': 'S',
'log_stream_name': 'S',
'action': 'S'
},
'required_fields': [
'task_id', 'created_at', 'author', 'invocation_id',
'log_stream_name', 'action'
],
},
}
AVAILABLE_ACTIONS_MAPPING = {}
CONTEXT_FIELDS_MAPPINGS = {
'author': 'function_name',
'invocation_id': 'aws_request_id',
'log_stream_name': 'log_stream_name'
}
def __init__(self, custom_config: Dict = None, **kwargs):
# Initialize config from default config
self.config = self.DEFAULT_CONFIG or {}
# Update config recursively from custom config
self.config = recursive_update(self.config, custom_config or {})
if self.config['write_meta_to_ddb']:
try:
self.dynamo_db_client = DynamoDbClient(
config=self.config['dynamo_db_config'])
except:
logging.exception(
"Failed to initialize MetaHandler DynamoDbClient")
self.dynamo_db_client = None
else:
self.dynamo_db_client = None
logger.info(
"Initialized MetaHandler with config %s and DynamoDbClient %s",
self.config, self.dynamo_db_client)
def post(self, task_id: str, action: str, **kwargs):
"""
Write row with meta data to sosw_tasks_meta DynamoDB Table if configured.
As long as collecting the meta data is optional, the MetaHandler will either save it
to DynamoDB or just log.
MetaHandler does not restrict the fields and actions you can post from your Worker Lambdas with the exception of
the following built-in fields. These ones are always taken from the current ``lambda_context`` automatically.
* ``'author'``
* ``'invocation_id'``
* ``'log_stream_name'``
"""
row = {
'task_id': task_id,
'created_at': time.time(),
'action': self._ma(action)
}
assert not set(kwargs).intersection(set(self.CONTEXT_FIELDS_MAPPINGS)), \
'args from lambda_context should not be passed'
for field, value in kwargs.items():
row[field] = value
for field, mapping in self.CONTEXT_FIELDS_MAPPINGS.items():
row[field] = getattr(global_vars.lambda_context, mapping)
if self.dynamo_db_client:
self.dynamo_db_client.create(row=row)
else:
logger.info(
"DynamoDB client/table is not configured for meta_handler. Skip saving task meta data: %s",
row)
def _ma(self, field_name):
# FIXMEONEDAY: implement mappings for the actions names
return str(field_name)
class dynamodb_client_UnitTestCase(unittest.TestCase):
TEST_CONFIG = {
'row_mapper': {
'lambda_name': 'S',
'invocation_id': 'S',
'en_time': 'N',
'hash_col': 'S',
'range_col': 'N',
'other_col': 'S',
'new_col': 'S',
'some_col': 'S',
'some_counter': 'N',
'some_bool': 'BOOL',
'some_bool2': 'BOOL',
'some_map': 'M',
'some_list': 'L'
},
'required_fields': ['lambda_name'],
'table_name': 'autotest_dynamo_db'
}
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.patcher = patch("boto3.client")
self.paginator_mock = MagicMock()
self.dynamo_mock = MagicMock()
self.dynamo_mock.get_paginator.return_value = self.paginator_mock
self.boto3_client_patch = self.patcher.start()
self.boto3_client_patch.return_value = self.dynamo_mock
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def tearDown(self):
self.patcher.stop()
def test_dict_to_dynamo_strict(self):
dict_row = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': True,
'some_bool2': 'True',
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row)
expected = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'some_bool': {
'BOOL': True
},
'some_bool2': {
'BOOL': True
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_not_strict(self):
dict_row = {
'name': 'cat',
'age': 3,
'other_bool': False,
'other_bool2': 'False',
'other_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {
'name': {
'S': 'cat'
},
'age': {
'N': '3'
},
'other_bool': {
'BOOL': False
},
'other_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo__not_strict__map_type(self):
dict_row = {
'accept_mimetypes': {
'image/webp': 1,
'image/apng': 1,
'image/*': 1,
'*/*': 0.8
},
}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row, strict=False)
expected = {}
logging.info(f"dynamo_row: {dynamo_row}")
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dict_to_dynamo_prefix(self):
dict_row = {'hash_col': 'cat', 'range_col': '123', 'some_col': 'no'}
dynamo_row = self.dynamo_client.dict_to_dynamo(dict_row,
add_prefix="#")
expected = {
'#hash_col': {
'S': 'cat'
},
'#range_col': {
'N': '123'
},
'#some_col': {
'S': 'no'
}
}
for key in expected.keys():
self.assertDictEqual(expected[key], dynamo_row[key])
def test_dynamo_to_dict(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key': {
'N': '42'
},
'some_bool': {
'BOOL': False
},
'some_map': {
'M': {
'a': {
'N': '1'
},
'b': {
'S': 'b1'
},
'c': {
'M': {
'test': {
'BOOL': True
}
}
}
}
},
'some_list': {
'L': [{
'S': 'x'
}, {
'S': 'y'
}]
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'some_bool': False,
'some_map': {
'a': 1,
'b': 'b1',
'c': {
'test': True
}
},
'some_list': ['x', 'y']
}
self.assertDictEqual(expected, dict_row)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
for k, v in dict_row['some_map'].items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict_no_strict_row_mapper(self):
dynamo_row = {
'lambda_name': {
'S': 'test_name'
},
'invocation_id': {
'S': 'test_id'
},
'en_time': {
'N': '123456'
},
'extra_key_n': {
'N': '42'
},
'extra_key_s': {
'S': 'wowie'
},
'other_bool': {
'BOOL': True
}
}
dict_row = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'lambda_name': 'test_name',
'invocation_id': 'test_id',
'en_time': 123456,
'extra_key_n': 42,
'extra_key_s': 'wowie',
'other_bool': True
}
self.assertDictEqual(dict_row, expected)
for k, v in dict_row.items():
self.assertNotIsInstance(v, Decimal)
def test_dynamo_to_dict__dont_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = True
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}',
'duck_quack': '{"quack": "duck"}'
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': '{"how many": 300}'
}
self.assertDictEqual(res, expected)
def test_dynamo_to_dict__do_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = False
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
},
'duck_quack': {
"quack": "duck"
}
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
}
}
self.assertDictEqual(res, expected)
def test_get_by_query__validates_comparison(self):
self.assertRaises(AssertionError,
self.dynamo_client.get_by_query,
keys={'k': '1'},
comparisons={'k': 'unsupported'})
def test_get_by_query__between(self):
keys = {
'hash_col': 'cat',
'st_between_range_col': '3',
'en_between_range_col': '6'
}
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys)
# print(f"Call_args for paginate: {self.paginator_mock.paginate.call_args}")
args, kwargs = self.paginator_mock.paginate.call_args
# print(kwargs)
self.assertEqual(len(kwargs['ExpressionAttributeValues']), 3)
self.assertIn(
'range_col between :st_between_range_col and :en_between_range_col',
kwargs['KeyConditionExpression'])
def test_get_by_query__return_count(self):
# Make sure dynamo paginator is mocked.
self.paginator_mock.paginate.return_value = [{
'Count': 24,
'LastEvaluatedKey': 'bzz'
}, {
'Count': 12
}]
self.dynamo_client.dynamo_client.get_paginator.return_value = self.paginator_mock
# Call the manager
result = self.dynamo_client.get_by_query(keys={'a': 'b'},
return_count=True)
# Validate result
self.assertEqual(
result, 36,
f"Result from 2 pages should be 24 + 12, but we received: {result}"
)
# Make sure the paginator was called
self.dynamo_client.dynamo_client.get_paginator.assert_called()
def test__parse_filter_expression(self):
TESTS = {
'key = 42': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
' key = 42 ': ("key = :filter_key", {
":filter_key": {
'N': '42'
}
}),
'cat = meaw': ("cat = :filter_cat", {
":filter_cat": {
'S': 'meaw'
}
}),
'magic between 41 and 42':
("magic between :st_between_magic and :en_between_magic", {
":st_between_magic": {
'N': '41'
},
":en_between_magic": {
'N': '42'
}
}),
'attribute_not_exists boo': ("attribute_not_exists (boo)", {})
}
for data, expected in TESTS.items():
self.assertEqual(self.dynamo_client._parse_filter_expression(data),
expected)
def test__parse_filter_expression__raises(self):
TESTS = [
{
'k': 1
},
[1, 2],
None, # Invalid input types
'key == 42',
'foo ~ 1',
'foo3 <=> 0',
'key between 42', # Invalid operators
'key between 23, 25',
'key between [23, 25]',
'key 23 between 21', # Invalid between formats.
]
for data in TESTS:
self.assertRaises((AssertionError, ValueError),
self.dynamo_client._parse_filter_expression,
data)
def test_create__calls_put(self):
row = {'hash_col': 'cat', 'range_key': 'test', 'another_col': 'wow'}
self.dynamo_client.put = MagicMock(return_value=None)
self.dynamo_client.create(row)
self.dynamo_client.put.assert_called_once_with(
row, None, overwrite_existing=False)
def test_batch_get_items_one_table__strict(self):
# Strict - returns only fields that are in the row mapper
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=False)
self.assertEqual(result, [{'hash_col': 'b', 'range_col': 10}])
def test_batch_get_items_one_table__not_strict(self):
# Not strict - returns all fields
db_items = [{
'hash_col': {
'S': 'b'
},
'range_col': {
'N': '10'
},
'unknown_col': {
'S': 'not_strict'
}
}]
db_result = {'Responses': {'autotest_dynamo_db': db_items}}
self.dynamo_client.dynamo_client.batch_get_item = Mock(
return_value=db_result)
result = self.dynamo_client.batch_get_items_one_table(
keys_list=[{
'hash_col': 'b'
}], fetch_all_fields=True)
self.assertEqual(result, [{
'hash_col': 'b',
'range_col': 10,
'unknown_col': 'not_strict'
}])
def test_get_by_query__max_items_and_count__raises(self):
with self.assertRaises(Exception) as e:
self.dynamo_client.get_by_query({'hash_col': 'key'},
table_name=self.table_name,
max_items=3,
return_count=True)
expected_msg = "DynamoDbCLient.get_by_query does not support `max_items` and `return_count` together"
self.assertEqual(e.exception.args[0], expected_msg)