def test_get_by_query__comparison_begins_with(self):
self.table_name = 'autotest_config_component' # This table has a string range key
self.HASH_KEY = ('env', 'S')
self.RANGE_KEY = ('config_name', 'S')
self.KEYS = ('env', 'config_name')
config = {
'row_mapper': {
'env': 'S',
'config_name': 'S',
'config_value': 'S'
},
'required_fields': ['env', 'config_name', 'config_value'],
'table_name': 'autotest_config_component'
}
self.dynamo_client = DynamoDbClient(config=config)
row1 = {'env': 'cat', 'config_name': 'testzing', 'config_value': 'abc123'}
row2 = {'env': 'cat', 'config_name': 'dont_get_this', 'config_value': 'abc123'}
row3 = {'env': 'cat', 'config_name': 'testzer', 'config_value': 'abc124'}
self.dynamo_client.put(row1, self.table_name)
self.dynamo_client.put(row2, self.table_name)
self.dynamo_client.put(row3, self.table_name)
keys = {'env': 'cat', 'config_name': 'testz'}
result = self.dynamo_client.get_by_query(
keys=keys,
table_name=self.table_name,
comparisons={'config_name': 'begins_with'}
)
self.assertEqual(len(result), 2)
self.assertTrue(row1 in result)
self.assertTrue(row3 in result)
def setUp(self):
"""
We keep copies of main parameters here, because they may differ from test to test and cleanup needs them.
This is responsibility of the test author to update these values if required from test.
"""
self.config = self.TEST_CONFIG.copy()
self.patcher = patch("sosw.app.get_config")
self.get_config_patch = self.patcher.start()
self.HASH_KEY = ('task_id', 'S')
self.RANGE_KEY = ('labourer_id', 'S')
self.NOW_TIME = 100000
self.table_name = self.config['dynamo_db_config']['table_name']
self.completed_tasks_table = self.config['sosw_closed_tasks_table']
self.retry_tasks_table = self.config['sosw_retry_tasks_table']
self.clean_task_tables()
self.dynamo_client = DynamoDbClient(
config=self.config['dynamo_db_config'])
self.manager = TaskManager(custom_config=self.config)
self.manager.ecology_client = MagicMock()
self.labourer = deepcopy(self.LABOURER)
def test_create__raises__if_no_hash_col_configured(self):
bad_config = deepcopy(self.TEST_CONFIG)
del bad_config['hash_key']
dynamo_client = DynamoDbClient(config=bad_config)
row = {self.HASH_KEY: 'cat', self.RANGE_KEY: '123'}
self.assertRaises(AssertionError, dynamo_client.create, row,
self.table_name)
def _get_dynamo_client(self):
if self.dynamo_client is None:
dynamo_config = self.config.get('dynamo_client_config')
if self.test:
dynamo_config['table_name'] = 'autotest_config_component'
self.dynamo_client = DynamoDbClient(dynamo_config)
return self.dynamo_client
def setUp(self):
self.HASH_COL = 'hash_col'
self.HASH_KEY = (self.HASH_COL, 'S')
self.RANGE_COL = 'range_col'
self.RANGE_COL_TYPE = 'N'
self.RANGE_KEY = (self.RANGE_COL, self.RANGE_COL)
self.KEYS = (self.HASH_COL, self.RANGE_COL)
self.table_name = 'autotest_dynamo_db'
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def test_get_by_query__expr_attr(self):
keys = {'st_between_range_col': '3', 'en_between_range_col': '6', 'session': 'ses1'}
expr_attrs_names = ['range_col', 'session']
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys, expr_attrs_names=expr_attrs_names)
args, kwargs = self.paginator_mock.paginate.call_args
self.assertIn('#range_col', kwargs['ExpressionAttributeNames'])
self.assertIn('#session', kwargs['ExpressionAttributeNames'])
self.assertIn('#range_col between :st_between_range_col and :en_between_range_col AND #session = :session',
kwargs['KeyConditionExpression'])
def test_get_by_query__between(self):
keys = {'hash_col': 'cat', 'st_between_range_col': '3', 'en_between_range_col': '6'}
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
self.dynamo_client.get_by_query(keys=keys)
# print(f"Call_args for paginate: {self.paginator_mock.paginate.call_args}")
args, kwargs = self.paginator_mock.paginate.call_args
# print(kwargs)
self.assertEqual(len(kwargs['ExpressionAttributeValues']), 3)
self.assertIn('range_col between :st_between_range_col and :en_between_range_col',
kwargs['KeyConditionExpression'])
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.patcher = patch("boto3.client")
self.paginator_mock = MagicMock()
self.dynamo_mock = MagicMock()
self.dynamo_mock.get_paginator.return_value = self.paginator_mock
self.boto3_client_patch = self.patcher.start()
self.boto3_client_patch.return_value = self.dynamo_mock
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)
def setUp(self):
"""
We keep copies of main parameters here, because they may differ from test to test and cleanup needs them.
This is responsibility of the test author to update these values if required from test.
"""
self.config = self.TEST_CONFIG.copy()
self.table_name = self.config['dynamo_db_config']['table_name']
self.HASH_KEY = ('task_id', 'S')
self.clean_task_tables()
self.dynamo_client = DynamoDbClient(
config=self.config['dynamo_db_config'])
self.assistant = WorkerAssistant(custom_config={'test': 1})
def setUp(self):
self.TEST_CONFIG['init_clients'] = ['DynamoDb']
self.scavenger = Scavenger(self.TEST_CONFIG)
self.dynamo_client = DynamoDbClient(
config=self.TEST_CONFIG['dynamo_db_config'])
self.table_name = self.TEST_CONFIG['dynamo_db_config']['table_name']
_ = self.scavenger.get_db_field_name
self.task = {
_('task_id'): '123',
_('labourer_id'): 'lambda1',
_('greenfield'): 8888,
_('attempts'): 2
}
def test_dynamo_to_dict__do_json_loads(self):
config = self.TEST_CONFIG.copy()
config['dont_json_loads_results'] = False
self.dynamo_client = DynamoDbClient(config=config)
dynamo_row = {
'hash_col': {
'S': 'aaa'
},
'range_col': {
'N': '123'
},
'other_col': {
'S': '{"how many": 300}'
},
'duck_quack': {
'S': '{"quack": "duck"}'
}
}
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=True)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
},
'duck_quack': {
"quack": "duck"
}
}
self.assertDictEqual(res, expected)
res = self.dynamo_client.dynamo_to_dict(dynamo_row,
fetch_all_fields=False)
expected = {
'hash_col': 'aaa',
'range_col': 123,
'other_col': {
"how many": 300
}
}
self.assertDictEqual(res, expected)
def setUp(self):
"""
We keep copies of main parameters here, because they may differ from test to test and cleanup needs them.
This is responsibility of the test author to update these values if required from test.
"""
self.config = self.TEST_CONFIG.copy()
self.patcher = patch("sosw.app.get_config")
self.get_config_patch = self.patcher.start()
self.get_config_patch.return_value = {}
self.table_name = self.config['dynamo_db_config']['table_name']
self.HASH_KEY = ('task_id', 'S')
self.clean_task_tables()
self.dynamo_client = DynamoDbClient(
config=self.config['dynamo_db_config'])
self.assistant = WorkerAssistant(
custom_config=TEST_WORKER_ASSISTANT_CONFIG)
def __init__(self, custom_config: Dict = None, **kwargs):
# Initialize config from default config
self.config = self.DEFAULT_CONFIG or {}
# Update config recursively from custom config
self.config = recursive_update(self.config, custom_config or {})
if self.config['write_meta_to_ddb']:
try:
self.dynamo_db_client = DynamoDbClient(
config=self.config['dynamo_db_config'])
except:
logging.exception(
"Failed to initialize MetaHandler DynamoDbClient")
self.dynamo_db_client = None
else:
self.dynamo_db_client = None
logger.info(
"Initialized MetaHandler with config %s and DynamoDbClient %s",
self.config, self.dynamo_db_client)
def setUp(self):
config = self.TEST_CONFIG.copy()
self.dynamo_client = DynamoDbClient(config)
self.dynamo_config = DynamoConfig(test=True)
def setUp(self):
self.HASH_KEY = ('hash_col', 'S')
self.RANGE_KEY = ('range_col', 'N')
self.KEYS = ('hash_col', 'range_col')
self.table_name = 'autotest_dynamo_db'
self.dynamo_client = DynamoDbClient(config=self.TEST_CONFIG)