def test_chunk_job__max_items_per_batch(self):
"""
Tests that `max_products_per_batch` will actually make chunks of products of specific size.
Here we have a tricky case:
`section_weddings` has 3 different `stores`. In `store_music` we have 5 `products`.
For example with max_products_per_batch = 2, we should have:
- store_1
- store_2
- store_3, products 1 + 2
- store_3, products 3 + 4
- store_3, products 5
"""
pl = deepcopy(self.PAYLOAD)
pl['sections']['section_weddings']['stores']['store_music'][
'max_products_per_batch'] = 2
# pl['sections']['section_funerals']['isolate_stores'] = True
# pl['isolate_sections'] = True
# pl['isolate_stores'] = True
response = self.scheduler.chunk_job(job=pl)
NUMBER_TASKS_EXPECTED = [
('sections', 'section_weddings', 5),
('sections', 'section_funerals', 1),
('sections', 'section_conversions', 1),
('sections', 'section_gifts', 1),
]
# for row in response:
# pprint.pprint(row)
# print('\n')
self.check_number_of_tasks(NUMBER_TASKS_EXPECTED, response)
batches = [
x['products'] for x in response
if x.get('stores') == ['store_music']
]
print(batches)
self.assertEqual(
batches,
list(
chunks(
pl['sections']['section_weddings']['stores']['store_music']
['products'], 2)))
def test_batch_get_items_one_table(self):
# If you want to stress test batch_get_items_one_table, use bigger numbers
num_of_items = 5
query_from = 2
query_till = 4
expected_items = query_till - query_from
# Write items
operations = []
query_keys = []
for i in range(num_of_items):
item = {self.HASH_COL: f'cat{i%2}', self.RANGE_COL: i}
operations.append({'Put': self.dynamo_client.build_put_query(item)})
query_keys.append(item)
for operations_chunk in chunks(operations, 10):
self.dynamo_client.dynamo_client.transact_write_items(TransactItems=operations_chunk)
time.sleep(1) # cause the table has 10 write/sec capacity
# Batch get items
results = self.dynamo_client.batch_get_items_one_table(keys_list=query_keys[query_from:query_till])
self.assertEqual(expected_items, len(results))
def get_credentials_by_prefix(self, prefix):
"""
Retrieve the credentials with given `prefix` from AWS SSM ParameterStore and return as a dictionary.
In ParameterStore the values `Name` must begin with `prefix_` and they must have Tag:Environment `(production|dev)`.
The type of elements is expected to be SecureString. Regular strings could work, but not guaranteed.
:param str prefix: prefix of records to extract
:rtype: dict
:return: Some credentials
"""
env_tag = 'production' if not self.test else 'dev'
prefix = prefix if prefix.endswith('_') else prefix + '_'
describe_params_response = self.call_boto_with_pagination(
'describe_parameters',
ParameterFilters=[{
"Key": "tag:Environment",
"Values": [env_tag]
}, {
'Key': 'Name',
'Option': 'BeginsWith',
'Values': [prefix]
}])
logging.debug(
f"SSM.describe_parameters(prefix={prefix}) received response: {describe_params_response}"
)
params = [
param for obj in describe_params_response
for param in obj['Parameters']
]
names = [param['Name'] for param in params]
if not names:
logging.warning(
f"No credentials found in SSM ParameterStore with prefix {prefix} for Environment: {env_tag}"
)
return dict()
# This is supposed to work fine if you ask multiple keys even if some are not encrypted.
# Anyway you should encrypt everything.
decryption_required = any(
[True for param in params if param['Type'] == 'SecureString'])
result = dict()
for chunk_of_names in chunks(names, 10):
get_params_response = self.call_boto_with_pagination(
'get_parameters',
Names=chunk_of_names,
WithDecryption=decryption_required)
logging.debug(
f"SSM.get_parameters(names={chunk_of_names}) received response: {get_params_response}"
)
# Update keys and values from this page of response to result. Removes the prefix away for keys.
params = [
param for obj in get_params_response
for param in obj['Parameters']
]
if params:
result.update(
dict([(x['Name'].replace(prefix, ''),
x['Value'] if x['Value'] != 'None' else None)
for x in params]))
return result
def push_list_chunks():
""" Appends chunks of lists using current skeleton and vals to chunk. """
for v in chunks(vals, batch_size):
data.append({**task_skeleton, **{plural(attr): v}})