def __init__(self, config):
assert isinstance(config, dict), "Config must be provided during DynamoDbClient initialization"
# If this is a test, make sure the table is a test table
if os.environ.get('STAGE') == 'test' and 'table_name' in config:
assert config['table_name'].startswith('autotest_') or config['table_name'] == 'config', \
f"Bad table name {config['table_name']} in autotest"
self.config = config
if not str(config.get('table_name')).startswith('autotest_mock_'):
self.dynamo_client = boto3.client('dynamodb')
else:
logger.info(f"Initialized DynamoClient without boto3 client for table {config.get('table_name')}")
# storage for table description(s)
self._table_descriptions: Optional[Dict[str, Dict]] = {}
# initialize table store
self._table_capacity = {}
self.identify_dynamo_capacity(table_name=self.config['table_name'])
self.stats = defaultdict(int)
if not hasattr(self, 'row_mapper'):
self.row_mapper = self.config.get('row_mapper')
self.type_serializer = TypeSerializer()
self.type_deserializer = TypeDeserializer()
def updateData(self, numRows, start_key, field_key, field_value):
client = self.dynamodb_client()
deserializer = TypeDeserializer()
serializer = TypeSerializer()
table_configs = self.expected_table_config()
for table in table_configs:
LOGGER.info('Updating %s Items by setting field with key %s to the value %s, with start_key %s, for table %s', numRows, field_key, field_value, start_key, table['TableName'])
for item in table['generator'](numRows, start_key):
record = deserializer.deserialize(item)
hashKey = table['HashKey']
key = {
hashKey: serializer.serialize(record[hashKey])
}
serializedFieldValue = serializer.serialize(field_value)
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.update_item
client.update_item(
TableName=table['TableName'],
Key=key,
UpdateExpression='set {}=:v'.format(field_key),
ExpressionAttributeValues={
':v': serializedFieldValue,
},
)
def _load_dice_pools(self):
if self.pools is not None:
return
response = self._client.query(
TableName='DicePools',
Select='ALL_ATTRIBUTES',
Limit=1,
ScanIndexForward=False,
KeyConditionExpression='game = :game_name',
ExpressionAttributeValues={':game_name': {
"S": "Shadowrun"
}})
logger.debug("DB Response: {}".format(response))
des = TypeDeserializer()
if 'Items' in response and len(
response['Items']) > 0 and 'pools' in response['Items'][0]:
self.pools = des.deserialize(response['Items'][0]['pools'])
else:
logger.error("Dice Pools not found in DynamoDB.")
logger.error(response)
raise Exception("No values found in DynamoDB.")
logger.debug("Pools: {}".format(self.pools))
if self._gm not in self.pools:
self.pools[self._gm] = 0
return self.pools
def get_start_over(intent, session):
"""
Go back to the beginning of the instruction set
"""
sesh_attr = persist_attributes(session)
userID = session['user']['userId']
if 'last_step' in sesh_attr and 'recipe' in sesh_attr:
recipe = sesh_attr['recipe']
else:
ds = TypeDeserializer()
full_last_step = db_get_last_step(userID)
recipe = ds.deserialize(full_last_step['recipe'])
next_step = recipe['recipe'][0]
# log to database for cross-session use
db_log_step(userID, recipe, next_step)
sesh_attr.update({'last_step': next_step, 'recipe': recipe})
return build_response(
sesh_attr,
build_speechlet_response("Starting Over",
step_to_ssml(next_step),
reprompt_text=done_this_step(),
should_end_session=False,
show_card=False))
def get_previous(intent, session):
"""
Go back to previous step in instructions
"""
sesh_attr = persist_attributes(session)
userID = session['user']['userId']
if 'last_step' in sesh_attr and 'recipe' in sesh_attr:
recipe = sesh_attr['recipe']
last_step = sesh_attr['last_step']
else:
ds = TypeDeserializer()
full_last_step = db_get_last_step(userID)
recipe = ds.deserialize(full_last_step['recipe'])
last_step = ds.deserialize(full_last_step['step'])
next_step = recipe_prior_step(recipe, last_step)
# log to database for cross-session use
db_log_step(userID, recipe, next_step)
sesh_attr.update({'last_step': next_step, 'recipe': recipe})
return build_response(
sesh_attr,
build_speechlet_response("Going Back",
step_to_ssml(next_step),
reprompt_text=done_this_step(),
should_end_session=False,
show_card=False))
def deserialize_dynamo_db(dynamo_format_dict):
deserializer = TypeDeserializer()
python_data = {
k: deserializer.deserialize(v)
for k, v in dynamo_format_dict.items()
}
return replace_decimals(python_data)
def _deserialize(dynamodb_data):
"""Static method to convert dynamodb data type to python data type
Types convention between DynamoDB and Python.
Reference link: http://boto3.readthedocs.io/en/latest/_modules/boto3/dynamodb/types.html
DynamoDB Python
-------- ------
{'NULL': True} None
{'BOOL': True/False} True/False
{'N': str(value)} Decimal(str(value))
{'S': string} string
{'B': bytes} Binary(bytes)
{'NS': [str(value)]} set([Decimal(str(value))])
{'SS': [string]} set([string])
{'BS': [bytes]} set([bytes])
{'L': list} list
{'M': dict} dict
Args:
dynamodb_data (list): Contains IOC info with DynamoDB types
Returns:
list: A list of Python dictionary type containing ioc_value and ioc_type
"""
result = []
if not dynamodb_data:
return result
deserializer = TypeDeserializer()
for raw_data in dynamodb_data:
python_data = {}
for key, val in raw_data.iteritems():
python_data[key] = deserializer.deserialize(val)
result.append(python_data)
return result
def lambda_handler(event, context):
deserializer = TypeDeserializer()
try:
body = []
resources = (
boto3.client("dynamodb")
.scan(TableName=os.environ.get("WHITELISTTABLE"))
.get("Items")
)
for resource in resources:
item = {}
for key, value in resource.items():
item[key] = str(deserializer.deserialize(value))
body.append(item)
return get_return(
200,
"Whitelist retrieved",
None,
{"whitelist": sorted(body, key=itemgetter("resource_id", "expiration"))},
)
except Exception as error:
print(f"[ERROR] {error}")
return get_return(400, "Could not retrieve whitelist", None, None)
def lambda_handler(event, context):
client = boto3.client("dynamodb")
paginator = client.get_paginator("scan")
deserializer = TypeDeserializer()
try:
body = []
page_iterator = paginator.paginate(
TableName=os.environ.get("WHITELISTTABLE"), )
for page in page_iterator:
for item in page["Items"]:
record = {}
for key, value in item.items():
record[key] = str(deserializer.deserialize(value))
body.append(record)
return get_return(
200,
"Whitelist retrieved",
None,
{
"whitelist":
sorted(body, key=itemgetter("resource_id", "expiration"))
},
)
except Exception as error:
print(f"[ERROR] {error}")
return get_return(400, "Could not retrieve whitelist", None, None)
def deconstruct_from_dynamo_event(event):
"""
{'Records':
[
{
'eventID': 'f2552522f15ac49bae72cc751973ac34',
'eventName': 'MODIFY',
'eventVersion': '1.1',
'eventSource': 'aws:dynamodb',
'awsRegion': 'us-east-2',
'dynamodb': {
'ApproximateCreatedDateTime': 1559833090.0,
'Keys': {
'id': {'S': 'b722c9e0-839e-11e9-8ff3-3a9ce2fecb55'}
},
'NewImage': {... object of the new image to be "deserialized"...},
'SequenceNumber': '27489800000000001219353382',
'SizeBytes': 304,
'StreamViewType': 'NEW_IMAGE'
},
'eventSourceARN': 'arn:aws:dynamodb:us-east-2:475285711284:table/Tab_DEV_AddFundReferenceTable/stream/2019-06-03T12:14:21.372'
}
]
}
:param event:
:return:
"""
deserializer = TypeDeserializer()
res = []
for r in event['Records']:
if 'dynamodb' in r.keys():
res.append(
deserializer.deserialize({'M': r['dynamodb']['NewImage']}))
return res
def test_fake_dynamo_flat_dict_output(self):
# This profile must have ldap groups and staff data
p = fake_profile.FakeUser(seed=44)
ddb = p.as_dynamo_flat_dict()
deserializer = TypeDeserializer()
res = {k: deserializer.deserialize(v) for k, v in ddb.items()}
assert len(res["access_information"]["ldap"]) > 0
assert len(res["staff_information"]["office_location"]) > 0
def deseralize(ddb_record):
# This is probablt a semi-dangerous hack.
# https://github.com/boto/boto3/blob/e353ecc219497438b955781988ce7f5cf7efae25/boto3/dynamodb/types.py#L233
ds = TypeDeserializer()
output = {}
for k, v in ddb_record.items():
output[k] = ds.deserialize(v)
return (output)
def deserialize_output(value):
try:
td = TypeDeserializer()
for k, v in dict(value).items():
value[k] = td.deserialize(v)
except BaseException:
pass
return value
def deserialize_ddb_data(self, ddb_data):
"""
Convert a DynamoDB item to a regular dictionary
"""
deserializer = TypeDeserializer()
deserialized_ddb_data = {k: deserializer.deserialize(v) for k, v in ddb_data.items()}
return deserialized_ddb_data
def dump_table(table, profile, **opts):
limit = opts.get('limit')
delay = opts.get('delay', 30) # seconds
session = botocore.session.Session(profile=profile)
client = session.create_client('dynamodb')
deser = TypeDeserializer()
more_data = True
i = 0
n = 0
start_key = None
while more_data:
if debug:
print(
f'{bcolors.GREY20}fetch {i} n:{n} with delay:{delay}{bcolors.ENDC}',
file=sys.stderr)
scan_opts = {}
if limit: scan_opts['Limit'] = limit - n
if start_key: scan_opts['ExclusiveStartKey'] = start_key
try:
r = client.scan(TableName=table, **scan_opts)
# pp.pprint(r)
# print("=====")
for d in r['Items']:
n += 1
# print(f'\n{bcolors.GREY10}record {n}:{bcolors.ENDC}')
# pp.pprint(json.loads(d['Data']))
# pp.pprint(d)
# pp.pprint(deser.deserialize({'M': d}))
print(
json.dumps(deser.deserialize({'M': d}),
default=defaultencode))
start_key = r.get('LastEvaluatedKey', {})
except client.exceptions.ProvisionedThroughputExceededException as e:
print(f'{bcolors.WARNING}caught {e}{bcolors.ENDC}',
file=sys.stderr)
delay = delay * 2
if not start_key:
more_data = False
elif limit and n >= limit:
more_data = False
else:
# Sleep to avoid exceeding 5-calls-per-second limit. This
# rate-limit is applied across all consumers, and clients will
# receive ProvisionedThroughputExceededException if it is
# exceeded.
time.sleep(delay)
i += 1
def __init__(self,
dynamodb_table_resource=None,
dynamodb_client=None,
transactions=True):
"""Take a dynamodb table resource to use for operations."""
self.table = dynamodb_table_resource
self.client = dynamodb_client
self.transactions = transactions
self.deserializer = TypeDeserializer()
def deserializeItem(record):
print(record)
dynamodb = AwsHelper().getResource('dynamodb')
deserializer = TypeDeserializer()
deserializedItem = {
k: deserializer.deserialize(v)
for k, v in record.items()
}
return deserializedItem
def convert_dynamo_json_to_py_data(input_json: list) -> list:
"""
:param input_json: DynamoDB compatible JSON with included attributes
:return: py_data: DynamoDB incompatible JSON with removed attributes
"""
py_data = []
deserializer = TypeDeserializer()
for item in input_json:
json_data = {k: deserializer.deserialize(v) for k, v in item.items()}
py_data.append(json_data)
logger.debug(f"Stripped attributes result: {py_data}")
return py_data
def lambda_handler(event, context):
deserializer = TypeDeserializer()
for record in event['Records']:
dynamodb_item = record['dynamodb']['NewImage']
deserialized = {
k: deserializer.deserialize(v)
for k, v in dynamodb_item.items()
}
print(deserialized)
def is_item_exists(table, item_to_get):
print("getting item:", item_to_get)
deserializer = TypeDeserializer()
response = table.get_item(Key={
k: deserializer.deserialize(v)
for k, v in item_to_get.iteritems()
},
AttributesToGet=['id'],
ConsistentRead=False)
print(response)
if 'Item' in response:
return True
print("not found {} in restored table".format(item_to_get))
return False
def ddb_to_dict(item):
# type: (Dict[str, Any]) -> Dict[str, Any]
# TODO: narrow these types down
"""Converts a raw DynamoDB item to a native Python dictionary.
:param dict item: DynamoDB item
:returns: Native item
:rtype: dict
"""
deserializer = TypeDeserializer()
return {
key: deserializer.deserialize(value)
for key, value in item.items()
}
def ddb_to_dict(item):
# type: (Dict[str, Any]) -> Dict[str, Any]
# narrow these types down
# https://github.com/aws/aws-dynamodb-encryption-python/issues/66
"""Converts a raw DynamoDB item to a native Python dictionary.
:param dict item: DynamoDB item
:returns: Native item
:rtype: dict
"""
deserializer = TypeDeserializer()
return {
key: deserializer.deserialize(value)
for key, value in item.items()
}
def store_item_hash(self, item):
"""
Store the item hash in the metadata.
:param item: The item to store the hash for.
"""
deserializer = TypeDeserializer()
# Try to deserialize the data in order to remove dynamoDB data types.
for key in item:
try:
item[key] = deserializer.deserialize(item[key])
except (TypeError, AttributeError):
break
self.resource['metadata']['item_hash'] = hashlib.md5(
json.dumps(item, sort_keys=True).encode('utf-8')).hexdigest()
def get_items(self, table_name, item_ids):
keys = list([{'id': {'S': item_id}} for item_id in item_ids])
response = self.client.batch_get_item(
RequestItems={table_name: {
'Keys': keys,
'ConsistentRead': True
}})
type_deserializer = TypeDeserializer()
items = response['Responses'][table_name]
for item in items:
for key, value in item.items():
value = type_deserializer.deserialize(value)
item[key] = value
return {'Items': items}
def __get_dynamo_schedule(ctx, table_name):
cmd = "aws {} dynamodb scan --table-name {}".format(profile_arg(), table_name)
res = ctx.run(cmd, hide=True).stdout
res = json.loads(res)
from boto3.dynamodb.types import TypeDeserializer
tds = TypeDeserializer()
current_schedule = {}
for item in res["Items"]:
item = { k: tds.deserialize(v) for k, v in item.items() }
current_schedule[item["zoom_series_id"]] = item
return current_schedule
def get_all_object_links_from_db(paginator, url: str) -> dict:
"""
Gets all objects from a db
:param paginator: a dynamodb scan paginator
:param url: the base url for an api get request
:return: the object
"""
database = paginator # type: botostubs.DynamoDB.ScanOutput
items = []
deserializer = TypeDeserializer()
for page in paginator:
for item in page["Items"]:
item = {k: deserializer.deserialize(v) for k, v in item.items()}
items.append({"url": f"{url}/{item['uid']}"})
return items
def _validate_keys(dynamodb_data):
"""Helper method to check if query key empty or duplicated"""
result = []
if not dynamodb_data:
err_msg = {'Error': {'Code': 403, 'Message': 'Empty query keys'}}
raise ParamValidationError(report=err_msg)
deserializer = TypeDeserializer()
for raw_data in dynamodb_data:
for _, val in raw_data.iteritems():
python_data = deserializer.deserialize(val).lower()
if not python_data or python_data in result:
err_msg = {'Error': {'Code': 403, 'Message': 'Parameter Validation Error'}}
raise ParamValidationError(report=err_msg)
result.append(python_data)
def quest_intent_handler(handler_input):
slots = handler_input.request_envelope.request.intent.slots
slot = slots['name']
quest_name = slot.value.title()
response = client.get_item(
Key={
'name': {
'S': quest_name.title(),
}
},
TableName='Runescape_Quests'
)
if 'Item' not in response:
speech_text = 'This quest is currently not supported. Please try again later.'
handler_input.response_builder.speak(speech_text).set_card(
SimpleCard('Unknown Quest', speech_text)).set_should_end_session(True)
return handler_input.response_builder.response
quest = response['Item']
deserializer = TypeDeserializer()
if 'requiredItems' in quest or 'requiredSkills' in quest:
speech_text = "For %s, you will need" % quest_name
if 'requiredItems' in quest:
speech_text = speech_text + ' ' + deserializer.deserialize(quest['requiredItems'])
if 'requiredItems' in quest and 'requiredSkills' in quest:
speech_text = speech_text + ' and '
if 'requiredSkills' in quest:
speech_text = speech_text + ' ' + deserializer.deserialize(quest['requiredSkills'])
speech_text = speech_text + '. '
else:
speech_text = "%s does not require any item or skill. " % quest_name
speech_text = speech_text + 'Do you want to start or cancel?'
handler_input.response_builder.speak(speech_text).set_card(
SimpleCard(quest_name, speech_text)).set_should_end_session(False)
handler_input.attributes_manager.session_attributes['step'] = -1
handler_input.attributes_manager.session_attributes['steps'] = deserializer.deserialize(quest['steps'])
handler_input.attributes_manager.session_attributes['quest_name'] = quest_name
return handler_input.response_builder.response
def read_Table(self, ModelObj):
#conn=self.connect(ModelObj)
#response = conn.scan()
data = []
#data = response['Items']
#while response.get('LastEvaluatedKey'):
# response = self.dynamoTable.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
# data.extend(response['Items'])
client = boto3.client(
'dynamodb',
region_name=self.region_name,
aws_access_key_id=settings.DYNAMO_AWS_ACCESS['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=settings.
DYNAMO_AWS_ACCESS['AWS_SECRET_ACCESS_KEY'],
endpoint_url=settings.DYNAMO['URL'])
paginator = client.get_paginator('scan')
service_model = client._service_model.operation_model('Scan')
operation_parameters = {
'TableName': ModelObj.Meta.table_name,
}
trans = TransformationInjector(deserializer=TypeDeserializer())
for page in paginator.paginate(**operation_parameters):
trans.inject_attribute_value_output(page, service_model)
while page.get('LastEvaluatedKey'):
page = self.dynamoTable.scan(
ExclusiveStartKey=page['LastEvaluatedKey'])
data.extend(page['Items'])
else:
data.extend(page['Items'])
return data
def deserialize_dynamo_data(input_data):
"""
Given a dict containing the "serialized" data format used by the low-level
dynamodb APIs, convert it to a standard python dictionary
Args:
input_data: (dict) - "serialized" form of a dynamodb record
Returns:
(dict) - the "deserialized" form of the input data
"""
deserializer = TypeDeserializer()
output_data = {}
for k, v in input_data.items():
output_data[k] = deserializer.deserialize(v)
return output_data
def setUp(self):
self.deserializer = TypeDeserializer()
class TestDeserializer(unittest.TestCase):
def setUp(self):
self.deserializer = TypeDeserializer()
def test_deserialize_invalid_type(self):
with self.assertRaisesRegexp(TypeError, 'FOO is not supported'):
self.deserializer.deserialize({'FOO': 'bar'})
def test_deserialize_empty_structure(self):
with self.assertRaisesRegexp(TypeError, 'Value must be a nonempty'):
self.assertEqual(self.deserializer.deserialize({}), {})
def test_deserialize_null(self):
self.assertEqual(self.deserializer.deserialize({"NULL": True}), None)
def test_deserialize_boolean(self):
self.assertEqual(self.deserializer.deserialize({"BOOL": False}), False)
def test_deserialize_integer(self):
self.assertEqual(
self.deserializer.deserialize({'N': '1'}), Decimal('1'))
def test_deserialize_decimal(self):
self.assertEqual(
self.deserializer.deserialize({'N': '1.25'}), Decimal('1.25'))
def test_deserialize_string(self):
self.assertEqual(
self.deserializer.deserialize({'S': 'foo'}), 'foo')
def test_deserialize_binary(self):
self.assertEqual(
self.deserializer.deserialize({'B': b'\x00'}), Binary(b'\x00'))
def test_deserialize_number_set(self):
self.assertEqual(
self.deserializer.deserialize(
{'NS': ['1', '1.25']}), set([Decimal('1'), Decimal('1.25')]))
def test_deserialize_string_set(self):
self.assertEqual(
self.deserializer.deserialize(
{'SS': ['foo', 'bar']}), set(['foo', 'bar']))
def test_deserialize_binary_set(self):
self.assertEqual(
self.deserializer.deserialize(
{'BS': [b'\x00', b'\x01']}),
set([Binary(b'\x00'), Binary(b'\x01')]))
def test_deserialize_list(self):
self.assertEqual(
self.deserializer.deserialize(
{'L': [{'N': '1'}, {'S': 'foo'}, {'L': [{'N': '1.25'}]}]}),
[Decimal('1'), 'foo', [Decimal('1.25')]])
def test_deserialize_map(self):
self.assertEqual(
self.deserializer.deserialize(
{'M': {'foo': {'S': 'mystring'},
'bar': {'M': {'baz': {'N': '1'}}}}}),
{'foo': 'mystring', 'bar': {'baz': Decimal('1')}}
)
