def __init__(self, config):
assert isinstance(config, dict), "Config must be provided during DynamoDbClient initialization"
# If this is a test, make sure the table is a test table
if os.environ.get('STAGE') == 'test' and 'table_name' in config:
assert config['table_name'].startswith('autotest_') or config['table_name'] == 'config', \
f"Bad table name {config['table_name']} in autotest"
self.config = config
if not str(config.get('table_name')).startswith('autotest_mock_'):
self.dynamo_client = boto3.client('dynamodb')
else:
logger.info(f"Initialized DynamoClient without boto3 client for table {config.get('table_name')}")
# storage for table description(s)
self._table_descriptions: Optional[Dict[str, Dict]] = {}
# initialize table store
self._table_capacity = {}
self.identify_dynamo_capacity(table_name=self.config['table_name'])
self.stats = defaultdict(int)
if not hasattr(self, 'row_mapper'):
self.row_mapper = self.config.get('row_mapper')
self.type_serializer = TypeSerializer()
self.type_deserializer = TypeDeserializer()
def updateData(self, numRows, start_key, field_key, field_value):
client = self.dynamodb_client()
deserializer = TypeDeserializer()
serializer = TypeSerializer()
table_configs = self.expected_table_config()
for table in table_configs:
LOGGER.info('Updating %s Items by setting field with key %s to the value %s, with start_key %s, for table %s', numRows, field_key, field_value, start_key, table['TableName'])
for item in table['generator'](numRows, start_key):
record = deserializer.deserialize(item)
hashKey = table['HashKey']
key = {
hashKey: serializer.serialize(record[hashKey])
}
serializedFieldValue = serializer.serialize(field_value)
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.update_item
client.update_item(
TableName=table['TableName'],
Key=key,
UpdateExpression='set {}=:v'.format(field_key),
ExpressionAttributeValues={
':v': serializedFieldValue,
},
)
def getSpecificProductInformation(self, keyValue):
print("In getSpecificProductInformation() method !")
try:
table = self.dynamodb.Table('Product')
resp = table.get_item(Key=keyValue)
# resp returns following value.
print(resp)
# {'Item': {'Id': Decimal('1'), 'ProductCategoryId': Decimal('1'), 'Name': 'IPhone 8\n'}, 'ResponseMetadata': {'RequestId': 'NMDA30M1HHF514MJVVUIBCELEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'Server', 'date': 'Mon, 05 Aug 2019 17:12:19 GMT', 'content-type': 'application/x-amz-json-1.0', 'content-length': '81', 'connection': 'keep-alive', 'x-amzn-requestid': 'NMDA30M1HHF514MJVVUIBCELEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'x-amz-crc32': '3394591035'}, 'RetryAttempts': 0}}
if resp and resp["ResponseMetadata"]["HTTPStatusCode"] == 200:
deserializer = TypeDeserializer()
serializer = TypeSerializer()
if resp["Item"]:
print(resp["Item"])
# {'Id': Decimal('1'), 'ProductCategoryId': Decimal('1'), 'Name': 'IPhone 8\n'}
data = {k: serializer.serialize(v) for k, v in resp["Item"].items()}
else:
data = {}
return data
else:
raise Exception
except Exception as error:
print("Not able to fetch item !")
raise error
class UserRepository:
serializer = TypeSerializer()
deserializer = TypeDeserializer()
usersTable: any
def __init__(self):
dynamodb = boto3.resource('dynamodb')
self.usersTable = dynamodb.Table('Users')
def getAll(self):
users = []
response = self.usersTable.scan()
for item in response['Items']:
dictionary = {
k: self.deserializer.deserialize(v)
for k, v in item.items() if k != 'userId'
}
dictionary["userId"] = int(item["userId"])
if not 'customReason' in dictionary:
dictionary['customReason'] = ''
user = User.Schema().load(dictionary)
users.append(user)
return users
def save(self, user: User):
json = {
k: self.serializer.serialize(v)
for k, v in User.Schema().dump(user).items() if v != ""
}
json['userId'] = user.userId
self.usersTable.put_item(Item=json)
return
def delete(self, user: User):
self.usersTable.delete_item(Key={'userId': user.userId})
def get_previous(intent, session):
"""
Go back to previous step in instructions
"""
sesh_attr = persist_attributes(session)
userID = session['user']['userId']
if 'last_step' in sesh_attr and 'recipe' in sesh_attr:
recipe = sesh_attr['recipe']
last_step = sesh_attr['last_step']
else:
ds = TypeDeserializer()
full_last_step = db_get_last_step(userID)
recipe = ds.deserialize(full_last_step['recipe'])
last_step = ds.deserialize(full_last_step['step'])
next_step = recipe_prior_step(recipe, last_step)
# log to database for cross-session use
db_log_step(userID, recipe, next_step)
sesh_attr.update({'last_step': next_step, 'recipe': recipe})
return build_response(
sesh_attr,
build_speechlet_response("Going Back",
step_to_ssml(next_step),
reprompt_text=done_this_step(),
should_end_session=False,
show_card=False))
def deconstruct_from_dynamo_event(event):
"""
{'Records':
[
{
'eventID': 'f2552522f15ac49bae72cc751973ac34',
'eventName': 'MODIFY',
'eventVersion': '1.1',
'eventSource': 'aws:dynamodb',
'awsRegion': 'us-east-2',
'dynamodb': {
'ApproximateCreatedDateTime': 1559833090.0,
'Keys': {
'id': {'S': 'b722c9e0-839e-11e9-8ff3-3a9ce2fecb55'}
},
'NewImage': {... object of the new image to be "deserialized"...},
'SequenceNumber': '27489800000000001219353382',
'SizeBytes': 304,
'StreamViewType': 'NEW_IMAGE'
},
'eventSourceARN': 'arn:aws:dynamodb:us-east-2:475285711284:table/Tab_DEV_AddFundReferenceTable/stream/2019-06-03T12:14:21.372'
}
]
}
:param event:
:return:
"""
deserializer = TypeDeserializer()
res = []
for r in event['Records']:
if 'dynamodb' in r.keys():
res.append(
deserializer.deserialize({'M': r['dynamodb']['NewImage']}))
return res
def history(tableName: str = 'wcmc-data-stasis-ecs-dev'):
import boto3
from boto3.dynamodb.types import TypeDeserializer
from boto3.dynamodb.transform import TransformationInjector
client = boto3.client('dynamodb')
paginator = client.get_paginator('scan')
operation_model = client._service_model.operation_model('Scan')
trans = TransformationInjector(deserializer=TypeDeserializer())
operation_parameters = {'TableName': tableName}
items = []
for page in paginator.paginate(**operation_parameters):
has_last_key = 'LastEvaluatedKey' in page
if has_last_key:
last_key = page['LastEvaluatedKey'].copy()
trans.inject_attribute_value_output(page, operation_model)
if has_last_key:
page['LastEvaluatedKey'] = last_key
for x in page['Items']:
items.append(x)
return items
def get_start_over(intent, session):
"""
Go back to the beginning of the instruction set
"""
sesh_attr = persist_attributes(session)
userID = session['user']['userId']
if 'last_step' in sesh_attr and 'recipe' in sesh_attr:
recipe = sesh_attr['recipe']
else:
ds = TypeDeserializer()
full_last_step = db_get_last_step(userID)
recipe = ds.deserialize(full_last_step['recipe'])
next_step = recipe['recipe'][0]
# log to database for cross-session use
db_log_step(userID, recipe, next_step)
sesh_attr.update({'last_step': next_step, 'recipe': recipe})
return build_response(
sesh_attr,
build_speechlet_response("Starting Over",
step_to_ssml(next_step),
reprompt_text=done_this_step(),
should_end_session=False,
show_card=False))
def lambda_handler(event, context):
deserializer = TypeDeserializer()
try:
body = []
resources = (
boto3.client("dynamodb")
.scan(TableName=os.environ.get("WHITELISTTABLE"))
.get("Items")
)
for resource in resources:
item = {}
for key, value in resource.items():
item[key] = str(deserializer.deserialize(value))
body.append(item)
return get_return(
200,
"Whitelist retrieved",
None,
{"whitelist": sorted(body, key=itemgetter("resource_id", "expiration"))},
)
except Exception as error:
print(f"[ERROR] {error}")
return get_return(400, "Could not retrieve whitelist", None, None)
def _deserialize(dynamodb_data):
"""Static method to convert dynamodb data type to python data type
Types convention between DynamoDB and Python.
Reference link: http://boto3.readthedocs.io/en/latest/_modules/boto3/dynamodb/types.html
DynamoDB Python
-------- ------
{'NULL': True} None
{'BOOL': True/False} True/False
{'N': str(value)} Decimal(str(value))
{'S': string} string
{'B': bytes} Binary(bytes)
{'NS': [str(value)]} set([Decimal(str(value))])
{'SS': [string]} set([string])
{'BS': [bytes]} set([bytes])
{'L': list} list
{'M': dict} dict
Args:
dynamodb_data (list): Contains IOC info with DynamoDB types
Returns:
list: A list of Python dictionary type containing ioc_value and ioc_type
"""
result = []
if not dynamodb_data:
return result
deserializer = TypeDeserializer()
for raw_data in dynamodb_data:
python_data = {}
for key, val in raw_data.iteritems():
python_data[key] = deserializer.deserialize(val)
result.append(python_data)
return result
def deserialize(value):
"""
boto3.dynamodb.types.TypeDeserializer을 이용하여 각 속성 값을 직렬화
:param value: ddb field dict, ex) {"S":"foobar"}
:return:
"""
return TypeDeserializer().deserialize(value)
def deserialize_dynamo_db(dynamo_format_dict):
deserializer = TypeDeserializer()
python_data = {
k: deserializer.deserialize(v)
for k, v in dynamo_format_dict.items()
}
return replace_decimals(python_data)
def _load_dice_pools(self):
if self.pools is not None:
return
response = self._client.query(
TableName='DicePools',
Select='ALL_ATTRIBUTES',
Limit=1,
ScanIndexForward=False,
KeyConditionExpression='game = :game_name',
ExpressionAttributeValues={':game_name': {
"S": "Shadowrun"
}})
logger.debug("DB Response: {}".format(response))
des = TypeDeserializer()
if 'Items' in response and len(
response['Items']) > 0 and 'pools' in response['Items'][0]:
self.pools = des.deserialize(response['Items'][0]['pools'])
else:
logger.error("Dice Pools not found in DynamoDB.")
logger.error(response)
raise Exception("No values found in DynamoDB.")
logger.debug("Pools: {}".format(self.pools))
if self._gm not in self.pools:
self.pools[self._gm] = 0
return self.pools
def lambda_handler(event, context):
client = boto3.client("dynamodb")
paginator = client.get_paginator("scan")
deserializer = TypeDeserializer()
try:
body = []
page_iterator = paginator.paginate(
TableName=os.environ.get("WHITELISTTABLE"), )
for page in page_iterator:
for item in page["Items"]:
record = {}
for key, value in item.items():
record[key] = str(deserializer.deserialize(value))
body.append(record)
return get_return(
200,
"Whitelist retrieved",
None,
{
"whitelist":
sorted(body, key=itemgetter("resource_id", "expiration"))
},
)
except Exception as error:
print(f"[ERROR] {error}")
return get_return(400, "Could not retrieve whitelist", None, None)
def read_Table(self, ModelObj):
#conn=self.connect(ModelObj)
#response = conn.scan()
data = []
#data = response['Items']
#while response.get('LastEvaluatedKey'):
# response = self.dynamoTable.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
# data.extend(response['Items'])
client = boto3.client(
'dynamodb',
region_name=self.region_name,
aws_access_key_id=settings.DYNAMO_AWS_ACCESS['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=settings.
DYNAMO_AWS_ACCESS['AWS_SECRET_ACCESS_KEY'],
endpoint_url=settings.DYNAMO['URL'])
paginator = client.get_paginator('scan')
service_model = client._service_model.operation_model('Scan')
operation_parameters = {
'TableName': ModelObj.Meta.table_name,
}
trans = TransformationInjector(deserializer=TypeDeserializer())
for page in paginator.paginate(**operation_parameters):
trans.inject_attribute_value_output(page, service_model)
while page.get('LastEvaluatedKey'):
page = self.dynamoTable.scan(
ExclusiveStartKey=page['LastEvaluatedKey'])
data.extend(page['Items'])
else:
data.extend(page['Items'])
return data
def _doc_to_api(event):
'''Serialize a DDB API Item to a DDB Document'''
if event.get('Item') is not None:
ddb_items = event.get('Item')
event['Item'] = {
k: TypeDeserializer().deserialize(v)
for k, v in ddb_items.items()
}
if event.get('Key') is not None:
ddb_key = event.get('Key')
event['Key'] = {
k: TypeDeserializer().deserialize(v)
for k, v in ddb_key.items()
}
return event
def test_fake_dynamo_flat_dict_output(self):
# This profile must have ldap groups and staff data
p = fake_profile.FakeUser(seed=44)
ddb = p.as_dynamo_flat_dict()
deserializer = TypeDeserializer()
res = {k: deserializer.deserialize(v) for k, v in ddb.items()}
assert len(res["access_information"]["ldap"]) > 0
assert len(res["staff_information"]["office_location"]) > 0
def deserialize_ddb_data(self, ddb_data):
"""
Convert a DynamoDB item to a regular dictionary
"""
deserializer = TypeDeserializer()
deserialized_ddb_data = {k: deserializer.deserialize(v) for k, v in ddb_data.items()}
return deserialized_ddb_data
def deserialize_output(value):
try:
td = TypeDeserializer()
for k, v in dict(value).items():
value[k] = td.deserialize(v)
except BaseException:
pass
return value
def deseralize(ddb_record):
# This is probablt a semi-dangerous hack.
# https://github.com/boto/boto3/blob/e353ecc219497438b955781988ce7f5cf7efae25/boto3/dynamodb/types.py#L233
ds = TypeDeserializer()
output = {}
for k, v in ddb_record.items():
output[k] = ds.deserialize(v)
return (output)
def ddb_deserialize(o: dict, **kw) -> Any:
'''
Deserialize DDB value
Usage:
>>> ddb_deserialize({'N': '200'})
Decimal('200')
'''
return TypeDeserializer(**kw).deserialize(o)
def dynamo_extend_items_with_schedule(items_list, full=False, df_format=False):
df = pd.DataFrame(items_list)
# Extract items primary keys and format it for getitem
extract = df[["day_train_num", "station_id"]]
extract.station_id = extract.station_id.apply(str)
# Serialize in dynamo types
seres = TypeSerializer()
extract_ser = extract.applymap(seres.serialize)
items_keys = extract_ser.to_dict(orient="records")
# Submit requests
responses = dynamo_submit_batch_getitem_request(items_keys,
dynamo_sched_dep)
# Deserialize into clean dataframe
resp_df = pd.DataFrame(responses)
deser = TypeDeserializer()
resp_df = resp_df.applymap(deser.deserialize)
# Select columns to keep:
all_columns = [
'arrival_time', 'block_id', 'day_train_num', 'direction_id',
'drop_off_type', 'pickup_type', 'route_id', 'route_short_name',
'scheduled_departure_day', 'scheduled_departure_time', 'service_id',
'station_id', 'stop_headsign', 'stop_id', 'stop_sequence', 'train_num',
'trip_headsign', 'trip_id'
]
columns_to_keep = [
'day_train_num', 'station_id', 'scheduled_departure_time', 'trip_id',
'service_id', 'route_short_name', 'trip_headsign', 'stop_sequence'
]
if full:
resp_df = resp_df[all_columns]
else:
resp_df = resp_df[columns_to_keep]
# Merge to add response dataframe to initial dataframe
# We use left jointure to keep items even if we couldn't find schedule
index_cols = ["day_train_num", "station_id"]
df_updated = df.merge(resp_df, on=index_cols, how="left")
# Compute delay
df_updated.loc[:, "delay"] = df_updated.apply(lambda x: compute_delay(
x["scheduled_departure_time"], x["expected_passage_time"]),
axis=1)
# Inform
logger.info(
"Asked to find schedule and trip_id for %d items, we found %d of them."
% (len(df), len(resp_df)))
if df_format:
return df_updated
# Safe json serializable python dict
df_updated = df_updated.applymap(str)
items_updated = json.loads(df_updated.to_json(orient='records'))
return items_updated
def dump_table(table, profile, **opts):
limit = opts.get('limit')
delay = opts.get('delay', 30) # seconds
session = botocore.session.Session(profile=profile)
client = session.create_client('dynamodb')
deser = TypeDeserializer()
more_data = True
i = 0
n = 0
start_key = None
while more_data:
if debug:
print(
f'{bcolors.GREY20}fetch {i} n:{n} with delay:{delay}{bcolors.ENDC}',
file=sys.stderr)
scan_opts = {}
if limit: scan_opts['Limit'] = limit - n
if start_key: scan_opts['ExclusiveStartKey'] = start_key
try:
r = client.scan(TableName=table, **scan_opts)
# pp.pprint(r)
# print("=====")
for d in r['Items']:
n += 1
# print(f'\n{bcolors.GREY10}record {n}:{bcolors.ENDC}')
# pp.pprint(json.loads(d['Data']))
# pp.pprint(d)
# pp.pprint(deser.deserialize({'M': d}))
print(
json.dumps(deser.deserialize({'M': d}),
default=defaultencode))
start_key = r.get('LastEvaluatedKey', {})
except client.exceptions.ProvisionedThroughputExceededException as e:
print(f'{bcolors.WARNING}caught {e}{bcolors.ENDC}',
file=sys.stderr)
delay = delay * 2
if not start_key:
more_data = False
elif limit and n >= limit:
more_data = False
else:
# Sleep to avoid exceeding 5-calls-per-second limit. This
# rate-limit is applied across all consumers, and clients will
# receive ProvisionedThroughputExceededException if it is
# exceeded.
time.sleep(delay)
i += 1
def __init__(self,
dynamodb_table_resource=None,
dynamodb_client=None,
transactions=True):
"""Take a dynamodb table resource to use for operations."""
self.table = dynamodb_table_resource
self.client = dynamodb_client
self.transactions = transactions
self.deserializer = TypeDeserializer()
def deserializeItem(record):
print(record)
dynamodb = AwsHelper().getResource('dynamodb')
deserializer = TypeDeserializer()
deserializedItem = {
k: deserializer.deserialize(v)
for k, v in record.items()
}
return deserializedItem
def image(self):
"""deserialized data"""
des = {}
if self._image:
for key in self._image:
des[key] = TypeDeserializer().deserialize(self._image[key])
n = 0
des = self.__decimal_to_integer_or_float(n, **des)
logger.info('deserialized: {}'.format(des))
return des
def updateTemplate(task, interesting_tasks):
stepfunctions = boto3.client('stepfunctions')
deserializer = TypeDeserializer()
accountId = task['accountId']
parameterOverrides = []
encryptionparams = {}
newregions = []
for overridekey in task['diff']['new'].keys():
if overridekey in interesting_tasks:
if overridekey == 'enabledregions':
overrideelement = {}
overrideelement['ParameterKey'] = overridekey
overrideelement['ParameterValue'] = ",".join(
task['diff']['new'][overridekey])
parameterOverrides.append(overrideelement)
else:
overrideelement = {}
overrideelement['ParameterKey'] = overridekey
overrideelement['ParameterValue'] = task['diff']['new'][
overridekey]
parameterOverrides.append(overrideelement)
if 'enabledregions' in task['diff']['new'].keys():
newregions = set(task['diff']['new']['enabledregions']) - set(
task['diff']['old']['enabledregions'])
encryptionparams = {
'accountId': accountId,
'enabledregions': ",".join(list(newregions))
}
if 'awsconfigenabled' in task['diff']['new'].keys():
newregions = []
for r in task['newImage']['enabledregions']['L']:
newregions.append(r['S'])
encryptionparams = {
'accountId': accountId,
'enabledregions': ",".join(list(newregions))
}
inputjson = json.dumps({
'accountId': accountId,
'encryptionparams': encryptionparams,
'overrides': parameterOverrides,
'newregions': list(newregions),
'newregionscount': len(newregions),
'newImage': task['newImage']
})
print(inputjson)
statemachinearn = 'arn:aws:states:eu-central-1:' + os.environ[
'accountid'] + ':stateMachine:enableRegion-' + os.environ['branchname']
response = stepfunctions.start_execution(stateMachineArn=statemachinearn,
name='{}-{}'.format(
accountId, str(uuid.uuid4())),
input=inputjson)
print(response)
def db_get_last_step(userID):
"""Get the most recent step that a user has executed
"""
dynamo = db_connect()
ds = TypeDeserializer()
response = dynamo.get_item(TableName=os.environ['STEP_LAST_TABLE'],
Key={'userID': {
'S': userID
}})
return response['Item']
def is_item_exists(table, item_to_get):
print("getting item:", item_to_get)
deserializer = TypeDeserializer()
response = table.get_item(Key={
k: deserializer.deserialize(v)
for k, v in item_to_get.iteritems()
},
AttributesToGet=['id'],
ConsistentRead=False)
print(response)
if 'Item' in response:
return True
print("not found {} in restored table".format(item_to_get))
return False
def ddb_to_dict(item):
# type: (Dict[str, Any]) -> Dict[str, Any]
# TODO: narrow these types down
"""Converts a raw DynamoDB item to a native Python dictionary.
:param dict item: DynamoDB item
:returns: Native item
:rtype: dict
"""
deserializer = TypeDeserializer()
return {
key: deserializer.deserialize(value)
for key, value in item.items()
}
