def getSpecificProductInformation(self, keyValue):
print("In getSpecificProductInformation() method !")
try:
table = self.dynamodb.Table('Product')
resp = table.get_item(Key=keyValue)
# resp returns following value.
print(resp)
# {'Item': {'Id': Decimal('1'), 'ProductCategoryId': Decimal('1'), 'Name': 'IPhone 8\n'}, 'ResponseMetadata': {'RequestId': 'NMDA30M1HHF514MJVVUIBCELEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'Server', 'date': 'Mon, 05 Aug 2019 17:12:19 GMT', 'content-type': 'application/x-amz-json-1.0', 'content-length': '81', 'connection': 'keep-alive', 'x-amzn-requestid': 'NMDA30M1HHF514MJVVUIBCELEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'x-amz-crc32': '3394591035'}, 'RetryAttempts': 0}}
if resp and resp["ResponseMetadata"]["HTTPStatusCode"] == 200:
deserializer = TypeDeserializer()
serializer = TypeSerializer()
if resp["Item"]:
print(resp["Item"])
# {'Id': Decimal('1'), 'ProductCategoryId': Decimal('1'), 'Name': 'IPhone 8\n'}
data = {k: serializer.serialize(v) for k, v in resp["Item"].items()}
else:
data = {}
return data
else:
raise Exception
except Exception as error:
print("Not able to fetch item !")
raise error
def convert_json_to_dynamo_json(input_json: list) -> list:
"""
Re-Serializes the data into DynamoDB compatible JSON that can be used to put items into the dynamo table
"""
# HACK: When serializing the JSON without dynamodb attribute types included, it wants to convert
# The DynamoDB 'String Set' objects to DynamoDB List objects because python loads the data as lists and not sets.
# I am choosing to go with DynamoDB attribute string sets, because I do not want duplicate entries for periods,
# and it is easier to parse visually. The only drawback I have seen so far is that sets are unordered,
# but since we are not evaluating the period string set responses in any particular order that should not matter.
# Can we change this to use cls instead of for loops?
serializer = TypeSerializer()
py_data, json_data = [], []
logger.info(f"Converting JSON config to DynamoDB compatible JSON.")
# Loop through JSON file data looking for Python object type list
# Convert the list object into a set of strings
# Store new data types as python object
for data in input_json:
for k, v in data.items():
if isinstance(v, list):
data[k] = set(v)
py_data.append(data)
# Serialize previously modified python object data into DynamoDB JSON
for data in py_data:
dynamo_data = {k: serializer.serialize(v) for k, v in data.items()}
json_data.append(dynamo_data)
return json_data
def updateData(self, numRows, start_key, field_key, field_value):
client = self.dynamodb_client()
deserializer = TypeDeserializer()
serializer = TypeSerializer()
table_configs = self.expected_table_config()
for table in table_configs:
LOGGER.info('Updating %s Items by setting field with key %s to the value %s, with start_key %s, for table %s', numRows, field_key, field_value, start_key, table['TableName'])
for item in table['generator'](numRows, start_key):
record = deserializer.deserialize(item)
hashKey = table['HashKey']
key = {
hashKey: serializer.serialize(record[hashKey])
}
serializedFieldValue = serializer.serialize(field_value)
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.update_item
client.update_item(
TableName=table['TableName'],
Key=key,
UpdateExpression='set {}=:v'.format(field_key),
ExpressionAttributeValues={
':v': serializedFieldValue,
},
)
def __init__(self, config):
assert isinstance(config, dict), "Config must be provided during DynamoDbClient initialization"
# If this is a test, make sure the table is a test table
if os.environ.get('STAGE') == 'test' and 'table_name' in config:
assert config['table_name'].startswith('autotest_') or config['table_name'] == 'config', \
f"Bad table name {config['table_name']} in autotest"
self.config = config
if not str(config.get('table_name')).startswith('autotest_mock_'):
self.dynamo_client = boto3.client('dynamodb')
else:
logger.info(f"Initialized DynamoClient without boto3 client for table {config.get('table_name')}")
# storage for table description(s)
self._table_descriptions: Optional[Dict[str, Dict]] = {}
# initialize table store
self._table_capacity = {}
self.identify_dynamo_capacity(table_name=self.config['table_name'])
self.stats = defaultdict(int)
if not hasattr(self, 'row_mapper'):
self.row_mapper = self.config.get('row_mapper')
self.type_serializer = TypeSerializer()
self.type_deserializer = TypeDeserializer()
def as_dynamo_flat_dict(self):
"""
Flattens out User.as_dict() output into a simple structure without any signature or metadata.
Effectively, this outputs something like this:
```{'uuid': '11c8a5c8-0305-4524-8b41-95970baba84c', 'user_id': 'email|c3cbf9f5830f1358e28d6b68a3e4bf15', ...```
`flatten()` is recursive.
Note that this form cannot be verified or validated back since it's missing all attributes!
Return: dynamodb serialized low level dict of user in a "flattened" form for dynamodb consumption in particular
"""
user = self._clean_dict()
def flatten(attrs, field=None):
flat = {}
for f in attrs:
# Skip "schema"
if isinstance(attrs[f], str):
continue
if not set(["value", "values"]).isdisjoint(set(attrs[f])):
res = attrs[f].get("value", attrs[f].get("values"))
if res is not None and res != "":
flat[f] = res
else:
flat[f] = flatten(attrs[f])
return flat
serializer = TypeSerializer()
return {k: serializer.serialize(v) for k, v in flatten(user).items()}
def construct_dynamo_type_dict(d: dict):
"""
DynamoDB transactions need a different way of specifying transaction.
The structure has to be recursively implemented as:
'string': {
'S': 'string',
'N': 'string',
'B': b'bytes',
'SS': [
'string',
],
'NS': [
'string',
],
'BS': [
b'bytes',
],
'M': {
'string': {'... recursive ...'}
},
'L': [
{'... recursive ...'},
],
'NULL': True|False,
'BOOL': True|False
}
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.transact_write_items
TypeSerializer and TypeDeserializer needs to be used to convert to appropriate representations for DynamoDB.
serialize(d)['M'] has been used in line with the documentation.
"""
serializer = TypeSerializer()
return serializer.serialize(d)['M']
def boto3_serializer(python_dict):
serializer = TypeSerializer()
return {
k: serializer.serialize(
v if not isinstance(v, float) else Decimal(str(v)))
for k, v in python_dict.items()
}
def generate_items(self, num_items):
serializer = TypeSerializer()
for i in range(num_items):
record = {
'int_id':
int(i / 10.0),
'decimal_field':
decimal.Decimal(str(i) + '.00000000001'),
'string_field':
str(i),
'byte_field':
b'some_bytes',
'int_list_field': [i, i + 1, i + 2],
'int_set_field':
set([i, i + 1, i + 2]),
'map_field': {
'map_entry_1': 'map_value_1',
'map_entry_2': 'map_value_2'
},
'string_list': [
self.random_string_generator(),
self.random_string_generator(),
self.random_string_generator()
],
'boolean_field':
True,
'other_boolean_field':
False,
'null_field':
None
}
yield serializer.serialize(record)
def serialize_output(value):
try:
td = TypeSerializer()
for k, v in dict(value).items():
value[k] = td.serialize(v)
except BaseException:
pass
return value
def dynamodb_put_item(ddb_client, table_name: str, item: dict):
serializer = TypeSerializer()
serialized_item = serializer.serialize(item)['M']
try:
ddb_client.put_item(TableName=table_name, Item=serialized_item)
except ddb_client.exceptions.ResourceNotFoundException:
raise TyphoonResourceNotFoundError(
f'Table {table_name} does not exist in DynamoDB')
def generate_items(self, num_items, start_key=0):
serializer = TypeSerializer()
for i in range(start_key, start_key + num_items):
record = {
'int_id': i,
'string_field': self.random_string_generator(),
'boolean_field': True,
}
yield serializer.serialize(record)
def dict_to_ddb(item):
# type: (Dict[str, Any]) -> Dict[str, Any]
# TODO: narrow these types down
"""Converts a native Python dictionary to a raw DynamoDB item.
:param dict item: Native item
:returns: DynamoDB item
:rtype: dict
"""
serializer = TypeSerializer()
return {key: serializer.serialize(value) for key, value in item.items()}
def _save_dice_pools(self):
if self.pools is None:
raise Exception("Tried to save dice pools before loading them.")
ser = TypeSerializer()
item = {
'game': ser.serialize('Shadowrun'),
'timestamp': ser.serialize(Decimal(time.time())),
'pools': ser.serialize(self.pools)
}
logger.debug("Item before put_item()ing: {}".format(item))
self._client.put_item(TableName='DicePools', Item=item)
def test_validate_products_multiple(lambda_module, product):
"""
Test validate_products() with multiple DynamoDB calls
"""
products = []
for i in range(0, 105):
product_temp = copy.deepcopy(product)
product_temp["productId"] += str(i)
products.append(product_temp)
# Stub boto3
dynamodb = stub.Stubber(lambda_module.dynamodb)
response = {
"Responses": {
lambda_module.TABLE_NAME: [{k: TypeSerializer().serialize(v) for k, v in p.items()} for p in products[0:100]]
}
}
expected_params = {
"RequestItems": {
lambda_module.TABLE_NAME: {
"Keys": [{"productId": {"S": p["productId"]}} for p in products[0:100]],
"ProjectionExpression": stub.ANY,
"ExpressionAttributeNames": stub.ANY
}
}
}
dynamodb.add_response("batch_get_item", response, expected_params)
response = {
"Responses": {
lambda_module.TABLE_NAME: [{k: TypeSerializer().serialize(v) for k, v in p.items()} for p in products[100:]]
}
}
expected_params = {
"RequestItems": {
lambda_module.TABLE_NAME: {
"Keys": [{"productId": {"S": p["productId"]}} for p in products[100:]],
"ProjectionExpression": stub.ANY,
"ExpressionAttributeNames": stub.ANY
}
}
}
dynamodb.add_response("batch_get_item", response, expected_params)
dynamodb.activate()
# Run command
retval = lambda_module.validate_products(products)
print(retval)
assert len(retval) == 2
assert len(retval[0]) == 0
assert isinstance(retval[1], str)
dynamodb.deactivate()
def lambda_handler(event, context):
accountId = str(event['accountId'])
accountName = str(event['description'])
accountRole = "customer"
confidentialKMSKey = 'alias/TSI_Base_ConfidentialS3Key'
internalKMSKey = 'alias/TSI_Base_InternalS3Key'
if 'customermasteraccountid' in event:
masteraccountId = event['customermasteraccountid']
else:
masteraccountId = os.environ['accountid']
readonlyRole = 'TSI_Base_ReadOnlySwitchRole'
securityEmail = str(event['email'])
accountemail = str(event['accountemail'])
enabledregions = event['enabledregions'].split(',')
supportenabled = 'false'
awsconfigenabled = 'false'
if 'config' in event:
config = event['config']
else:
config = "disabled"
if 'support' in event:
support = event['support']
else:
config = "disabled"
ouname = event['ouname']
terraformVersion = '1.0'
writeRole = 'TSI_Base_FullAccess'
featureLevel = 'full'
dynamoentry = {
'accountId': accountId,
'accountName': accountName,
'config': config,
'support': support,
'ouname': ouname,
'accountRole': accountRole,
'confidentialKMSKey': confidentialKMSKey,
'internalKMSKey': internalKMSKey,
'masteraccountId': masteraccountId,
'readonlyRole': readonlyRole,
'securityEmail': securityEmail,
'accountemail': accountemail,
'enabledregions': enabledregions,
'supportenabled': supportenabled,
'awsconfigenabled': awsconfigenabled,
'terraformVersion': terraformVersion,
'writeRole': writeRole,
'featureLevel': featureLevel
}
serializer = TypeSerializer()
print(json.dumps(serializer.serialize(dynamoentry)['M']))
dynamoclient = boto3.client('dynamodb')
return (dynamoclient.put_item(TableName='accounts',
Item=serializer.serialize(dynamoentry)['M']))
def dict_to_ddb(item):
# type: (Dict[str, Any]) -> Dict[str, Any]
# narrow these types down
# https://github.com/aws/aws-dynamodb-encryption-python/issues/66
"""Converts a native Python dictionary to a raw DynamoDB item.
:param dict item: Native item
:returns: DynamoDB item
:rtype: dict
"""
serializer = TypeSerializer()
return {key: serializer.serialize(value) for key, value in item.items()}
def as_dynamo_flat_dict(self):
"""
Flattens out User.as_dict() output into a simple structure without any signature or metadata.
Effectively, this outputs something like this:
```{'uuid': '11c8a5c8-0305-4524-8b41-95970baba84c', 'user_id': 'email|c3cbf9f5830f1358e28d6b68a3e4bf15', ...```
`flatten()` is recursive.
Note that this form cannot be verified or validated back since it's missing all attributes!
Return: dynamodb serialized low level dict of user in a "flattened" form for dynamodb consumption in particular
"""
user = self._clean_dict()
def sanitize(attrs):
# Types whose values need no sanitization to serialize.
supported_base_types = [type(None), bool, int, float]
# Empty strings cannot be sanitized.
def is_nonempty_str(s):
return isinstance(s, str) and len(s) > 0
def not_empty_str(v):
return not isinstance(v, str) or is_nonempty_str(v)
if type(attrs) in supported_base_types or is_nonempty_str(attrs):
return attrs
# We want to remove empty strings from lists and sanitize everything else.
if isinstance(attrs, list):
cleaned = filter(not_empty_str, attrs)
return list(map(sanitize, cleaned))
# We are dealing with a dictionary.
cleaned = {
key: sanitize(value)
for key, value in attrs.items()
if not_empty_str(key) and not_empty_str(value)
}
# If we have a dictionary, we want to ensure it only has one of either
# the "value" key or "values" key.
has_value = "value" in cleaned
has_values = "values" in cleaned
if (has_value and not has_values) or (has_values
and not has_value):
return cleaned.get("value", cleaned.get("values"))
return cleaned
serializer = TypeSerializer()
return {k: serializer.serialize(v) for k, v in sanitize(user).items()}
def dumps(dct, as_dict=False, **kwargs):
""" Dump the dict to json in DynamoDB Format
You can use any other simplejson or json options
:param dct - the dict to dump
:param as_dict - returns the result as python dict (useful for DynamoDB boto3 library) or as json sting
:returns: DynamoDB json format.
"""
result_ = TypeSerializer().serialize(
json.loads(json.dumps(dct, default=json_serial), use_decimal=True))
if as_dict:
return result_.iteritems().next()[1]
else:
return json.dumps(result_.iteritems().next()[1], **kwargs)
def log_finish_to_dynamodb(self, event, context):
dynamodb.update_item(
TableName=self.dynamodb_table,
Key={
'request_id':
TypeSerializer().serialize(str(context.aws_request_id)),
},
UpdateExpression='set lambda_status = :s, finish_time = :t',
ExpressionAttributeValues={
':s':
TypeSerializer().serialize('done'),
':t':
TypeSerializer().serialize(
int(datetime.datetime.now().timestamp())),
})
def build_shopify_url(self):
#Get page number:
ts = TypeSerializer()
pageNumber = int(ts.serialize(self.params['pages'])['N'])
query_params = [{
"url":
"{}products.json?page={}".format(self.params['url'], p),
"country":
self.params['country'],
"current_datetime":
self.current_datetime,
"id_shop":
self.idTable
} for p in range(1, pageNumber + 1)]
return query_params
def insert_data(order):
record = {
"awsRegion": "us-east-1",
"dynamodb": {
"Keys": {
"orderId": {
"S": order["orderId"]
}
},
"NewImage":
{k: TypeSerializer().serialize(v)
for k, v in order.items()},
"SequenceNumber": "1234567890123456789012345",
"SizeBytes": 123,
"StreamViewType": "NEW_AND_OLD_IMAGES"
},
"eventID": str(uuid.uuid4()),
"eventName": "INSERT",
"eventSource": "aws:dynamodb",
"eventVersion": "1.0"
}
event = {
"Source": "ecommerce.orders",
"Resources": [order["orderId"]],
"DetailType": "OrderCreated",
"Detail": json.dumps(order),
"EventBusName": "EVENT_BUS_NAME"
}
return {"record": record, "event": event}
def test_validate_products(lambda_module, product):
"""
Test validate_products() against an incorrect product
"""
product_incorrect = copy.deepcopy(product)
product_incorrect["price"] += 200
# Stub boto3
table = stub.Stubber(lambda_module.table.meta.client)
response = {
"Item": {k: TypeSerializer().serialize(v) for k, v in product.items()}
}
expected_params = {
"Key": {"productId": product["productId"]},
"ProjectionExpression": stub.ANY,
"ExpressionAttributeNames": stub.ANY,
"TableName": lambda_module.TABLE_NAME
}
table.add_response("get_item", response, expected_params)
table.activate()
# Run command
retval = lambda_module.validate_products([product_incorrect])
assert len(retval) == 2
assert len(retval[0]) == 1
assert isinstance(retval[1], str)
table.deactivate()
def convert_to_dyn_objects(user_msg_list, tnow):
tnow_dec = quantize_tstamp(decimal.Decimal(tnow))
ts = TypeSerializer()
msg_d = {}
def build_item(user_id, msg):
# hash of message and timestamp
item = msg.copy()
item['userId'] = user_id
item['created'] = tnow_dec
item = common.floats_to_decimals(item)
# filter out item values
item_dyn = dict([(k, ts.serialize(v)) for k, v in item.iteritems()])
item_dyn = dict(trim_empty_leafs(item_dyn))
return ((item['userId'], item['messageId']), {
'PutRequest': {
'Item': item_dyn
}
})
# adding to dict will ensure only latest message kept for each user_id/message_id
for user_id, m in user_msg_list:
key, dyn_item = build_item(user_id, m)
msg_d[key] = dyn_item
return msg_d.values()
def test_get_order(lambda_module, order):
"""
Test get_order()
"""
# Stub boto3
table = stub.Stubber(lambda_module.table.meta.client)
response = {
"Item": {k: TypeSerializer().serialize(v)
for k, v in order.items()},
# We do not use ConsumedCapacity
"ConsumedCapacity": {}
}
expected_params = {
"TableName": lambda_module.TABLE_NAME,
"Key": {
"orderId": order["orderId"]
}
}
table.add_response("get_item", response, expected_params)
table.activate()
# Gather orders
ddb_order = lambda_module.get_order(order["orderId"])
# Remove stub
table.assert_no_pending_responses()
table.deactivate()
# Check response
compare_dict(order, ddb_order)
class UserRepository:
serializer = TypeSerializer()
deserializer = TypeDeserializer()
usersTable: any
def __init__(self):
dynamodb = boto3.resource('dynamodb')
self.usersTable = dynamodb.Table('Users')
def getAll(self):
users = []
response = self.usersTable.scan()
for item in response['Items']:
dictionary = {
k: self.deserializer.deserialize(v)
for k, v in item.items() if k != 'userId'
}
dictionary["userId"] = int(item["userId"])
if not 'customReason' in dictionary:
dictionary['customReason'] = ''
user = User.Schema().load(dictionary)
users.append(user)
return users
def save(self, user: User):
json = {
k: self.serializer.serialize(v)
for k, v in User.Schema().dump(user).items() if v != ""
}
json['userId'] = user.userId
self.usersTable.put_item(Item=json)
return
def delete(self, user: User):
self.usersTable.delete_item(Key={'userId': user.userId})
def test_handler(lambda_module, apigateway_event, order, context):
"""
Test handler()
"""
# Stub boto3
table = stub.Stubber(lambda_module.table.meta.client)
response = {
"Item": {k: TypeSerializer().serialize(v)
for k, v in order.items()},
# We do not use ConsumedCapacity
"ConsumedCapacity": {}
}
expected_params = {
"TableName": lambda_module.TABLE_NAME,
"Key": {
"orderId": order["orderId"]
}
}
table.add_response("get_item", response, expected_params)
table.activate()
# Send request
response = lambda_module.handler(apigateway_event, context)
# Remove stub
table.assert_no_pending_responses()
table.deactivate()
assert response["statusCode"] == 200
assert "body" in response
body = json.loads(response["body"])
compare_dict(order, body)
def test_validate_products(lambda_module, product):
"""
Test validate_products() against a correct product
"""
# Stub boto3
dynamodb = stub.Stubber(lambda_module.dynamodb)
response = {
"Responses": {
lambda_module.TABLE_NAME: [{k: TypeSerializer().serialize(v) for k, v in product.items()}]
}
}
expected_params = {
"RequestItems": {
lambda_module.TABLE_NAME: {
"Keys": [{"productId": {"S": product["productId"]}}],
"ProjectionExpression": stub.ANY,
"ExpressionAttributeNames": stub.ANY
}
}
}
dynamodb.add_response("batch_get_item", response, expected_params)
dynamodb.activate()
# Run command
retval = lambda_module.validate_products([product])
print(retval)
assert len(retval) == 2
assert len(retval[0]) == 0
assert isinstance(retval[1], str)
dynamodb.deactivate()
def _api_to_doc(event):
'''Serialize a DDB Document to a DDB API Item'''
if event.get('Item') is not None:
ddb_items = event.get('Item')
event['Item'] = {
k: TypeSerializer().serialize(v)
for k, v in ddb_items.items()
}
if event.get('Key') is not None:
ddb_key = event.get('Key')
event['Key'] = {
k: TypeSerializer().serialize(v)
for k, v in ddb_key.items()
}
return event
def dynamo_extend_items_with_schedule(items_list, full=False, df_format=False):
df = pd.DataFrame(items_list)
# Extract items primary keys and format it for getitem
extract = df[["day_train_num", "station_id"]]
extract.station_id = extract.station_id.apply(str)
# Serialize in dynamo types
seres = TypeSerializer()
extract_ser = extract.applymap(seres.serialize)
items_keys = extract_ser.to_dict(orient="records")
# Submit requests
responses = dynamo_submit_batch_getitem_request(items_keys,
dynamo_sched_dep)
# Deserialize into clean dataframe
resp_df = pd.DataFrame(responses)
deser = TypeDeserializer()
resp_df = resp_df.applymap(deser.deserialize)
# Select columns to keep:
all_columns = [
'arrival_time', 'block_id', 'day_train_num', 'direction_id',
'drop_off_type', 'pickup_type', 'route_id', 'route_short_name',
'scheduled_departure_day', 'scheduled_departure_time', 'service_id',
'station_id', 'stop_headsign', 'stop_id', 'stop_sequence', 'train_num',
'trip_headsign', 'trip_id'
]
columns_to_keep = [
'day_train_num', 'station_id', 'scheduled_departure_time', 'trip_id',
'service_id', 'route_short_name', 'trip_headsign', 'stop_sequence'
]
if full:
resp_df = resp_df[all_columns]
else:
resp_df = resp_df[columns_to_keep]
# Merge to add response dataframe to initial dataframe
# We use left jointure to keep items even if we couldn't find schedule
index_cols = ["day_train_num", "station_id"]
df_updated = df.merge(resp_df, on=index_cols, how="left")
# Compute delay
df_updated.loc[:, "delay"] = df_updated.apply(lambda x: compute_delay(
x["scheduled_departure_time"], x["expected_passage_time"]),
axis=1)
# Inform
logger.info(
"Asked to find schedule and trip_id for %d items, we found %d of them."
% (len(df), len(resp_df)))
if df_format:
return df_updated
# Safe json serializable python dict
df_updated = df_updated.applymap(str)
items_updated = json.loads(df_updated.to_json(orient='records'))
return items_updated
def update_item_from_dict(table_name, key, dictionary, client):
"""
Update the item identified by `key` in the DynamoDB `table` by adding
all of the attributes in the `dictionary`.
Args:
table_name (str):
key (dict):
dictionary (dict):
client:
Returns:
dict
"""
serializer = TypeSerializer()
deserializer = TypeDeserializer()
# Prepare data by generating an alphanumeric version of the key
working_data = {k: [pattern.sub("", k), v] for k, v in dictionary.items()}
updates_string = ', '.join(
[f'#{v[0]} = :{v[0]}' for v in working_data.values()])
update_expression = f'SET {updates_string}'
attribute_names = {f'#{v[0]}': k for k, v in working_data.items()}
attribute_values = {
f':{v[0]}': serializer.serialize(v[1])
for k, v in working_data.items()
}
item = client.update_item(
TableName=table_name,
Key={k: serializer.serialize(v)
for k, v in key.items()},
UpdateExpression=update_expression,
ExpressionAttributeNames=attribute_names,
ExpressionAttributeValues=attribute_values,
ReturnValues='ALL_NEW',
)
if item:
result_data = item.get('Attributes', {})
output_data = {}
for k, v in result_data.items():
output_data[k] = deserializer.deserialize(v)
return output_data
else:
return None
def _get_dict(self):
'''
Parses self.__dict__ and returns only those objects that should be stored in the database.
:rtype: dict
'''
ts = TypeSerializer()
d = {}
for a in [k for k in dir(self) if k not in dir(type(self))]:
# limited to only instance attributes, not class attributes
try:
if not inspect.ismethod(getattr(self,a)) and not inspect.isfunction(getattr(self,a)) and not a[0] == '_' and not (hasattr(type(self),a) and isinstance(getattr(type(self),a), property)):
if not isinstance(getattr(self,a), Object):
# if DDB will choke on the data type, this will throw an error and prevent it from getting added to the dict
# however, we convert Objects to foreign-key references before saving, so don't do this if it's one of ours.
ts.serialize(getattr(self,a))
d[a] = getattr(self,a)
except Exception as e:
pass
# logger.exception("Exception occured while parsing attr {} of object {}. NOT STORING.".format(str(a), str(self)))
return d
class TestSerializer(unittest.TestCase):
def setUp(self):
self.serializer = TypeSerializer()
def test_serialize_unsupported_type(self):
with self.assertRaisesRegexp(TypeError, 'Unsupported type'):
self.serializer.serialize(object())
def test_serialize_null(self):
self.assertEqual(self.serializer.serialize(None), {'NULL': True})
def test_serialize_boolean(self):
self.assertEqual(self.serializer.serialize(False), {'BOOL': False})
def test_serialize_integer(self):
self.assertEqual(self.serializer.serialize(1), {'N': '1'})
def test_serialize_decimal(self):
self.assertEqual(
self.serializer.serialize(Decimal('1.25')), {'N': '1.25'})
def test_serialize_float_error(self):
with self.assertRaisesRegexp(
TypeError,
'Float types are not supported. Use Decimal types instead'):
self.serializer.serialize(1.25)
def test_serialize_NaN_error(self):
with self.assertRaisesRegexp(
TypeError,
'Infinity and NaN not supported'):
self.serializer.serialize(Decimal('NaN'))
def test_serialize_string(self):
self.assertEqual(self.serializer.serialize('foo'), {'S': 'foo'})
def test_serialize_binary(self):
self.assertEqual(self.serializer.serialize(
Binary(b'\x01')), {'B': b'\x01'})
def test_serialize_bytearray(self):
self.assertEqual(self.serializer.serialize(bytearray([1])),
{'B': b'\x01'})
@unittest.skipIf(six.PY2,
'This is a test when using python3 version of bytes')
def test_serialize_bytes(self):
self.assertEqual(self.serializer.serialize(b'\x01'), {'B': b'\x01'})
def test_serialize_number_set(self):
serialized_value = self.serializer.serialize(set([1, 2, 3]))
self.assertEqual(len(serialized_value), 1)
self.assertIn('NS', serialized_value)
self.assertCountEqual(serialized_value['NS'], ['1', '2', '3'])
def test_serialize_string_set(self):
serialized_value = self.serializer.serialize(set(['foo', 'bar']))
self.assertEqual(len(serialized_value), 1)
self.assertIn('SS', serialized_value)
self.assertCountEqual(serialized_value['SS'], ['foo', 'bar'])
def test_serialize_binary_set(self):
serialized_value = self.serializer.serialize(
set([Binary(b'\x01'), Binary(b'\x02')]))
self.assertEqual(len(serialized_value), 1)
self.assertIn('BS', serialized_value)
self.assertCountEqual(serialized_value['BS'], [b'\x01', b'\x02'])
def test_serialize_list(self):
serialized_value = self.serializer.serialize(['foo', 1, [1]])
self.assertEqual(len(serialized_value), 1)
self.assertIn('L', serialized_value)
self.assertCountEqual(
serialized_value['L'],
[{'S': 'foo'}, {'N': '1'}, {'L': [{'N': '1'}]}]
)
def test_serialize_map(self):
serialized_value = self.serializer.serialize(
{'foo': 'bar', 'baz': {'biz': 1}})
self.assertEqual(
serialized_value,
{'M': {'foo': {'S': 'bar'}, 'baz': {'M': {'biz': {'N': '1'}}}}})
def setUp(self):
self.serializer = TypeSerializer()
