def test_validate_products_multiple(lambda_module, product): """ Test validate_products() with multiple DynamoDB calls """ products = [] for i in range(0, 105): product_temp = copy.deepcopy(product) product_temp["productId"] += str(i) products.append(product_temp) # Stub boto3 dynamodb = stub.Stubber(lambda_module.dynamodb) response = { "Responses": { lambda_module.TABLE_NAME: [{k: TypeSerializer().serialize(v) for k, v in p.items()} for p in products[0:100]] } } expected_params = { "RequestItems": { lambda_module.TABLE_NAME: { "Keys": [{"productId": {"S": p["productId"]}} for p in products[0:100]], "ProjectionExpression": stub.ANY, "ExpressionAttributeNames": stub.ANY } } } dynamodb.add_response("batch_get_item", response, expected_params) response = { "Responses": { lambda_module.TABLE_NAME: [{k: TypeSerializer().serialize(v) for k, v in p.items()} for p in products[100:]] } } expected_params = { "RequestItems": { lambda_module.TABLE_NAME: { "Keys": [{"productId": {"S": p["productId"]}} for p in products[100:]], "ProjectionExpression": stub.ANY, "ExpressionAttributeNames": stub.ANY } } } dynamodb.add_response("batch_get_item", response, expected_params) dynamodb.activate() # Run command retval = lambda_module.validate_products(products) print(retval) assert len(retval) == 2 assert len(retval[0]) == 0 assert isinstance(retval[1], str) dynamodb.deactivate()
def log_finish_to_dynamodb(self, event, context): dynamodb.update_item( TableName=self.dynamodb_table, Key={ 'request_id': TypeSerializer().serialize(str(context.aws_request_id)), }, UpdateExpression='set lambda_status = :s, finish_time = :t', ExpressionAttributeValues={ ':s': TypeSerializer().serialize('done'), ':t': TypeSerializer().serialize( int(datetime.datetime.now().timestamp())), })
def boto3_serializer(python_dict): serializer = TypeSerializer() return { k: serializer.serialize( v if not isinstance(v, float) else Decimal(str(v))) for k, v in python_dict.items() }
class UserRepository: serializer = TypeSerializer() deserializer = TypeDeserializer() usersTable: any def __init__(self): dynamodb = boto3.resource('dynamodb') self.usersTable = dynamodb.Table('Users') def getAll(self): users = [] response = self.usersTable.scan() for item in response['Items']: dictionary = { k: self.deserializer.deserialize(v) for k, v in item.items() if k != 'userId' } dictionary["userId"] = int(item["userId"]) if not 'customReason' in dictionary: dictionary['customReason'] = '' user = User.Schema().load(dictionary) users.append(user) return users def save(self, user: User): json = { k: self.serializer.serialize(v) for k, v in User.Schema().dump(user).items() if v != "" } json['userId'] = user.userId self.usersTable.put_item(Item=json) return def delete(self, user: User): self.usersTable.delete_item(Key={'userId': user.userId})
def getSpecificProductInformation(self, keyValue): print("In getSpecificProductInformation() method !") try: table = self.dynamodb.Table('Product') resp = table.get_item(Key=keyValue) # resp returns following value. print(resp) # {'Item': {'Id': Decimal('1'), 'ProductCategoryId': Decimal('1'), 'Name': 'IPhone 8\n'}, 'ResponseMetadata': {'RequestId': 'NMDA30M1HHF514MJVVUIBCELEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'Server', 'date': 'Mon, 05 Aug 2019 17:12:19 GMT', 'content-type': 'application/x-amz-json-1.0', 'content-length': '81', 'connection': 'keep-alive', 'x-amzn-requestid': 'NMDA30M1HHF514MJVVUIBCELEFVV4KQNSO5AEMVJF66Q9ASUAAJG', 'x-amz-crc32': '3394591035'}, 'RetryAttempts': 0}} if resp and resp["ResponseMetadata"]["HTTPStatusCode"] == 200: deserializer = TypeDeserializer() serializer = TypeSerializer() if resp["Item"]: print(resp["Item"]) # {'Id': Decimal('1'), 'ProductCategoryId': Decimal('1'), 'Name': 'IPhone 8\n'} data = {k: serializer.serialize(v) for k, v in resp["Item"].items()} else: data = {} return data else: raise Exception except Exception as error: print("Not able to fetch item !") raise error
def updateData(self, numRows, start_key, field_key, field_value): client = self.dynamodb_client() deserializer = TypeDeserializer() serializer = TypeSerializer() table_configs = self.expected_table_config() for table in table_configs: LOGGER.info('Updating %s Items by setting field with key %s to the value %s, with start_key %s, for table %s', numRows, field_key, field_value, start_key, table['TableName']) for item in table['generator'](numRows, start_key): record = deserializer.deserialize(item) hashKey = table['HashKey'] key = { hashKey: serializer.serialize(record[hashKey]) } serializedFieldValue = serializer.serialize(field_value) # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.update_item client.update_item( TableName=table['TableName'], Key=key, UpdateExpression='set {}=:v'.format(field_key), ExpressionAttributeValues={ ':v': serializedFieldValue, }, )
def _api_to_doc(event): '''Serialize a DDB Document to a DDB API Item''' if event.get('Item') is not None: ddb_items = event.get('Item') event['Item'] = { k: TypeSerializer().serialize(v) for k, v in ddb_items.items() } if event.get('Key') is not None: ddb_key = event.get('Key') event['Key'] = { k: TypeSerializer().serialize(v) for k, v in ddb_key.items() } return event
def __init__(self, config): assert isinstance(config, dict), "Config must be provided during DynamoDbClient initialization" # If this is a test, make sure the table is a test table if os.environ.get('STAGE') == 'test' and 'table_name' in config: assert config['table_name'].startswith('autotest_') or config['table_name'] == 'config', \ f"Bad table name {config['table_name']} in autotest" self.config = config if not str(config.get('table_name')).startswith('autotest_mock_'): self.dynamo_client = boto3.client('dynamodb') else: logger.info(f"Initialized DynamoClient without boto3 client for table {config.get('table_name')}") # storage for table description(s) self._table_descriptions: Optional[Dict[str, Dict]] = {} # initialize table store self._table_capacity = {} self.identify_dynamo_capacity(table_name=self.config['table_name']) self.stats = defaultdict(int) if not hasattr(self, 'row_mapper'): self.row_mapper = self.config.get('row_mapper') self.type_serializer = TypeSerializer() self.type_deserializer = TypeDeserializer()
def as_dynamo_flat_dict(self): """ Flattens out User.as_dict() output into a simple structure without any signature or metadata. Effectively, this outputs something like this: ```{'uuid': '11c8a5c8-0305-4524-8b41-95970baba84c', 'user_id': 'email|c3cbf9f5830f1358e28d6b68a3e4bf15', ...``` `flatten()` is recursive. Note that this form cannot be verified or validated back since it's missing all attributes! Return: dynamodb serialized low level dict of user in a "flattened" form for dynamodb consumption in particular """ user = self._clean_dict() def flatten(attrs, field=None): flat = {} for f in attrs: # Skip "schema" if isinstance(attrs[f], str): continue if not set(["value", "values"]).isdisjoint(set(attrs[f])): res = attrs[f].get("value", attrs[f].get("values")) if res is not None and res != "": flat[f] = res else: flat[f] = flatten(attrs[f]) return flat serializer = TypeSerializer() return {k: serializer.serialize(v) for k, v in flatten(user).items()}
def insert_data(order): record = { "awsRegion": "us-east-1", "dynamodb": { "Keys": { "orderId": { "S": order["orderId"] } }, "NewImage": {k: TypeSerializer().serialize(v) for k, v in order.items()}, "SequenceNumber": "1234567890123456789012345", "SizeBytes": 123, "StreamViewType": "NEW_AND_OLD_IMAGES" }, "eventID": str(uuid.uuid4()), "eventName": "INSERT", "eventSource": "aws:dynamodb", "eventVersion": "1.0" } event = { "Source": "ecommerce.orders", "Resources": [order["orderId"]], "DetailType": "OrderCreated", "Detail": json.dumps(order), "EventBusName": "EVENT_BUS_NAME" } return {"record": record, "event": event}
def db_log_step(userID, recipe, step): """Log the most recent step that a user has been given """ dynamo = db_connect() ts = TypeSerializer() put_resp = dynamo.put_item(TableName=os.environ['STEP_HISTORY_TABLE'], Item={ 'userID': { 'S': userID }, 'time': { 'N': str(time.time()) }, 'step': ts.serialize(step), 'recipe': ts.serialize(recipe) }) upd_resp = dynamo.update_item(TableName=os.environ['STEP_LAST_TABLE'], Key={'userID': { 'S': userID }}, AttributeUpdates={ 'step': { 'Action': 'PUT', 'Value': ts.serialize(step) }, 'recipe': { 'Action': 'PUT', 'Value': ts.serialize(recipe) } }) return (put_resp, upd_resp)
def test_validate_products(lambda_module, product): """ Test validate_products() against an incorrect product """ product_incorrect = copy.deepcopy(product) product_incorrect["price"] += 200 # Stub boto3 table = stub.Stubber(lambda_module.table.meta.client) response = { "Item": {k: TypeSerializer().serialize(v) for k, v in product.items()} } expected_params = { "Key": {"productId": product["productId"]}, "ProjectionExpression": stub.ANY, "ExpressionAttributeNames": stub.ANY, "TableName": lambda_module.TABLE_NAME } table.add_response("get_item", response, expected_params) table.activate() # Run command retval = lambda_module.validate_products([product_incorrect]) assert len(retval) == 2 assert len(retval[0]) == 1 assert isinstance(retval[1], str) table.deactivate()
def convert_to_dyn_objects(user_msg_list, tnow): tnow_dec = quantize_tstamp(decimal.Decimal(tnow)) ts = TypeSerializer() msg_d = {} def build_item(user_id, msg): # hash of message and timestamp item = msg.copy() item['userId'] = user_id item['created'] = tnow_dec item = common.floats_to_decimals(item) # filter out item values item_dyn = dict([(k, ts.serialize(v)) for k, v in item.iteritems()]) item_dyn = dict(trim_empty_leafs(item_dyn)) return ((item['userId'], item['messageId']), { 'PutRequest': { 'Item': item_dyn } }) # adding to dict will ensure only latest message kept for each user_id/message_id for user_id, m in user_msg_list: key, dyn_item = build_item(user_id, m) msg_d[key] = dyn_item return msg_d.values()
def test_get_order(lambda_module, order): """ Test get_order() """ # Stub boto3 table = stub.Stubber(lambda_module.table.meta.client) response = { "Item": {k: TypeSerializer().serialize(v) for k, v in order.items()}, # We do not use ConsumedCapacity "ConsumedCapacity": {} } expected_params = { "TableName": lambda_module.TABLE_NAME, "Key": { "orderId": order["orderId"] } } table.add_response("get_item", response, expected_params) table.activate() # Gather orders ddb_order = lambda_module.get_order(order["orderId"]) # Remove stub table.assert_no_pending_responses() table.deactivate() # Check response compare_dict(order, ddb_order)
def test_handler(lambda_module, apigateway_event, order, context): """ Test handler() """ # Stub boto3 table = stub.Stubber(lambda_module.table.meta.client) response = { "Item": {k: TypeSerializer().serialize(v) for k, v in order.items()}, # We do not use ConsumedCapacity "ConsumedCapacity": {} } expected_params = { "TableName": lambda_module.TABLE_NAME, "Key": { "orderId": order["orderId"] } } table.add_response("get_item", response, expected_params) table.activate() # Send request response = lambda_module.handler(apigateway_event, context) # Remove stub table.assert_no_pending_responses() table.deactivate() assert response["statusCode"] == 200 assert "body" in response body = json.loads(response["body"]) compare_dict(order, body)
def generate_items(self, num_items): serializer = TypeSerializer() for i in range(num_items): record = { 'int_id': int(i / 10.0), 'decimal_field': decimal.Decimal(str(i) + '.00000000001'), 'string_field': str(i), 'byte_field': b'some_bytes', 'int_list_field': [i, i + 1, i + 2], 'int_set_field': set([i, i + 1, i + 2]), 'map_field': { 'map_entry_1': 'map_value_1', 'map_entry_2': 'map_value_2' }, 'string_list': [ self.random_string_generator(), self.random_string_generator(), self.random_string_generator() ], 'boolean_field': True, 'other_boolean_field': False, 'null_field': None } yield serializer.serialize(record)
def convert_json_to_dynamo_json(input_json: list) -> list: """ Re-Serializes the data into DynamoDB compatible JSON that can be used to put items into the dynamo table """ # HACK: When serializing the JSON without dynamodb attribute types included, it wants to convert # The DynamoDB 'String Set' objects to DynamoDB List objects because python loads the data as lists and not sets. # I am choosing to go with DynamoDB attribute string sets, because I do not want duplicate entries for periods, # and it is easier to parse visually. The only drawback I have seen so far is that sets are unordered, # but since we are not evaluating the period string set responses in any particular order that should not matter. # Can we change this to use cls instead of for loops? serializer = TypeSerializer() py_data, json_data = [], [] logger.info(f"Converting JSON config to DynamoDB compatible JSON.") # Loop through JSON file data looking for Python object type list # Convert the list object into a set of strings # Store new data types as python object for data in input_json: for k, v in data.items(): if isinstance(v, list): data[k] = set(v) py_data.append(data) # Serialize previously modified python object data into DynamoDB JSON for data in py_data: dynamo_data = {k: serializer.serialize(v) for k, v in data.items()} json_data.append(dynamo_data) return json_data
def construct_dynamo_type_dict(d: dict): """ DynamoDB transactions need a different way of specifying transaction. The structure has to be recursively implemented as: 'string': { 'S': 'string', 'N': 'string', 'B': b'bytes', 'SS': [ 'string', ], 'NS': [ 'string', ], 'BS': [ b'bytes', ], 'M': { 'string': {'... recursive ...'} }, 'L': [ {'... recursive ...'}, ], 'NULL': True|False, 'BOOL': True|False } https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.transact_write_items TypeSerializer and TypeDeserializer needs to be used to convert to appropriate representations for DynamoDB. serialize(d)['M'] has been used in line with the documentation. """ serializer = TypeSerializer() return serializer.serialize(d)['M']
def test_validate_products(lambda_module, product): """ Test validate_products() against a correct product """ # Stub boto3 dynamodb = stub.Stubber(lambda_module.dynamodb) response = { "Responses": { lambda_module.TABLE_NAME: [{k: TypeSerializer().serialize(v) for k, v in product.items()}] } } expected_params = { "RequestItems": { lambda_module.TABLE_NAME: { "Keys": [{"productId": {"S": product["productId"]}}], "ProjectionExpression": stub.ANY, "ExpressionAttributeNames": stub.ANY } } } dynamodb.add_response("batch_get_item", response, expected_params) dynamodb.activate() # Run command retval = lambda_module.validate_products([product]) print(retval) assert len(retval) == 2 assert len(retval[0]) == 0 assert isinstance(retval[1], str) dynamodb.deactivate()
def dynamodb_put_item(ddb_client, table_name: str, item: dict): serializer = TypeSerializer() serialized_item = serializer.serialize(item)['M'] try: ddb_client.put_item(TableName=table_name, Item=serialized_item) except ddb_client.exceptions.ResourceNotFoundException: raise TyphoonResourceNotFoundError( f'Table {table_name} does not exist in DynamoDB')
def serialize_output(value): try: td = TypeSerializer() for k, v in dict(value).items(): value[k] = td.serialize(v) except BaseException: pass return value
def dynamo_extend_items_with_schedule(items_list, full=False, df_format=False): df = pd.DataFrame(items_list) # Extract items primary keys and format it for getitem extract = df[["day_train_num", "station_id"]] extract.station_id = extract.station_id.apply(str) # Serialize in dynamo types seres = TypeSerializer() extract_ser = extract.applymap(seres.serialize) items_keys = extract_ser.to_dict(orient="records") # Submit requests responses = dynamo_submit_batch_getitem_request(items_keys, dynamo_sched_dep) # Deserialize into clean dataframe resp_df = pd.DataFrame(responses) deser = TypeDeserializer() resp_df = resp_df.applymap(deser.deserialize) # Select columns to keep: all_columns = [ 'arrival_time', 'block_id', 'day_train_num', 'direction_id', 'drop_off_type', 'pickup_type', 'route_id', 'route_short_name', 'scheduled_departure_day', 'scheduled_departure_time', 'service_id', 'station_id', 'stop_headsign', 'stop_id', 'stop_sequence', 'train_num', 'trip_headsign', 'trip_id' ] columns_to_keep = [ 'day_train_num', 'station_id', 'scheduled_departure_time', 'trip_id', 'service_id', 'route_short_name', 'trip_headsign', 'stop_sequence' ] if full: resp_df = resp_df[all_columns] else: resp_df = resp_df[columns_to_keep] # Merge to add response dataframe to initial dataframe # We use left jointure to keep items even if we couldn't find schedule index_cols = ["day_train_num", "station_id"] df_updated = df.merge(resp_df, on=index_cols, how="left") # Compute delay df_updated.loc[:, "delay"] = df_updated.apply(lambda x: compute_delay( x["scheduled_departure_time"], x["expected_passage_time"]), axis=1) # Inform logger.info( "Asked to find schedule and trip_id for %d items, we found %d of them." % (len(df), len(resp_df))) if df_format: return df_updated # Safe json serializable python dict df_updated = df_updated.applymap(str) items_updated = json.loads(df_updated.to_json(orient='records')) return items_updated
def generate_items(self, num_items, start_key=0): serializer = TypeSerializer() for i in range(start_key, start_key + num_items): record = { 'int_id': i, 'string_field': self.random_string_generator(), 'boolean_field': True, } yield serializer.serialize(record)
def modify_data(order): new_order = copy.deepcopy(order) new_order["status"] = "COMPLETED" record = { "awsRegion": "us-east-1", "dynamodb": { "Keys": { "orderId": { "S": order["orderId"] } }, "OldImage": {k: TypeSerializer().serialize(v) for k, v in order.items()}, "NewImage": {k: TypeSerializer().serialize(v) for k, v in new_order.items()}, "SequenceNumber": "1234567890123456789012345", "SizeBytes": 123, "StreamViewType": "NEW_AND_OLD_IMAGES" }, "eventID": str(uuid.uuid4()), "eventName": "REMOVE", "eventSource": "aws:dynamodb", "eventVersion": "1.0" } event = { "Source": "ecommerce.orders", "Resources": [order["orderId"]], "DetailType": "OrderDeleted", "Detail": json.dumps({ "old": order, "new": new_order, "changed": ["status"] }), "EventBusName": "EVENT_BUS_NAME" } return {"record": record, "event": event}
def dict_to_ddb(item): # type: (Dict[str, Any]) -> Dict[str, Any] # TODO: narrow these types down """Converts a native Python dictionary to a raw DynamoDB item. :param dict item: Native item :returns: DynamoDB item :rtype: dict """ serializer = TypeSerializer() return {key: serializer.serialize(value) for key, value in item.items()}
def lambda_handler(event, context): accountId = str(event['accountId']) accountName = str(event['description']) accountRole = "customer" confidentialKMSKey = 'alias/TSI_Base_ConfidentialS3Key' internalKMSKey = 'alias/TSI_Base_InternalS3Key' if 'customermasteraccountid' in event: masteraccountId = event['customermasteraccountid'] else: masteraccountId = os.environ['accountid'] readonlyRole = 'TSI_Base_ReadOnlySwitchRole' securityEmail = str(event['email']) accountemail = str(event['accountemail']) enabledregions = event['enabledregions'].split(',') supportenabled = 'false' awsconfigenabled = 'false' if 'config' in event: config = event['config'] else: config = "disabled" if 'support' in event: support = event['support'] else: config = "disabled" ouname = event['ouname'] terraformVersion = '1.0' writeRole = 'TSI_Base_FullAccess' featureLevel = 'full' dynamoentry = { 'accountId': accountId, 'accountName': accountName, 'config': config, 'support': support, 'ouname': ouname, 'accountRole': accountRole, 'confidentialKMSKey': confidentialKMSKey, 'internalKMSKey': internalKMSKey, 'masteraccountId': masteraccountId, 'readonlyRole': readonlyRole, 'securityEmail': securityEmail, 'accountemail': accountemail, 'enabledregions': enabledregions, 'supportenabled': supportenabled, 'awsconfigenabled': awsconfigenabled, 'terraformVersion': terraformVersion, 'writeRole': writeRole, 'featureLevel': featureLevel } serializer = TypeSerializer() print(json.dumps(serializer.serialize(dynamoentry)['M'])) dynamoclient = boto3.client('dynamodb') return (dynamoclient.put_item(TableName='accounts', Item=serializer.serialize(dynamoentry)['M']))
def _save_dice_pools(self): if self.pools is None: raise Exception("Tried to save dice pools before loading them.") ser = TypeSerializer() item = { 'game': ser.serialize('Shadowrun'), 'timestamp': ser.serialize(Decimal(time.time())), 'pools': ser.serialize(self.pools) } logger.debug("Item before put_item()ing: {}".format(item)) self._client.put_item(TableName='DicePools', Item=item)
def log_start_to_dynamodb(self, event, context): data = { 'start_time': int(datetime.datetime.now().timestamp()), 'lambda_status': 'started', 'request_id': str(context.aws_request_id) } dynamodb.put_item( TableName=self.dynamodb_table, Item={k: TypeSerializer().serialize(v) for k, v in data.items()}, )
def dict_to_ddb(item): # type: (Dict[str, Any]) -> Dict[str, Any] # narrow these types down # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Converts a native Python dictionary to a raw DynamoDB item. :param dict item: Native item :returns: DynamoDB item :rtype: dict """ serializer = TypeSerializer() return {key: serializer.serialize(value) for key, value in item.items()}
def as_dynamo_flat_dict(self): """ Flattens out User.as_dict() output into a simple structure without any signature or metadata. Effectively, this outputs something like this: ```{'uuid': '11c8a5c8-0305-4524-8b41-95970baba84c', 'user_id': 'email|c3cbf9f5830f1358e28d6b68a3e4bf15', ...``` `flatten()` is recursive. Note that this form cannot be verified or validated back since it's missing all attributes! Return: dynamodb serialized low level dict of user in a "flattened" form for dynamodb consumption in particular """ user = self._clean_dict() def sanitize(attrs): # Types whose values need no sanitization to serialize. supported_base_types = [type(None), bool, int, float] # Empty strings cannot be sanitized. def is_nonempty_str(s): return isinstance(s, str) and len(s) > 0 def not_empty_str(v): return not isinstance(v, str) or is_nonempty_str(v) if type(attrs) in supported_base_types or is_nonempty_str(attrs): return attrs # We want to remove empty strings from lists and sanitize everything else. if isinstance(attrs, list): cleaned = filter(not_empty_str, attrs) return list(map(sanitize, cleaned)) # We are dealing with a dictionary. cleaned = { key: sanitize(value) for key, value in attrs.items() if not_empty_str(key) and not_empty_str(value) } # If we have a dictionary, we want to ensure it only has one of either # the "value" key or "values" key. has_value = "value" in cleaned has_values = "values" in cleaned if (has_value and not has_values) or (has_values and not has_value): return cleaned.get("value", cleaned.get("values")) return cleaned serializer = TypeSerializer() return {k: serializer.serialize(v) for k, v in sanitize(user).items()}