def write_to_table(self, table: Table):
"""
Write operations for this object need to be special cased (to avoid overwritting)
Therefore we do not implement `to_dynamodb_item` however basically the body of that
method is used in this class's impl of `write_to_table_if_not_found`
"""
# put_item does not support UpdateExpression
table.update_item(
Key={
"PK": self.get_dynamodb_content_key(self.content_id),
"SK": self.get_dynamodb_content_type_key(),
},
# If ContentRef exists it needs to match or BAD THING(tm) can happen...
ConditionExpression=Or(
Attr("ContentRef").not_exists(),
Attr("ContentRef").eq(self.content_ref)), # type: ignore
# Unfortunately while prod is happy with this on multiple lines pytest breaks...
UpdateExpression=
"""SET ContentType = :ct, ContentRef = :cr, ContentRefType = :crt, SubmissionTimes = list_append(if_not_exists(SubmissionTimes, :empty_list), :s), CreatedAt = if_not_exists(CreatedAt, :c), UpdatedAt = :u ADD AdditionalFields :af""",
ExpressionAttributeValues={
":ct": self.content_type.get_name(),
":cr": self.content_ref,
":crt": self.content_ref_type.value,
":af": self.additional_fields if self.additional_fields else
{self.ADDITIONAL_FIELDS_PLACE_HOLDER},
":s": [s.isoformat() for s in self.submission_times],
":c": self.created_at.isoformat(),
":u": self.updated_at.isoformat(),
":empty_list": [],
},
)
def save(self, force=False, save_if_missing=True, save_if_existing=True):
_TRACE("DynamoObject.save reached")
if not save_if_missing and not save_if_existing:
raise RuntimeError(
"At least one of save_if_missing and save_if_existing must be true."
)
old_version = DECIMAL(self.get(VERSION_KEY, -1))
create_condition = Attr(VERSION_KEY).not_exists()
if force:
update_condition = Attr(VERSION_KEY).exists()
else:
update_condition = Attr(VERSION_KEY).eq(old_version)
CE = None
if force and save_if_missing and save_if_existing:
pass
elif save_if_missing and save_if_existing:
CE = Or(create_condition, update_condition)
elif save_if_existing:
CE = update_condition
else:
# If we're here, we know that create_condition=True
CE = create_condition
try:
self[VERSION_KEY] = old_version + 1
if CE:
self._store(CE)
else:
self._store()
return self
except ClientError as e:
self[VERSION_KEY] = old_version
raise e
def _save(self, force=False, save_if_missing=True, save_if_existing=True, only_if_updated=False):
if not save_if_missing and not save_if_existing:
raise RuntimeError("At least one of save_if_missing and save_if_existing must be true.")
if only_if_updated and not self._obj_updates:
return self
old_version = getattr(self, VERSION_KEY)
create_condition = Attr(VERSION_KEY).not_exists()
if force:
update_condition = Attr(VERSION_KEY).exists()
else:
update_condition = Attr(VERSION_KEY).eq(old_version)
CE = None
if force and save_if_missing and save_if_existing:
pass
elif save_if_missing and save_if_existing:
CE = Or(create_condition, update_condition)
elif save_if_existing:
CE = update_condition
else:
# If we're here, we know that create_condition=True
CE = create_condition
try:
setattr(self, VERSION_KEY, old_version+1)
if CE:
self._store(CE)
else:
self._store()
self._clear_update_record()
self._in_db = True
return self
except ClientError as e:
setattr(self, VERSION_KEY, old_version)
raise e
def search_votes(search_form):
keywords = search_form.vote_topic.data
dynamodb = current_app.extensions['dynamo']
keyword = re.sub(
' ', " ",
re.sub(
"[\u0060|\u0021-\u002c|\u002e-\u002f|\u003a-\u003f|\u2200-\u22ff|\uFB00-\uFFFD|\u2E80-\u33FF]",
' ', keywords)).split(' ')
filter_expression_list = []
while ("" in keyword):
keyword.remove("")
if len(keyword) > 1:
for i in range(len(keyword)):
filter_expression_list.append(
Attr("topic").contains("{}".format(keyword[i])))
expression = Or(*filter_expression_list)
elif len(keyword) == 1:
expression = Attr("topic").contains("{}".format(keyword[0]))
else:
return False
results = dynamodb.tables['votes'].scan(FilterExpression=expression)
return results["Items"]
def test_or(self):
cond1 = Equals(self.value, self.value2)
cond2 = Equals(self.value, self.value2)
or_cond = Or(cond1, cond2)
self.build_and_assert_expression(
or_cond,
{'format': '({0} {operator} {1})',
'operator': 'OR', 'values': (cond1, cond2)})
def test_update(mocker):
mocker.patch.object(checkpoint, 'TABLE')
checkpoint.update(5)
checkpoint.TABLE.put_item.assert_called_with(Item={
'id': checkpoint.RECORD_KEY,
'since_id': 5
},
ConditionExpression=Or(
Attr('id').not_exists(),
Attr('since_id').lt(5)))
def update(since_id):
"""Update checkpoint to given tweet id."""
try:
TABLE.put_item(Item={
'id': RECORD_KEY,
'since_id': since_id
},
ConditionExpression=Or(
Attr('id').not_exists(),
Attr('since_id').lt(since_id)))
except ClientError as e:
if e.response['Error']['Code'] != 'ConditionalCheckFailedException':
raise
def scan_schedule_data(prefix, weekday_ja):
schedules_table_name = os.environ['SCHEDULES_TABLE_NAME']
try:
filter_exp = And(
Attr('weekday').eq(weekday_ja),
Or(Attr('prefix').eq(0),
Attr('prefix').eq(prefix)))
response = dynamodb.Table(schedules_table_name).scan(
FilterExpression=filter_exp)
return response['Items']
except ClientError as e:
print(e.response['Error']['Message'])
return []
def lambda_handler(event, context):
query_parameters = event["queryStringParameters"]
tags = []
[tags.append(value) for value in query_parameters.values()]
if len(tags) == 1:
filter_expression = Attr("tag").contains(tags[0])
else:
filter_expression = Or(*[(Attr("tag").contains(value))
for value in tags])
dynamodb = boto3.resource("dynamodb")
table = dynamodb.Table("FIT5225Assignment2")
response = table.scan(FilterExpression=filter_expression)
result = {"links": []}
[result["links"].append(item["link"]) for item in response["Items"]]
return {
"statusCode": 200,
"body": json.dumps(result),
"isBase64Encoded": "false"
}
def write_to_table(self, table: Table):
"""
Write operations for this object need to be special cased (to avoid overwritting)
Therefore we do not implement `to_dynamodb_item`
If you're curious it would ~look like this:
def to_dynamodb_item(self) -> dict:
return {
"PK": self.get_dynamodb_content_key(self.content_id),
"SK": self.get_dynamodb_content_type_key(self.content_type),
"ContentRef": self.content_ref,
"ContentRefType": self.content_ref_type,
"AdditionalFields": self.additional_fields,
"SubmissionTimes": [s.isoformat() for s in self.submission_times],
"CreatedOn": self.created_at.isoformat(),
"UpdatedAt": self.updated_at.isoformat(),
}
"""
# put_item does not support UpdateExpression
table.update_item(
Key={
"PK": self.get_dynamodb_content_key(self.content_id),
"SK": self.get_dynamodb_content_type_key(self.content_type),
},
# If ContentRef exists it needs to match or BAD THING(tm) can happen...
ConditionExpression=Or(
Attr("ContentRef").not_exists(),
Attr("ContentRef").eq(self.content_ref)), # type: ignore
# Unfortunately while prod is happy with this on multiple lines pytest breaks...
UpdateExpression=
"""SET ContentRef = :cr, ContentRefType = :crt, SubmissionTimes = list_append(if_not_exists(SubmissionTimes, :empty_list), :s), CreatedAt = if_not_exists(CreatedAt, :c), UpdatedAt = :u ADD AdditionalFields :af""",
ExpressionAttributeValues={
":cr": self.content_ref,
":crt": self.content_ref_type.value,
":af": self.additional_fields
if self.additional_fields else {"Placeholder"},
":s": [s.isoformat() for s in self.submission_times],
":c": self.created_at.isoformat(),
":u": self.updated_at.isoformat(),
":empty_list": [],
},
)
def get_from_dynamo(name: str):
dynamodb = boto3.resource("dynamodb", region_name='us-east-1')
table_name = os.environ.get('tableName', None)
all_responses = []
if table_name is None:
return all_responses
table = dynamodb.Table(table_name)
fe = Or(Attr('name').eq(name), Attr("location").eq(name))
pe = "#g_id, #d_id, #name, #is_group, #delay, #loc"
ean = {
"#g_id": "group_id",
"#d_id": "device_id",
"#name": "name",
"#is_group": "is_group",
"#delay": 'delay',
"#loc": "location"
}
response = table.scan(FilterExpression=fe,
ProjectionExpression=pe,
ExpressionAttributeNames=ean)
for i in response['Items']:
all_responses.append(i)
while 'LastEvaluatedKey' in response:
response = table.scan(ProjectionExpression=pe,
FilterExpression=fe,
ExpressionAttributeNames=ean,
ExclusiveStartKey=response['LastEvaluatedKey'])
for i in response['Items']:
all_responses.append(i)
print(all_responses)
return all_responses
def test_or_operator(self):
cond1 = Equals(self.value, self.value2)
cond2 = Equals(self.value, self.value2)
self.assertEqual(cond1 | cond2, Or(cond1, cond2))
def lambda_handler(event, context):
"""Sample pure Lambda function
Parameters
----------
event: dict, required
API Gateway Lambda Proxy Input Format
Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context: object, required
Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
API Gateway Lambda Proxy Output Format: dict
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
# try:
# ip = requests.get("http://checkip.amazonaws.com/")
# except requests.RequestException as e:
# # Send some context about this error to Lambda Logs
# print(e)
# raise e
print(event)
method = event['httpMethod']
print(f"method={method}")
print(f"table_name={table_name}")
myTriggerType = 'instrument_price'
if method == "DELETE":
#path=event['path']
trigger_id = event['pathParameters']['trigger_id']
print(f"triggerId={trigger_id}")
try:
#see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.delete_item
response = table.delete_item(
Key={
'PK': f"TR#{myTriggerType}#{trigger_id}",
"SK": f"TR#{myTriggerType}#{trigger_id}"
},
ConditionExpression=And(
Attr('PK').eq(Attr('SK')),
Attr('triggerType').eq(myTriggerType)),
)
except ClientError as e:
print(f"clientError={e}")
if e.response['Error'][
'Code'] == 'ConditionalCheckFailedException':
return iftttError(404, "item not found")
raise
print(f"response={response}")
return {
"statusCode": 200,
"body": "",
}
elif method == "POST":
body = json.loads(event['body'])
trigger_id = body['trigger_identity']
print(f"triggerId={trigger_id}")
response = table.get_item(
Key={
'PK': f"TR#{myTriggerType}#{trigger_id}",
"SK": f"TR#{myTriggerType}#{trigger_id}"
},
ProjectionExpression="triggerEvents, triggerType",
)
print(f"response={response}")
if "Item" not in response:
#brand new
print(f"inserting {trigger_id}")
if 'triggerFields' not in body:
return iftttError(400, "triggerFields missing from request")
triggerFields = body['triggerFields']
#todo validate trigger fields
try:
response = table.put_item(
Item={
'PK': f"TR#{myTriggerType}#{trigger_id}",
"SK": f"TR#{myTriggerType}#{trigger_id}",
'triggerId': trigger_id,
#hacky string way to avoid having multiple columns
'triggerFields': json.dumps(triggerFields),
'triggerType': myTriggerType,
},
ConditionExpression=Or(
Attr('triggerType').eq(myTriggerType),
Attr('triggerType').not_exists()))
except ClientError as e:
print(f"clientError={e}")
#somehow got created with someone elses triggerType
if e.response['Error'][
'Code'] == 'ConditionalCheckFailedException':
return iftttError(404, "item not found")
raise
print("response ", response)
triggered = []
elif response['Item'].get("triggerType",
myTriggerType) != myTriggerType:
#it exists but it is someone elses
return iftttError(404, "item not found")
else:
item = response['Item']
print(f"found {item} ")
#hacky string way to avoid having multiple columns
#TODO: change this to use a Map? (will allow to add without overwrite)
events = json.loads(item.get("triggerEvents", "[]"))
triggered = []
for event in events:
#TODO: implement limit (not needed now becasue I expect only up to one events)
triggered.append(event['data'])
return {
"statusCode":
200,
"body":
json.dumps({
"data": triggered,
# "location": ip.text.replace("\n", "")
}),
}
else:
return iftttError(400, f"unexpected httpMethod {method}")
def test_or_operator(self):
cond1 = Equals(self.value, self.value2)
cond2 = Equals(self.value, self.value2)
assert cond1 | cond2 == Or(cond1, cond2)
def lambda_handler(event, context):
"""Sample pure Lambda function
Parameters
----------
event: dict, required
API Gateway Lambda Proxy Input Format
Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context: object, required
Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
API Gateway Lambda Proxy Output Format: dict
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
# try:
# ip = requests.get("http://checkip.amazonaws.com/")
# except requests.RequestException as e:
# # Send some context about this error to Lambda Logs
# print(e)
# raise e
print(event)
method=event['httpMethod']
print(f"method={method}")
print(f"table_name={table_name}")
myTriggerType='prev_day_change' # TODO: get from path
if method == "DELETE":
#path=event['path']
trigger_id=event['pathParameters']['trigger_id']
print(f"triggerId={trigger_id}")
try:
#see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.delete_item
response = table.update_item(
Key={'PK':f"TR#{myTriggerType}#{trigger_id}", "SK":f"TR#{myTriggerType}#{trigger_id}"},
UpdateExpression="SET expiresOn = :val1",
ExpressionAttributeValues={
':val1': calendar.timegm(time.gmtime()),
},
ConditionExpression=And(
And(Attr('PK').exists(),Attr('expiresOn').not_exists()),
Attr('triggerType').eq(myTriggerType)),
)
except ClientError as e:
print(f"clientError={e}")
if e.response['Error']['Code']=='ConditionalCheckFailedException':
return iftttError(404,"item not found")
raise
print(f"response={response}")
return {
"statusCode": 200,
"body":"",
}
elif method == "POST":
body=json.loads(event['body'])
trigger_id=body['trigger_identity']
limit = body.get('limit',50)
print(f"triggerId={trigger_id}")
###########
# for a PK TR#1 with events EV#1 EV#2 EV#N it will load:
# TR#1,TR#1
# TR#1,EV#N
# TR#1,EV#N-1
# ....
# TR#1,EV#N-(limit-1)
response = table.query(
KeyConditionExpression=Key("PK").eq(f"TR#{myTriggerType}#{trigger_id}")
#.__and__(Key("SK").begins_with(f"TR#{myTriggerType}#").__or__(Key("SK").begins_with(f"EV#")))
#parked for now but how do I filter for keys begining with X or y? (probably with a query filter?)
#TODO: filter query on SK, how do I do that?
,
ScanIndexForward=False, #the latest X events + trigger (trigger sorts after events)
Limit=limit + 1, #+1 for Trigger row
ProjectionExpression="SK, triggerEvent, expiresOn",
)
#no need to itterate as we do not expect to filter out anything
print(f"response={response}")
items = response["Items"]
if 0 == len(items) \
or (not items[0]['SK'].startswith("TR#") )\
or 'expiresOn' in items[0]:
#brand new
print(f"inserting {trigger_id}")
if 'triggerFields' not in body:
return iftttError(400, "triggerFields missing from request")
triggerFields=body['triggerFields']
#todo validate trigger fields
try:
response = table.put_item(
Item={
'PK':f"TR#{myTriggerType}#{trigger_id}",
"SK":f"TR#{myTriggerType}#{trigger_id}",
'triggerId': trigger_id,
#hacky string way to avoid having multiple columns
'triggerFields': json.dumps(triggerFields),
'triggerType': myTriggerType,
},
ConditionExpression=Or(
Attr('expiresOn').exists(),#previously deleted item # TODO: in this case we 'resurect' the old events. This should not happen
Attr('PK').not_exists() # brand new item
),
)
except ClientError as e:
print(f"clientError={e}")
if e.response['Error']['Code']=='ConditionalCheckFailedException':
return iftttError(404,"item not found") #
raise
print("response ",response)
triggered=[]
else:
events = items[1:]
print(f"found {events} ")
#hacky string way to avoid having multiple columns
#TODO: change this to use a Map? (will allow to add without overwrite)
#events = json.loads(item.get("triggerEvents","[]"))
triggered= []
now=calendar.timegm(time.gmtime())
for event in events:
if now< event.get('expiresOn',now+1):
triggered.append(json.loads(event['triggerEvent']))
return {
"statusCode": 200,
"body": json.dumps({
"data": triggered,
# "location": ip.text.replace("\n", "")
}),
}
else :
return iftttError(400, f"unexpected httpMethod {method}")