def remove(self, expense, user_uid):
"""
:param expense: expense object
:param user_uid:
:return: None
:raises NoExpenseWithThisId
:returns void
"""
expense = expense.copy()
try:
self.expenses_table.delete_item(
Key={
'user_uid': user_uid,
'timestamp_utc': expense['timestamp_utc']
},
# ensure that the expense at rest has the same `id` AND that the expense at rest exists at all
ConditionExpression=And(
Attr('id').eq(expense['id']),
Attr(self.RANGE_KEY).exists()))
except Exception as err:
if "ConditionalCheckFailedException" in str(err):
# either there's not expense with such Key,or the expense at rest has different `id`.
raise NoExpenseWithThisId()
else:
raise err
def _get_run_metrics(self, run_id, metric_key=None):
dynamodb = self._get_dynamodb_resource()
table_name = "_".join([self.table_prefix, DynamodbStore.METRICS_TABLE])
table = dynamodb.Table(table_name)
condition = Key("run_id").eq(run_id)
if metric_key:
condition = And(condition, Key("key").eq(metric_key))
# TODO: Can't just take the first metric in the list (need to sort by step/timestamp)
# TODO: Refactor to have metrics under experiment then filter for top N
if self.use_projections:
response = table.query(
ProjectionExpression=
"#key, #metrics[0].#value, #metrics[0].#timestamp",
ExpressionAttributeNames={
"#key": "key",
"#metrics": "metrics",
"#value": "value",
"#timestamp": "timestamp",
},
ConsistentRead=True,
KeyConditionExpression=condition,
ReturnConsumedCapacity="TOTAL",
)
else:
response = table.query(
ConsistentRead=True,
KeyConditionExpression=condition,
ReturnConsumedCapacity="TOTAL",
)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
if "Items" in response:
return response["Items"]
return []
def EZQuery(tbl,
key_attr,
key,
index_name=None,
range_attr=None,
op=None,
range=None,
reverse=False):
kce = Key(key_attr).eq(key)
if range_attr:
op_func = getattr(Key(range_attr), op)
range_op = op_func(*range) \
if type(range) in (tuple, list) \
else op_func(range)
kce = And(kce, range_op)
kwargs = {'KeyConditionExpression': kce, 'ScanIndexForward': not reverse}
if index_name:
kwargs['IndexName'] = index_name
items = []
while True:
result = tbl.query(**kwargs)
items += result['Items']
if not 'LastEvaluatedKey' in result:
break
kwargs['ExclusiveStartKey'] = result['LastEvaluatedKey']
return items
def _update_field_in_table_if_exists(
self, table: Table, field_value: t.Any, field_name: str
) -> bool:
"""
Only write the field for object in table if the objects with matchig PK and SK already exist
(also updates updated_at).
Returns true if object existed and therefore update was successful otherwise false.
"""
try:
table.update_item(
Key={
"PK": self.get_dynamodb_signal_key(
self.SIGNAL_SOURCE_SHORTCODE, self.signal_id
),
"SK": self.get_sort_key(self.privacy_group_id),
},
# service_resource.Table.update_item's ConditionExpression params is not typed to use its own objects here...
ConditionExpression=And(Attr("PK").exists(), Attr("SK").exists()), # type: ignore
ExpressionAttributeValues={
":f": field_value,
":u": self.updated_at.isoformat(),
},
ExpressionAttributeNames={
"#F": field_name,
"#U": "UpdatedAt",
},
UpdateExpression="SET #F = :f, #U = :u",
)
except ClientError as e:
if e.response["Error"]["Code"] != "ConditionalCheckFailedException":
raise e
return False
return True
def get(self, **kwargs) -> List[DatapumpConfig]:
filter_expr = And(
*[Attr(key).eq(value) for key, value in kwargs.items()])
response = self._client.scan(FilterExpression=filter_expr)
items = response["Items"]
return [DatapumpConfig(**item) for item in items]
def test_and(self):
cond1 = Equals(self.value, self.value2)
cond2 = Equals(self.value, self.value2)
and_cond = And(cond1, cond2)
self.build_and_assert_expression(
and_cond,
{'format': '({0} {operator} {1})',
'operator': 'AND', 'values': (cond1, cond2)})
def scanAllItems(self,
*,
attributes_returned_list=[],
filter_expression_list=[]):
if attributes_returned_list == [] and filter_expression_list == []:
return self._table.scan()['Items']
elif attributes_returned_list != [] and filter_expression_list == []:
if len(attributes_returned_list) == 1:
return self._table.scan(
ProjectionExpression=attributes_returned_list[0])['Items']
elif len(attributes_returned_list) > 1:
return self._table.scan(ProjectionExpression=','.join(
attributes_returned_list))['Items']
elif attributes_returned_list == [] and filter_expression_list != []:
if len(filter_expression_list) == 1:
return self._table.scan(
FilterExpression=filter_expression_list[0])['Items']
elif len(filter_expression_list) > 1:
return self._table.scan(FilterExpression=And(
*filter_expression_list))['Items']
elif attributes_returned_list != [] and filter_expression_list != []:
if len(attributes_returned_list) == 1 and len(
filter_expression_list) == 1:
return self._table.scan(
ProjectionExpression=attributes_returned_list[0],
FilterExpression=filter_expression_list[0])['Items']
elif len(attributes_returned_list
) == 1 and len(filter_expression_list) > 1:
return self._table.scan(
ProjectionExpression=attributes_returned_list[0],
FilterExpression=And(*filter_expression_list))['Items']
elif len(attributes_returned_list) > 1 and len(
filter_expression_list) == 1:
return self._table.scan(
ProjectionExpression=','.join(attributes_returned_list),
FilterExpression=filter_expression_list[0])['Items']
elif len(attributes_returned_list) > 1 and len(
filter_expression_list) > 1:
return self._table.scan(
ProjectionExpression=','.join(attributes_returned_list),
FilterExpression=And(*filter_expression_list))['Items']
def query(self, part_value, lat, long, precision=7, **kwargs):
hash = encode(lat, long, precision)
res = self.table.query(IndexName=self.index_name,
KeyConditionExpression=And(
Key(self.part_key).eq(part_value),
Key(self.range_key).begins_with(hash)),
**kwargs)
for item in res['Items']:
lats, longs = decode(item[self.range_key])
item['_lat'] = float(lats)
item['_long'] = float(longs)
return res
def is_user_have_pitch(user, oid_pitch_ignore=None):
if user:
if oid_pitch_ignore:
filter_ = And(
Attr('oid_user').eq(user.oid),
Attr('oid').ne(oid_pitch_ignore))
else:
filter_ = Attr('oid_user').eq(user.oid)
table = _get_table()
query = table.scan(FilterExpression=filter_)
items = query['Items']
return len(items) > 0
def scan_schedule_data(prefix, weekday_ja):
schedules_table_name = os.environ['SCHEDULES_TABLE_NAME']
try:
filter_exp = And(
Attr('weekday').eq(weekday_ja),
Or(Attr('prefix').eq(0),
Attr('prefix').eq(prefix)))
response = dynamodb.Table(schedules_table_name).scan(
FilterExpression=filter_exp)
return response['Items']
except ClientError as e:
print(e.response['Error']['Message'])
return []
def _get_run_params(self, run_id, param_name=None):
dynamodb = self._get_dynamodb_resource()
table_name = "_".join([self.table_prefix, DynamodbStore.PARAMS_TABLE])
table = dynamodb.Table(table_name)
condition = Key("run_id").eq(run_id)
if param_name:
condition = And(condition, Key("key").eq(param_name))
response = table.query(
ConsistentRead=True,
KeyConditionExpression=condition,
ReturnConsumedCapacity="TOTAL",
)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
if "Items" in response:
return response["Items"]
return []
def _find(username, **kwargs):
query_params = {
'IndexName': 'username-index',
'KeyConditionExpression': Key('username').eq(username),
}
if kwargs:
filter_condition_expressions = [
Attr(key).eq(value) for key, value in kwargs.items()
]
if len(filter_condition_expressions) > 1:
query_params['FilterExpression'] = And(
*filter_condition_expressions)
elif len(filter_condition_expressions) == 1:
query_params['FilterExpression'] = filter_condition_expressions[0]
data = contact_list.query(**query_params)
return data['Items']
def lambda_handler(event, context):
query_parameters = event["queryStringParameters"] #get the query
tags = []
[tags.append(value) for value in query_parameters.values()]
if len(tags) == 1:
filter_expression = Attr("tag").contains(tags[0])
else:
filter_expression = And(*[(Attr("tag").contains(value))
for value in tags])
dynamodb = boto3.resource("dynamodb")
table = dynamodb.Table("todos")
response = table.scan(FilterExpression=filter_expression)
result = {"links": []}
[result["links"].append(item["link"]) for item in response["Items"]]
return {
"statusCode": 200,
"body": json.dumps(result),
"isBase64Encoded": "false"
}
def parse_filters(self, filters, doc_class):
index_name = None
filter_expression_list = []
query_params = {}
for idx, filter in enumerate(filters):
prop_name, prop_value = filter.split(':')[3:5]
if idx == 0:
prop = doc_class()._base_properties[prop_name]
index_name = prop.kwargs.get(self.index_field_name, None) or \
self.default_index_name.format(prop_name)
query_params['KeyConditionExpression'] = Key(prop_name).eq(prop_value)
else:
filter_expression_list.append(Attr(prop_name).eq(prop_value))
if len(filter_expression_list) > 1:
query_params['FilterExpression'] = And(*filter_expression_list)
elif len(filter_expression_list) == 1:
query_params['FilterExpression'] = filter_expression_list[0]
if index_name != '_id':
query_params['IndexName'] = index_name
return query_params
def get_data(from_timestamp, to_timestamp, interval):
"""
Fetch weather data readings between the specified timestamps
"""
start_time = time.time()
resource = boto3.resource(
'dynamodb',
config=Config(region_name='eu-west-2'),
)
table = resource.Table(os.environ.get('TABLE_NAME'))
minutes = [i for i in range(60) if i % interval == 0]
query_params = {}
if interval != 1:
query_params['FilterExpression'] = Attr('minute').is_in(minutes)
inc = datetime.timedelta(days=1)
data = []
current_date = from_timestamp.date()
stop = to_timestamp.date()
while current_date <= stop:
response = table.query(
KeyConditionExpression=And(
Key('date').eq(str(current_date)),
Key('timestamp').between(
int(from_timestamp.timestamp()),
int(to_timestamp.timestamp()),
),
),
**query_params,
)
data.extend([utils.translate_item(i) for i in response['Items']])
current_date += inc
finish_time = time.time()
elapsed_time = finish_time - start_time
print(f'Time spent fetching data from DynamoDB: {elapsed_time} seconds')
return data
def _list_runs_ids(self,
experiment_id,
view_type=None,
max_results=SEARCH_MAX_RESULTS_DEFAULT):
dynamodb = self._get_dynamodb_resource()
table_name = "_".join([self.table_prefix, DynamodbStore.RUN_TABLE])
table = dynamodb.Table(table_name)
# Filter on active/deleted with optional experiment_id
condition = None
if self.use_gsi and view_type and view_type != ViewType.ALL:
if view_type == ViewType.ACTIVE_ONLY:
condition = Key("lifecycle_stage").eq(LifecycleStage.ACTIVE)
elif view_type == ViewType.DELETED_ONLY:
condition = Key("lifecycle_stage").eq(LifecycleStage.DELETED)
if experiment_id:
condition = And(condition,
Key("experiment_id").eq(experiment_id))
response = table.query(
IndexName="LifeCycleStage",
KeyConditionExpression=condition,
ProjectionExpression="run_id, experiment_id, lifecycle_stage",
ReturnConsumedCapacity="TOTAL",
)
elif experiment_id:
condition = Key("experiment_id").eq(experiment_id)
response = table.scan(
FilterExpression=condition,
ReturnConsumedCapacity="TOTAL",
Limit=max_results,
)
else:
response = table.scan(ReturnConsumedCapacity="TOTAL", )
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
if "Items" in response:
return _filter_run(response["Items"], view_type, experiment_id)
return []
def update_item(self, table_name, keys, update_values, conditions=None):
"""
:param table_name: Table to update
:param keys: Primary key of the item to update
This is a dict with format {key1: value1, key2: value2}
:param update_values: Attributes in the item to update. Attributes can be existing or new.
This is a dict where key is an attribute name and value is the value to assign
:param conditions: Values that must match in order to perform the update
e.g. condition={'a': 1} means that attribute 'a' must be equal to 1 to perform update
:return: Updated item, or None if item was not found
"""
expression_params = dict()
if len(update_values) > 0: # allow update even if no values are given
expression_attribute_values = {}
expression_terms = []
for attr, value in update_values.items():
expression_attribute_values[f':{attr}'] = value
expression_terms.append(f'{attr} = :{attr}')
expression_params['UpdateExpression'] = 'SET ' + ', '.join(
expression_terms)
expression_params[
'ExpressionAttributeValues'] = expression_attribute_values
if conditions:
condition_terms = [
Attr(key).eq(value) for key, value in conditions.items()
]
if len(condition_terms) > 1:
expression_params['ConditionExpression'] = And(
*condition_terms)
else:
expression_params['ConditionExpression'] = condition_terms[0]
try:
return self.get_table(table_name).update_item(
TableName=table_name,
Key=keys,
ReturnValues='ALL_NEW',
**expression_params).get('Attributes')
except self.dynamo_client.meta.client.exceptions.ConditionalCheckFailedException:
raise ConditionalUpdateItemError(table_name, keys, update_values)
def statistics(self, from_dt, to_dt, user_uid):
query_kwargs = {
# **projection_expr_expenseONLY_attrs,
"ProjectionExpression":
"currency,amount,timestamp_utc",
"Select":
"SPECIFIC_ATTRIBUTES",
"ConsistentRead":
False,
"ScanIndexForward":
False, # descending order
"KeyConditionExpression":
And(
Key("timestamp_utc").between(from_dt, to_dt),
Key('user_uid').eq(user_uid))
}
result = self.expenses_table.query(**query_kwargs)['Items']
if result and result[0][
'timestamp_utc'] == to_dt: # simulate exclusive lte (between() is inclusive on both sides)
result.pop(0)
return self._groupStatisticsItems(result)
def search(location, cuisine, dining_date, dining_time, num_people, phone):
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('yelp_restaurant')
filters = dict()
filters['cuisine'] = cuisine
filters['price'] = '$$'
response = table.scan(FilterExpression=And(
*[(Key(key).eq(value)) for key, value in filters.items()]))
print(len(response['Items']))
responseList = []
for i in range(0, 10):
name = response['Items'][i]['name']
address = response['Items'][i]['address']
address = ", ".join(address)
num_reviews = str(response['Items'][i]['review_count'])
rating = str(response['Items'][i]['rating'])
cuisine = cuisine
phone = str(response['Items'][i]['phone'])
responseList.append(
[name, phone, address, num_reviews, rating, cuisine])
return responseList
def test_and_operator(self):
cond1 = Equals(self.value, self.value2)
cond2 = Equals(self.value, self.value2)
self.assertEqual(cond1 & cond2, And(cond1, cond2))
def lambda_handler(event, context):
"""Sample pure Lambda function
Parameters
----------
event: dict, required
API Gateway Lambda Proxy Input Format
Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context: object, required
Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
API Gateway Lambda Proxy Output Format: dict
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
# try:
# ip = requests.get("http://checkip.amazonaws.com/")
# except requests.RequestException as e:
# # Send some context about this error to Lambda Logs
# print(e)
# raise e
print(event)
method=event['httpMethod']
print(f"method={method}")
print(f"table_name={table_name}")
myTriggerType='prev_day_change' # TODO: get from path
if method == "DELETE":
#path=event['path']
trigger_id=event['pathParameters']['trigger_id']
print(f"triggerId={trigger_id}")
try:
#see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.delete_item
response = table.update_item(
Key={'PK':f"TR#{myTriggerType}#{trigger_id}", "SK":f"TR#{myTriggerType}#{trigger_id}"},
UpdateExpression="SET expiresOn = :val1",
ExpressionAttributeValues={
':val1': calendar.timegm(time.gmtime()),
},
ConditionExpression=And(
And(Attr('PK').exists(),Attr('expiresOn').not_exists()),
Attr('triggerType').eq(myTriggerType)),
)
except ClientError as e:
print(f"clientError={e}")
if e.response['Error']['Code']=='ConditionalCheckFailedException':
return iftttError(404,"item not found")
raise
print(f"response={response}")
return {
"statusCode": 200,
"body":"",
}
elif method == "POST":
body=json.loads(event['body'])
trigger_id=body['trigger_identity']
limit = body.get('limit',50)
print(f"triggerId={trigger_id}")
###########
# for a PK TR#1 with events EV#1 EV#2 EV#N it will load:
# TR#1,TR#1
# TR#1,EV#N
# TR#1,EV#N-1
# ....
# TR#1,EV#N-(limit-1)
response = table.query(
KeyConditionExpression=Key("PK").eq(f"TR#{myTriggerType}#{trigger_id}")
#.__and__(Key("SK").begins_with(f"TR#{myTriggerType}#").__or__(Key("SK").begins_with(f"EV#")))
#parked for now but how do I filter for keys begining with X or y? (probably with a query filter?)
#TODO: filter query on SK, how do I do that?
,
ScanIndexForward=False, #the latest X events + trigger (trigger sorts after events)
Limit=limit + 1, #+1 for Trigger row
ProjectionExpression="SK, triggerEvent, expiresOn",
)
#no need to itterate as we do not expect to filter out anything
print(f"response={response}")
items = response["Items"]
if 0 == len(items) \
or (not items[0]['SK'].startswith("TR#") )\
or 'expiresOn' in items[0]:
#brand new
print(f"inserting {trigger_id}")
if 'triggerFields' not in body:
return iftttError(400, "triggerFields missing from request")
triggerFields=body['triggerFields']
#todo validate trigger fields
try:
response = table.put_item(
Item={
'PK':f"TR#{myTriggerType}#{trigger_id}",
"SK":f"TR#{myTriggerType}#{trigger_id}",
'triggerId': trigger_id,
#hacky string way to avoid having multiple columns
'triggerFields': json.dumps(triggerFields),
'triggerType': myTriggerType,
},
ConditionExpression=Or(
Attr('expiresOn').exists(),#previously deleted item # TODO: in this case we 'resurect' the old events. This should not happen
Attr('PK').not_exists() # brand new item
),
)
except ClientError as e:
print(f"clientError={e}")
if e.response['Error']['Code']=='ConditionalCheckFailedException':
return iftttError(404,"item not found") #
raise
print("response ",response)
triggered=[]
else:
events = items[1:]
print(f"found {events} ")
#hacky string way to avoid having multiple columns
#TODO: change this to use a Map? (will allow to add without overwrite)
#events = json.loads(item.get("triggerEvents","[]"))
triggered= []
now=calendar.timegm(time.gmtime())
for event in events:
if now< event.get('expiresOn',now+1):
triggered.append(json.loads(event['triggerEvent']))
return {
"statusCode": 200,
"body": json.dumps({
"data": triggered,
# "location": ip.text.replace("\n", "")
}),
}
else :
return iftttError(400, f"unexpected httpMethod {method}")
def test_and_operator(self):
cond1 = Equals(self.value, self.value2)
cond2 = Equals(self.value, self.value2)
assert cond1 & cond2 == And(cond1, cond2)
def lambda_handler(event, context):
"""Sample pure Lambda function
Parameters
----------
event: dict, required
API Gateway Lambda Proxy Input Format
Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context: object, required
Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
API Gateway Lambda Proxy Output Format: dict
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
# try:
# ip = requests.get("http://checkip.amazonaws.com/")
# except requests.RequestException as e:
# # Send some context about this error to Lambda Logs
# print(e)
# raise e
print(event)
method = event['httpMethod']
print(f"method={method}")
print(f"table_name={table_name}")
myTriggerType = 'instrument_price'
if method == "DELETE":
#path=event['path']
trigger_id = event['pathParameters']['trigger_id']
print(f"triggerId={trigger_id}")
try:
#see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.delete_item
response = table.delete_item(
Key={
'PK': f"TR#{myTriggerType}#{trigger_id}",
"SK": f"TR#{myTriggerType}#{trigger_id}"
},
ConditionExpression=And(
Attr('PK').eq(Attr('SK')),
Attr('triggerType').eq(myTriggerType)),
)
except ClientError as e:
print(f"clientError={e}")
if e.response['Error'][
'Code'] == 'ConditionalCheckFailedException':
return iftttError(404, "item not found")
raise
print(f"response={response}")
return {
"statusCode": 200,
"body": "",
}
elif method == "POST":
body = json.loads(event['body'])
trigger_id = body['trigger_identity']
print(f"triggerId={trigger_id}")
response = table.get_item(
Key={
'PK': f"TR#{myTriggerType}#{trigger_id}",
"SK": f"TR#{myTriggerType}#{trigger_id}"
},
ProjectionExpression="triggerEvents, triggerType",
)
print(f"response={response}")
if "Item" not in response:
#brand new
print(f"inserting {trigger_id}")
if 'triggerFields' not in body:
return iftttError(400, "triggerFields missing from request")
triggerFields = body['triggerFields']
#todo validate trigger fields
try:
response = table.put_item(
Item={
'PK': f"TR#{myTriggerType}#{trigger_id}",
"SK": f"TR#{myTriggerType}#{trigger_id}",
'triggerId': trigger_id,
#hacky string way to avoid having multiple columns
'triggerFields': json.dumps(triggerFields),
'triggerType': myTriggerType,
},
ConditionExpression=Or(
Attr('triggerType').eq(myTriggerType),
Attr('triggerType').not_exists()))
except ClientError as e:
print(f"clientError={e}")
#somehow got created with someone elses triggerType
if e.response['Error'][
'Code'] == 'ConditionalCheckFailedException':
return iftttError(404, "item not found")
raise
print("response ", response)
triggered = []
elif response['Item'].get("triggerType",
myTriggerType) != myTriggerType:
#it exists but it is someone elses
return iftttError(404, "item not found")
else:
item = response['Item']
print(f"found {item} ")
#hacky string way to avoid having multiple columns
#TODO: change this to use a Map? (will allow to add without overwrite)
events = json.loads(item.get("triggerEvents", "[]"))
triggered = []
for event in events:
#TODO: implement limit (not needed now becasue I expect only up to one events)
triggered.append(event['data'])
return {
"statusCode":
200,
"body":
json.dumps({
"data": triggered,
# "location": ip.text.replace("\n", "")
}),
}
else:
return iftttError(400, f"unexpected httpMethod {method}")
def make_query(self,
table_name,
key_conditions,
filters=None,
index_name=None,
select=None,
limit=None,
consistent_read: bool = False,
exclusive_start_key=None):
"""
Make a query and get results one page at a time. This method handles the pagination logic so the caller can
process each page at a time without having to re-query
:param table_name: DynamoDB table to query
:param key_conditions: Primary key conditions to query for
This is a dict with format {key1: value1, key2: value2} where the resulting query will be
'key1 = value1 AND key2 = value2'
:param filters: Optional conditions on non-primary key attributes
Follow the same format as key conditions
:param index_name: Name of secondary index to use; Use None to use primary index
:param select: The list of selected attributes
:param limit: Maximum number of items per query (used for testing pagination)
:param consistent_read: Flag for consistent read
:param exclusive_start_key: DynamoDB exclusive start key
:yield: results of a single query call
"""
query_params = dict(ConsistentRead=consistent_read)
key_expression = [
Key(attr).eq(value) for attr, value in key_conditions.items()
]
if len(key_conditions) == 0:
raise ValueError('At least one key condition must be given')
elif len(key_conditions) > 1:
query_params['KeyConditionExpression'] = And(*key_expression)
else:
query_params['KeyConditionExpression'] = key_expression[0]
if filters is not None:
filter_expression = [
Attr(attr).eq(value) for attr, value in filters.items()
]
if len(filter_expression) > 1:
query_params['FilterExpression'] = And(*filter_expression)
else:
query_params['FilterExpression'] = filter_expression[0]
if index_name is not None:
query_params['IndexName'] = index_name
if select:
query_params['Select'] = 'SPECIFIC_ATTRIBUTES'
query_params['ProjectionExpression'] = ', '.join(select)
if limit is not None:
query_params['Limit'] = limit
if index_name is not None:
query_params['IndexName'] = index_name
if exclusive_start_key is not None:
query_params['ExclusiveStartKey'] = exclusive_start_key
while True:
query_result = self.get_table(table_name).query(**query_params)
last_evaluated_key = query_result.get('LastEvaluatedKey')
yield Page(query_result.get('Items'), query_result.get('Count'),
last_evaluated_key)
if last_evaluated_key is None:
break
query_params['ExclusiveStartKey'] = last_evaluated_key
def _list_experiments(self, view_type=None, name=None):
dynamodb = self._get_dynamodb_resource()
table_name = "_".join(
[self.table_prefix, DynamodbStore.EXPERIMENT_TABLE])
table = dynamodb.Table(table_name)
# Filter on active/deleted with optional name
condition = None
if self.use_gsi and view_type and view_type != ViewType.ALL:
if view_type == ViewType.ACTIVE_ONLY:
condition = Key("lifecycle_stage").eq(LifecycleStage.ACTIVE)
elif view_type == ViewType.DELETED_ONLY:
condition = Key("lifecycle_stage").eq(LifecycleStage.DELETED)
if name:
condition = And(condition, Key("name").eq(name))
response = table.query(
IndexName="LifeCycleStage",
KeyConditionExpression=condition,
ReturnConsumedCapacity="TOTAL",
)
elif name:
condition = Key("name").eq(name)
response = table.scan(
FilterExpression=condition,
ReturnConsumedCapacity="TOTAL",
)
else:
response = table.scan(ReturnConsumedCapacity="TOTAL", )
items = []
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
if "Items" in response:
items += _filter_experiment(response["Items"], view_type, name)
# Keep fetching results if there are more than the limit
while "LastEvaluatedKey" in response:
print("more", response["LastEvaluatedKey"])
if self.use_gsi and view_type and view_type != ViewType.ALL:
response = table.query(
IndexName="LifeCycleStage",
KeyConditionExpression=condition,
ReturnConsumedCapacity="TOTAL",
ExclusiveStartKey=response["LastEvaluatedKey"],
)
elif name:
response = table.scan(
FilterExpression=condition,
ReturnConsumedCapacity="TOTAL",
ExclusiveStartKey=response["LastEvaluatedKey"],
)
else:
response = table.scan(
ReturnConsumedCapacity="TOTAL",
ExclusiveStartKey=response["LastEvaluatedKey"],
)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
if "Items" in response:
items += _filter_experiment(response["Items"], view_type, name)
return items
def get_list(self,
property_value,
user_uid,
property_name='timestamp_utc',
ordering_direction: OrderingDirection = DEFAULT_ORDERING,
batch_size=25,
inclusive_start=False):
"""
:param property_value: the value of the property `property_name`. could be None
:param property_name: which expense property to use as sort key
:param user_uid:
:param ordering_direction: OrderingDirection instance
:param batch_size: int
:param inclusive_start - only valid if property_value is set. if true, `property_value` will be included in the results
:return: list of expense objects
:raises UnindexedPropertySelected if `property_name` is not a property that can be used to query
"""
# validation
self.validate_get_list(property_name, ordering_direction, batch_size)
batch_size = min(batch_size, MAX_BATCH_SIZE)
property_is_main_sort_key = index_for_property[property_name] is None
# configure the query
query_kwargs = {
**projection_expr_expenseONLY_attrs,
"Select":
"SPECIFIC_ATTRIBUTES",
"Limit":
batch_size,
"ConsistentRead":
False,
"ScanIndexForward":
True if ordering_direction is OrderingDirection.asc else False,
}
# choose which index to query, if applicable
if property_name in index_for_property.keys():
if not property_is_main_sort_key:
query_kwargs['IndexName']: index_for_property[property_name]
query_kwargs['TableName'] = self.EXPENSES_TABLE_NAME
# configure the hash and sort keys
if not property_value:
# e.g. if querying via timestamp_utc, desc - search will start from the newest items
query_kwargs['KeyConditionExpression'] = Key(
self.HASH_KEY).eq(user_uid)
else:
greater = Key(property_name).gte if inclusive_start else Key(
property_name).gt
less = Key(property_name).lte if inclusive_start else Key(
property_name).lt
# e.g. if querying via timestamp_utc, desc - search will start(exclusive) from the given property_value
sort_key_cond = greater if ordering_direction is OrderingDirection.asc else less
query_kwargs['KeyConditionExpression'] = And(
Key(self.HASH_KEY).eq(user_uid), sort_key_cond(property_value))
result = self.expenses_table.query(**query_kwargs)['Items']
return [self.converter.convertFromDbFormat(e) for e in result]
def lambda_handler(event, context):
try:
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(os.environ['TABLE_NAME'])
except Exception as e:
return {
"statusCode":
500,
"body":
json.dumps({
"msg":
f"Failed to connect to dynamodb, table {os.environ['TABLE_NAME']}",
"error": str(e)
})
}
if event["httpMethod"] == "POST":
event_id = event["requestContext"].get("requestId", uuid.uuid4())
ts = event["requestContext"].get(
"requestTimeEpoch", int(datetime.now().timestamp() * 1000))
try:
table.put_item(Item={
"id": event_id,
"timestamp": ts,
"event": event["body"]
})
return {
"statusCode":
200,
"body":
json.dumps({
"msg": "Event added",
"event": {
"id": event_id,
"timestamp": ts,
"body": event["body"]
},
})
}
except Exception as e:
return {
"statusCode":
500,
"body":
json.dumps({
"msg":
f"Failed to insert event into table {os.environ['TABLE_NAME']}",
"error": str(e)
})
}
elif event["httpMethod"] == "GET":
query = event.get("queryStringParameters", None)
if query == "None" or query is None:
return {
"statusCode":
500,
"body":
json.dumps({
"msg": "Empty query string, search aborted",
"event": event
})
}
try:
# BOOOOM mindblown
if len(query) == 1:
FilterExpression = Attr(f'event.{list(query)[0]}').eq(
list(query.values())[0])
else:
FilterExpression = And(*[(Attr(f'event.{key}').eq(value))
for key, value in query.items()])
res = table.scan(FilterExpression=FilterExpression)
return {
"statusCode": 200,
"body": json.dumps({
"msg": "Event found",
"events": res['Items']
})
}
except Exception as e:
return {
"statusCode":
500,
"body":
json.dumps({
"msg": f"Failed to scan table {os.environ['TABLE_NAME']}",
"query": query,
"error": str(e)
})
}
else:
return {
"statusCode": 500,
"body": json.dumps({
"msg": "Method Unsupported. Use GET or POST",
}),
}