def test_reset(self):
a = Attr("myattr")
self.assert_condition_expression_build(a.eq("foo"), "#n0 = :v0", {"#n0": "myattr"}, {":v0": "foo"})
self.assert_condition_expression_build(a.eq("foo"), "#n1 = :v1", {"#n1": "myattr"}, {":v1": "foo"})
self.builder.reset()
self.assert_condition_expression_build(a.eq("foo"), "#n0 = :v0", {"#n0": "myattr"}, {":v0": "foo"})
def test_build_or(self):
a = Attr("myattr")
a2 = Attr("myattr2")
self.assert_condition_expression_build(
a.eq("foo") | a2.eq("bar"),
"(#n0 = :v0 OR #n1 = :v1)",
{"#n0": "myattr", "#n1": "myattr2"},
{":v0": "foo", ":v1": "bar"},
)
def test_build_double_nested_and_or(self):
a = Attr('myattr')
a2 = Attr('myattr2')
self.assert_condition_expression_build(
(a.eq('foo') & a2.eq('foo2')) | (a.eq('bar') & a2.eq('bar2')),
'((#n0 = :v0 AND #n1 = :v1) OR (#n2 = :v2 AND #n3 = :v3))',
{'#n0': 'myattr', '#n1': 'myattr2', '#n2': 'myattr',
'#n3': 'myattr2'},
{':v0': 'foo', ':v1': 'foo2', ':v2': 'bar', ':v3': 'bar2'})
def test_build_double_nested_and_or(self):
a = Attr('myattr')
a2 = Attr('myattr2')
self.assert_condition_expression_build(
(a.eq('foo') & a2.eq('foo2')) | (a.eq('bar') & a2.eq('bar2')),
'((#n0 = :v0 AND #n1 = :v1) OR (#n2 = :v2 AND #n3 = :v3))',
{'#n0': 'myattr', '#n1': 'myattr2', '#n2': 'myattr',
'#n3': 'myattr2'},
{':v0': 'foo', ':v1': 'foo2', ':v2': 'bar', ':v3': 'bar2'})
def test_build_double_nested_and_or(self):
a = Attr("myattr")
a2 = Attr("myattr2")
self.assert_condition_expression_build(
(a.eq("foo") & a2.eq("foo2")) | (a.eq("bar") & a2.eq("bar2")),
"((#n0 = :v0 AND #n1 = :v1) OR (#n2 = :v2 AND #n3 = :v3))",
{"#n0": "myattr", "#n1": "myattr2", "#n2": "myattr", "#n3": "myattr2"},
{":v0": "foo", ":v1": "foo2", ":v2": "bar", ":v3": "bar2"},
)
def test_reset(self):
a = Attr('myattr')
self.assert_condition_expression_build(
a.eq('foo'), '#n0 = :v0', {'#n0': 'myattr'}, {':v0': 'foo'})
self.assert_condition_expression_build(
a.eq('foo'), '#n1 = :v1', {'#n1': 'myattr'}, {':v1': 'foo'})
self.builder.reset()
self.assert_condition_expression_build(
a.eq('foo'), '#n0 = :v0', {'#n0': 'myattr'}, {':v0': 'foo'})
def test_reset(self):
a = Attr('myattr')
self.assert_condition_expression_build(
a.eq('foo'), '#n0 = :v0', {'#n0': 'myattr'}, {':v0': 'foo'})
self.assert_condition_expression_build(
a.eq('foo'), '#n1 = :v1', {'#n1': 'myattr'}, {':v1': 'foo'})
self.builder.reset()
self.assert_condition_expression_build(
a.eq('foo'), '#n0 = :v0', {'#n0': 'myattr'}, {':v0': 'foo'})
def test_build_or(self):
a = Attr('myattr')
a2 = Attr('myattr2')
self.assert_condition_expression_build(
a.eq('foo') | a2.eq('bar'), '(#n0 = :v0 OR #n1 = :v1)', {
'#n0': 'myattr',
'#n1': 'myattr2'
}, {
':v0': 'foo',
':v1': 'bar'
})
def __publish_pre_persist_records_to_cloudwatch(table_name, batch_id,
is_historical):
"""
:param table_name: Not used
:param batch_id:
:param is_historical:
:return:
"""
try:
dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
table = dynamodb.Table(
os.environ["CURATION_MANIFEST_TABLE"]
) # dev-CurationManifestFilesTable, dev-BatchId-TableName-index
filter_expression = Attr('IsHistorical')
filter_expression = filter_expression.eq(is_historical)
response = table.query(
IndexName=os.environ["CURATION_MANIFEST_TABLE_BATCH_INDX"],
KeyConditionExpression=Key('BatchId').eq(batch_id),
FilterExpression=filter_expression)
for item in response['Items']:
total_curated_records_by_state = item["TotalCuratedRecordsByState"]
__publish_pre_persist_custom_metrics_to_cloudwatch(
item["TableName"], total_curated_records_by_state)
except Exception as e:
LoggerUtility.log_info("Failed to persist get status for batch "
" - {} with exception - {}".format(batch_id, e))
raise
def test_build_attribute_with_attr_value(self):
a = Attr('myattr')
value = Attr('myreference')
self.assert_condition_expression_build(a.eq(value), '#n0 = #n1', {
'#n0': 'myattr',
'#n1': 'myreference'
}, {})
def test_build_nested_attr_map_list(self):
a = Attr('MyMap.MyList[2].MyElement')
self.assert_condition_expression_build(a.eq('foo'),
'#n0.#n1[2].#n2 = :v0', {
'#n0': 'MyMap',
'#n1': 'MyList',
'#n2': 'MyElement'
}, {':v0': 'foo'})
def test_build_attr_map(self):
a = Attr('MyMap.MyKey')
self.assert_condition_expression_build(
a.eq('foo'),
'#n0.#n1 = :v0',
{
'#n0': 'MyMap',
'#n1': 'MyKey'
},
{':v0': 'foo'},
)
def generate_filter_expression(filter_key: str, filter_cond: str,
filter_value: str) -> dict:
filter_attr = Attr(filter_key)
if filter_cond == "eq":
return {"FilterExpression": filter_attr.eq(filter_value)}
elif filter_cond == "ne":
return {"FilterExpression": filter_attr.eq(filter_value)}
elif filter_cond == "gt":
return {"FilterExpression": filter_attr.gt(filter_value)}
elif filter_cond == "ge":
return {"FilterExpression": filter_attr.gte(filter_value)}
elif filter_cond == "lt":
return {"FilterExpression": filter_attr.lt(filter_value)}
elif filter_cond == "le":
return {"FilterExpression": filter_attr.lte(filter_value)}
elif filter_cond == "begins_with":
return {"FilterExpression": filter_attr.begins_with(filter_value)}
elif filter_cond == "between":
return {"FilterExpression": filter_attr.between(*[filter_value])}
elif filter_cond == "contains":
return {"FilterExpression": filter_attr.contains(filter_value)}
else:
raise AttributeError("filter condition missing")
def generateAttr(filter_val):
if len(filter_val) > 1:
to_search = ''
for i in range(2, len(filter_val)):
if i == 2:
to_search += filter_val[i]
else:
to_search += ' ' + filter_val[i]
attr_filter = Attr(filter_val[0])
if filter_val[1] == 'eq':
attr_filter = attr_filter.eq(to_search)
elif filter_val[1] == 'contains':
attr_filter = attr_filter.contains(to_search)
elif filter_val[1] == 'lt':
attr_filter = attr_filter.lt(int(filter_val[2]))
elif filter_val[1] == 'gt':
attr_filter = attr_filter.gt(int(filter_val[2]))
return attr_filter
else:
return -1
def _get_targets(self, parsed_targets, db):
overall_cond = None
for i, (key, (invert, type_, value)) in enumerate(parsed_targets.items()):
cond = None
attr = Attr(key)
if type_ == "=":
cond = attr.eq(value)
elif type_ == "<":
cond = attr.lt(value)
elif type_ == ">":
cond = attr.lte(value)
elif type_ == "{":
cond = attr.is_in(value)
elif type_ == "}":
cond = attr.contains(value)
if invert:
cond = ~cond
if overall_cond is None:
overall_cond = cond
else:
overall_cond &= cond
# Return async generator
return db.scan_users(overall_cond, "via_instance, user_id")
def test_build_with_is_key_condition_throws_error(self):
a = Attr('myattr')
with self.assertRaises(DynamoDBNeedsKeyConditionError):
self.builder.build_expression(a.eq('foo'), is_key_condition=True)
def test_build_attribute_with_attr_value(self):
a = Attr("myattr")
value = Attr("myreference")
self.assert_condition_expression_build(a.eq(value), "#n0 = #n1", {"#n0": "myattr", "#n1": "myreference"}, {})
def test_build_not(self):
a = Attr('myattr')
self.assert_condition_expression_build(~a.eq('foo'), '(NOT #n0 = :v0)',
{'#n0': 'myattr'},
{':v0': 'foo'})
def test_build_attr_map(self):
a = Attr("MyMap.MyKey")
self.assert_condition_expression_build(
a.eq("foo"), "#n0.#n1 = :v0", {"#n0": "MyMap", "#n1": "MyKey"}, {":v0": "foo"}
)
def test_build_nested_attr_map_list(self):
a = Attr('MyMap.MyList[2].MyElement')
self.assert_condition_expression_build(
a.eq('foo'), '#n0.#n1[2].#n2 = :v0',
{'#n0': 'MyMap', '#n1': 'MyList', '#n2': 'MyElement'},
{':v0': 'foo'})
def test_build_nested_attr_map_list(self):
a = Attr("MyMap.MyList[2].MyElement")
self.assert_condition_expression_build(
a.eq("foo"), "#n0.#n1[2].#n2 = :v0", {"#n0": "MyMap", "#n1": "MyList", "#n2": "MyElement"}, {":v0": "foo"}
)
def test_build_attr_map(self):
a = Attr('MyMap.MyKey')
self.assert_condition_expression_build(
a.eq('foo'), '#n0.#n1 = :v0', {'#n0': 'MyMap', '#n1': 'MyKey'},
{':v0': 'foo'})
def test_build_not(self):
a = Attr("myattr")
self.assert_condition_expression_build(~a.eq("foo"), "(NOT #n0 = :v0)", {"#n0": "myattr"}, {":v0": "foo"})
def test_build_attribute_with_attr_value(self):
a = Attr('myattr')
value = Attr('myreference')
self.assert_condition_expression_build(
a.eq(value), '#n0 = #n1',
{'#n0': 'myattr', '#n1': 'myreference'}, {})
def test_build_not(self):
a = Attr('myattr')
self.assert_condition_expression_build(
~a.eq('foo'), '(NOT #n0 = :v0)',
{'#n0': 'myattr'}, {':v0': 'foo'})
def test_build_or(self):
a = Attr('myattr')
a2 = Attr('myattr2')
self.assert_condition_expression_build(
a.eq('foo') | a2.eq('bar'), '(#n0 = :v0 OR #n1 = :v1)',
{'#n0': 'myattr', '#n1': 'myattr2'}, {':v0': 'foo', ':v1': 'bar'})
def doquery(self, queryspec):
db = boto3.resource('dynamodb')
table = db.Table(queryspec.DataType)
data = None
if len(queryspec.Filters) == 0:
response = table.scan()
data = response['Items']
while 'LastEvaluatedKey' in response:
response = table.scan(
ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(response['Items'])
else:
for prop in queryspec.Filters:
if data is None and queryspec.Filters[prop][2]: # queryable
key = Key(prop)
if queryspec.Filters[prop][0] == '=':
expr = key.eq(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '<':
expr = key.lt(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '>':
expr = key.gt(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '<=':
expr = key.lte(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '>=':
expr = key.gte(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == 'btw':
expr = key.between(queryspec.Filters[prop][1][0],
queryspec.Filters[prop][1][0])
response = table.query(KeyConditionExpression=expr)
data = response['Items']
else: # queryable = False
if data is None:
attr = Attr(prop)
if queryspec.Filters[prop][0] == '=':
expr = attr.eq(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '<':
expr = attr.lt(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '>':
expr = attr.gt(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '<=':
expr = attr.lte(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == '>=':
expr = attr.gte(queryspec.Filters[prop][1])
elif queryspec.Filters[prop][0] == 'btw':
expr = attr.between(queryspec.Filters[prop][1][0],
queryspec.Filters[prop][1][0])
response = table.scan(FilterExpression=expr)
data = response['Items']
while 'LastEvaluatedKey' in response:
response = table.scan(
ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(response['Items'])
else:
pass # TODO: Secondary value
if data is None or len(data) == 0 or len(queryspec.Sorts) == 0:
return data
sortproperty = next(iter(queryspec.Sorts))
sorteddata = sorted(data, key=lambda item: item[sortproperty])
return sorteddata
def test_build_attr_list(self):
a = Attr('MyList[0]')
self.assert_condition_expression_build(a.eq('foo'), '#n0[0] = :v0',
{'#n0': 'MyList'},
{':v0': 'foo'})
def test_build_attr_list(self):
a = Attr('MyList[0]')
self.assert_condition_expression_build(
a.eq('foo'), '#n0[0] = :v0', {'#n0': 'MyList'}, {':v0': 'foo'})
def test_build_with_is_key_condition_throws_error(self):
a = Attr('myattr')
with self.assertRaises(DynamoDBNeedsKeyConditionError):
self.builder.build_expression(a.eq('foo'), is_key_condition=True)
def test_build_attr_list(self):
a = Attr("MyList[0]")
self.assert_condition_expression_build(a.eq("foo"), "#n0[0] = :v0", {"#n0": "MyList"}, {":v0": "foo"})
