def test_short_circuiting_or_complete(self):
# tests whether right-hand side is bypassed completely with or
tup_val_exp_l = TupleValueExpression(col_name=0)
tup_val_exp_r = TupleValueExpression(col_name=1)
comp_exp_l = ComparisonExpression(
ExpressionType.COMPARE_EQUAL,
tup_val_exp_l,
tup_val_exp_r
)
comp_exp_r = Mock(spec=ComparisonExpression)
logical_exp = LogicalExpression(
ExpressionType.LOGICAL_OR,
comp_exp_l,
comp_exp_r
)
tuples = Batch(pd.DataFrame(
{0: [1, 2, 3], 1: [1, 2, 3]}))
self.assertEqual(
[True, True, True],
logical_exp.evaluate(tuples).frames[0].tolist()
)
comp_exp_r.evaluate.assert_not_called()
def test_short_circuiting_or_partial(self):
# tests whether right-hand side is partially executed with or
tup_val_exp_l = TupleValueExpression(col_name=0)
tup_val_exp_r = TupleValueExpression(col_name=1)
comp_exp_l = ComparisonExpression(
ExpressionType.COMPARE_EQUAL,
tup_val_exp_l,
tup_val_exp_r
)
comp_exp_r = Mock(spec=ComparisonExpression)
comp_exp_r.evaluate = Mock(return_value=Mock(frames=[[True], [False]]))
logical_exp = LogicalExpression(
ExpressionType.LOGICAL_OR,
comp_exp_l,
comp_exp_r
)
tuples = Batch(pd.DataFrame(
{0: [1, 2, 3, 4], 1: [5, 6, 3, 4]}))
self.assertEqual(
[True, False, True, True],
logical_exp.evaluate(tuples).frames[0].tolist()
)
comp_exp_r.evaluate.assert_called_once_with(tuples, mask=[0, 1])
def test_logical_or(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_left = ComparisonExpression(
ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp)
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER, tpl_exp, const_exp)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR,
comparison_expression_left,
comparison_expression_right)
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
],
info=None)
expected_value = [[True], [True], [True]]
output_value = logical_expr.evaluate(input_batch)
self.assertEqual(expected_value, output_value)
def test_should_visit_select_if_nested_query(self, mock_p, mock_c, mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT id FROM (SELECT data, id FROM video \
WHERE data > 2) WHERE id>3;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
plans = [LogicalProject([TupleValueExpression('id')])]
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('id'),
ConstantValueExpression(3))))
plans.append(LogicalQueryDerivedGet())
plans.append(
LogicalProject(
[TupleValueExpression('data'),
TupleValueExpression('id')]))
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('data'),
ConstantValueExpression(2))))
plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expected_plan = None
for plan in reversed(plans):
if expected_plan:
plan.append_child(expected_plan)
expected_plan = plan
self.assertEqual(expected_plan, actual_plan)
wrong_plan = plans[0]
for plan in plans[1:]:
wrong_plan.append_child(plan)
self.assertNotEqual(wrong_plan, actual_plan)
def test_should_return_sorted_frames(self):
"""
data (3 batches):
'A' 'B' 'C'
[1, 1, 1]
----------
[1, 5, 6]
[4, 7, 10]
----------
[2, 9, 7]
[4, 1, 2]
[4, 2, 4]
"""
df1 = pd.DataFrame(np.array([[1, 1, 1]]), columns=['A', 'B', 'C'])
df2 = pd.DataFrame(np.array([[1, 5, 6], [4, 7, 10]]),
columns=['A', 'B', 'C'])
df3 = pd.DataFrame(np.array([[2, 9, 7], [4, 1, 2], [4, 2, 4]]),
columns=['A', 'B', 'C'])
batches = [Batch(frames=df) for df in [df1, df2, df3]]
"query: .... ORDER BY A ASC, B DESC "
plan = OrderByPlan([
(TupleValueExpression('A'), ParserOrderBySortType.ASC),
(TupleValueExpression('B'), ParserOrderBySortType.DESC)
])
orderby_executor = OrderByExecutor(plan)
orderby_executor.append_child(DummyExecutor(batches))
sorted_batches = list(orderby_executor.exec())
"""
A B C
0 1 5 6
1 1 1 1
2 2 9 7
3 4 7 10
4 4 2 4
5 4 1 2
"""
expected_df1 = pd.DataFrame(np.array([[1, 5, 6]]),
columns=['A', 'B', 'C'])
expected_df2 = pd.DataFrame(np.array([[1, 1, 1], [2, 9, 7]]),
columns=['A', 'B', 'C'])
expected_df3 = pd.DataFrame(np.array([[4, 7, 10], [4, 2, 4], [4, 1,
2]]),
columns=['A', 'B', 'C'])
expected_batches = [
Batch(frames=df)
for df in [expected_df1, expected_df2, expected_df3]
]
self.assertEqual(expected_batches[0], sorted_batches[0])
self.assertEqual(expected_batches[1], sorted_batches[1])
self.assertEqual(expected_batches[2], sorted_batches[2])
def test_should_return_top_frames_after_sorting(self):
"""
Checks if limit returns the top 2 rows from the data
after sorting
data (3 batches):
'A' 'B' 'C'
[1, 1, 1]
----------
[1, 5, 6]
[4, 7, 10]
----------
[2, 9, 7]
[4, 1, 2]
[4, 2, 4]
"""
df1 = pd.DataFrame(
np.array([[1, 1, 1]]), columns=['A', 'B', 'C'])
df2 = pd.DataFrame(
np.array([[1, 5, 6], [4, 7, 10]]), columns=['A', 'B', 'C'])
df3 = pd.DataFrame(
np.array([[2, 9, 7], [4, 1, 2],
[4, 2, 4]]), columns=['A', 'B', 'C'])
batches = [Batch(frames=df) for df in [df1, df2, df3]]
"query: .... ORDER BY A ASC, B DESC limit 2"
plan = OrderByPlan(
[(TupleValueExpression('A'), ParserOrderBySortType.ASC),
(TupleValueExpression('B'), ParserOrderBySortType.DESC)])
orderby_executor = OrderByExecutor(plan)
orderby_executor.append_child(DummyExecutor(batches))
sorted_batches = list(orderby_executor.exec())
limit_value = 2
plan = LimitPlan(ConstantValueExpression(limit_value))
limit_executor = LimitExecutor(plan)
limit_executor.append_child(DummyExecutor(sorted_batches))
reduced_batches = list(limit_executor.exec())
# merge everything into one batch
aggregated_batch = Batch.concat(reduced_batches, copy=False)
"""
A B C
0 1 5 6
1 1 1 1
"""
expected_df1 = pd.DataFrame(
np.array([[1, 5, 6], [1, 1, 1]]), columns=['A', 'B', 'C'])
expected_batches = [Batch(frames=df) for df in [expected_df1]]
self.assertEqual(expected_batches[0], aggregated_batch)
def test_should_return_correct_plan_tree_for_orderby_logical_tree(self):
# SELECT data, id FROM video ORDER BY data, id DESC;
logical_plan = LogicalOrderBy([
(TupleValueExpression('data'), ParserOrderBySortType.ASC),
(TupleValueExpression('id'), ParserOrderBySortType.DESC)
])
plan = ScanGenerator().build(logical_plan)
self.assertTrue(isinstance(plan, OrderByPlan))
self.assertEqual(TupleValueExpression('data'), plan.orderby_list[0][0])
self.assertEqual(ParserOrderBySortType.ASC, plan.orderby_list[0][1])
self.assertEqual(TupleValueExpression('id'), plan.orderby_list[1][0])
self.assertEqual(ParserOrderBySortType.DESC, plan.orderby_list[1][1])
def test_comparison_compare_greater(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_left = ComparisonExpression(
ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp)
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER, tpl_exp, const_exp)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR,
comparison_expression_left,
comparison_expression_right)
tuple1 = [[1], 2, 3]
self.assertEqual([True], logical_expr.evaluate(tuple1, None))
def bind_tuple_value_expr(expr: TupleValueExpression, column_mapping):
if not column_mapping:
# TODO: Remove this and bring uniform interface throughout the system.
_old_bind_tuple_value_expr(expr)
return
expr.col_object = column_mapping.get(expr.col_name.lower(), None)
def visitFullColumnName(self, ctx: evaql_parser.FullColumnNameContext):
# dotted id not supported yet
column_name = self.visit(ctx.uid())
if column_name is not None:
return TupleValueExpression(col_name=column_name)
else:
warnings.warn("Column Name Missing", SyntaxWarning)
def test_if_expr_tree_is_equal(self):
const_exp1 = ConstantValueExpression(0)
const_exp2 = ConstantValueExpression(0)
columnName1 = TupleValueExpression(col_name='DATA')
columnName2 = TupleValueExpression(col_name='DATA')
aggr_expr1 = AggregationExpression(ExpressionType.AGGREGATION_AVG,
None, columnName1)
aggr_expr2 = AggregationExpression(ExpressionType.AGGREGATION_AVG,
None, columnName2)
cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_NEQ,
aggr_expr1, const_exp1)
cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_NEQ,
aggr_expr2, const_exp2)
self.assertEqual(cmpr_exp1, cmpr_exp2)
def test_comparison_compare_lesser(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(2)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LESSER, tpl_exp,
const_exp)
tuple1 = [1, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def test_comparison_compare_greater(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_GREATER,
tpl_exp, const_exp)
tuple1 = [2, 1, 1]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def visitUidList(self, ctx: evaql_parser.UidListContext):
uid_list = []
uid_list_length = len(ctx.uid())
for uid_index in range(uid_list_length):
uid = self.visit(ctx.uid(uid_index))
uid_expr = TupleValueExpression(uid)
uid_list.append(uid_expr)
return uid_list
def visitFullColumnName(self, ctx: evaql_parser.FullColumnNameContext):
# Adding support for a.b
# Will restrict implementation to raise error for a.b.c
dottedIds = []
if ctx.dottedId():
if len(ctx.dottedId()) != 1:
LoggingManager().log("Only tablename.colname syntax supported",
LoggingLevel.ERROR)
return
for id in ctx.dottedId():
dottedIds.append(self.visit(id))
uid = self.visit(ctx.uid())
if len(dottedIds):
return TupleValueExpression(table_name=uid, col_name=dottedIds[0])
else:
return TupleValueExpression(col_name=uid)
def test_aggregation_max(self):
columnName = TupleValueExpression(col_name=0)
aggr_expr = AggregationExpression(
ExpressionType.AGGREGATION_MAX,
None,
columnName
)
tuples = Batch(pd.DataFrame({0: [1, 2, 3], 1: [2, 3, 4], 2: [3, 4, 5]}))
self.assertEqual(3, aggr_expr.evaluate(tuples, None))
def test_divide(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_DIVIDE,
tpl_exp, const_exp)
tuple1 = [5, 2, 3]
# 5/5 = 1
self.assertEqual(1, cmpr_exp.evaluate(tuple1, None))
def test_multiply(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_MULTIPLY,
tpl_exp, const_exp)
tuple1 = [5, 2, 3]
# 5*5 = 25
self.assertEqual(25, cmpr_exp.evaluate(tuple1, None))
def test_addition(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_ADD, tpl_exp,
const_exp)
tuple1 = [5, 2, 3]
# 5+5 = 10
self.assertEqual(10, cmpr_exp.evaluate(tuple1, None))
def test_insert_statement(self):
parser = Parser()
insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path)
VALUES (1, '/mnt/frames/1.png');
"""
expected_stmt = InsertTableStatement(TableRef(TableInfo('MyVideo')), [
TupleValueExpression('Frame_ID'),
TupleValueExpression('Frame_Path')
], [
ConstantValueExpression(1),
ConstantValueExpression('/mnt/frames/1.png')
])
eva_statement_list = parser.parse(insert_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.INSERT)
insert_stmt = eva_statement_list[0]
self.assertEqual(insert_stmt, expected_stmt)
def test_subtraction(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_SUBTRACT,
tpl_exp, const_exp)
tuple1 = [5, 2, 3]
# 5-5 = 0
self.assertEqual(0, cmpr_exp.evaluate(tuple1, None))
def test_comparison_compare_equal(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_EQUAL, tpl_exp,
const_exp)
# ToDo implement a generic tuple class
# to fetch the tuple from table
tuple1 = [1, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def visitUidList(self, ctx: evaql_parser.UidListContext):
uid_list = []
for child in ctx.children:
# Skippping commas
if not isinstance(child, TerminalNode):
uid = self.visit(child)
uid_expr = TupleValueExpression(uid)
uid_list.append(uid_expr)
return uid_list
def test_aggregation_min(self):
columnName = TupleValueExpression(col_name=0)
aggr_expr = AggregationExpression(
ExpressionType.AGGREGATION_MIN,
None,
columnName
)
tuples = Batch(pd.DataFrame(
{0: [1, 2, 3], 1: [2, 3, 4], 2: [3, 4, 5]}))
batch = aggr_expr.evaluate(tuples, None)
self.assertEqual(1, batch.frames.iloc[0][0])
def test_visit_select_orderby(self, mock_p, mock_c, mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT data, id FROM video \
WHERE data > 2 ORDER BY data, id DESC;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
plans = []
plans.append(
LogicalOrderBy([
(TupleValueExpression('data'), ParserOrderBySortType.ASC),
(TupleValueExpression('id'), ParserOrderBySortType.DESC)
]))
plans.append(
LogicalProject(
[TupleValueExpression('data'),
TupleValueExpression('id')]))
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('data'),
ConstantValueExpression(2))))
plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expected_plan = None
for plan in reversed(plans):
if expected_plan:
plan.append_child(expected_plan)
expected_plan = plan
self.assertEqual(expected_plan, actual_plan)
wrong_plan = plans[0]
for plan in plans[1:]:
wrong_plan.append_child(plan)
self.assertNotEqual(wrong_plan, actual_plan)
def test_should_visit_select_union_if_union_query(self, mock_p, mock_c,
mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT id FROM video WHERE id>3
UNION ALL
SELECT id FROM video WHERE id<=3;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
left_plans = [LogicalProject([TupleValueExpression('id')])]
left_plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('id'),
ConstantValueExpression(3))))
left_plans.append(LogicalGet(TableRef(TableInfo('video')), m))
def reverse_plan(plans):
return_plan = None
for plan in reversed(plans):
if return_plan:
plan.append_child(return_plan)
return_plan = plan
return return_plan
expect_left_plan = reverse_plan(left_plans)
right_plans = [LogicalProject([TupleValueExpression('id')])]
right_plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_LEQ,
TupleValueExpression('id'),
ConstantValueExpression(3))))
right_plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expect_right_plan = reverse_plan(right_plans)
expected_plan = LogicalUnion(True)
expected_plan.append_child(expect_right_plan)
expected_plan.append_child(expect_left_plan)
self.assertEqual(expected_plan, actual_plan)
def test_comparison_compare_leq(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(2)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LEQ, tpl_exp,
const_exp)
# checking lesser
tuple1 = [1, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
# checking equal
tuple2 = [2, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple2, None))
def test_comparison_compare_neq(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_NEQ, tpl_exp,
const_exp)
# checking not equal
tuple1 = [2, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
tuple1 = [3, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def test_compare_doesnt_broadcast_when_rhs_is_list(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression([1])
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_EQUAL, tpl_exp,
const_exp)
compare = type("compare", (), {
"value": 1,
"__eq__": lambda s, x: s.value == x
})
tuple1 = [[compare()], 2, 3]
self.assertEqual([True], cmpr_exp.evaluate(tuple1, None))
def test_comparison_compare_equal(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_EQUAL, tpl_exp,
const_exp)
# ToDo implement a generic tuple class
# to fetch the tuple from table
compare = type("compare", (BasePrediction, ), {
"value": 1,
"__eq__": lambda s, x: s.value == x
})
tuple1 = [[compare()], 2, 3]
self.assertEqual([True], cmpr_exp.evaluate(tuple1, None))