def test_logical_or(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_left = ComparisonExpression(
ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp)
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER, tpl_exp, const_exp)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR,
comparison_expression_left,
comparison_expression_right)
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
],
info=None)
expected_value = [[True], [True], [True]]
output_value = logical_expr.evaluate(input_batch)
self.assertEqual(expected_value, output_value)
def test_logical_or(self):
const_exp1 = ConstantValueExpression(1)
const_exp2 = ConstantValueExpression(1)
comparison_expression_left = ComparisonExpression(
ExpressionType.COMPARE_EQUAL,
const_exp1,
const_exp2
)
const_exp1 = ConstantValueExpression(1)
const_exp2 = ConstantValueExpression(2)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER,
const_exp1,
const_exp2
)
logical_expr = LogicalExpression(
ExpressionType.LOGICAL_OR,
comparison_expression_left,
comparison_expression_right
)
self.assertEqual(
[True],
logical_expr.evaluate(None).frames[0].tolist()
)
def test_should_visit_select_if_nested_query(self, mock_p, mock_c, mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT id FROM (SELECT data, id FROM video \
WHERE data > 2) WHERE id>3;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
plans = [LogicalProject([TupleValueExpression('id')])]
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('id'),
ConstantValueExpression(3))))
plans.append(LogicalQueryDerivedGet())
plans.append(
LogicalProject(
[TupleValueExpression('data'),
TupleValueExpression('id')]))
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('data'),
ConstantValueExpression(2))))
plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expected_plan = None
for plan in reversed(plans):
if expected_plan:
plan.append_child(expected_plan)
expected_plan = plan
self.assertEqual(expected_plan, actual_plan)
wrong_plan = plans[0]
for plan in plans[1:]:
wrong_plan.append_child(plan)
self.assertNotEqual(wrong_plan, actual_plan)
def test_comparison_compare_lesser(self):
const_exp1 = ConstantValueExpression(0)
const_exp2 = ConstantValueExpression(2)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LESSER,
const_exp1, const_exp2)
self.assertEqual([True], cmpr_exp.evaluate(None).frames[0].tolist())
def visitConstant(self, ctx: evaql_parser.ConstantContext):
if ctx.REAL_LITERAL() is not None:
return ConstantValueExpression(float(ctx.getText()))
if ctx.decimalLiteral() is not None:
return ConstantValueExpression(self.visit(ctx.decimalLiteral()))
return self.visitChildren(ctx)
def test_divide(self):
const_exp1 = ConstantValueExpression(5)
const_exp2 = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_DIVIDE,
const_exp1, const_exp2)
self.assertEqual([1], cmpr_exp.evaluate(None).frames[0].tolist())
def test_multiply(self):
const_exp1 = ConstantValueExpression(3)
const_exp2 = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_MULTIPLY,
const_exp1, const_exp2)
self.assertEqual([15], cmpr_exp.evaluate(None).frames[0].tolist())
def test_subtraction(self):
const_exp1 = ConstantValueExpression(5)
const_exp2 = ConstantValueExpression(2)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_SUBTRACT,
const_exp1, const_exp2)
self.assertEqual([3], cmpr_exp.evaluate(None).frames[0].tolist())
def test_logical_not(self):
const_exp1 = ConstantValueExpression(0)
const_exp2 = ConstantValueExpression(1)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER, const_exp1, const_exp2)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_NOT, None,
comparison_expression_right)
self.assertEqual([True], logical_expr.evaluate(None))
def test_comparison_compare_greater(self):
const_exp1 = ConstantValueExpression(1)
const_exp2 = ConstantValueExpression(0)
cmpr_exp = ComparisonExpression(
ExpressionType.COMPARE_GREATER,
const_exp1,
const_exp2
)
self.assertEqual([True], cmpr_exp.evaluate(None))
def test_addition(self):
const_exp1 = ConstantValueExpression(2)
const_exp2 = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(
ExpressionType.ARITHMETIC_ADD,
const_exp1,
const_exp2
)
self.assertEqual(7, cmpr_exp.evaluate(None))
def test_comparison_compare_contains(self):
const_exp1 = ConstantValueExpression([1, 2], ColumnType.NDARRAY)
const_exp2 = ConstantValueExpression([1, 5], ColumnType.NDARRAY)
const_exp3 = ConstantValueExpression([1, 2, 3, 4], ColumnType.NDARRAY)
cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_CONTAINS,
const_exp3, const_exp1)
self.assertEqual([True], cmpr_exp1.evaluate(None).frames[0].tolist())
cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_CONTAINS,
const_exp3, const_exp2)
self.assertEqual([False], cmpr_exp2.evaluate(None).frames[0].tolist())
def test_comparison_compare_geq(self):
const_exp1 = ConstantValueExpression(1)
const_exp2 = ConstantValueExpression(1)
const_exp3 = ConstantValueExpression(0)
cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_GEQ,
const_exp1, const_exp2)
cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_GEQ,
const_exp1, const_exp3)
# checking equal
self.assertEqual([True], cmpr_exp1.evaluate(None).frames[0].tolist())
# checking greater equal
self.assertEqual([True], cmpr_exp2.evaluate(None).frames[0].tolist())
def test_comparison_compare_greater(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_left = ComparisonExpression(
ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp)
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER, tpl_exp, const_exp)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR,
comparison_expression_left,
comparison_expression_right)
tuple1 = [[1], 2, 3]
self.assertEqual([True], logical_expr.evaluate(tuple1, None))
def test_should_return_limit_greater_than_size(self):
""" This should return the exact same data
if the limit value is greater than what is present.
This will also leave a warning """
dfs = [pd.DataFrame(np.random.randint(0, 100, size=(100, 4)),
columns=list('ABCD')) for _ in range(4)]
batches = [Batch(frames=df) for df in dfs]
previous_total_size = 0
for batch in batches:
previous_total_size += batch.batch_size
limit_value = 500
plan = LimitPlan(ConstantValueExpression(limit_value))
limit_executor = LimitExecutor(plan)
limit_executor.append_child(DummyExecutor(batches))
reduced_batches = list(limit_executor.exec())
after_total_size = 0
for batch in reduced_batches:
after_total_size += batch.batch_size
self.assertEqual(previous_total_size, after_total_size)
def test_select_statement_sample_class(self):
'''Testing sample frequency '''
parser = Parser()
select_query = "SELECT CLASS, REDNESS FROM TAIPAI SAMPLE 5;"
eva_statement_list = parser.parse(select_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.SELECT)
select_stmt = eva_statement_list[0]
# target List
self.assertIsNotNone(select_stmt.target_list)
self.assertEqual(len(select_stmt.target_list), 2)
self.assertEqual(
select_stmt.target_list[0].etype, ExpressionType.TUPLE_VALUE)
self.assertEqual(
select_stmt.target_list[1].etype, ExpressionType.TUPLE_VALUE)
# from_table
self.assertIsNotNone(select_stmt.from_table)
self.assertIsInstance(select_stmt.from_table, TableRef)
self.assertEqual(
select_stmt.from_table.table.table_name, 'TAIPAI')
# sample_freq
self.assertEqual(select_stmt.from_table.sample_freq,
ConstantValueExpression(5))
def test_if_expr_tree_is_equal(self):
const_exp1 = ConstantValueExpression(0)
const_exp2 = ConstantValueExpression(0)
columnName1 = TupleValueExpression(col_name='DATA')
columnName2 = TupleValueExpression(col_name='DATA')
aggr_expr1 = AggregationExpression(ExpressionType.AGGREGATION_AVG,
None, columnName1)
aggr_expr2 = AggregationExpression(ExpressionType.AGGREGATION_AVG,
None, columnName2)
cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_NEQ,
aggr_expr1, const_exp1)
cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_NEQ,
aggr_expr2, const_exp2)
self.assertEqual(cmpr_exp1, cmpr_exp2)
def test_should_return_smaller_num_rows(self):
dfs = [
pd.DataFrame(np.random.randint(0, 100, size=(100, 4)),
columns=list('ABCD')) for _ in range(4)
]
batches = [Batch(frames=df) for df in dfs]
sample_value = 3
plan = SamplePlan(ConstantValueExpression(sample_value))
sample_executor = SampleExecutor(plan)
sample_executor.append_child(DummyExecutor(batches))
reduced_batches = list(sample_executor.exec())
original = Batch.concat(batches)
filter = range(0, len(original), sample_value)
original = original._get_frames_from_indices(filter)
original = Batch.concat([original])
reduced = Batch.concat(reduced_batches)
self.assertEqual(len(original), len(reduced))
self.assertEqual(original, reduced)
def test_comparison_compare_greater(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_GREATER,
tpl_exp, const_exp)
tuple1 = [2, 1, 1]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def test_comparison_compare_lesser(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(2)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LESSER, tpl_exp,
const_exp)
tuple1 = [1, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def visitStringLiteral(self, ctx: evaql_parser.StringLiteralContext):
# Fix a bug here; 'VAN' Literal gets converted to "'VAN'";
# Multiple quotes should be removed
if ctx.STRING_LITERAL() is not None:
return ConstantValueExpression(ctx.getText())
# todo handle other types
return self.visitChildren(ctx)
def test_should_return_top_frames_after_sorting(self):
"""
Checks if limit returns the top 2 rows from the data
after sorting
data (3 batches):
'A' 'B' 'C'
[1, 1, 1]
----------
[1, 5, 6]
[4, 7, 10]
----------
[2, 9, 7]
[4, 1, 2]
[4, 2, 4]
"""
df1 = pd.DataFrame(
np.array([[1, 1, 1]]), columns=['A', 'B', 'C'])
df2 = pd.DataFrame(
np.array([[1, 5, 6], [4, 7, 10]]), columns=['A', 'B', 'C'])
df3 = pd.DataFrame(
np.array([[2, 9, 7], [4, 1, 2],
[4, 2, 4]]), columns=['A', 'B', 'C'])
batches = [Batch(frames=df) for df in [df1, df2, df3]]
"query: .... ORDER BY A ASC, B DESC limit 2"
plan = OrderByPlan(
[(TupleValueExpression('A'), ParserOrderBySortType.ASC),
(TupleValueExpression('B'), ParserOrderBySortType.DESC)])
orderby_executor = OrderByExecutor(plan)
orderby_executor.append_child(DummyExecutor(batches))
sorted_batches = list(orderby_executor.exec())
limit_value = 2
plan = LimitPlan(ConstantValueExpression(limit_value))
limit_executor = LimitExecutor(plan)
limit_executor.append_child(DummyExecutor(sorted_batches))
reduced_batches = list(limit_executor.exec())
# merge everything into one batch
aggregated_batch = Batch.concat(reduced_batches, copy=False)
"""
A B C
0 1 5 6
1 1 1 1
"""
expected_df1 = pd.DataFrame(
np.array([[1, 5, 6], [1, 1, 1]]), columns=['A', 'B', 'C'])
expected_batches = [Batch(frames=df) for df in [expected_df1]]
self.assertEqual(expected_batches[0], aggregated_batch)
def test_divide(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_DIVIDE,
tpl_exp, const_exp)
tuple1 = [5, 2, 3]
# 5/5 = 1
self.assertEqual(1, cmpr_exp.evaluate(tuple1, None))
def test_insert_statement(self):
parser = Parser()
insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path)
VALUES (1, '/mnt/frames/1.png');
"""
expected_stmt = InsertTableStatement(TableRef(TableInfo('MyVideo')), [
TupleValueExpression('Frame_ID'),
TupleValueExpression('Frame_Path')
], [
ConstantValueExpression(1),
ConstantValueExpression('/mnt/frames/1.png')
])
eva_statement_list = parser.parse(insert_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.INSERT)
insert_stmt = eva_statement_list[0]
self.assertEqual(insert_stmt, expected_stmt)
def test_comparison_compare_equal(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_EQUAL, tpl_exp,
const_exp)
# ToDo implement a generic tuple class
# to fetch the tuple from table
tuple1 = [1, 2, 3]
self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def test_subtraction(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_SUBTRACT,
tpl_exp, const_exp)
tuple1 = [5, 2, 3]
# 5-5 = 0
self.assertEqual(0, cmpr_exp.evaluate(tuple1, None))
def test_should_return_correct_plan_tree_for_limit_logical_tree(self):
# SELECT data, id FROM video limit 5;
logical_plan = LogicalLimit(ConstantValueExpression(5))
plan = ScanGenerator().build(logical_plan)
self.assertTrue(isinstance(plan, LimitPlan))
self.assertTrue(
isinstance(plan.limit_expression, ConstantValueExpression))
self.assertEqual(plan.limit_value, 5)
def test_multiply(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_MULTIPLY,
tpl_exp, const_exp)
tuple1 = [5, 2, 3]
# 5*5 = 25
self.assertEqual(25, cmpr_exp.evaluate(tuple1, None))
def test_should_return_false_for_unequal_expressions(self):
const_exp1 = ConstantValueExpression(0)
const_exp2 = ConstantValueExpression(1)
func_expr = FunctionExpression(lambda x: x + 1)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_NEQ, const_exp1,
const_exp2)
tuple_expr = TupleValueExpression(col_name='id')
aggr_expr = AggregationExpression(ExpressionType.AGGREGATION_MAX, None,
tuple_expr)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR, cmpr_exp,
cmpr_exp)
self.assertNotEqual(const_exp1, const_exp2)
self.assertNotEqual(cmpr_exp, const_exp1)
self.assertNotEqual(func_expr, cmpr_exp)
self.assertNotEqual(tuple_expr, aggr_expr)
self.assertNotEqual(aggr_expr, tuple_expr)
self.assertNotEqual(tuple_expr, cmpr_exp)
self.assertNotEqual(logical_expr, cmpr_exp)
def test_addition(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_ADD, tpl_exp,
const_exp)
tuple1 = [5, 2, 3]
# 5+5 = 10
self.assertEqual(10, cmpr_exp.evaluate(tuple1, None))
