def test_logical_or(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(1) comparison_expression_left = ComparisonExpression( ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp) tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(1) comparison_expression_right = ComparisonExpression( ExpressionType.COMPARE_GREATER, tpl_exp, const_exp) logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR, comparison_expression_left, comparison_expression_right) frame_1 = Frame(1, np.ones((1, 1)), None) frame_2 = Frame(2, 2 * np.ones((1, 1)), None) frame_3 = Frame(3, 3 * np.ones((1, 1)), None) input_batch = FrameBatch(frames=[ frame_1, frame_2, frame_3, ], info=None) expected_value = [[True], [True], [True]] output_value = logical_expr.evaluate(input_batch) self.assertEqual(expected_value, output_value)
def test_logical_or(self): const_exp1 = ConstantValueExpression(1) const_exp2 = ConstantValueExpression(1) comparison_expression_left = ComparisonExpression( ExpressionType.COMPARE_EQUAL, const_exp1, const_exp2 ) const_exp1 = ConstantValueExpression(1) const_exp2 = ConstantValueExpression(2) comparison_expression_right = ComparisonExpression( ExpressionType.COMPARE_GREATER, const_exp1, const_exp2 ) logical_expr = LogicalExpression( ExpressionType.LOGICAL_OR, comparison_expression_left, comparison_expression_right ) self.assertEqual( [True], logical_expr.evaluate(None).frames[0].tolist() )
def test_should_visit_select_if_nested_query(self, mock_p, mock_c, mock_d): m = MagicMock() mock_p.return_value = mock_c.return_value = mock_d.return_value = m stmt = Parser().parse(""" SELECT id FROM (SELECT data, id FROM video \ WHERE data > 2) WHERE id>3;""")[0] converter = StatementToPlanConvertor() actual_plan = converter.visit(stmt) plans = [LogicalProject([TupleValueExpression('id')])] plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('id'), ConstantValueExpression(3)))) plans.append(LogicalQueryDerivedGet()) plans.append( LogicalProject( [TupleValueExpression('data'), TupleValueExpression('id')])) plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('data'), ConstantValueExpression(2)))) plans.append(LogicalGet(TableRef(TableInfo('video')), m)) expected_plan = None for plan in reversed(plans): if expected_plan: plan.append_child(expected_plan) expected_plan = plan self.assertEqual(expected_plan, actual_plan) wrong_plan = plans[0] for plan in plans[1:]: wrong_plan.append_child(plan) self.assertNotEqual(wrong_plan, actual_plan)
def test_comparison_compare_lesser(self): const_exp1 = ConstantValueExpression(0) const_exp2 = ConstantValueExpression(2) cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LESSER, const_exp1, const_exp2) self.assertEqual([True], cmpr_exp.evaluate(None).frames[0].tolist())
def visitConstant(self, ctx: evaql_parser.ConstantContext): if ctx.REAL_LITERAL() is not None: return ConstantValueExpression(float(ctx.getText())) if ctx.decimalLiteral() is not None: return ConstantValueExpression(self.visit(ctx.decimalLiteral())) return self.visitChildren(ctx)
def test_divide(self): const_exp1 = ConstantValueExpression(5) const_exp2 = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_DIVIDE, const_exp1, const_exp2) self.assertEqual([1], cmpr_exp.evaluate(None).frames[0].tolist())
def test_multiply(self): const_exp1 = ConstantValueExpression(3) const_exp2 = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_MULTIPLY, const_exp1, const_exp2) self.assertEqual([15], cmpr_exp.evaluate(None).frames[0].tolist())
def test_subtraction(self): const_exp1 = ConstantValueExpression(5) const_exp2 = ConstantValueExpression(2) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_SUBTRACT, const_exp1, const_exp2) self.assertEqual([3], cmpr_exp.evaluate(None).frames[0].tolist())
def test_logical_not(self): const_exp1 = ConstantValueExpression(0) const_exp2 = ConstantValueExpression(1) comparison_expression_right = ComparisonExpression( ExpressionType.COMPARE_GREATER, const_exp1, const_exp2) logical_expr = LogicalExpression(ExpressionType.LOGICAL_NOT, None, comparison_expression_right) self.assertEqual([True], logical_expr.evaluate(None))
def test_comparison_compare_greater(self): const_exp1 = ConstantValueExpression(1) const_exp2 = ConstantValueExpression(0) cmpr_exp = ComparisonExpression( ExpressionType.COMPARE_GREATER, const_exp1, const_exp2 ) self.assertEqual([True], cmpr_exp.evaluate(None))
def test_addition(self): const_exp1 = ConstantValueExpression(2) const_exp2 = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression( ExpressionType.ARITHMETIC_ADD, const_exp1, const_exp2 ) self.assertEqual(7, cmpr_exp.evaluate(None))
def test_comparison_compare_contains(self): const_exp1 = ConstantValueExpression([1, 2], ColumnType.NDARRAY) const_exp2 = ConstantValueExpression([1, 5], ColumnType.NDARRAY) const_exp3 = ConstantValueExpression([1, 2, 3, 4], ColumnType.NDARRAY) cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_CONTAINS, const_exp3, const_exp1) self.assertEqual([True], cmpr_exp1.evaluate(None).frames[0].tolist()) cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_CONTAINS, const_exp3, const_exp2) self.assertEqual([False], cmpr_exp2.evaluate(None).frames[0].tolist())
def test_comparison_compare_geq(self): const_exp1 = ConstantValueExpression(1) const_exp2 = ConstantValueExpression(1) const_exp3 = ConstantValueExpression(0) cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_GEQ, const_exp1, const_exp2) cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_GEQ, const_exp1, const_exp3) # checking equal self.assertEqual([True], cmpr_exp1.evaluate(None).frames[0].tolist()) # checking greater equal self.assertEqual([True], cmpr_exp2.evaluate(None).frames[0].tolist())
def test_comparison_compare_greater(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(1) comparison_expression_left = ComparisonExpression( ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp) tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(1) comparison_expression_right = ComparisonExpression( ExpressionType.COMPARE_GREATER, tpl_exp, const_exp) logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR, comparison_expression_left, comparison_expression_right) tuple1 = [[1], 2, 3] self.assertEqual([True], logical_expr.evaluate(tuple1, None))
def test_should_return_limit_greater_than_size(self): """ This should return the exact same data if the limit value is greater than what is present. This will also leave a warning """ dfs = [pd.DataFrame(np.random.randint(0, 100, size=(100, 4)), columns=list('ABCD')) for _ in range(4)] batches = [Batch(frames=df) for df in dfs] previous_total_size = 0 for batch in batches: previous_total_size += batch.batch_size limit_value = 500 plan = LimitPlan(ConstantValueExpression(limit_value)) limit_executor = LimitExecutor(plan) limit_executor.append_child(DummyExecutor(batches)) reduced_batches = list(limit_executor.exec()) after_total_size = 0 for batch in reduced_batches: after_total_size += batch.batch_size self.assertEqual(previous_total_size, after_total_size)
def test_select_statement_sample_class(self): '''Testing sample frequency ''' parser = Parser() select_query = "SELECT CLASS, REDNESS FROM TAIPAI SAMPLE 5;" eva_statement_list = parser.parse(select_query) self.assertIsInstance(eva_statement_list, list) self.assertEqual(len(eva_statement_list), 1) self.assertEqual(eva_statement_list[0].stmt_type, StatementType.SELECT) select_stmt = eva_statement_list[0] # target List self.assertIsNotNone(select_stmt.target_list) self.assertEqual(len(select_stmt.target_list), 2) self.assertEqual( select_stmt.target_list[0].etype, ExpressionType.TUPLE_VALUE) self.assertEqual( select_stmt.target_list[1].etype, ExpressionType.TUPLE_VALUE) # from_table self.assertIsNotNone(select_stmt.from_table) self.assertIsInstance(select_stmt.from_table, TableRef) self.assertEqual( select_stmt.from_table.table.table_name, 'TAIPAI') # sample_freq self.assertEqual(select_stmt.from_table.sample_freq, ConstantValueExpression(5))
def test_if_expr_tree_is_equal(self): const_exp1 = ConstantValueExpression(0) const_exp2 = ConstantValueExpression(0) columnName1 = TupleValueExpression(col_name='DATA') columnName2 = TupleValueExpression(col_name='DATA') aggr_expr1 = AggregationExpression(ExpressionType.AGGREGATION_AVG, None, columnName1) aggr_expr2 = AggregationExpression(ExpressionType.AGGREGATION_AVG, None, columnName2) cmpr_exp1 = ComparisonExpression(ExpressionType.COMPARE_NEQ, aggr_expr1, const_exp1) cmpr_exp2 = ComparisonExpression(ExpressionType.COMPARE_NEQ, aggr_expr2, const_exp2) self.assertEqual(cmpr_exp1, cmpr_exp2)
def test_should_return_smaller_num_rows(self): dfs = [ pd.DataFrame(np.random.randint(0, 100, size=(100, 4)), columns=list('ABCD')) for _ in range(4) ] batches = [Batch(frames=df) for df in dfs] sample_value = 3 plan = SamplePlan(ConstantValueExpression(sample_value)) sample_executor = SampleExecutor(plan) sample_executor.append_child(DummyExecutor(batches)) reduced_batches = list(sample_executor.exec()) original = Batch.concat(batches) filter = range(0, len(original), sample_value) original = original._get_frames_from_indices(filter) original = Batch.concat([original]) reduced = Batch.concat(reduced_batches) self.assertEqual(len(original), len(reduced)) self.assertEqual(original, reduced)
def test_comparison_compare_greater(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(1) cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_GREATER, tpl_exp, const_exp) tuple1 = [2, 1, 1] self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def test_comparison_compare_lesser(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(2) cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LESSER, tpl_exp, const_exp) tuple1 = [1, 2, 3] self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def visitStringLiteral(self, ctx: evaql_parser.StringLiteralContext): # Fix a bug here; 'VAN' Literal gets converted to "'VAN'"; # Multiple quotes should be removed if ctx.STRING_LITERAL() is not None: return ConstantValueExpression(ctx.getText()) # todo handle other types return self.visitChildren(ctx)
def test_should_return_top_frames_after_sorting(self): """ Checks if limit returns the top 2 rows from the data after sorting data (3 batches): 'A' 'B' 'C' [1, 1, 1] ---------- [1, 5, 6] [4, 7, 10] ---------- [2, 9, 7] [4, 1, 2] [4, 2, 4] """ df1 = pd.DataFrame( np.array([[1, 1, 1]]), columns=['A', 'B', 'C']) df2 = pd.DataFrame( np.array([[1, 5, 6], [4, 7, 10]]), columns=['A', 'B', 'C']) df3 = pd.DataFrame( np.array([[2, 9, 7], [4, 1, 2], [4, 2, 4]]), columns=['A', 'B', 'C']) batches = [Batch(frames=df) for df in [df1, df2, df3]] "query: .... ORDER BY A ASC, B DESC limit 2" plan = OrderByPlan( [(TupleValueExpression('A'), ParserOrderBySortType.ASC), (TupleValueExpression('B'), ParserOrderBySortType.DESC)]) orderby_executor = OrderByExecutor(plan) orderby_executor.append_child(DummyExecutor(batches)) sorted_batches = list(orderby_executor.exec()) limit_value = 2 plan = LimitPlan(ConstantValueExpression(limit_value)) limit_executor = LimitExecutor(plan) limit_executor.append_child(DummyExecutor(sorted_batches)) reduced_batches = list(limit_executor.exec()) # merge everything into one batch aggregated_batch = Batch.concat(reduced_batches, copy=False) """ A B C 0 1 5 6 1 1 1 1 """ expected_df1 = pd.DataFrame( np.array([[1, 5, 6], [1, 1, 1]]), columns=['A', 'B', 'C']) expected_batches = [Batch(frames=df) for df in [expected_df1]] self.assertEqual(expected_batches[0], aggregated_batch)
def test_divide(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_DIVIDE, tpl_exp, const_exp) tuple1 = [5, 2, 3] # 5/5 = 1 self.assertEqual(1, cmpr_exp.evaluate(tuple1, None))
def test_insert_statement(self): parser = Parser() insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path) VALUES (1, '/mnt/frames/1.png'); """ expected_stmt = InsertTableStatement(TableRef(TableInfo('MyVideo')), [ TupleValueExpression('Frame_ID'), TupleValueExpression('Frame_Path') ], [ ConstantValueExpression(1), ConstantValueExpression('/mnt/frames/1.png') ]) eva_statement_list = parser.parse(insert_query) self.assertIsInstance(eva_statement_list, list) self.assertEqual(len(eva_statement_list), 1) self.assertEqual(eva_statement_list[0].stmt_type, StatementType.INSERT) insert_stmt = eva_statement_list[0] self.assertEqual(insert_stmt, expected_stmt)
def test_comparison_compare_equal(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(1) cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp) # ToDo implement a generic tuple class # to fetch the tuple from table tuple1 = [1, 2, 3] self.assertEqual(True, cmpr_exp.evaluate(tuple1, None))
def test_subtraction(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_SUBTRACT, tpl_exp, const_exp) tuple1 = [5, 2, 3] # 5-5 = 0 self.assertEqual(0, cmpr_exp.evaluate(tuple1, None))
def test_should_return_correct_plan_tree_for_limit_logical_tree(self): # SELECT data, id FROM video limit 5; logical_plan = LogicalLimit(ConstantValueExpression(5)) plan = ScanGenerator().build(logical_plan) self.assertTrue(isinstance(plan, LimitPlan)) self.assertTrue( isinstance(plan.limit_expression, ConstantValueExpression)) self.assertEqual(plan.limit_value, 5)
def test_multiply(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_MULTIPLY, tpl_exp, const_exp) tuple1 = [5, 2, 3] # 5*5 = 25 self.assertEqual(25, cmpr_exp.evaluate(tuple1, None))
def test_should_return_false_for_unequal_expressions(self): const_exp1 = ConstantValueExpression(0) const_exp2 = ConstantValueExpression(1) func_expr = FunctionExpression(lambda x: x + 1) cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_NEQ, const_exp1, const_exp2) tuple_expr = TupleValueExpression(col_name='id') aggr_expr = AggregationExpression(ExpressionType.AGGREGATION_MAX, None, tuple_expr) logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR, cmpr_exp, cmpr_exp) self.assertNotEqual(const_exp1, const_exp2) self.assertNotEqual(cmpr_exp, const_exp1) self.assertNotEqual(func_expr, cmpr_exp) self.assertNotEqual(tuple_expr, aggr_expr) self.assertNotEqual(aggr_expr, tuple_expr) self.assertNotEqual(tuple_expr, cmpr_exp) self.assertNotEqual(logical_expr, cmpr_exp)
def test_addition(self): tpl_exp = TupleValueExpression(0) const_exp = ConstantValueExpression(5) cmpr_exp = ArithmeticExpression(ExpressionType.ARITHMETIC_ADD, tpl_exp, const_exp) tuple1 = [5, 2, 3] # 5+5 = 10 self.assertEqual(10, cmpr_exp.evaluate(tuple1, None))