def test_visit_table_ref_populates_column_mapping(self, mock, mock_lget): converter = StatementToPlanConvertor() converter._populate_column_map = MagicMock() table_ref = TableRef(TableInfo("test")) converter.visit_table_ref(table_ref) converter._populate_column_map.assert_called_with(mock.return_value)
def test_should_visit_select_if_nested_query(self, mock_p, mock_c, mock_d): m = MagicMock() mock_p.return_value = mock_c.return_value = mock_d.return_value = m stmt = Parser().parse(""" SELECT id FROM (SELECT data, id FROM video \ WHERE data > 2) WHERE id>3;""")[0] converter = StatementToPlanConvertor() actual_plan = converter.visit(stmt) plans = [LogicalProject([TupleValueExpression('id')])] plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('id'), ConstantValueExpression(3)))) plans.append(LogicalQueryDerivedGet()) plans.append( LogicalProject( [TupleValueExpression('data'), TupleValueExpression('id')])) plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('data'), ConstantValueExpression(2)))) plans.append(LogicalGet(TableRef(TableInfo('video')), m)) expected_plan = None for plan in reversed(plans): if expected_plan: plan.append_child(expected_plan) expected_plan = plan self.assertEqual(expected_plan, actual_plan) wrong_plan = plans[0] for plan in plans[1:]: wrong_plan.append_child(plan) self.assertNotEqual(wrong_plan, actual_plan)
def test_should_insert_row_in_table(self): dummy_info = TableInfo('MyVideo') dummy_table = TableRef(dummy_info) columns = [ DataFrameColumn('Frame_ID', ColumnType.INTEGER), DataFrameColumn('Frame_Path', ColumnType.TEXT, array_dimensions=50) ] plan_node = CreatePlan(dummy_table, columns, False) createExec = CreateExecutor(plan_node) url = createExec.exec() parser = Parser() insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path) VALUES (1, '/mnt/frames/1.png'); """ eva_statement_list = parser.parse(insert_query) insert_stmt = eva_statement_list[0] convertor = StatementToPlanConvertor() convertor.visit(insert_stmt) logical_plan_node = convertor.plan print("logical", logical_plan_node) phy_plan_node = InsertPlan(logical_plan_node.video_catalog_id, logical_plan_node.column_list, logical_plan_node.value_list) insertExec = InsertExecutor(phy_plan_node) insertExec.exec() # test if we have a added the in our storage df = load_dataframe(url) self.assertEqual(df.collect()[0][0], 1) self.assertEqual(df.collect()[0][1], "'/mnt/frames/1.png'")
def test_visit_table_ref_should_create_logical_get_opr( self, mock, mock_lget): converter = StatementToPlanConvertor() table_ref = TableRef(TableInfo("test")) converter.visit_table_ref(table_ref) mock.assert_called_with(table_ref.table_info) mock_lget.assert_called_with(table_ref, mock.return_value) self.assertEqual(mock_lget.return_value, converter._plan)
def test_visit_should_call_create_udf(self): stmt = MagicMock(spec=CreateUDFStatement) convertor = StatementToPlanConvertor() mock = MagicMock() convertor.visit_create_udf = mock actual = convertor.visit(stmt) mock.assert_called_once() mock.assert_called_with(stmt)
def test_visit_should_call_create(self): stmt = MagicMock(spec=CreateTableStatement) convertor = StatementToPlanConvertor() mock = MagicMock() convertor.visit_create = mock convertor.visit(stmt) mock.assert_called_once() mock.assert_called_with(stmt)
def test_visit_should_call_load_data(self): stmt = MagicMock(spec=LoadDataStatement) convertor = StatementToPlanConvertor() mock = MagicMock() convertor.visit_load_data = mock convertor.visit(stmt) mock.assert_called_once() mock.assert_called_with(stmt)
def test_visit_should_call_insert(self): stmt = MagicMock(spec=InsertTableStatement) convertor = StatementToPlanConvertor() mock = MagicMock() convertor.visit_insert = mock actual = convertor.visit(stmt) mock.assert_called_once() mock.assert_called_with(stmt)
def test_visit_select_predicate_should_add_logical_filter( self, mock, mock_lfilter): converter = StatementToPlanConvertor() select_predicate = MagicMock() converter._visit_select_predicate(select_predicate) mock_lfilter.assert_called_with(select_predicate) mock.assert_called_with(select_predicate, converter._column_map) mock_lfilter.return_value.append_child.assert_called() self.assertEqual(mock_lfilter.return_value, converter._plan)
def test_visit_projection_should_add_logical_predicate( self, mock, mock_lproject): converter = StatementToPlanConvertor() projects = MagicMock() converter._visit_projection(projects) mock_lproject.assert_called_with(projects) mock.assert_called_with(projects, converter._column_map) mock_lproject.return_value.append_child.assert_called() self.assertEqual(mock_lproject.return_value, converter._plan)
def test_populate_column_map_should_populate_correctly(self): converter = StatementToPlanConvertor() dataset = MagicMock() dataset.columns = [MagicMock() for i in range(5)] expected = {} for i, column in enumerate(dataset.columns): column.name = "NAME" + str(i) expected[column.name.lower()] = column converter._populate_column_map(dataset) self.assertEqual(converter._column_map, expected)
def handle_request(transport, request_message): """ Reads a request from a client and processes it If user inputs 'quit' stops the event loop otherwise just echoes user input """ LoggingManager().log('Receive request: --|' + str(request_message) + '|--') output_batch = None response = None try: stmt = Parser().parse(request_message)[0] l_plan = StatementToPlanConvertor().visit(stmt) p_plan = PlanGenerator().build(l_plan) output_batch = PlanExecutor(p_plan).execute_plan() except Exception as e: LoggingManager().log(e, LoggingLevel.WARNING) response = Response(status=ResponseStatus.FAIL, batch=None) if response is None: response = Response(status=ResponseStatus.SUCCESS, batch=output_batch) responseData = response.to_json() # Send data length, because response can be very large data = (str(len(responseData)) + '|' + responseData).encode('ascii') LoggingManager().log('Response to client: --|' + str(response) + '|--\n' + 'Length: ' + str(len(responseData))) transport.write(data) return response
def test_visit_create_udf(self, mock, l_create_udf_mock): convertor = StatementToPlanConvertor() stmt = MagicMock() stmt.name = 'name' stmt.if_not_exists = True stmt.inputs = ['inp'] stmt.outputs = ['out'] stmt.impl_path = 'tmp.py' stmt.udf_type = 'classification' mock.side_effect = ['inp', 'out'] convertor.visit_create_udf(stmt) mock.assert_any_call(stmt.inputs, True) mock.assert_any_call(stmt.outputs, False) l_create_udf_mock.assert_called_once() l_create_udf_mock.assert_called_with(stmt.name, stmt.if_not_exists, 'inp', 'out', stmt.impl_path, stmt.udf_type)
def execute_query(query) -> Iterator[Batch]: """ Execute the query and return a result generator. """ stmt = Parser().parse(query)[0] l_plan = StatementToPlanConvertor().visit(stmt) p_plan = PlanGenerator().build(l_plan) return PlanExecutor(p_plan).execute_plan()
def test_visit_load_data_when_bind_returns_None(self, mock_create, mock_bind, mock_load): mock_bind.return_value = None table_ref = TableRef(TableInfo("test")) stmt = MagicMock(table=table_ref, path='path') StatementToPlanConvertor().visit_load_data(stmt) mock_create.assert_called_once_with(table_ref.table_info.table_name) mock_bind.assert_called_with(table_ref.table_info) mock_load.assert_called_with(mock_create.return_value, 'path')
def test_visit_select_orderby(self, mock_p, mock_c, mock_d): m = MagicMock() mock_p.return_value = mock_c.return_value = mock_d.return_value = m stmt = Parser().parse(""" SELECT data, id FROM video \ WHERE data > 2 ORDER BY data, id DESC;""")[0] converter = StatementToPlanConvertor() actual_plan = converter.visit(stmt) plans = [] plans.append( LogicalOrderBy([ (TupleValueExpression('data'), ParserOrderBySortType.ASC), (TupleValueExpression('id'), ParserOrderBySortType.DESC) ])) plans.append( LogicalProject( [TupleValueExpression('data'), TupleValueExpression('id')])) plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('data'), ConstantValueExpression(2)))) plans.append(LogicalGet(TableRef(TableInfo('video')), m)) expected_plan = None for plan in reversed(plans): if expected_plan: plan.append_child(expected_plan) expected_plan = plan self.assertEqual(expected_plan, actual_plan) wrong_plan = plans[0] for plan in plans[1:]: wrong_plan.append_child(plan) self.assertNotEqual(wrong_plan, actual_plan)
def test_should_visit_select_union_if_union_query(self, mock_p, mock_c, mock_d): m = MagicMock() mock_p.return_value = mock_c.return_value = mock_d.return_value = m stmt = Parser().parse(""" SELECT id FROM video WHERE id>3 UNION ALL SELECT id FROM video WHERE id<=3;""")[0] converter = StatementToPlanConvertor() actual_plan = converter.visit(stmt) left_plans = [LogicalProject([TupleValueExpression('id')])] left_plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('id'), ConstantValueExpression(3)))) left_plans.append(LogicalGet(TableRef(TableInfo('video')), m)) def reverse_plan(plans): return_plan = None for plan in reversed(plans): if return_plan: plan.append_child(return_plan) return_plan = plan return return_plan expect_left_plan = reverse_plan(left_plans) right_plans = [LogicalProject([TupleValueExpression('id')])] right_plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_LEQ, TupleValueExpression('id'), ConstantValueExpression(3)))) right_plans.append(LogicalGet(TableRef(TableInfo('video')), m)) expect_right_plan = reverse_plan(right_plans) expected_plan = LogicalUnion(True) expected_plan.append_child(expect_right_plan) expected_plan.append_child(expect_left_plan) self.assertEqual(expected_plan, actual_plan)
def test_visit_select_should_not_call_visits_for_null_values(self): converter = StatementToPlanConvertor() converter.visit_table_ref = MagicMock() converter._visit_projection = MagicMock() converter._visit_select_predicate = MagicMock() converter._visit_union = MagicMock() statement = SelectStatement() converter.visit_select(statement) converter.visit_table_ref.assert_not_called() converter._visit_projection.assert_not_called() converter._visit_select_predicate.assert_not_called()
def test_visit_select_should_call_appropriate_visit_methods(self): converter = StatementToPlanConvertor() converter.visit_table_ref = MagicMock() converter._visit_projection = MagicMock() converter._visit_select_predicate = MagicMock() converter._visit_union = MagicMock() statement = MagicMock() statement.from_table = MagicMock(spec=TableRef) converter.visit_select(statement) converter.visit_table_ref.assert_called_with(statement.from_table) converter._visit_projection.assert_called_with(statement.target_list) converter._visit_select_predicate.assert_called_with( statement.where_clause)
def handle_request(transport, query): """ Reads a request from a client and processes it If user inputs 'quit' stops the event loop otherwise just echoes user input """ parser = Parser() eva_statement = parser.parse(query) insert_statement = eva_statement[0] LoggingManager().log("Result from the parser: " + str(insert_statement)) convertor = StatementToPlanConvertor() convertor.visit(insert_statement) convertor.plan eva_statement = parser.parse(query) response_message = "" for i in range(len(eva_statement)): LoggingManager().log("Parse Tree: " + str(eva_statement[i])) logical_plan = StatementToPlanConvertor().visit(eva_statement[i]) physical_plan = PlanGenerator().build(logical_plan) df = PlanExecutor(physical_plan).execute_plan() statement_response_message = ','.join(map(str, df.collect())) response_message += statement_response_message LoggingManager().log('Response to client: --|' + str(response_message) + '|--') data = response_message.encode('ascii') transport.write(data) return response_message
def test_visit_select_sample(self, mock_p, mock_c, mock_d): m = MagicMock() mock_p.return_value = mock_c.return_value = mock_d.return_value = m stmt = Parser().parse(""" SELECT data, id FROM video SAMPLE 2 \ WHERE id > 2 LIMIT 3;""")[0] converter = StatementToPlanConvertor() actual_plan = converter.visit(stmt) plans = [] plans.append(LogicalLimit(ConstantValueExpression(3))) plans.append( LogicalProject( [TupleValueExpression('data'), TupleValueExpression('id')])) plans.append( LogicalFilter( ComparisonExpression(ExpressionType.COMPARE_GREATER, TupleValueExpression('id'), ConstantValueExpression(2)))) plans.append(LogicalSample(ConstantValueExpression(2))) plans.append( LogicalGet( TableRef(TableInfo('video'), ConstantValueExpression(2)), m)) expected_plan = None for plan in reversed(plans): if expected_plan: plan.append_child(expected_plan) expected_plan = plan self.assertEqual(expected_plan, actual_plan)
def handle_request(transport, query): """ Reads a request from a client and processes it If user inputs 'quit' stops the event loop otherwise just echoes user input """ parser = Parser() eva_statement = parser.parse(query) for i in range(len(eva_statement)): LoggingManager().log("Result from the parser: " + str(eva_statement[i])) logical_plan = StatementToPlanConvertor().visit(eva_statement[i]) physical_plan = PlanGenerator().build(logical_plan) PlanExecutor(physical_plan).execute_plan() table_name = 'MyVideo' file_url = os.path.join(tempfile.gettempdir(), table_name) file_url = 'file://' + file_url df = load_dataframe(file_url) response_message = ','.join(map(str, df.collect())) LoggingManager().log("Response received" + str(response_message)) LoggingManager().log('Response to client: --|' + str(response_message) + '|--') data = response_message.encode('ascii') transport.write(data) return response_message
def perform_query(query): stmt = Parser().parse(query)[0] l_plan = StatementToPlanConvertor().visit(stmt) p_plan = PlanGenerator().build(l_plan) return PlanExecutor(p_plan).execute_plan()