def test_should_return_the_new_path_after_execution(self, mock_class): class_instatnce = mock_class.return_value dummy_expr = type('dummy_expr', (), {"evaluate": lambda x=None: [True, False, True]}) # Build plan tree video = DataFrameMetadata("dataset", "dummy.avi") batch_1 = Batch(pd.DataFrame({'data': [1, 2, 3]})) batch_2 = Batch(pd.DataFrame({'data': [4, 5, 6]})) class_instatnce.load.return_value = map(lambda x: x, [batch_1, batch_2]) storage_plan = StoragePlan(video) seq_scan = SeqScanPlan(predicate=dummy_expr, column_ids=[]) seq_scan.append_child(storage_plan) # Execute the plan executor = PlanExecutor(seq_scan) actual = executor.execute_plan() expected = batch_1[::2] + batch_2[::2] mock_class.assert_called_once() self.assertEqual(expected, actual)
def test_execute_plan_for_create_insert_load_plans(self, mock_batch, mock_clean, mock_build): mock_batch.return_value = [] # CreateExecutor tree = MagicMock(node=CreatePlan(None, [], False)) mock_build.return_value = tree actual = PlanExecutor(None).execute_plan() tree.exec.assert_called_once() mock_batch.assert_called_once() assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame()) mock_build.assert_called_once_with(None) mock_clean.assert_called_once() self.assertEqual(actual, []) # InsertExecutor mock_batch.reset_mock() mock_build.reset_mock() mock_clean.reset_mock() tree = MagicMock(node=InsertPlan(0, [], [])) mock_build.return_value = tree actual = PlanExecutor(None).execute_plan() tree.exec.assert_called_once() mock_batch.assert_called_once() assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame()) mock_build.assert_called_once_with(None) mock_clean.assert_called_once() self.assertEqual(actual, []) # CreateUDFExecutor mock_batch.reset_mock() mock_build.reset_mock() mock_clean.reset_mock() tree = MagicMock(node=CreateUDFPlan(None, False, [], [], None)) mock_build.return_value = tree actual = PlanExecutor(None).execute_plan() tree.exec.assert_called_once() mock_batch.assert_called_once() assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame()) mock_build.assert_called_once_with(None) mock_clean.assert_called_once() self.assertEqual(actual, []) # LoadDataExecutor mock_batch.reset_mock() mock_build.reset_mock() mock_clean.reset_mock() tree = MagicMock(node=LoadDataPlan(None, None)) mock_build.return_value = tree actual = PlanExecutor(None).execute_plan() tree.exec.assert_called_once() mock_batch.assert_called_once() assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame()) mock_build.assert_called_once_with(None) mock_clean.assert_called_once() self.assertEqual(actual, [])
def handle_request(transport, request_message): """ Reads a request from a client and processes it If user inputs 'quit' stops the event loop otherwise just echoes user input """ LoggingManager().log('Receive request: --|' + str(request_message) + '|--') output_batch = None response = None try: stmt = Parser().parse(request_message)[0] l_plan = StatementToPlanConvertor().visit(stmt) p_plan = PlanGenerator().build(l_plan) output_batch = PlanExecutor(p_plan).execute_plan() except Exception as e: LoggingManager().log(e, LoggingLevel.WARNING) response = Response(status=ResponseStatus.FAIL, batch=None) if response is None: response = Response(status=ResponseStatus.SUCCESS, batch=output_batch) responseData = response.to_json() # Send data length, because response can be very large data = (str(len(responseData)) + '|' + responseData).encode('ascii') LoggingManager().log('Response to client: --|' + str(response) + '|--\n' + 'Length: ' + str(len(responseData))) transport.write(data) return response
def execute_query(query) -> Iterator[Batch]: """ Execute the query and return a result generator. """ stmt = Parser().parse(query)[0] l_plan = StatementToPlanConvertor().visit(stmt) p_plan = PlanGenerator().build(l_plan) return PlanExecutor(p_plan).execute_plan()
def test_build_execution_tree_should_create_correct_exec_node(self): # SequentialScanExecutor plan = SeqScanPlan(MagicMock(), []) executor = PlanExecutor(plan)._build_execution_tree(plan) self.assertIsInstance(executor, SequentialScanExecutor) # PPExecutor plan = PPScanPlan(MagicMock()) executor = PlanExecutor(plan)._build_execution_tree(plan) self.assertIsInstance(executor, PPExecutor) # CreateExecutor plan = CreatePlan(MagicMock(), [], False) executor = PlanExecutor(plan)._build_execution_tree(plan) self.assertIsInstance(executor, CreateExecutor) # InsertExecutor plan = InsertPlan(0, [], []) executor = PlanExecutor(plan)._build_execution_tree(plan) self.assertIsInstance(executor, InsertExecutor) # CreateUDFExecutor plan = CreateUDFPlan('test', False, [], [], MagicMock(), None) executor = PlanExecutor(plan)._build_execution_tree(plan) self.assertIsInstance(executor, CreateUDFExecutor) # LoadDataExecutor plan = LoadDataPlan(MagicMock(), MagicMock()) executor = PlanExecutor(plan)._build_execution_tree(plan) self.assertIsInstance(executor, LoadDataExecutor)
def handle_request(transport, query): """ Reads a request from a client and processes it If user inputs 'quit' stops the event loop otherwise just echoes user input """ parser = Parser() eva_statement = parser.parse(query) insert_statement = eva_statement[0] LoggingManager().log("Result from the parser: " + str(insert_statement)) convertor = StatementToPlanConvertor() convertor.visit(insert_statement) convertor.plan eva_statement = parser.parse(query) response_message = "" for i in range(len(eva_statement)): LoggingManager().log("Parse Tree: " + str(eva_statement[i])) logical_plan = StatementToPlanConvertor().visit(eva_statement[i]) physical_plan = PlanGenerator().build(logical_plan) df = PlanExecutor(physical_plan).execute_plan() statement_response_message = ','.join(map(str, df.collect())) response_message += statement_response_message LoggingManager().log('Response to client: --|' + str(response_message) + '|--') data = response_message.encode('ascii') transport.write(data) return response_message
def test_execute_plan_for_pp_scan_plan(self, mock_clean, mock_build): # PPExecutor tree = MagicMock(node=PPScanPlan(None)) tree.exec.return_value = [ Batch(pd.DataFrame([1])), Batch(pd.DataFrame([2])), Batch(pd.DataFrame([3])) ] mock_build.return_value = tree actual = PlanExecutor(None).execute_plan() mock_build.assert_called_once_with(None) mock_clean.assert_called_once() tree.exec.assert_called_once() self.assertEqual(actual, Batch(pd.DataFrame([[1], [2], [3]])))
def test_execute_plan_for_pp_scan_plan(self, mock_batch, mock_clean, mock_build): # PPExecutor mock_batch.return_value = [] tree = MagicMock(node=PPScanPlan(None)) tree.exec.return_value = [[1], [2], [3]] mock_build.return_value = tree actual = PlanExecutor(None).execute_plan() mock_batch.assert_called_once() assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame()) mock_build.assert_called_once_with(None) mock_clean.assert_called_once() tree.exec.assert_called_once() self.assertEqual(actual, [1, 2, 3])
def test_execute_plan_for_seq_scan_plan(self, mock_clean, mock_build): batch_list = [ Batch(pd.DataFrame([1])), Batch(pd.DataFrame([2])), Batch(pd.DataFrame([3])) ] # SequentialScanExecutor tree = MagicMock(node=SeqScanPlan(None, [])) tree.exec.return_value = batch_list mock_build.return_value = tree actual = list(PlanExecutor(None).execute_plan()) mock_build.assert_called_once_with(None) mock_clean.assert_called_once() tree.exec.assert_called_once() self.assertEqual(actual, batch_list)
def test_tree_structure_for_build_execution_tree(self): """ Build an Abastract Plan with nodes: ß root / | \ c1 c2 c3 / c1_1 """ predicate = None root_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[]) child_1_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[]) child_2_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[]) child_3_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[]) child_1_1_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[]) root_abs_plan.append_child(child_1_abs_plan) root_abs_plan.append_child(child_2_abs_plan) root_abs_plan.append_child(child_3_abs_plan) child_1_abs_plan.append_child(child_1_1_abs_plan) '''Build Execution Tree and check the nodes are of the same type''' root_abs_executor = PlanExecutor( plan=root_abs_plan)._build_execution_tree(plan=root_abs_plan) # Root Nodes self.assertEqual(root_abs_plan.node_type, root_abs_executor._node.node_type) # Children of Root for child_abs, child_exec in zip(root_abs_plan.children, root_abs_executor.children): self.assertEqual(child_abs.node_type, child_exec._node.node_type) # Grand Children of Root for gc_abs, gc_exec in zip(child_abs.children, child_exec.children): self.assertEqual(gc_abs.node_type, gc_exec._node.node_type)
def handle_request(transport, query): """ Reads a request from a client and processes it If user inputs 'quit' stops the event loop otherwise just echoes user input """ parser = Parser() eva_statement = parser.parse(query) for i in range(len(eva_statement)): LoggingManager().log("Result from the parser: " + str(eva_statement[i])) logical_plan = StatementToPlanConvertor().visit(eva_statement[i]) physical_plan = PlanGenerator().build(logical_plan) PlanExecutor(physical_plan).execute_plan() table_name = 'MyVideo' file_url = os.path.join(tempfile.gettempdir(), table_name) file_url = 'file://' + file_url df = load_dataframe(file_url) response_message = ','.join(map(str, df.collect())) LoggingManager().log("Response received" + str(response_message)) LoggingManager().log('Response to client: --|' + str(response_message) + '|--') data = response_message.encode('ascii') transport.write(data) return response_message
def perform_query(query): stmt = Parser().parse(query)[0] l_plan = StatementToPlanConvertor().visit(stmt) p_plan = PlanGenerator().build(l_plan) return PlanExecutor(p_plan).execute_plan()