def test_should_return_the_new_path_after_execution(self, mock_class):
class_instatnce = mock_class.return_value
dummy_expr = type('dummy_expr', (),
{"evaluate": lambda x=None: [True, False, True]})
# Build plan tree
video = DataFrameMetadata("dataset", "dummy.avi")
batch_1 = Batch(pd.DataFrame({'data': [1, 2, 3]}))
batch_2 = Batch(pd.DataFrame({'data': [4, 5, 6]}))
class_instatnce.load.return_value = map(lambda x: x,
[batch_1, batch_2])
storage_plan = StoragePlan(video)
seq_scan = SeqScanPlan(predicate=dummy_expr, column_ids=[])
seq_scan.append_child(storage_plan)
# Execute the plan
executor = PlanExecutor(seq_scan)
actual = executor.execute_plan()
expected = batch_1[::2] + batch_2[::2]
mock_class.assert_called_once()
self.assertEqual(expected, actual)
def test_execute_plan_for_create_insert_load_plans(self, mock_batch,
mock_clean, mock_build):
mock_batch.return_value = []
# CreateExecutor
tree = MagicMock(node=CreatePlan(None, [], False))
mock_build.return_value = tree
actual = PlanExecutor(None).execute_plan()
tree.exec.assert_called_once()
mock_batch.assert_called_once()
assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame())
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
self.assertEqual(actual, [])
# InsertExecutor
mock_batch.reset_mock()
mock_build.reset_mock()
mock_clean.reset_mock()
tree = MagicMock(node=InsertPlan(0, [], []))
mock_build.return_value = tree
actual = PlanExecutor(None).execute_plan()
tree.exec.assert_called_once()
mock_batch.assert_called_once()
assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame())
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
self.assertEqual(actual, [])
# CreateUDFExecutor
mock_batch.reset_mock()
mock_build.reset_mock()
mock_clean.reset_mock()
tree = MagicMock(node=CreateUDFPlan(None, False, [], [], None))
mock_build.return_value = tree
actual = PlanExecutor(None).execute_plan()
tree.exec.assert_called_once()
mock_batch.assert_called_once()
assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame())
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
self.assertEqual(actual, [])
# LoadDataExecutor
mock_batch.reset_mock()
mock_build.reset_mock()
mock_clean.reset_mock()
tree = MagicMock(node=LoadDataPlan(None, None))
mock_build.return_value = tree
actual = PlanExecutor(None).execute_plan()
tree.exec.assert_called_once()
mock_batch.assert_called_once()
assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame())
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
self.assertEqual(actual, [])
def handle_request(transport, request_message):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
LoggingManager().log('Receive request: --|' + str(request_message) + '|--')
output_batch = None
response = None
try:
stmt = Parser().parse(request_message)[0]
l_plan = StatementToPlanConvertor().visit(stmt)
p_plan = PlanGenerator().build(l_plan)
output_batch = PlanExecutor(p_plan).execute_plan()
except Exception as e:
LoggingManager().log(e, LoggingLevel.WARNING)
response = Response(status=ResponseStatus.FAIL, batch=None)
if response is None:
response = Response(status=ResponseStatus.SUCCESS, batch=output_batch)
responseData = response.to_json()
# Send data length, because response can be very large
data = (str(len(responseData)) + '|' + responseData).encode('ascii')
LoggingManager().log('Response to client: --|' + str(response) + '|--\n' +
'Length: ' + str(len(responseData)))
transport.write(data)
return response
def execute_query(query) -> Iterator[Batch]:
"""
Execute the query and return a result generator.
"""
stmt = Parser().parse(query)[0]
l_plan = StatementToPlanConvertor().visit(stmt)
p_plan = PlanGenerator().build(l_plan)
return PlanExecutor(p_plan).execute_plan()
def test_build_execution_tree_should_create_correct_exec_node(self):
# SequentialScanExecutor
plan = SeqScanPlan(MagicMock(), [])
executor = PlanExecutor(plan)._build_execution_tree(plan)
self.assertIsInstance(executor, SequentialScanExecutor)
# PPExecutor
plan = PPScanPlan(MagicMock())
executor = PlanExecutor(plan)._build_execution_tree(plan)
self.assertIsInstance(executor, PPExecutor)
# CreateExecutor
plan = CreatePlan(MagicMock(), [], False)
executor = PlanExecutor(plan)._build_execution_tree(plan)
self.assertIsInstance(executor, CreateExecutor)
# InsertExecutor
plan = InsertPlan(0, [], [])
executor = PlanExecutor(plan)._build_execution_tree(plan)
self.assertIsInstance(executor, InsertExecutor)
# CreateUDFExecutor
plan = CreateUDFPlan('test', False, [], [], MagicMock(), None)
executor = PlanExecutor(plan)._build_execution_tree(plan)
self.assertIsInstance(executor, CreateUDFExecutor)
# LoadDataExecutor
plan = LoadDataPlan(MagicMock(), MagicMock())
executor = PlanExecutor(plan)._build_execution_tree(plan)
self.assertIsInstance(executor, LoadDataExecutor)
def handle_request(transport, query):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
parser = Parser()
eva_statement = parser.parse(query)
insert_statement = eva_statement[0]
LoggingManager().log("Result from the parser: " + str(insert_statement))
convertor = StatementToPlanConvertor()
convertor.visit(insert_statement)
convertor.plan
eva_statement = parser.parse(query)
response_message = ""
for i in range(len(eva_statement)):
LoggingManager().log("Parse Tree: " + str(eva_statement[i]))
logical_plan = StatementToPlanConvertor().visit(eva_statement[i])
physical_plan = PlanGenerator().build(logical_plan)
df = PlanExecutor(physical_plan).execute_plan()
statement_response_message = ','.join(map(str, df.collect()))
response_message += statement_response_message
LoggingManager().log('Response to client: --|' +
str(response_message) +
'|--')
data = response_message.encode('ascii')
transport.write(data)
return response_message
def test_execute_plan_for_pp_scan_plan(self, mock_clean, mock_build):
# PPExecutor
tree = MagicMock(node=PPScanPlan(None))
tree.exec.return_value = [
Batch(pd.DataFrame([1])),
Batch(pd.DataFrame([2])),
Batch(pd.DataFrame([3]))
]
mock_build.return_value = tree
actual = PlanExecutor(None).execute_plan()
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
tree.exec.assert_called_once()
self.assertEqual(actual, Batch(pd.DataFrame([[1], [2], [3]])))
def test_execute_plan_for_pp_scan_plan(self, mock_batch, mock_clean,
mock_build):
# PPExecutor
mock_batch.return_value = []
tree = MagicMock(node=PPScanPlan(None))
tree.exec.return_value = [[1], [2], [3]]
mock_build.return_value = tree
actual = PlanExecutor(None).execute_plan()
mock_batch.assert_called_once()
assert_frame_equal(mock_batch.call_args[0][0], pd.DataFrame())
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
tree.exec.assert_called_once()
self.assertEqual(actual, [1, 2, 3])
def test_execute_plan_for_seq_scan_plan(self, mock_clean, mock_build):
batch_list = [
Batch(pd.DataFrame([1])),
Batch(pd.DataFrame([2])),
Batch(pd.DataFrame([3]))
]
# SequentialScanExecutor
tree = MagicMock(node=SeqScanPlan(None, []))
tree.exec.return_value = batch_list
mock_build.return_value = tree
actual = list(PlanExecutor(None).execute_plan())
mock_build.assert_called_once_with(None)
mock_clean.assert_called_once()
tree.exec.assert_called_once()
self.assertEqual(actual, batch_list)
def test_tree_structure_for_build_execution_tree(self):
"""
Build an Abastract Plan with nodes:
ß root
/ | \
c1 c2 c3
/
c1_1
"""
predicate = None
root_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[])
child_1_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[])
child_2_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[])
child_3_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[])
child_1_1_abs_plan = SeqScanPlan(predicate=predicate, column_ids=[])
root_abs_plan.append_child(child_1_abs_plan)
root_abs_plan.append_child(child_2_abs_plan)
root_abs_plan.append_child(child_3_abs_plan)
child_1_abs_plan.append_child(child_1_1_abs_plan)
'''Build Execution Tree and check the nodes
are of the same type'''
root_abs_executor = PlanExecutor(
plan=root_abs_plan)._build_execution_tree(plan=root_abs_plan)
# Root Nodes
self.assertEqual(root_abs_plan.node_type,
root_abs_executor._node.node_type)
# Children of Root
for child_abs, child_exec in zip(root_abs_plan.children,
root_abs_executor.children):
self.assertEqual(child_abs.node_type, child_exec._node.node_type)
# Grand Children of Root
for gc_abs, gc_exec in zip(child_abs.children,
child_exec.children):
self.assertEqual(gc_abs.node_type, gc_exec._node.node_type)
def handle_request(transport, query):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
parser = Parser()
eva_statement = parser.parse(query)
for i in range(len(eva_statement)):
LoggingManager().log("Result from the parser: " +
str(eva_statement[i]))
logical_plan = StatementToPlanConvertor().visit(eva_statement[i])
physical_plan = PlanGenerator().build(logical_plan)
PlanExecutor(physical_plan).execute_plan()
table_name = 'MyVideo'
file_url = os.path.join(tempfile.gettempdir(), table_name)
file_url = 'file://' + file_url
df = load_dataframe(file_url)
response_message = ','.join(map(str, df.collect()))
LoggingManager().log("Response received" + str(response_message))
LoggingManager().log('Response to client: --|' + str(response_message) +
'|--')
data = response_message.encode('ascii')
transport.write(data)
return response_message
def perform_query(query):
stmt = Parser().parse(query)[0]
l_plan = StatementToPlanConvertor().visit(stmt)
p_plan = PlanGenerator().build(l_plan)
return PlanExecutor(p_plan).execute_plan()