def test_visit_table_ref_populates_column_mapping(self, mock, mock_lget):
converter = StatementToPlanConvertor()
converter._populate_column_map = MagicMock()
table_ref = TableRef(TableInfo("test"))
converter.visit_table_ref(table_ref)
converter._populate_column_map.assert_called_with(mock.return_value)
def test_should_visit_select_if_nested_query(self, mock_p, mock_c, mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT id FROM (SELECT data, id FROM video \
WHERE data > 2) WHERE id>3;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
plans = [LogicalProject([TupleValueExpression('id')])]
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('id'),
ConstantValueExpression(3))))
plans.append(LogicalQueryDerivedGet())
plans.append(
LogicalProject(
[TupleValueExpression('data'),
TupleValueExpression('id')]))
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('data'),
ConstantValueExpression(2))))
plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expected_plan = None
for plan in reversed(plans):
if expected_plan:
plan.append_child(expected_plan)
expected_plan = plan
self.assertEqual(expected_plan, actual_plan)
wrong_plan = plans[0]
for plan in plans[1:]:
wrong_plan.append_child(plan)
self.assertNotEqual(wrong_plan, actual_plan)
def test_should_insert_row_in_table(self):
dummy_info = TableInfo('MyVideo')
dummy_table = TableRef(dummy_info)
columns = [
DataFrameColumn('Frame_ID', ColumnType.INTEGER),
DataFrameColumn('Frame_Path', ColumnType.TEXT, array_dimensions=50)
]
plan_node = CreatePlan(dummy_table, columns, False)
createExec = CreateExecutor(plan_node)
url = createExec.exec()
parser = Parser()
insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path)
VALUES (1, '/mnt/frames/1.png');
"""
eva_statement_list = parser.parse(insert_query)
insert_stmt = eva_statement_list[0]
convertor = StatementToPlanConvertor()
convertor.visit(insert_stmt)
logical_plan_node = convertor.plan
print("logical", logical_plan_node)
phy_plan_node = InsertPlan(logical_plan_node.video_catalog_id,
logical_plan_node.column_list,
logical_plan_node.value_list)
insertExec = InsertExecutor(phy_plan_node)
insertExec.exec()
# test if we have a added the in our storage
df = load_dataframe(url)
self.assertEqual(df.collect()[0][0], 1)
self.assertEqual(df.collect()[0][1], "'/mnt/frames/1.png'")
def test_visit_table_ref_should_create_logical_get_opr(
self, mock, mock_lget):
converter = StatementToPlanConvertor()
table_ref = TableRef(TableInfo("test"))
converter.visit_table_ref(table_ref)
mock.assert_called_with(table_ref.table_info)
mock_lget.assert_called_with(table_ref, mock.return_value)
self.assertEqual(mock_lget.return_value, converter._plan)
def test_visit_should_call_create_udf(self):
stmt = MagicMock(spec=CreateUDFStatement)
convertor = StatementToPlanConvertor()
mock = MagicMock()
convertor.visit_create_udf = mock
actual = convertor.visit(stmt)
mock.assert_called_once()
mock.assert_called_with(stmt)
def test_visit_should_call_create(self):
stmt = MagicMock(spec=CreateTableStatement)
convertor = StatementToPlanConvertor()
mock = MagicMock()
convertor.visit_create = mock
convertor.visit(stmt)
mock.assert_called_once()
mock.assert_called_with(stmt)
def test_visit_should_call_load_data(self):
stmt = MagicMock(spec=LoadDataStatement)
convertor = StatementToPlanConvertor()
mock = MagicMock()
convertor.visit_load_data = mock
convertor.visit(stmt)
mock.assert_called_once()
mock.assert_called_with(stmt)
def test_visit_should_call_insert(self):
stmt = MagicMock(spec=InsertTableStatement)
convertor = StatementToPlanConvertor()
mock = MagicMock()
convertor.visit_insert = mock
actual = convertor.visit(stmt)
mock.assert_called_once()
mock.assert_called_with(stmt)
def test_visit_select_predicate_should_add_logical_filter(
self, mock, mock_lfilter):
converter = StatementToPlanConvertor()
select_predicate = MagicMock()
converter._visit_select_predicate(select_predicate)
mock_lfilter.assert_called_with(select_predicate)
mock.assert_called_with(select_predicate, converter._column_map)
mock_lfilter.return_value.append_child.assert_called()
self.assertEqual(mock_lfilter.return_value, converter._plan)
def test_visit_projection_should_add_logical_predicate(
self, mock, mock_lproject):
converter = StatementToPlanConvertor()
projects = MagicMock()
converter._visit_projection(projects)
mock_lproject.assert_called_with(projects)
mock.assert_called_with(projects, converter._column_map)
mock_lproject.return_value.append_child.assert_called()
self.assertEqual(mock_lproject.return_value, converter._plan)
def test_populate_column_map_should_populate_correctly(self):
converter = StatementToPlanConvertor()
dataset = MagicMock()
dataset.columns = [MagicMock() for i in range(5)]
expected = {}
for i, column in enumerate(dataset.columns):
column.name = "NAME" + str(i)
expected[column.name.lower()] = column
converter._populate_column_map(dataset)
self.assertEqual(converter._column_map, expected)
def handle_request(transport, request_message):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
LoggingManager().log('Receive request: --|' + str(request_message) + '|--')
output_batch = None
response = None
try:
stmt = Parser().parse(request_message)[0]
l_plan = StatementToPlanConvertor().visit(stmt)
p_plan = PlanGenerator().build(l_plan)
output_batch = PlanExecutor(p_plan).execute_plan()
except Exception as e:
LoggingManager().log(e, LoggingLevel.WARNING)
response = Response(status=ResponseStatus.FAIL, batch=None)
if response is None:
response = Response(status=ResponseStatus.SUCCESS, batch=output_batch)
responseData = response.to_json()
# Send data length, because response can be very large
data = (str(len(responseData)) + '|' + responseData).encode('ascii')
LoggingManager().log('Response to client: --|' + str(response) + '|--\n' +
'Length: ' + str(len(responseData)))
transport.write(data)
return response
def test_visit_create_udf(self, mock, l_create_udf_mock):
convertor = StatementToPlanConvertor()
stmt = MagicMock()
stmt.name = 'name'
stmt.if_not_exists = True
stmt.inputs = ['inp']
stmt.outputs = ['out']
stmt.impl_path = 'tmp.py'
stmt.udf_type = 'classification'
mock.side_effect = ['inp', 'out']
convertor.visit_create_udf(stmt)
mock.assert_any_call(stmt.inputs, True)
mock.assert_any_call(stmt.outputs, False)
l_create_udf_mock.assert_called_once()
l_create_udf_mock.assert_called_with(stmt.name, stmt.if_not_exists,
'inp', 'out', stmt.impl_path,
stmt.udf_type)
def execute_query(query) -> Iterator[Batch]:
"""
Execute the query and return a result generator.
"""
stmt = Parser().parse(query)[0]
l_plan = StatementToPlanConvertor().visit(stmt)
p_plan = PlanGenerator().build(l_plan)
return PlanExecutor(p_plan).execute_plan()
def test_visit_load_data_when_bind_returns_None(self, mock_create,
mock_bind, mock_load):
mock_bind.return_value = None
table_ref = TableRef(TableInfo("test"))
stmt = MagicMock(table=table_ref, path='path')
StatementToPlanConvertor().visit_load_data(stmt)
mock_create.assert_called_once_with(table_ref.table_info.table_name)
mock_bind.assert_called_with(table_ref.table_info)
mock_load.assert_called_with(mock_create.return_value, 'path')
def test_visit_select_orderby(self, mock_p, mock_c, mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT data, id FROM video \
WHERE data > 2 ORDER BY data, id DESC;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
plans = []
plans.append(
LogicalOrderBy([
(TupleValueExpression('data'), ParserOrderBySortType.ASC),
(TupleValueExpression('id'), ParserOrderBySortType.DESC)
]))
plans.append(
LogicalProject(
[TupleValueExpression('data'),
TupleValueExpression('id')]))
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('data'),
ConstantValueExpression(2))))
plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expected_plan = None
for plan in reversed(plans):
if expected_plan:
plan.append_child(expected_plan)
expected_plan = plan
self.assertEqual(expected_plan, actual_plan)
wrong_plan = plans[0]
for plan in plans[1:]:
wrong_plan.append_child(plan)
self.assertNotEqual(wrong_plan, actual_plan)
def test_should_visit_select_union_if_union_query(self, mock_p, mock_c,
mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT id FROM video WHERE id>3
UNION ALL
SELECT id FROM video WHERE id<=3;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
left_plans = [LogicalProject([TupleValueExpression('id')])]
left_plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('id'),
ConstantValueExpression(3))))
left_plans.append(LogicalGet(TableRef(TableInfo('video')), m))
def reverse_plan(plans):
return_plan = None
for plan in reversed(plans):
if return_plan:
plan.append_child(return_plan)
return_plan = plan
return return_plan
expect_left_plan = reverse_plan(left_plans)
right_plans = [LogicalProject([TupleValueExpression('id')])]
right_plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_LEQ,
TupleValueExpression('id'),
ConstantValueExpression(3))))
right_plans.append(LogicalGet(TableRef(TableInfo('video')), m))
expect_right_plan = reverse_plan(right_plans)
expected_plan = LogicalUnion(True)
expected_plan.append_child(expect_right_plan)
expected_plan.append_child(expect_left_plan)
self.assertEqual(expected_plan, actual_plan)
def test_visit_select_should_not_call_visits_for_null_values(self):
converter = StatementToPlanConvertor()
converter.visit_table_ref = MagicMock()
converter._visit_projection = MagicMock()
converter._visit_select_predicate = MagicMock()
converter._visit_union = MagicMock()
statement = SelectStatement()
converter.visit_select(statement)
converter.visit_table_ref.assert_not_called()
converter._visit_projection.assert_not_called()
converter._visit_select_predicate.assert_not_called()
def test_visit_select_should_call_appropriate_visit_methods(self):
converter = StatementToPlanConvertor()
converter.visit_table_ref = MagicMock()
converter._visit_projection = MagicMock()
converter._visit_select_predicate = MagicMock()
converter._visit_union = MagicMock()
statement = MagicMock()
statement.from_table = MagicMock(spec=TableRef)
converter.visit_select(statement)
converter.visit_table_ref.assert_called_with(statement.from_table)
converter._visit_projection.assert_called_with(statement.target_list)
converter._visit_select_predicate.assert_called_with(
statement.where_clause)
def handle_request(transport, query):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
parser = Parser()
eva_statement = parser.parse(query)
insert_statement = eva_statement[0]
LoggingManager().log("Result from the parser: " + str(insert_statement))
convertor = StatementToPlanConvertor()
convertor.visit(insert_statement)
convertor.plan
eva_statement = parser.parse(query)
response_message = ""
for i in range(len(eva_statement)):
LoggingManager().log("Parse Tree: " + str(eva_statement[i]))
logical_plan = StatementToPlanConvertor().visit(eva_statement[i])
physical_plan = PlanGenerator().build(logical_plan)
df = PlanExecutor(physical_plan).execute_plan()
statement_response_message = ','.join(map(str, df.collect()))
response_message += statement_response_message
LoggingManager().log('Response to client: --|' +
str(response_message) +
'|--')
data = response_message.encode('ascii')
transport.write(data)
return response_message
def test_visit_select_sample(self, mock_p, mock_c, mock_d):
m = MagicMock()
mock_p.return_value = mock_c.return_value = mock_d.return_value = m
stmt = Parser().parse(""" SELECT data, id FROM video SAMPLE 2 \
WHERE id > 2 LIMIT 3;""")[0]
converter = StatementToPlanConvertor()
actual_plan = converter.visit(stmt)
plans = []
plans.append(LogicalLimit(ConstantValueExpression(3)))
plans.append(
LogicalProject(
[TupleValueExpression('data'),
TupleValueExpression('id')]))
plans.append(
LogicalFilter(
ComparisonExpression(ExpressionType.COMPARE_GREATER,
TupleValueExpression('id'),
ConstantValueExpression(2))))
plans.append(LogicalSample(ConstantValueExpression(2)))
plans.append(
LogicalGet(
TableRef(TableInfo('video'), ConstantValueExpression(2)), m))
expected_plan = None
for plan in reversed(plans):
if expected_plan:
plan.append_child(expected_plan)
expected_plan = plan
self.assertEqual(expected_plan, actual_plan)
def handle_request(transport, query):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
parser = Parser()
eva_statement = parser.parse(query)
for i in range(len(eva_statement)):
LoggingManager().log("Result from the parser: " +
str(eva_statement[i]))
logical_plan = StatementToPlanConvertor().visit(eva_statement[i])
physical_plan = PlanGenerator().build(logical_plan)
PlanExecutor(physical_plan).execute_plan()
table_name = 'MyVideo'
file_url = os.path.join(tempfile.gettempdir(), table_name)
file_url = 'file://' + file_url
df = load_dataframe(file_url)
response_message = ','.join(map(str, df.collect()))
LoggingManager().log("Response received" + str(response_message))
LoggingManager().log('Response to client: --|' + str(response_message) +
'|--')
data = response_message.encode('ascii')
transport.write(data)
return response_message
def perform_query(query):
stmt = Parser().parse(query)[0]
l_plan = StatementToPlanConvertor().visit(stmt)
p_plan = PlanGenerator().build(l_plan)
return PlanExecutor(p_plan).execute_plan()
