def test_parsing_and_to_dict(self): _property = {'type': 'property', 'name': 'x'} constant = {'type': 'constant', 'value': 1} function = {'type': 'function', 'name': 'add', 'arguments': [deepcopy(_property), deepcopy(constant)]} self.assertEqual(Value.parse(deepcopy(constant)).to_dict(), constant) self.assertEqual(Value.parse(deepcopy(_property)).to_dict(), _property) self.assertEqual(Value.parse(deepcopy(function)).to_dict(), function) kronos = {'type': 'data_access', 'source': 'kronos', 'stream': 'mystream', 'start_time': 100, 'end_time': 200} self.assertEqual(Operator.parse(deepcopy(kronos)).to_dict(), kronos) condition_lt = {'op': 'lt', 'left': deepcopy(_property), 'right': deepcopy(constant)} condition_eq = {'op': 'eq', 'left': deepcopy(function), 'right': deepcopy(_property)} condition_or = {'type': 'or', 'conditions': [deepcopy(condition_lt), deepcopy(condition_eq)]} self.assertEqual(Condition.parse(deepcopy(condition_lt)).to_dict(), condition_lt) self.assertEqual(Condition.parse(deepcopy(condition_eq)).to_dict(), condition_eq) self.assertEqual(Condition.parse(deepcopy(condition_or)).to_dict(), condition_or) avg = {'op': 'avg', 'arguments': [deepcopy(_property)], 'alias': 'myavg'} count = {'op': 'count', 'alias': 'mycount'} self.assertEqual(Aggregator.parse(deepcopy(avg)).to_dict(), avg) self.assertEqual(Aggregator.parse(deepcopy(count)).to_dict(), count) group_by = deepcopy(function) group_by['alias'] = 'mygroup' group_by = [group_by] self.assertEqual(GroupBy.parse(deepcopy(group_by)).to_dict(), group_by) project = {'type': 'project', 'fields': [deepcopy(_property)], 'source': deepcopy(kronos)} _filter = {'type': 'filter', 'condition': condition_lt, 'source': deepcopy(project)} aggregate = {'type': 'aggregate', 'group_by': deepcopy(group_by), 'aggregates': [deepcopy(avg), deepcopy(count)], 'source': deepcopy(_filter)} join = {'type': 'join', 'left': deepcopy(aggregate), 'right': deepcopy(project), 'condition': deepcopy(condition_or)} self.assertEqual(Operator.parse(deepcopy(project)).to_dict(), project) self.assertEqual(Operator.parse(deepcopy(_filter)).to_dict(), _filter) self.assertEqual(Operator.parse(deepcopy(aggregate)).to_dict(), aggregate) self.assertEqual(Operator.parse(deepcopy(join)).to_dict(), join)
def parse(plan): return Operator.parse(plan)
def execute_compute_task(plan): spark_context = get_spark_context() return Operator.parse(plan).get_rdd(spark_context).collect()