def test_parsing_and_to_dict(self):
_property = {'type': 'property', 'name': 'x'}
constant = {'type': 'constant', 'value': 1}
function = {'type': 'function', 'name': 'add',
'arguments': [deepcopy(_property), deepcopy(constant)]}
self.assertEqual(Value.parse(deepcopy(constant)).to_dict(), constant)
self.assertEqual(Value.parse(deepcopy(_property)).to_dict(), _property)
self.assertEqual(Value.parse(deepcopy(function)).to_dict(), function)
kronos = {'type': 'data_access', 'source': 'kronos', 'stream': 'mystream',
'start_time': 100, 'end_time': 200}
self.assertEqual(Operator.parse(deepcopy(kronos)).to_dict(), kronos)
condition_lt = {'op': 'lt', 'left': deepcopy(_property),
'right': deepcopy(constant)}
condition_eq = {'op': 'eq', 'left': deepcopy(function),
'right': deepcopy(_property)}
condition_or = {'type': 'or', 'conditions': [deepcopy(condition_lt),
deepcopy(condition_eq)]}
self.assertEqual(Condition.parse(deepcopy(condition_lt)).to_dict(),
condition_lt)
self.assertEqual(Condition.parse(deepcopy(condition_eq)).to_dict(),
condition_eq)
self.assertEqual(Condition.parse(deepcopy(condition_or)).to_dict(),
condition_or)
avg = {'op': 'avg', 'arguments': [deepcopy(_property)], 'alias': 'myavg'}
count = {'op': 'count', 'alias': 'mycount'}
self.assertEqual(Aggregator.parse(deepcopy(avg)).to_dict(), avg)
self.assertEqual(Aggregator.parse(deepcopy(count)).to_dict(), count)
group_by = deepcopy(function)
group_by['alias'] = 'mygroup'
group_by = [group_by]
self.assertEqual(GroupBy.parse(deepcopy(group_by)).to_dict(), group_by)
project = {'type': 'project', 'fields': [deepcopy(_property)],
'source': deepcopy(kronos)}
_filter = {'type': 'filter', 'condition': condition_lt,
'source': deepcopy(project)}
aggregate = {'type': 'aggregate',
'group_by': deepcopy(group_by),
'aggregates': [deepcopy(avg), deepcopy(count)],
'source': deepcopy(_filter)}
join = {'type': 'join', 'left': deepcopy(aggregate),
'right': deepcopy(project), 'condition': deepcopy(condition_or)}
self.assertEqual(Operator.parse(deepcopy(project)).to_dict(), project)
self.assertEqual(Operator.parse(deepcopy(_filter)).to_dict(), _filter)
self.assertEqual(Operator.parse(deepcopy(aggregate)).to_dict(), aggregate)
self.assertEqual(Operator.parse(deepcopy(join)).to_dict(), join)
def parse(plan):
return Operator.parse(plan)
def execute_compute_task(plan): spark_context = get_spark_context() return Operator.parse(plan).get_rdd(spark_context).collect()
def parse(plan): return Operator.parse(plan)