Example #1
0
  def test_parsing_and_to_dict(self):
    _property = {'type': 'property', 'name': 'x'}
    constant = {'type': 'constant', 'value': 1}
    function = {'type': 'function', 'name': 'add',
                'arguments': [deepcopy(_property), deepcopy(constant)]}
    self.assertEqual(Value.parse(deepcopy(constant)).to_dict(), constant)
    self.assertEqual(Value.parse(deepcopy(_property)).to_dict(), _property)
    self.assertEqual(Value.parse(deepcopy(function)).to_dict(), function)

    kronos = {'type': 'data_access', 'source': 'kronos', 'stream': 'mystream',
              'start_time': 100, 'end_time': 200}
    self.assertEqual(Operator.parse(deepcopy(kronos)).to_dict(), kronos)

    condition_lt = {'op': 'lt', 'left': deepcopy(_property),
                    'right': deepcopy(constant)}
    condition_eq = {'op': 'eq', 'left': deepcopy(function),
                    'right': deepcopy(_property)}
    condition_or = {'type': 'or', 'conditions': [deepcopy(condition_lt),
                                                 deepcopy(condition_eq)]}
    self.assertEqual(Condition.parse(deepcopy(condition_lt)).to_dict(),
                     condition_lt)
    self.assertEqual(Condition.parse(deepcopy(condition_eq)).to_dict(),
                     condition_eq)
    self.assertEqual(Condition.parse(deepcopy(condition_or)).to_dict(),
                     condition_or)

    avg = {'op': 'avg', 'arguments': [deepcopy(_property)], 'alias': 'myavg'}
    count = {'op': 'count', 'alias': 'mycount'}
    self.assertEqual(Aggregator.parse(deepcopy(avg)).to_dict(), avg)
    self.assertEqual(Aggregator.parse(deepcopy(count)).to_dict(), count)

    group_by = deepcopy(function)
    group_by['alias'] = 'mygroup'
    group_by = [group_by]
    self.assertEqual(GroupBy.parse(deepcopy(group_by)).to_dict(), group_by)

    project = {'type': 'project', 'fields': [deepcopy(_property)],
               'source': deepcopy(kronos)}
    _filter = {'type': 'filter', 'condition': condition_lt,
               'source': deepcopy(project)}
    aggregate = {'type': 'aggregate',
                 'group_by': deepcopy(group_by),
                 'aggregates': [deepcopy(avg), deepcopy(count)],
                 'source': deepcopy(_filter)}
    join = {'type': 'join', 'left': deepcopy(aggregate),
            'right': deepcopy(project), 'condition': deepcopy(condition_or)}
    self.assertEqual(Operator.parse(deepcopy(project)).to_dict(), project)
    self.assertEqual(Operator.parse(deepcopy(_filter)).to_dict(), _filter)
    self.assertEqual(Operator.parse(deepcopy(aggregate)).to_dict(), aggregate)
    self.assertEqual(Operator.parse(deepcopy(join)).to_dict(), join)
Example #2
0
def parse(plan):
    return Operator.parse(plan)
Example #3
0
def execute_compute_task(plan):
  spark_context = get_spark_context()
  return Operator.parse(plan).get_rdd(spark_context).collect()
Example #4
0
def parse(plan):
  return Operator.parse(plan)