def reduce_group(self, operator, types, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :param types: The type of the resulting DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f operator._set_grouping_keys(self._child_chain[0][_Fields.KEYS]) operator._set_sort_ops([(x[_Fields.FIELD], x[_Fields.ORDER]) for x in self._child_chain[1:]]) child = dict() child_set = OperatorSet(self._env, child) child[_Fields.IDENTIFIER] = _Identifier.GROUPREDUCE child[_Fields.PARENT] = self._info child[_Fields.OPERATOR] = copy.deepcopy(operator) child[_Fields.OPERATOR]._combine = False child[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__) child[_Fields.TYPES] = types child[_Fields.COMBINE] = combinable child[_Fields.COMBINEOP] = operator child[_Fields.COMBINEOP]._combine = True child[_Fields.NAME] = "PythonGroupReduce" self._info[_Fields.CHILDREN].append(child) self._env._sets.append(child) return child_set
def reduce_group(self, operator, types, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :param types: The type of the resulting DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f operator._set_grouping_keys(self._child_chain[0].keys) operator._set_sort_ops([(x.field, x.order) for x in self._child_chain[1:]]) child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = copy.deepcopy(operator) child.operator._combine = False child.meta = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__) child.types = types child.combine = combinable child.combineop = operator child.combineop._combine = True child.name = "PythonGroupReduce" self._info.children.append(child) self._env._sets.append(child) return child_set
def reduce_group(self, operator, types, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :param types: The type of the resulting DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f operator._set_grouping_keys(self._child_chain[0].keys) child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = operator child.types = types child.name = "PythonGroupReduce" self._info.children.append(child) self._env._sets.append(child) return child_set
def reduce_group(self, operator, types, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :param types: The type of the resulting DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f operator._set_grouping_keys(self._child_chain[0].keys) operator._set_sort_ops([(x.field, x.order) for x in self._child_chain[1:]]) child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = copy.deepcopy(operator) child.operator._combine = False child.meta = str(inspect.getmodule(operator)) + "|" + str( operator.__class__.__name__) child.types = types child.combine = combinable child.combineop = operator child.combineop._combine = True child.name = "PythonGroupReduce" self._info.children.append(child) self._env._sets.append(child) return child_set
def reduce_group(self, operator, types, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :param types: The type of the resulting DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f operator._set_grouping_keys(self._child_chain[0][_Fields.KEYS]) operator._set_sort_ops([(x[_Fields.FIELD], x[_Fields.ORDER]) for x in self._child_chain[1:]]) child = dict() child_set = OperatorSet(self._env, child) child[_Fields.IDENTIFIER] = _Identifier.GROUPREDUCE child[_Fields.PARENT] = self._info child[_Fields.OPERATOR] = copy.deepcopy(operator) child[_Fields.OPERATOR]._combine = False child[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str( operator.__class__.__name__) child[_Fields.TYPES] = types child[_Fields.COMBINE] = combinable child[_Fields.COMBINEOP] = operator child[_Fields.COMBINEOP]._combine = True child[_Fields.NAME] = "PythonGroupReduce" self._info[_Fields.CHILDREN].append(child) self._env._sets.append(child) return child_set