def reduce(self, operator): """ Applies a Reduce transformation on a non-grouped DataSet. The transformation consecutively calls a ReduceFunction until only a single element remains which is the result of the transformation. A ReduceFunction combines two elements into one new element of the same type. :param operator:The ReduceFunction that is applied on the DataSet. :return:A ReduceOperator that represents the reduced DataSet. """ self._finalize() if isinstance(operator, TYPES.FunctionType): f = operator operator = ReduceFunction() operator.reduce = f child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.REDUCE child.parent = self._info child.operator = operator child.name = "PythonReduce" child.types = _createArrayTypeInfo() child.key1 = self._child_chain[0].keys self._info.parallelism = child.parallelism self._info.children.append(child) self._env._sets.append(child) return child_set
def reduce(self, operator): """ Applies a Reduce transformation on a non-grouped DataSet. The transformation consecutively calls a ReduceFunction until only a single element remains which is the result of the transformation. A ReduceFunction combines two elements into one new element of the same type. :param operator:The ReduceFunction that is applied on the DataSet. :return:A ReduceOperator that represents the reduced DataSet. """ self._finalize() if isinstance(operator, TYPES.FunctionType): f = operator operator = ReduceFunction() operator.reduce = f child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.REDUCE child.parent = self._info child.operator = operator child.name = "PythonReduce" child.types = _createArrayTypeInfo() child.key1 = self._child_chain[0].keys self._info.parallelism = child.parallelism self._info.children.append(child) self._env._sets.append(child) return child_set
def reduce_group(self, operator, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ self._finalize() if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = operator child.types = _createArrayTypeInfo() child.name = "PythonGroupReduce" child.key1 = self._child_chain[0].keys self._info.parallelism = child.parallelism self._info.children.append(child) self._env._sets.append(child) return child_set
def reduce_group(self, operator, combinable=False): """ Applies a GroupReduce transformation. The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a non-grouped DataSet. The GroupReduceFunction can iterate over all elements of the DataSet and emit any number of output elements including none. :param operator: The GroupReduceFunction that is applied on the DataSet. :return:A GroupReduceOperator that represents the reduced DataSet. """ self._finalize() if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = operator child.types = _createArrayTypeInfo() child.name = "PythonGroupReduce" child.key1 = self._child_chain[0].keys self._info.children.append(child) self._env._sets.append(child) return child_set
def _reduce_group(self, operator, combinable=False): self._finalize() if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f child = OperationInfo() child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = operator child.types = _createArrayTypeInfo() child.name = "PythonGroupReduce" child.key1 = self._child_chain[0].keys return child
def _reduce_group(self, operator, combinable=False): self._finalize() if isinstance(operator, TYPES.FunctionType): f = operator operator = GroupReduceFunction() operator.reduce = f child = OperationInfo() child.identifier = _Identifier.GROUPREDUCE child.parent = self._info child.operator = operator child.types = _createArrayTypeInfo() child.name = "PythonGroupReduce" child.key1 = self._child_chain[0].keys return child