def reduce(self, operator):
"""
Applies a Reduce transformation on a non-grouped DataSet.
The transformation consecutively calls a ReduceFunction until only a single element remains which is the result
of the transformation. A ReduceFunction combines two elements into one new element of the same type.
:param operator:The ReduceFunction that is applied on the DataSet.
:return:A ReduceOperator that represents the reduced DataSet.
"""
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = ReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.REDUCE
child.parent = self._info
child.operator = operator
child.name = "PythonReduce"
child.types = _createArrayTypeInfo()
child.key1 = self._child_chain[0].keys
self._info.parallelism = child.parallelism
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def reduce(self, operator):
"""
Applies a Reduce transformation on a non-grouped DataSet.
The transformation consecutively calls a ReduceFunction until only a single element remains which is the result
of the transformation. A ReduceFunction combines two elements into one new element of the same type.
:param operator:The ReduceFunction that is applied on the DataSet.
:return:A ReduceOperator that represents the reduced DataSet.
"""
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = ReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.REDUCE
child.parent = self._info
child.operator = operator
child.name = "PythonReduce"
child.types = _createArrayTypeInfo()
child.key1 = self._child_chain[0].keys
self._info.parallelism = child.parallelism
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
child.key1 = self._child_chain[0].keys
self._info.parallelism = child.parallelism
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
child.key1 = self._child_chain[0].keys
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def _reduce_group(self, operator, combinable=False):
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
child.key1 = self._child_chain[0].keys
return child
def _reduce_group(self, operator, combinable=False):
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
child.key1 = self._child_chain[0].keys
return child