def reduce_group(self, operator, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
child.key1 = self._child_chain[0].keys
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
self._finalize()
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
child.key1 = self._child_chain[0].keys
self._info.parallelism = child.parallelism
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, types, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:param types: The type of the resulting DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = dict()
child_set = OperatorSet(self._env, child)
child[_Fields.IDENTIFIER] = _Identifier.GROUPREDUCE
child[_Fields.PARENT] = self._info
child[_Fields.OPERATOR] = copy.deepcopy(operator)
child[_Fields.OPERATOR]._combine = False
child[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__)
child[_Fields.TYPES] = types
child[_Fields.COMBINE] = combinable
child[_Fields.COMBINEOP] = operator
child[_Fields.COMBINEOP]._combine = True
child[_Fields.NAME] = "PythonGroupReduce"
self._info[_Fields.CHILDREN].append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, types, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:param types: The type of the resulting DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = copy.deepcopy(operator)
child.operator._combine = False
child.meta = str(inspect.getmodule(operator)) + "|" + str(
operator.__class__.__name__)
child.types = types
child.combine = combinable
child.combineop = operator
child.combineop._combine = True
child.name = "PythonGroupReduce"
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, types, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:param types: The type of the resulting DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = dict()
child_set = OperatorSet(self._env, child)
child[_Fields.IDENTIFIER] = _Identifier.GROUPREDUCE
child[_Fields.PARENT] = self._info
child[_Fields.OPERATOR] = copy.deepcopy(operator)
child[_Fields.OPERATOR]._combine = False
child[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(
operator.__class__.__name__)
child[_Fields.TYPES] = types
child[_Fields.COMBINE] = combinable
child[_Fields.COMBINEOP] = operator
child[_Fields.COMBINEOP]._combine = True
child[_Fields.NAME] = "PythonGroupReduce"
self._info[_Fields.CHILDREN].append(child)
self._env._sets.append(child)
return child_set
def reduce_group(self, operator, types, combinable=False):
"""
Applies a GroupReduce transformation.
The transformation calls a GroupReduceFunction once for each group of the DataSet, or one when applied on a
non-grouped DataSet.
The GroupReduceFunction can iterate over all elements of the DataSet and
emit any number of output elements including none.
:param operator: The GroupReduceFunction that is applied on the DataSet.
:param types: The type of the resulting DataSet.
:return:A GroupReduceOperator that represents the reduced DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child_set = OperatorSet(self._env, child)
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = copy.deepcopy(operator)
child.operator._combine = False
child.meta = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__)
child.types = types
child.combine = combinable
child.combineop = operator
child.combineop._combine = True
child.name = "PythonGroupReduce"
self._info.children.append(child)
self._env._sets.append(child)
return child_set
def _reduce_group(self, operator, combinable=False):
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
return child
def _reduce_group(self, operator, combinable=False):
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = GroupReduceFunction()
operator.reduce = f
child = OperationInfo()
child.identifier = _Identifier.GROUPREDUCE
child.parent = self._info
child.operator = operator
child.types = _createArrayTypeInfo()
child.name = "PythonGroupReduce"
return child
