def using(self, operator):
"""
Finalizes a CoGroup transformation.
Applies a CoGroupFunction to groups of elements with identical keys.
Each CoGroupFunction call returns an arbitrary number of keys.
:param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
:return:An CoGroupOperator that represents the co-grouped result DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = CoGroupFunction()
operator.co_group = f
new_set = OperatorSet(self._env, self._info)
self._info.key1 = tuple([x for x in range(len(self._info.key1))])
self._info.key2 = tuple([x for x in range(len(self._info.key2))])
operator._keys1 = self._info.key1
operator._keys2 = self._info.key2
self._info.parent.parallelism = self._info.parallelism
self._info.other.parallelism = self._info.parallelism
self._info.operator = operator
self._info.types = _createArrayTypeInfo()
self._info.name = "PythonCoGroup"
self._env._sets.append(self._info)
return new_set
def using(self, operator):
"""
Finalizes a CoGroup transformation.
Applies a CoGroupFunction to groups of elements with identical keys.
Each CoGroupFunction call returns an arbitrary number of keys.
:param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
:return:An CoGroupOperator that represents the co-grouped result DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = CoGroupFunction()
operator.co_group = f
new_set = OperatorSet(self._env, self._info)
self._info.key1 = tuple([x for x in range(len(self._info.key1))])
self._info.key2 = tuple([x for x in range(len(self._info.key2))])
operator._keys1 = self._info.key1
operator._keys2 = self._info.key2
self._info.parent.parallelism = self._info.parallelism
self._info.other.parallelism = self._info.parallelism
self._info.operator = operator
self._info.types = _createArrayTypeInfo()
self._info.name = "PythonCoGroup"
self._env._sets.append(self._info)
return new_set
def using(self, operator, types):
"""
Finalizes a CoGroup transformation.
Applies a CoGroupFunction to groups of elements with identical keys.
Each CoGroupFunction call returns an arbitrary number of keys.
:param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
:param types: The type of the resulting DataSet.
:return:An CoGroupOperator that represents the co-grouped result DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = CoGroupFunction()
operator.co_group = f
new_set = OperatorSet(self._env, self._info)
operator._keys1 = self._info.key1
operator._keys2 = self._info.key2
self._info.operator = operator
self._info.meta = str(inspect.getmodule(operator)) + "|" + str(
operator.__class__.__name__)
self._info.types = types
self._info.name = "PythonCoGroup"
self._env._sets.append(self._info)
return new_set
def using(self, operator, types):
"""
Finalizes a CoGroup transformation.
Applies a CoGroupFunction to groups of elements with identical keys.
Each CoGroupFunction call returns an arbitrary number of keys.
:param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
:param types: The type of the resulting DataSet.
:return:An CoGroupOperator that represents the co-grouped result DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = CoGroupFunction()
operator.co_group = f
tmp1 = self._parent[0]
tmp2 = self._parent[1]
key1 = self._info[_Fields.KEY1]
key2 = self._info[_Fields.KEY2]
use_key_selector = False
if isinstance(key1, (KeySelectorFunction, TYPES.FunctionType)):
if isinstance(key1, KeySelectorFunction):
f1 = key1
else:
f1 = KeySelectorFunction()
f1.get_key = key1
tmp1 = self._parent[0].map(f1, (self._info[_Fields.TYPE1], deduct_output_type(self._parent[0]._info)))
use_key_selector = True
operator._discard_key1 = True
key1 = (0,)
if isinstance(key2, (KeySelectorFunction, TYPES.FunctionType)):
if isinstance(key2, KeySelectorFunction):
f2 = key2
else:
f2 = KeySelectorFunction()
f2.get_key = key2
tmp2 = self._parent[1].map(f2, (self._info[_Fields.TYPE2], deduct_output_type(self._parent[1]._info)))
use_key_selector = True
operator._discard_key2 = True
key2 = (0,)
if use_key_selector:
return tmp1.co_group(tmp2).where(*key1).equal_to(*key2).using(operator, types)
else:
new_set = OperatorSet(self._env, self._info)
operator._keys1 = self._info[_Fields.KEY1]
operator._keys2 = self._info[_Fields.KEY2]
self._info[_Fields.OPERATOR] = operator
self._info[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__)
self._info[_Fields.TYPES] = types
self._info[_Fields.NAME] = "PythonCoGroup"
self._info[_Fields.PARENT][_Fields.CHILDREN].append(self._info)
self._info[_Fields.OTHER][_Fields.CHILDREN].append(self._info)
self._env._sets.append(self._info)
return new_set
def using(self, operator, types):
"""
Finalizes a CoGroup transformation.
Applies a CoGroupFunction to groups of elements with identical keys.
Each CoGroupFunction call returns an arbitrary number of keys.
:param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
:param types: The type of the resulting DataSet.
:return:An CoGroupOperator that represents the co-grouped result DataSet.
"""
if isinstance(operator, TYPES.FunctionType):
f = operator
operator = CoGroupFunction()
operator.co_group = f
new_set = OperatorSet(self._env, self._info)
operator._keys1 = self._info[_Fields.KEY1]
operator._keys2 = self._info[_Fields.KEY2]
self._info[_Fields.OPERATOR] = operator
self._info[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__)
self._info[_Fields.TYPES] = types
self._info[_Fields.NAME] = "PythonCoGroup"
self._env._sets.append(self._info)
return new_set
