def using(self, operator): """ Finalizes a CoGroup transformation. Applies a CoGroupFunction to groups of elements with identical keys. Each CoGroupFunction call returns an arbitrary number of keys. :param operator: The CoGroupFunction that is called for all groups of elements with identical keys. :return:An CoGroupOperator that represents the co-grouped result DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = CoGroupFunction() operator.co_group = f new_set = OperatorSet(self._env, self._info) self._info.key1 = tuple([x for x in range(len(self._info.key1))]) self._info.key2 = tuple([x for x in range(len(self._info.key2))]) operator._keys1 = self._info.key1 operator._keys2 = self._info.key2 self._info.parent.parallelism = self._info.parallelism self._info.other.parallelism = self._info.parallelism self._info.operator = operator self._info.types = _createArrayTypeInfo() self._info.name = "PythonCoGroup" self._env._sets.append(self._info) return new_set
def using(self, operator, types): """ Finalizes a CoGroup transformation. Applies a CoGroupFunction to groups of elements with identical keys. Each CoGroupFunction call returns an arbitrary number of keys. :param operator: The CoGroupFunction that is called for all groups of elements with identical keys. :param types: The type of the resulting DataSet. :return:An CoGroupOperator that represents the co-grouped result DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = CoGroupFunction() operator.co_group = f new_set = OperatorSet(self._env, self._info) operator._keys1 = self._info.key1 operator._keys2 = self._info.key2 self._info.operator = operator self._info.meta = str(inspect.getmodule(operator)) + "|" + str( operator.__class__.__name__) self._info.types = types self._info.name = "PythonCoGroup" self._env._sets.append(self._info) return new_set
def using(self, operator, types): """ Finalizes a CoGroup transformation. Applies a CoGroupFunction to groups of elements with identical keys. Each CoGroupFunction call returns an arbitrary number of keys. :param operator: The CoGroupFunction that is called for all groups of elements with identical keys. :param types: The type of the resulting DataSet. :return:An CoGroupOperator that represents the co-grouped result DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = CoGroupFunction() operator.co_group = f tmp1 = self._parent[0] tmp2 = self._parent[1] key1 = self._info[_Fields.KEY1] key2 = self._info[_Fields.KEY2] use_key_selector = False if isinstance(key1, (KeySelectorFunction, TYPES.FunctionType)): if isinstance(key1, KeySelectorFunction): f1 = key1 else: f1 = KeySelectorFunction() f1.get_key = key1 tmp1 = self._parent[0].map(f1, (self._info[_Fields.TYPE1], deduct_output_type(self._parent[0]._info))) use_key_selector = True operator._discard_key1 = True key1 = (0,) if isinstance(key2, (KeySelectorFunction, TYPES.FunctionType)): if isinstance(key2, KeySelectorFunction): f2 = key2 else: f2 = KeySelectorFunction() f2.get_key = key2 tmp2 = self._parent[1].map(f2, (self._info[_Fields.TYPE2], deduct_output_type(self._parent[1]._info))) use_key_selector = True operator._discard_key2 = True key2 = (0,) if use_key_selector: return tmp1.co_group(tmp2).where(*key1).equal_to(*key2).using(operator, types) else: new_set = OperatorSet(self._env, self._info) operator._keys1 = self._info[_Fields.KEY1] operator._keys2 = self._info[_Fields.KEY2] self._info[_Fields.OPERATOR] = operator self._info[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__) self._info[_Fields.TYPES] = types self._info[_Fields.NAME] = "PythonCoGroup" self._info[_Fields.PARENT][_Fields.CHILDREN].append(self._info) self._info[_Fields.OTHER][_Fields.CHILDREN].append(self._info) self._env._sets.append(self._info) return new_set
def using(self, operator, types): """ Finalizes a CoGroup transformation. Applies a CoGroupFunction to groups of elements with identical keys. Each CoGroupFunction call returns an arbitrary number of keys. :param operator: The CoGroupFunction that is called for all groups of elements with identical keys. :param types: The type of the resulting DataSet. :return:An CoGroupOperator that represents the co-grouped result DataSet. """ if isinstance(operator, TYPES.FunctionType): f = operator operator = CoGroupFunction() operator.co_group = f new_set = OperatorSet(self._env, self._info) operator._keys1 = self._info[_Fields.KEY1] operator._keys2 = self._info[_Fields.KEY2] self._info[_Fields.OPERATOR] = operator self._info[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__) self._info[_Fields.TYPES] = types self._info[_Fields.NAME] = "PythonCoGroup" self._env._sets.append(self._info) return new_set