Beispiel #1
0
    def using(self, operator):
        """
        Finalizes a CoGroup transformation.

        Applies a CoGroupFunction to groups of elements with identical keys.
        Each CoGroupFunction call returns an arbitrary number of keys.

        :param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
        :return:An CoGroupOperator that represents the co-grouped result DataSet.
        """
        if isinstance(operator, TYPES.FunctionType):
            f = operator
            operator = CoGroupFunction()
            operator.co_group = f
        new_set = OperatorSet(self._env, self._info)
        self._info.key1 = tuple([x for x in range(len(self._info.key1))])
        self._info.key2 = tuple([x for x in range(len(self._info.key2))])
        operator._keys1 = self._info.key1
        operator._keys2 = self._info.key2
        self._info.parent.parallelism = self._info.parallelism
        self._info.other.parallelism = self._info.parallelism
        self._info.operator = operator
        self._info.types = _createArrayTypeInfo()
        self._info.name = "PythonCoGroup"
        self._env._sets.append(self._info)
        return new_set
Beispiel #2
0
    def using(self, operator):
        """
        Finalizes a CoGroup transformation.

        Applies a CoGroupFunction to groups of elements with identical keys.
        Each CoGroupFunction call returns an arbitrary number of keys.

        :param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
        :return:An CoGroupOperator that represents the co-grouped result DataSet.
        """
        if isinstance(operator, TYPES.FunctionType):
            f = operator
            operator = CoGroupFunction()
            operator.co_group = f
        new_set = OperatorSet(self._env, self._info)
        self._info.key1 = tuple([x for x in range(len(self._info.key1))])
        self._info.key2 = tuple([x for x in range(len(self._info.key2))])
        operator._keys1 = self._info.key1
        operator._keys2 = self._info.key2
        self._info.parent.parallelism = self._info.parallelism
        self._info.other.parallelism = self._info.parallelism
        self._info.operator = operator
        self._info.types = _createArrayTypeInfo()
        self._info.name = "PythonCoGroup"
        self._env._sets.append(self._info)
        return new_set
Beispiel #3
0
    def using(self, operator, types):
        """
        Finalizes a CoGroup transformation.

        Applies a CoGroupFunction to groups of elements with identical keys.
        Each CoGroupFunction call returns an arbitrary number of keys.

        :param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
        :param types: The type of the resulting DataSet.
        :return:An CoGroupOperator that represents the co-grouped result DataSet.
        """
        if isinstance(operator, TYPES.FunctionType):
            f = operator
            operator = CoGroupFunction()
            operator.co_group = f
        new_set = OperatorSet(self._env, self._info)
        operator._keys1 = self._info.key1
        operator._keys2 = self._info.key2
        self._info.operator = operator
        self._info.meta = str(inspect.getmodule(operator)) + "|" + str(
            operator.__class__.__name__)
        self._info.types = types
        self._info.name = "PythonCoGroup"
        self._env._sets.append(self._info)
        return new_set
    def using(self, operator, types):
        """
        Finalizes a CoGroup transformation.

        Applies a CoGroupFunction to groups of elements with identical keys.
        Each CoGroupFunction call returns an arbitrary number of keys.

        :param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
        :param types: The type of the resulting DataSet.
        :return:An CoGroupOperator that represents the co-grouped result DataSet.
        """
        if isinstance(operator, TYPES.FunctionType):
            f = operator
            operator = CoGroupFunction()
            operator.co_group = f
        tmp1 = self._parent[0]
        tmp2 = self._parent[1]
        key1 = self._info[_Fields.KEY1]
        key2 = self._info[_Fields.KEY2]
        use_key_selector = False

        if isinstance(key1, (KeySelectorFunction, TYPES.FunctionType)):
            if isinstance(key1, KeySelectorFunction):
                f1 = key1
            else:
                f1 = KeySelectorFunction()
                f1.get_key = key1
            tmp1 = self._parent[0].map(f1, (self._info[_Fields.TYPE1], deduct_output_type(self._parent[0]._info)))
            use_key_selector = True
            operator._discard_key1 = True
            key1 = (0,)

        if isinstance(key2, (KeySelectorFunction, TYPES.FunctionType)):
            if isinstance(key2, KeySelectorFunction):
                f2 = key2
            else:
                f2 = KeySelectorFunction()
                f2.get_key = key2
            tmp2 = self._parent[1].map(f2, (self._info[_Fields.TYPE2], deduct_output_type(self._parent[1]._info)))
            use_key_selector = True
            operator._discard_key2 = True
            key2 = (0,)

        if use_key_selector:
            return tmp1.co_group(tmp2).where(*key1).equal_to(*key2).using(operator, types)
        else:
            new_set = OperatorSet(self._env, self._info)
            operator._keys1 = self._info[_Fields.KEY1]
            operator._keys2 = self._info[_Fields.KEY2]
            self._info[_Fields.OPERATOR] = operator
            self._info[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__)
            self._info[_Fields.TYPES] = types
            self._info[_Fields.NAME] = "PythonCoGroup"
            self._info[_Fields.PARENT][_Fields.CHILDREN].append(self._info)
            self._info[_Fields.OTHER][_Fields.CHILDREN].append(self._info)
            self._env._sets.append(self._info)
            return new_set
Beispiel #5
0
    def using(self, operator, types):
        """
        Finalizes a CoGroup transformation.

        Applies a CoGroupFunction to groups of elements with identical keys.
        Each CoGroupFunction call returns an arbitrary number of keys.

        :param operator: The CoGroupFunction that is called for all groups of elements with identical keys.
        :param types: The type of the resulting DataSet.
        :return:An CoGroupOperator that represents the co-grouped result DataSet.
        """
        if isinstance(operator, TYPES.FunctionType):
            f = operator
            operator = CoGroupFunction()
            operator.co_group = f
        new_set = OperatorSet(self._env, self._info)
        operator._keys1 = self._info[_Fields.KEY1]
        operator._keys2 = self._info[_Fields.KEY2]
        self._info[_Fields.OPERATOR] = operator
        self._info[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__)
        self._info[_Fields.TYPES] = types
        self._info[_Fields.NAME] = "PythonCoGroup"
        self._env._sets.append(self._info)
        return new_set