def flat_map(self, operator, types): """ Applies a FlatMap transformation on a DataSet. The transformation calls a FlatMapFunction for each element of the DataSet. Each FlatMapFunction call can return any number of elements including none. :param operator: The FlatMapFunction that is called for each element of the DataSet. :param types: The type of the resulting DataSet. :return:A FlatMapOperator that represents the transformed DataSe """ if isinstance(operator, TYPES.FunctionType): f = operator operator = FlatMapFunction() operator.flat_map = f child = dict() child_set = OperatorSet(self._env, child) child[_Fields.IDENTIFIER] = _Identifier.FLATMAP child[_Fields.PARENT] = self._info child[_Fields.OPERATOR] = operator child[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__) child[_Fields.TYPES] = types child[_Fields.NAME] = "PythonFlatMap" self._info[_Fields.CHILDREN].append(child) self._env._sets.append(child) return child_set
def flat_map(self, operator, types): """ Applies a FlatMap transformation on a DataSet. The transformation calls a FlatMapFunction for each element of the DataSet. Each FlatMapFunction call can return any number of elements including none. :param operator: The FlatMapFunction that is called for each element of the DataSet. :param types: The type of the resulting DataSet. :return:A FlatMapOperator that represents the transformed DataSe """ if isinstance(operator, TYPES.FunctionType): f = operator operator = FlatMapFunction() operator.flat_map = f child = dict() child_set = OperatorSet(self._env, child) child[_Fields.IDENTIFIER] = _Identifier.FLATMAP child[_Fields.PARENT] = self._info child[_Fields.OPERATOR] = operator child[_Fields.META] = str(inspect.getmodule(operator)) + "|" + str( operator.__class__.__name__) child[_Fields.TYPES] = types child[_Fields.NAME] = "PythonFlatMap" self._info[_Fields.CHILDREN].append(child) self._env._sets.append(child) return child_set
def flat_map(self, operator, types): """ Applies a FlatMap transformation on a DataSet. The transformation calls a FlatMapFunction for each element of the DataSet. Each FlatMapFunction call can return any number of elements including none. :param operator: The FlatMapFunction that is called for each element of the DataSet. :param types: The type of the resulting DataSet. :return:A FlatMapOperator that represents the transformed DataSe """ if isinstance(operator, TYPES.FunctionType): f = operator operator = FlatMapFunction() operator.flat_map = f child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.FLATMAP child.parent = self._info child.operator = operator child.meta = str(inspect.getmodule(operator)) + "|" + str( operator.__class__.__name__) child.types = types child.name = "PythonFlatMap" self._info.children.append(child) self._env._sets.append(child) return child_set
def flat_map(self, operator, types): """ Applies a FlatMap transformation on a DataSet. The transformation calls a FlatMapFunction for each element of the DataSet. Each FlatMapFunction call can return any number of elements including none. :param operator: The FlatMapFunction that is called for each element of the DataSet. :param types: The type of the resulting DataSet. :return:A FlatMapOperator that represents the transformed DataSe """ if isinstance(operator, TYPES.FunctionType): f = operator operator = FlatMapFunction() operator.flat_map = f child = OperationInfo() child_set = OperatorSet(self._env, child) child.identifier = _Identifier.FLATMAP child.parent = self._info child.operator = operator child.meta = str(inspect.getmodule(operator)) + "|" + str(operator.__class__.__name__) child.types = types child.name = "PythonFlatMap" self._info.children.append(child) self._env._sets.append(child) return child_set