Ejemplo n.º 1
0
def main():
    for i in range(2):
        gpool.apply(delay_func)
    print(time.time())
    gpool.join()
    print(time.time())
    print('----end---?')
    print(len(gpool))
    gpool.apply_async(add, args=[4, 2])
    gpool.join()
Ejemplo n.º 2
0
    def run_with_apply(cls):
        __greenlet_number = 5
        pool = Pool(size=__greenlet_number)
        for greenlet_index in range(__greenlet_number):
            params = {"index": greenlet_index}
            result = pool.apply(func=cls.main_run_fun, kwds=params)
            cls.__Greenlet_List.append(result)

        print("result: ", cls.__Greenlet_List)
Ejemplo n.º 3
0
class GreenThreadPoolStrategy(BaseGreenThreadStrategy, _PoolRunnableStrategy, _Resultable):

    _Strategy_Feature_Mode: _FeatureMode = _FeatureMode.GreenThread

    _GreenThread_Pool: Pool = None
    _GreenThread_List: List[Greenlet] = []
    # _GreenThread_Running_Result: List = []

    def __init__(self, pool_size: int):
        super().__init__(pool_size=pool_size)


    def initialization(self, queue_tasks: Optional[Union[_BaseQueueTask, _BaseList]] = None,
                       features: Optional[Union[_BaseFeatureAdapterFactory, _BaseList]] = None,
                       *args, **kwargs) -> None:

        super(GreenThreadPoolStrategy, self).initialization(queue_tasks=queue_tasks, features=features, *args, **kwargs)

        # Initialize and build the Processes Pool.
        # self._GreenThread_Pool = Pool(size=self.pool_size, greenlet_class=greenlet_class)
        self._GreenThread_Pool = Pool(size=self.pool_size)


    def apply(self, tasks_size: int, function: Callable, args: Tuple = (), kwargs: Dict = {}) -> None:
        self.reset_result()

        self._GreenThread_List = [
            self._GreenThread_Pool.apply(func=function, args=args, kwds=kwargs)
            for _ in range(tasks_size)]

        for _green_thread in self._GreenThread_List:
            __process_running_result = None
            __process_run_successful = None
            __exception = None

            try:
                __process_running_result = _green_thread
                __exception = None
                __process_run_successful = True
            except Exception as e:
                __exception = e
                __process_run_successful = False

            # Save Running result state and Running result value as dict
            self._result_saving(successful=__process_run_successful, result=__process_running_result, exception=__exception)


    def async_apply(self, tasks_size: int, function: Callable, args: Tuple = (),
                    kwargs: Dict = {}, callback: Callable = None, error_callback: Callable = None) -> None:

        self.reset_result()
        self._GreenThread_List = [
            self._GreenThread_Pool.apply_async(func=function,
                                               args=args,
                                               kwds=kwargs,
                                               callback=callback)
            for _ in range(tasks_size)]

        for process in self._GreenThread_List:
            __process_running_result = None
            __process_run_successful = None
            __exception = None

            try:
                __process_running_result = process.get()
                __process_run_successful = process.successful()
                __exception = None
            except Exception as e:
                __exception = e
                __process_running_result = None
                __process_run_successful = process.successful()

            # Save Running result state and Running result value as dict
            self._result_saving(successful=__process_run_successful, result=__process_running_result, exception=__exception)


    def apply_with_iter(self, functions_iter: List[Callable], args_iter: List[Tuple] = None, kwargs_iter: List[Dict] = None) -> None:
        self.reset_result()
        __process_running_result = None

        if args_iter is None:
            args_iter = [() for _ in functions_iter]

        if kwargs_iter is None:
            kwargs_iter = [{} for _ in functions_iter]

        self._GreenThread_List = [
            self._GreenThread_Pool.apply(func=_func, args=_args, kwds=_kwargs)
            for _func, _args, _kwargs in zip(functions_iter, args_iter, kwargs_iter)
        ]

        for _green_thread in self._GreenThread_List:
            __process_running_result = None
            __process_run_successful = None
            __exception = None

            try:
                __process_running_result = _green_thread
                __exception = None
                __process_run_successful = True
            except Exception as e:
                __exception = e
                __process_run_successful = False

            # Save Running result state and Running result value as dict
            self._result_saving(successful=__process_run_successful, result=__process_running_result, exception=__exception)


    def async_apply_with_iter(self, functions_iter: List[Callable], args_iter: List[Tuple] = None,
                              kwargs_iter: List[Dict] = None, callback_iter: List[Callable] = None,
                              error_callback_iter: List[Callable] = None) -> None:

        self.reset_result()

        if args_iter is None:
            args_iter = [() for _ in functions_iter]

        if kwargs_iter is None:
            kwargs_iter = [{} for _ in functions_iter]

        if callback_iter is None:
            callback_iter = [None for _ in functions_iter]

        if error_callback_iter is None:
            error_callback_iter = [None for _ in functions_iter]

        self._GreenThread_List = [
            self._GreenThread_Pool.apply_async(
                func=_func,
                args=_args,
                kwds=_kwargs,
                callback=_callback)
            for _func, _args, _kwargs, _callback in zip(functions_iter, args_iter, kwargs_iter, callback_iter)
        ]

        for process in self._GreenThread_List:
            _process_running_result = None
            _process_run_successful = None
            _exception = None

            try:
                _process_running_result = process.get()
                _process_run_successful = process.successful()
            except Exception as e:
                _exception = e
                _process_run_successful = False

            # Save Running result state and Running result value as dict
            self._result_saving(successful=_process_run_successful, result=_process_running_result, exception=_exception)


    def map(self, function: Callable, args_iter: IterableType = (), chunksize: int = None) -> None:
        self.reset_result()
        __process_running_result = None

        try:
            __process_running_result = self._GreenThread_Pool.map(
                func=function, iterable=args_iter)
            __exception = None
            __process_run_successful = True
        except Exception as e:
            __exception = e
            __process_run_successful = False

        # Save Running result state and Running result value as dict
        for __result in (__process_running_result or []):
            self._result_saving(successful=__process_run_successful, result=__result, exception=__exception)


    def async_map(self, function: Callable, args_iter: IterableType = (), chunksize: int = None,
                  callback: Callable = None, error_callback: Callable = None) -> None:

        self.reset_result()

        __process_running_result = None
        __process_run_successful = None
        __exception = None

        __map_result = self._GreenThread_Pool.map_async(
            func=function,
            iterable=args_iter,
            callback=callback)
        try:
            __process_running_result = __map_result.get()
            __process_run_successful = __map_result.successful()
        except Exception as e:
            __exception = e
            __process_running_result = None

        # Save Running result state and Running result value as dict
        for __result in (__process_running_result or []):
            self._result_saving(successful=__process_run_successful, result=__result, exception=__exception)


    def map_by_args(self, function: Callable, args_iter: IterableType[IterableType] = (), chunksize: int = None) -> None:
        """
        Note:
            For Green-Thread in Python (gevent), there isn't any methods
            could pass multiple parameters like multiprocessing.pool.starmap.
        :param function:
        :param args_iter:
        :param chunksize:
        :return:
        """
        self.reset_result()
        args_iter_set = set(args_iter)
        if len(args_iter_set) == 1:
            _arguments = args_iter[0]
            self._map_with_args(function=function, args=_arguments, size=len(args_iter), chunksize=chunksize)
        else:
            __results = []
            __process_run_successful = None

            try:
                for _args in args_iter:
                    _greenlet = self._GreenThread_Pool.spawn(function, *_args)
                    self._GreenThread_List.append(_greenlet)

                for _one_greenlet in self._GreenThread_List:
                    _one_greenlet.join()
                    _one_greenlet_value = _one_greenlet.value
                    __results.append(_one_greenlet_value)

                __process_run_successful = True
                __exception = None
            except Exception as e:
                __process_run_successful = False
                __exception = e

            # Save Running result state and Running result value as dict
            for __result in (__results or []):
                self._result_saving(successful=__process_run_successful, result=__result, exception=__exception)


    def _map_with_args(self, function: Callable, args: Iterable, size: int, chunksize: int) -> None:
        """
        Description:
            For passing multiple arguments into target function.
            That's the reason why initial a partial function first and then pass ONE parameter into it.
        :param function:
        :param args:
        :param size:
        :param chunksize:
        :return:
        """
        self.reset_result()
        _args = args[:-1]
        _last_args = args[-1:] * size
        partial_function = functools.partial(function, *_args)
        self.map(function=partial_function, args_iter=_last_args, chunksize=chunksize)


    def async_map_by_args(self, function: Callable, args_iter: IterableType[IterableType] = (),
                          chunksize: int = None, callback: Callable = None, error_callback: Callable = None) -> None:

        self.reset_result()
        args_iter_set = set(args_iter)
        if len(args_iter_set) == 1:
            _arguments = args_iter[0]
            self._async_map_with_args(function=function, args=_arguments, size=len(args_iter), chunksize=chunksize, callback=callback)
        else:
            __results = []
            __process_run_successful = None

            try:
                for _args in args_iter:
                    _greenlet = self._GreenThread_Pool.spawn(function, *_args)
                    self._GreenThread_List.append(_greenlet)

                for _one_greenlet in self._GreenThread_List:
                    _one_greenlet.join()
                    _one_greenlet_value = _one_greenlet.value
                    __results.append(_one_greenlet_value)

                __process_run_successful = True
                __exception = None
            except Exception as e:
                __process_run_successful = False
                __exception = e

            # Save Running result state and Running result value as dict
            for __result in (__results or []):
                self._result_saving(successful=__process_run_successful, result=__result, exception=__exception)


    def _async_map_with_args(self, function: Callable, args: Iterable, size: int, chunksize: int, callback: Callable) -> None:
        """
        Description:
            This is asynchronous version of function '_map_with_args'.
        :param function:
        :param args:
        :param size:
        :param chunksize:
        :param callback:
        :return:
        """
        self.reset_result()
        _args = args[:-1]
        _last_args = args[-1:] * size
        partial_function = functools.partial(function, *_args)
        self.async_map(function=partial_function, args_iter=_last_args, chunksize=chunksize, callback=callback)


    def imap(self, function: Callable, args_iter: IterableType = (), chunksize: int = 1) -> None:
        self.reset_result()
        __process_running_result = None

        try:
            imap_running_result = self._GreenThread_Pool.imap(function, args_iter)
            __process_running_result = [result for result in imap_running_result]
            __exception = None
            __process_run_successful = True
        except Exception as e:
            __exception = e
            __process_run_successful = False

        # Save Running result state and Running result value as dict
        for __result in (__process_running_result or []):
            self._result_saving(successful=__process_run_successful, result=__result, exception=__exception)


    def imap_unordered(self, function: Callable, args_iter: IterableType = (), chunksize: int = 1) -> None:
        self.reset_result()
        __process_running_result = None

        try:
            imap_running_result = self._GreenThread_Pool.imap_unordered(function, args_iter)
            __process_running_result = [result for result in imap_running_result]
            __exception = None
            __process_run_successful = True
        except Exception as e:
            __exception = e
            __process_run_successful = False

        # Save Running result state and Running result value as dict
        for __result in (__process_running_result or []):
            self._result_saving(successful=__process_run_successful, result=__result, exception=__exception)


    def _result_saving(self, successful: bool, result: List, exception: Exception) -> None:
        process_result = {"successful": successful, "result": result, "exception": exception}
        # Saving value into list
        self._GreenThread_Running_Result.append(process_result)


    def close(self) -> None:
        self._GreenThread_Pool.join()


    def terminal(self) -> None:
        pass
        # self._GreenThread_Pool.terminate()


    def get_result(self) -> List[_CoroutineResult]:
        return self.result()


    def _saving_process(self) -> List[_GreenThreadPoolResult]:
        _pool_results = []
        for __result in self._GreenThread_Running_Result:
            _pool_result = _GreenThreadPoolResult()
            _pool_result.is_successful = __result["successful"]
            _pool_result.data = __result["result"]
            _pool_result.exception = __result["exception"]
            _pool_results.append(_pool_result)
        return _pool_results