def test_async_apply_with_iter(self, simple_pool: SimplePool): TestSimplePool._initial() simple_pool.initial() simple_pool.async_apply_with_iter(functions_iter=target_funcs_iter()) TestSimplePool._chk_record() _results = simple_pool.get_result() TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def test_apply(self, simple_pool: SimplePool): TestSimplePool._initial() simple_pool.initial() simple_pool.apply(tasks_size=Task_Size, function=target_function) TestSimplePool._chk_blocking_record() _results = simple_pool.get_result() TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def test_initial_running_strategy_with_parallel(self, process_pool: SimplePool): process_pool._initial_running_strategy() from multirunnable.pool import Pool_Runnable_Strategy assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance." assert isinstance( Pool_Runnable_Strategy, ProcessPoolStrategy ), "It should be an sub-instance of 'ProcessPoolStrategy'."
def test_initial_running_strategy_with_coroutine( self, green_thread_pool: SimplePool): green_thread_pool._initial_running_strategy() from multirunnable.pool import Pool_Runnable_Strategy assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance." assert isinstance( Pool_Runnable_Strategy, GreenThreadPoolStrategy ), "It should be an sub-instance of 'GreenThreadPoolStrategy'."
def test_imap_unordered(self, simple_pool: SimplePool): TestSimplePool._initial() simple_pool.initial() simple_pool.imap_unordered(function=target_function_for_map, args_iter=Test_Function_Args) TestSimplePool._chk_map_record() _results = simple_pool.get_result() TestSimplePool.chk_results(results=_results, expected_size=len(Test_Function_Args))
def test_async_map_by_args(self, simple_pool: SimplePool): TestSimplePool._initial() simple_pool.initial() simple_pool.async_map_by_args(function=target_function_for_map, args_iter=Test_Function_Multiple_Args) TestSimplePool._chk_map_record() _results = simple_pool.get_result() TestSimplePool.chk_results( results=_results, expected_size=len(Test_Function_Multiple_Args))
def test_apply_with_iter_by_pykeyword_with(self, simple_pool: SimplePool): TestSimplePool._initial() with simple_pool as _pool: _pool.apply_with_iter(functions_iter=target_funcs_iter()) TestSimplePool._chk_blocking_record() _results = simple_pool.get_result() TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def test_async_apply_by_pykeyword_with(self, simple_pool: SimplePool): TestSimplePool._initial() with simple_pool as _pool: _pool.async_apply(tasks_size=Task_Size, function=target_function) TestSimplePool._chk_record() _results = simple_pool.get_result() TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def main_run(self): # test_dao = TestDao(connection_strategy=self.persistence_strategy()) test_dao = TestDao(db_driver="mysql", use_pool=False) # # # # Initial and instantiate feature object: Queue Task, Lock and Bounded Semaphore _queue_task = self.__init_queue() _features = self.__init_features() _pool = SimplePool( mode=RunningMode.Parallel, # mode=RunningMode.Concurrent, # mode=RunningMode.GreenThread, pool_size=self.__Pool_Size) _pool.initial(queue_tasks=_queue_task, features=_features) _pool.async_apply(function=test_dao.get_test_data, tasks_size=self.__Pool_Size) result = _pool.get_result() print("Result: ", result) for r in result: print(f"+====================================+") print("Result.data: ", r.data) print("Result.is_successful: ", r.is_successful) print("+====================================+\n") self.__save_to_files(result=result) end_time = time.time() self.__logger.info(f"Total taking time: {end_time - start_time} seconds")
def test_imap_by_pykeyword_with(self, simple_pool: SimplePool): TestSimplePool._initial() with simple_pool as _pool: _pool.imap(function=target_function_for_map, args_iter=Test_Function_Args) TestSimplePool._chk_map_record() _results = simple_pool.get_result() TestSimplePool.chk_results(results=_results, expected_size=len(Test_Function_Args))
def test_initial_running_strategy(self, simple_pool: SimplePool): simple_pool._initial_running_strategy() _rmode = get_current_mode(force=True) from multirunnable.pool import Pool_Runnable_Strategy if _rmode is RunningMode.Parallel: assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance." assert isinstance( Pool_Runnable_Strategy, ProcessPoolStrategy ), "It should be an sub-instance of 'ProcessPoolStrategy'." elif _rmode is RunningMode.Concurrent: assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance." assert isinstance( Pool_Runnable_Strategy, ThreadPoolStrategy ), "It should be an sub-instance of 'ThreadPoolStrategy'." elif _rmode is RunningMode.GreenThread: assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance." assert isinstance( Pool_Runnable_Strategy, GreenThreadPoolStrategy ), "It should be an sub-instance of 'GreenThreadPoolStrategy'." else: raise ValueError("The RunningMode has the unexpected mode.")
def test_close(self, thread_pool: SimplePool): try: thread_pool.initial() thread_pool.async_apply(tasks_size=Task_Size, function=lambda a: a + a, args=(1, )) thread_pool.close() except Exception as e: assert False, "It should work finely without any issue. Please check it." else: assert True, "It work finely without any issue."
def test_close(self, simple_pool: SimplePool): try: simple_pool.initial() simple_pool.async_apply(tasks_size=Task_Size, function=lambda a: a + a, args=(1, )) simple_pool.close() except Exception: assert False, f"It should work finely without any issue. Please check it.\n Error: {traceback.format_exc()}" else: assert True, "It work finely without any issue."
def main_run(self): # # # # Initial Pool object __pool = SimplePool(mode=RunningMode.Parallel, pool_size=self.__Pool_Size) # __pool = SimplePool(mode=RunningMode.Concurrent, pool_size=self.__Pool_Size) # __pool = SimplePool(mode=RunningMode.GreenThread, pool_size=self.__Pool_Size) __result = None with __pool as pool: # # # # Running Pool # pool.apply(function=self.__Example_Target.target_function, tasks_size=self.__Pool_Size) pool.async_apply(function=self.__Example_Target.crawl_function, kwargs={"sleep_time": random.randrange(10, 20)}, tasks_size=self.__Pool_Size) pool.map(function=self.__Example_Target.crawl_function, args_iter=(1, 2, 3)) # pool.map_by_args(function=self.__Example_Target.target_function, args_iter=[("index_1", "index_2.2"), ("index_3",), (1, 2, 3)]) # # # # Get result __result = pool.get_result() print("Result: ", __result)
def process_pool(): return SimplePool(mode=RunningMode.Parallel, pool_size=_Worker_Pool_Size)
def test_get_result(self, thread_pool: SimplePool): thread_pool.get_result()
def test_imap_unordered(self, thread_pool: SimplePool): TestSimplePool._initial() thread_pool.initial() thread_pool.imap_unordered(function=map_target_fun, args_iter=Test_Function_Args) TestSimplePool._chk_map_record()
def test_async_map_by_args(self, thread_pool: SimplePool): TestSimplePool._initial() thread_pool.initial() thread_pool.async_map_by_args(function=map_target_fun, args_iter=Test_Function_Multiple_Args) TestSimplePool._chk_map_record()
def test_async_apply_with_iter(self, thread_pool: SimplePool): TestSimplePool._initial() thread_pool.initial() thread_pool.async_apply_with_iter(functions_iter=target_funcs_iter()) TestSimplePool._chk_record()
def test_async_apply(self, thread_pool: SimplePool): TestSimplePool._initial() thread_pool.initial() thread_pool.async_apply(tasks_size=Task_Size, function=pool_target_fun) TestSimplePool._chk_record()
def simple_pool(request) -> SimplePool: return SimplePool(mode=request.param, pool_size=_Worker_Pool_Size)
def green_thread_pool(): return SimplePool(mode=RunningMode.GreenThread, pool_size=_Worker_Pool_Size)
def thread_pool(): return SimplePool(mode=RunningMode.Concurrent, pool_size=_Worker_Pool_Size)
# Import package multirunnable import pathlib import random import time import sys package_path = str(pathlib.Path(__file__).parent.parent.parent.absolute()) sys.path.append(package_path) from multirunnable import SimplePool, RunningMode def function(index): print(f"This isfunction with index {index}") time.sleep(3) return "Return Value" pool = SimplePool(mode=RunningMode.Parallel, pool_size=3) pool.initial() pool.async_apply(function=function, kwargs={"index": f"test_{random.randrange(1, 10)}"}, tasks_size=3) pool.close() result = pool.get_result() print(f"This is final result: {result}")