def test_async_apply_with_iter(self, simple_pool: SimplePool):
TestSimplePool._initial()
simple_pool.initial()
simple_pool.async_apply_with_iter(functions_iter=target_funcs_iter())
TestSimplePool._chk_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def test_apply(self, simple_pool: SimplePool):
TestSimplePool._initial()
simple_pool.initial()
simple_pool.apply(tasks_size=Task_Size, function=target_function)
TestSimplePool._chk_blocking_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def test_initial_running_strategy_with_parallel(self,
process_pool: SimplePool):
process_pool._initial_running_strategy()
from multirunnable.pool import Pool_Runnable_Strategy
assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance."
assert isinstance(
Pool_Runnable_Strategy, ProcessPoolStrategy
), "It should be an sub-instance of 'ProcessPoolStrategy'."
def test_initial_running_strategy_with_coroutine(
self, green_thread_pool: SimplePool):
green_thread_pool._initial_running_strategy()
from multirunnable.pool import Pool_Runnable_Strategy
assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance."
assert isinstance(
Pool_Runnable_Strategy, GreenThreadPoolStrategy
), "It should be an sub-instance of 'GreenThreadPoolStrategy'."
def test_imap_unordered(self, simple_pool: SimplePool):
TestSimplePool._initial()
simple_pool.initial()
simple_pool.imap_unordered(function=target_function_for_map,
args_iter=Test_Function_Args)
TestSimplePool._chk_map_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(results=_results,
expected_size=len(Test_Function_Args))
def test_async_map_by_args(self, simple_pool: SimplePool):
TestSimplePool._initial()
simple_pool.initial()
simple_pool.async_map_by_args(function=target_function_for_map,
args_iter=Test_Function_Multiple_Args)
TestSimplePool._chk_map_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(
results=_results, expected_size=len(Test_Function_Multiple_Args))
def test_apply_with_iter_by_pykeyword_with(self, simple_pool: SimplePool):
TestSimplePool._initial()
with simple_pool as _pool:
_pool.apply_with_iter(functions_iter=target_funcs_iter())
TestSimplePool._chk_blocking_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def test_async_apply_by_pykeyword_with(self, simple_pool: SimplePool):
TestSimplePool._initial()
with simple_pool as _pool:
_pool.async_apply(tasks_size=Task_Size, function=target_function)
TestSimplePool._chk_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(results=_results, expected_size=Task_Size)
def main_run(self):
# test_dao = TestDao(connection_strategy=self.persistence_strategy())
test_dao = TestDao(db_driver="mysql", use_pool=False)
# # # # Initial and instantiate feature object: Queue Task, Lock and Bounded Semaphore
_queue_task = self.__init_queue()
_features = self.__init_features()
_pool = SimplePool(
mode=RunningMode.Parallel,
# mode=RunningMode.Concurrent,
# mode=RunningMode.GreenThread,
pool_size=self.__Pool_Size)
_pool.initial(queue_tasks=_queue_task, features=_features)
_pool.async_apply(function=test_dao.get_test_data, tasks_size=self.__Pool_Size)
result = _pool.get_result()
print("Result: ", result)
for r in result:
print(f"+====================================+")
print("Result.data: ", r.data)
print("Result.is_successful: ", r.is_successful)
print("+====================================+\n")
self.__save_to_files(result=result)
end_time = time.time()
self.__logger.info(f"Total taking time: {end_time - start_time} seconds")
def test_imap_by_pykeyword_with(self, simple_pool: SimplePool):
TestSimplePool._initial()
with simple_pool as _pool:
_pool.imap(function=target_function_for_map,
args_iter=Test_Function_Args)
TestSimplePool._chk_map_record()
_results = simple_pool.get_result()
TestSimplePool.chk_results(results=_results,
expected_size=len(Test_Function_Args))
def test_initial_running_strategy(self, simple_pool: SimplePool):
simple_pool._initial_running_strategy()
_rmode = get_current_mode(force=True)
from multirunnable.pool import Pool_Runnable_Strategy
if _rmode is RunningMode.Parallel:
assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance."
assert isinstance(
Pool_Runnable_Strategy, ProcessPoolStrategy
), "It should be an sub-instance of 'ProcessPoolStrategy'."
elif _rmode is RunningMode.Concurrent:
assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance."
assert isinstance(
Pool_Runnable_Strategy, ThreadPoolStrategy
), "It should be an sub-instance of 'ThreadPoolStrategy'."
elif _rmode is RunningMode.GreenThread:
assert Pool_Runnable_Strategy is not None, "It should be assign running-strategy instance."
assert isinstance(
Pool_Runnable_Strategy, GreenThreadPoolStrategy
), "It should be an sub-instance of 'GreenThreadPoolStrategy'."
else:
raise ValueError("The RunningMode has the unexpected mode.")
def test_close(self, thread_pool: SimplePool):
try:
thread_pool.initial()
thread_pool.async_apply(tasks_size=Task_Size,
function=lambda a: a + a,
args=(1, ))
thread_pool.close()
except Exception as e:
assert False, "It should work finely without any issue. Please check it."
else:
assert True, "It work finely without any issue."
def test_close(self, simple_pool: SimplePool):
try:
simple_pool.initial()
simple_pool.async_apply(tasks_size=Task_Size,
function=lambda a: a + a,
args=(1, ))
simple_pool.close()
except Exception:
assert False, f"It should work finely without any issue. Please check it.\n Error: {traceback.format_exc()}"
else:
assert True, "It work finely without any issue."
def main_run(self):
# # # # Initial Pool object
__pool = SimplePool(mode=RunningMode.Parallel, pool_size=self.__Pool_Size)
# __pool = SimplePool(mode=RunningMode.Concurrent, pool_size=self.__Pool_Size)
# __pool = SimplePool(mode=RunningMode.GreenThread, pool_size=self.__Pool_Size)
__result = None
with __pool as pool:
# # # # Running Pool
# pool.apply(function=self.__Example_Target.target_function, tasks_size=self.__Pool_Size)
pool.async_apply(function=self.__Example_Target.crawl_function, kwargs={"sleep_time": random.randrange(10, 20)}, tasks_size=self.__Pool_Size)
pool.map(function=self.__Example_Target.crawl_function, args_iter=(1, 2, 3))
# pool.map_by_args(function=self.__Example_Target.target_function, args_iter=[("index_1", "index_2.2"), ("index_3",), (1, 2, 3)])
# # # # Get result
__result = pool.get_result()
print("Result: ", __result)
def process_pool():
return SimplePool(mode=RunningMode.Parallel, pool_size=_Worker_Pool_Size)
def test_get_result(self, thread_pool: SimplePool):
thread_pool.get_result()
def test_imap_unordered(self, thread_pool: SimplePool):
TestSimplePool._initial()
thread_pool.initial()
thread_pool.imap_unordered(function=map_target_fun,
args_iter=Test_Function_Args)
TestSimplePool._chk_map_record()
def test_async_map_by_args(self, thread_pool: SimplePool):
TestSimplePool._initial()
thread_pool.initial()
thread_pool.async_map_by_args(function=map_target_fun,
args_iter=Test_Function_Multiple_Args)
TestSimplePool._chk_map_record()
def test_async_apply_with_iter(self, thread_pool: SimplePool):
TestSimplePool._initial()
thread_pool.initial()
thread_pool.async_apply_with_iter(functions_iter=target_funcs_iter())
TestSimplePool._chk_record()
def test_async_apply(self, thread_pool: SimplePool):
TestSimplePool._initial()
thread_pool.initial()
thread_pool.async_apply(tasks_size=Task_Size, function=pool_target_fun)
TestSimplePool._chk_record()
def simple_pool(request) -> SimplePool:
return SimplePool(mode=request.param, pool_size=_Worker_Pool_Size)
def green_thread_pool():
return SimplePool(mode=RunningMode.GreenThread,
pool_size=_Worker_Pool_Size)
def thread_pool():
return SimplePool(mode=RunningMode.Concurrent, pool_size=_Worker_Pool_Size)
# Import package multirunnable
import pathlib
import random
import time
import sys
package_path = str(pathlib.Path(__file__).parent.parent.parent.absolute())
sys.path.append(package_path)
from multirunnable import SimplePool, RunningMode
def function(index):
print(f"This isfunction with index {index}")
time.sleep(3)
return "Return Value"
pool = SimplePool(mode=RunningMode.Parallel, pool_size=3)
pool.initial()
pool.async_apply(function=function,
kwargs={"index": f"test_{random.randrange(1, 10)}"},
tasks_size=3)
pool.close()
result = pool.get_result()
print(f"This is final result: {result}")
