def test_integration_success():
task = Square()
with PooledQueueProcessor(task, [2],
fill_at_construction=True,
max_queue_size=10) as processor:
for a, _ in zip(processor, range(30)):
pass
def test_integration_except_ctx():
task = Exc()
with pytest.raises(NotImplementedError) as exc_info:
task() # TODO: MP does not work in pytest
with PooledQueueProcessor(task, [2], max_queue_size=10) as processor:
for a, _ in zip(processor, range(30)):
pass
# print(a)
assert exc_info.type is NotImplementedError
def pqp_benchmark() -> None:
"""
:rtype: None
"""
task = Zeroes()
# task = Lamb #Error: cant be pickled
# task = Func
batch_size = 16
tensor_size = (9, 9, 9, 9, 9)
wait_time = 0.1
samples = 100
df = PooledQueueProcessor(
task,
args=[batch_size],
kwargs={"tensor_size": tensor_size},
max_queue_size=samples,
)
def get() -> None:
"""
:rtype: None
"""
return df.get()
def wait_get() -> None:
"""
:rtype: None
"""
time.sleep(wait_time)
return df.get()
def generate() -> None:
"""
:rtype: None
"""
return task(batch_size, tensor_size=tensor_size)
def wait_generate() -> None:
"""
:rtype: None
"""
time.sleep(wait_time)
return task(batch_size, tensor_size=tensor_size)
for func, discount in zip(
(get, wait_get, generate, wait_generate),
(0, samples * wait_time, 0, samples * wait_time),
):
t, res = benchmark_func(func, samples)
print(f"{func.__name__}: {t - discount} seconds")
def test_lambda_func():
task = lambda x: x
with PooledQueueProcessor(task, [2], max_queue_size=10) as processor:
for a, _ in zip(processor, range(30)):
pass
def test_integration_func():
task = identity
with PooledQueueProcessor(task, [2], max_queue_size=10) as processor:
for a, _ in zip(processor, range(30)):
pass