def test_KeyError():
with Pipeline() as pipeline:
with ParallelPipeline(4) as pp:
KeyErrorRaiser()
with pytest.raises(KeyError, match="foo"):
pipeline.run()
def test_speed():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
level2 = Unpack(range(N_STEPS))
Sleep()
with Timer("sequential") as t:
expected_result = [
(obj[level1], obj[level2]) for obj in pipeline.transform_stream()
]
elapsed_sequential = t.elapsed
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4) as pp:
level2 = Unpack(range(N_STEPS))
Sleep()
with Timer("parallel") as t:
result = [(obj[level1], obj[level2]) for obj in pipeline.transform_stream()]
elapsed_parallel = t.elapsed
assert result == expected_result
assert elapsed_parallel < elapsed_sequential
def test_num_workers(num_workers):
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(num_workers) as pp:
level2 = Unpack(range(N_STEPS))
pipeline.run()
def test_exception_worker():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4) as pp:
Sleep()
Raiser()
with pytest.raises(SomeException, match="foo"):
pipeline.run()
def test_worker_die():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4):
Call(lambda: os.kill(os.getpid(), signal.SIGKILL))
with pytest.raises(
RuntimeError,
match=r"Worker \d+ died unexpectedly. Exit code: -SIGKILL"):
pipeline.run()
def test_exception_main_thread():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4) as pp:
level2 = Unpack(range(N_STEPS))
Sleep()
stream = pipeline.transform_stream()
try:
with pytest.raises(SomeException):
for i, obj in enumerate(stream):
if i == 10:
raise SomeException()
finally:
stream.close()
base_path = Unpack(["/path/a", "/path/b", "/path/c"])
# Number the objects in the stream
running_number = Enumerate()
# Call calls regular Python functions.
# Here, a subpath is appended to base_path.
pattern = Call(os.path.join, base_path, "subpath/to/input/files/*.jpg")
# Corresponds to `for path in glob(pattern):`
path = Glob(pattern)
# Remove path and extension from the filename
source_basename = Call(lambda x: os.path.splitext(os.path.basename(x))[0], path)
with ParallelPipeline():
# The following operations are distributed among multiple
# worker processes to speed up the calculations.
# Read the image
image = ImageReader(path)
# Do some thresholding
mask = image < 128
# Find regions in the image
region = FindRegions(mask, image)
# Extract just the object
roi_image = region.intensity_image