def test_num_workers(num_workers):
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(num_workers) as pp:
level2 = Unpack(range(N_STEPS))
pipeline.run()
def test_speed():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
level2 = Unpack(range(N_STEPS))
Sleep()
with Timer("sequential") as t:
expected_result = [
(obj[level1], obj[level2]) for obj in pipeline.transform_stream()
]
elapsed_sequential = t.elapsed
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4) as pp:
level2 = Unpack(range(N_STEPS))
Sleep()
with Timer("parallel") as t:
result = [(obj[level1], obj[level2]) for obj in pipeline.transform_stream()]
elapsed_parallel = t.elapsed
assert result == expected_result
assert elapsed_parallel < elapsed_sequential
def test_exception_upstream():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
Raiser()
with ParallelPipeline(4) as pp:
level2 = Unpack(range(N_STEPS))
with pytest.raises(SomeException, match="foo"):
pipeline.run()
def test_FilterVariables():
values = list(range(10))
with Pipeline() as pipeline:
a = Unpack(values)
b = Unpack(values)
FilterVariables(b)
stream = list(pipeline.transform_stream())
for o in stream:
assert a not in o
assert b in o
def test_ecotaxa(tmp_path):
archive_fn = tmp_path / "ecotaxa.zip"
print(archive_fn)
# Create an archive
with Pipeline() as p:
i = Unpack(range(10))
meta = Call(dict, i=i, foo="Sömé UTF-8 ſtríng…")
image = BinaryBlobs()
image_name = Format("image_{}.png", i)
EcotaxaWriter(archive_fn, (image_name, image), meta)
result = [o.to_dict(meta=meta, image=image) for o in p.transform_stream()]
# Read the archive
with Pipeline() as p:
image, meta = EcotaxaReader(archive_fn)
roundtrip_result = [
o.to_dict(meta=meta, image=image) for o in p.transform_stream()
]
for meta_field in ("i", "foo"):
assert [o["meta"][meta_field] for o in result
] == [o["meta"][meta_field] for o in roundtrip_result]
assert_equal([o["image"] for o in result],
[o["image"] for o in roundtrip_result])
def test_exception_main_thread():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4) as pp:
level2 = Unpack(range(N_STEPS))
Sleep()
stream = pipeline.transform_stream()
try:
with pytest.raises(SomeException):
for i, obj in enumerate(stream):
if i == 10:
raise SomeException()
finally:
stream.close()
def test_StreamBuffer():
with Pipeline() as pipeline:
item = Unpack(range(10))
result = StreamBuffer(1)
stream = pipeline.transform_stream()
result = [o[item] for o in stream]
assert result == list(range(10))
def test_Enumerate():
with Pipeline() as pipeline:
a = Unpack(range(10))
i = Enumerate()
stream = pipeline.transform_stream()
for obj in stream:
assert obj[a] == obj[i]
def test_ImageProperties():
with Pipeline() as pipeline:
image = Unpack([skimage.data.camera()])
mask = ThresholdConst(image, 255)
region = ImageProperties(mask, image)
image2 = ExtractROI(image, region, 0)
for obj in pipeline.transform_stream():
assert_equal(obj[image], obj[image2])
def test_ThresholdConst():
images = [skimage.data.camera(), np.zeros((10, 10), np.uint8) + 255]
with Pipeline() as pipeline:
image = Unpack(images)
mask = ThresholdConst(image, 255)
objects = list(pipeline.transform_stream())
assert not objects[1][mask].any()
def test_ExtractROI():
with Pipeline() as pipeline:
image = Unpack([skimage.data.camera()])
mask = ThresholdConst(image, 255)
regions = FindRegions(mask, image)
ExtractROI(image, regions)
ExtractROI(image, regions, 0.5)
pipeline.run()
def test_vignette_corrector_no_channel(image_fns):
with Pipeline() as pipeline:
img_fn = Unpack(image_fns)
image = Call(imread, img_fn, as_gray=True)
result = VignettingCorrector(image)
stream = pipeline.transform_stream()
obj = next(stream)
def test_TQDM():
# Assert that the progress bar works with stream
with Pipeline() as pipeline:
item = Unpack(range(10))
result = TQDM("Description")
stream = pipeline.transform_stream()
result = [o[item] for o in stream]
assert result == list(range(10))
def test_StreamBuffer():
with Pipeline() as pipeline:
item = Unpack(range(10))
StreamBuffer(1)
stream = pipeline.transform_stream()
objects = [o for o in stream]
assert objects[0].n_remaining_hint == 10
assert [o[item] for o in objects] == list(range(10))
def test_worker_die():
with Pipeline() as pipeline:
level1 = Unpack(range(N_STEPS))
with ParallelPipeline(4):
Call(lambda: os.kill(os.getpid(), signal.SIGKILL))
with pytest.raises(
RuntimeError,
match=r"Worker \d+ died unexpectedly. Exit code: -SIGKILL"):
pipeline.run()
def test_Unpack():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
stream = pipeline.transform_stream()
result = [o[value] for o in stream]
assert values == result
def test_Unpack():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
stream = pipeline.transform_stream()
objects = [o for o in stream]
assert objects[0].n_remaining_hint == 10
assert values == [o[value] for o in objects]
def test_CalculateZooProcessFeatures(prefix):
with Pipeline() as p:
i = Unpack(range(10))
mask = BinaryBlobs()
image = NoiseImage(mask.shape)
regionprops = FindRegions(mask, image)
CalculateZooProcessFeatures(regionprops, prefix=prefix)
p.run()
def test_Filter():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
Filter(lambda obj: obj[value] % 2 == 0)
stream = pipeline.transform_stream()
result = [o[value] for o in stream]
assert [v for v in values if v % 2 == 0] == result
def test_Pack():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
values_packed = Pack(2, value)
stream = pipeline.transform_stream()
result = [o[values_packed] for o in stream]
assert [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)] == result
def test_Pack():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
values_packed = Pack(2, value)
objects = [o for o in pipeline.transform_stream()]
# TODO:
# assert objects[0].n_remaining_hint == 5
assert [(0, 1), (2, 3), (4, 5), (6, 7),
(8, 9)] == [o[values_packed] for o in objects]
def test_Filter():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
Filter(lambda obj: obj[value] % 2 == 0)
objects = [o for o in pipeline.transform_stream()]
assert [v for v in values if v % 2 == 0] == [o[value] for o in objects]
# n_remaining_hint of last object is 1
n_remaining = [o.n_remaining_hint for o in objects]
assert n_remaining[-1] == 1
def test_Filter_highlevel():
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
predicate = value % 2 == 0
Filter(predicate)
objects = [o for o in pipeline.transform_stream()]
assert [v for v in values if v % 2 == 0] == [o[value] for o in objects]
# n_remaining_hint of last object is 1
n_remaining = [o.n_remaining_hint for o in objects]
assert n_remaining[-1] == 1
def test_PrintObjects(capsys):
values = list(range(10))
with Pipeline() as pipeline:
value = Unpack(values)
PrintObjects(value)
# TODO: Capture output and compare
# https://docs.pytest.org/en/latest/capture.html#accessing-captured-output-from-a-test-function
# pipeline.run()
stream = pipeline.transform_stream()
result = [o[value] for o in stream]
captured = capsys.readouterr()
print(captured.out)
assert result == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
def test_FindRegions(warn_empty, recwarn):
images = [skimage.data.camera(), np.zeros((10, 10), np.uint8) + 255]
with Pipeline() as pipeline:
image = Unpack(images)
mask = ThresholdConst(image, 255)
result = FindRegions(mask,
image,
0,
100,
padding=10,
warn_empty=warn_empty)
pipeline.run()
if warn_empty:
w = recwarn.pop(UserWarning)
assert re.search(r"^(Image|foo) did not contain any objects.$",
str(w.message))
def test_ecotaxa(tmp_path, ext):
archive_fn = tmp_path / ("ecotaxa" + ext)
print(archive_fn)
# Create an archive
with Pipeline() as p:
i = Unpack(range(10))
meta = Call(dict, i=i, foo="Sömé UTF-8 ſtríng…")
image = BinaryBlobs()
image_name = Format("image_{}.png", i)
EcotaxaWriter(
archive_fn,
(image_name, image),
meta,
object_meta={"foo": 0},
acq_meta={"foo": 1},
process_meta={"foo": 2},
sample_meta={"foo": 3},
)
result = [o.to_dict(meta=meta, image=image) for o in p.transform_stream()]
# Read the archive
with Pipeline() as p:
image, meta = EcotaxaReader(archive_fn)
roundtrip_result = [
o.to_dict(meta=meta, image=image) for o in p.transform_stream()
]
for meta_field in ("i", "foo"):
assert [o["meta"][meta_field] for o in result
] == [o["meta"][meta_field] for o in roundtrip_result]
for i, prefix in enumerate(("object_", "acq_", "process_", "sample_")):
assert [o["meta"][prefix + "foo"]
for o in result] == [i for _ in roundtrip_result]
assert_equal([o["image"] for o in result],
[o["image"] for o in roundtrip_result])
def test_Progress(monkeypatch: pytest.MonkeyPatch):
# Monkeypatch tqdm so that we can extract tqdm instance attributes
tqdm_instance: List[Optional[tqdm.tqdm]] = [None]
tqdm_cls = tqdm.tqdm
def mock_tqdm(*args, **kwargs):
tqdm_instance[0] = tqdm_cls(*args, **kwargs)
return tqdm_instance[0]
monkeypatch.setattr(tqdm, "tqdm", mock_tqdm)
# Assert that the progress bar works with stream
with Pipeline() as pipeline:
item = Unpack(range(10))
result = Progress("Description")
stream = pipeline.transform_stream()
result = [o[item] for o in stream]
assert result == list(range(10))
assert tqdm_instance[0].total == 10
def test_Profile():
with Pipeline() as pipeline:
Unpack(range(N))
# Sleep beforehand to make sure that only the inner Sleep is profiled.
Sleep(DURATION_OUTER)
with Profile("Sleep") as profile_sleep:
Sleep(DURATION_INNER)
# Sleep afterwards to make sure that only the inner Sleep is profiled.
Sleep(DURATION_OUTER)
objects = list(pipeline.transform_stream())
assert len(objects) == N
overhead = profile_sleep._average - DURATION_INNER
print(f"Overhead {overhead:g}s")
assert DURATION_INNER <= profile_sleep._average < DURATION_OUTER
import os.path
from morphocut import Call, Pipeline
from morphocut.contrib.ecotaxa import EcotaxaWriter
from morphocut.contrib.zooprocess import CalculateZooProcessFeatures
from morphocut.file import Glob
from morphocut.image import FindRegions, ImageReader
from morphocut.parallel import ParallelPipeline
from morphocut.str import Format
from morphocut.stream import Enumerate, Unpack
# First, a Pipeline is defined that contains all operations
# that should be carried out on the objects of the stream.
with Pipeline() as p:
# Corresponds to `for base_path in ["/path/a", "/path/b", "/path/c"]:`
base_path = Unpack(["/path/a", "/path/b", "/path/c"])
# Number the objects in the stream
running_number = Enumerate()
# Call calls regular Python functions.
# Here, a subpath is appended to base_path.
pattern = Call(os.path.join, base_path, "subpath/to/input/files/*.jpg")
# Corresponds to `for path in glob(pattern):`
path = Glob(pattern)
# Remove path and extension from the filename
source_basename = Call(lambda x: os.path.splitext(os.path.basename(x))[0], path)
with ParallelPipeline():
