def test_augmentations_with_seed_match_for_images_and_keypoints(self):
augseq = iaa.AddElementwise((0, 255))
image = np.zeros((10, 10, 1), dtype=np.uint8)
# keypoints here will not be changed by augseq, but they will induce
# deterministic mode to start in augment_batches() as each batch
# contains images AND keypoints
kps = ia.KeypointsOnImage([ia.Keypoint(x=2, y=0)], shape=(10, 10, 1))
batch = ia.Batch(images=np.uint8([image, image]), keypoints=[kps, kps])
batches = [batch.deepcopy() for _ in sm.xrange(60)]
# seed=1
with multicore.Pool(augseq, processes=2, maxtasksperchild=30,
seed=1) as pool:
batches_aug1 = pool.map_batches(batches, chunksize=2)
# seed=1
with multicore.Pool(augseq, processes=2, seed=1) as pool:
batches_aug2 = pool.map_batches(batches, chunksize=1)
# seed=2
with multicore.Pool(augseq, processes=2, seed=2) as pool:
batches_aug3 = pool.map_batches(batches, chunksize=1)
assert len(batches_aug1) == 60
assert len(batches_aug2) == 60
assert len(batches_aug3) == 60
for batches_aug in [batches_aug1, batches_aug2, batches_aug3]:
for batch in batches_aug:
for keypoints_aug in batch.keypoints_aug:
assert keypoints_aug.keypoints[0].x == 2
assert keypoints_aug.keypoints[0].y == 0
for b1, b2, b3 in zip(batches_aug1, batches_aug2, batches_aug3):
# images were augmented
assert not np.array_equal(b1.images_unaug, b1.images_aug)
assert not np.array_equal(b2.images_unaug, b2.images_aug)
assert not np.array_equal(b3.images_unaug, b3.images_aug)
# original images still the same
assert np.array_equal(b1.images_unaug, batch.images_unaug)
assert np.array_equal(b2.images_unaug, batch.images_unaug)
assert np.array_equal(b3.images_unaug, batch.images_unaug)
# augmentations for same seed are the same
assert np.array_equal(b1.images_aug, b2.images_aug)
# augmentations for different seeds are different
assert not np.array_equal(b1.images_aug, b3.images_aug)
# make sure that batches for the two pools with same seed did not
# repeat within results (only between the results of the two pools)
for batches_aug in [batches_aug1, batches_aug2, batches_aug3]:
self._assert_each_augmentation_not_more_than_once(batches_aug)
def test_augmentations_without_seed_differ(self):
augseq = iaa.AddElementwise((0, 255))
image = np.zeros((10, 10, 1), dtype=np.uint8)
batch = ia.Batch(images=np.uint8([image, image]))
batches = [batch.deepcopy() for _ in sm.xrange(20)]
with multicore.Pool(augseq, processes=2, maxtasksperchild=5) as pool:
batches_aug = pool.map_batches(batches, chunksize=2)
with multicore.Pool(augseq, processes=2) as pool:
batches_aug.extend(pool.map_batches(batches, chunksize=1))
assert len(batches_aug) == 2 * 20
self._assert_each_augmentation_not_more_than_once(batches_aug)
def test_inputs_not_lost(self):
"""Test to make sure that inputs (e.g. images) are never lost."""
def _assert_contains_all_ids(batches_aug):
# batch.images_unaug
ids = set()
for batch_aug in batches_aug:
ids.add(int(batch_aug.images_unaug.flat[0]))
ids.add(int(batch_aug.images_unaug.flat[1]))
for idx in sm.xrange(2 * 100):
assert idx in ids
assert len(ids) == 200
# batch.images_aug
ids = set()
for batch_aug in batches_aug:
ids.add(int(batch_aug.images_aug.flat[0]))
ids.add(int(batch_aug.images_aug.flat[1]))
for idx in sm.xrange(2 * 100):
assert idx in ids
assert len(ids) == 200
augseq = iaa.Identity()
image = np.zeros((1, 1, 1), dtype=np.uint8)
# creates batches containing images with ids from 0 to 199 (one pair
# of consecutive ids per batch)
batches = [
ia.Batch(
images=np.uint8([image + b_idx * 2, image + b_idx * 2 + 1]))
for b_idx in sm.xrange(100)
]
with multicore.Pool(augseq, processes=2, maxtasksperchild=25) as pool:
batches_aug = pool.map_batches(batches)
_assert_contains_all_ids(batches_aug)
with multicore.Pool(augseq, processes=2, maxtasksperchild=25,
seed=1) as pool:
batches_aug = pool.map_batches(batches)
_assert_contains_all_ids(batches_aug)
with multicore.Pool(augseq, processes=3, seed=2) as pool:
batches_aug = pool.map_batches(batches)
_assert_contains_all_ids(batches_aug)
with multicore.Pool(augseq, processes=2, seed=None) as pool:
batches_aug = pool.map_batches(batches)
_assert_contains_all_ids(batches_aug)
batches_aug = pool.map_batches(batches)
_assert_contains_all_ids(batches_aug)
def test_processes(self):
augseq = iaa.Identity()
mock_Pool = mock.MagicMock()
mock_cpu_count = mock.Mock()
patch_pool = mock.patch("multiprocessing.Pool", mock_Pool)
patch_cpu_count = mock.patch("multiprocessing.cpu_count",
mock_cpu_count)
with patch_pool, patch_cpu_count:
# (cpu cores available, processes requested, processes started)
combos = [(1, 1, 1), (2, 1, 1), (3, 1, 1), (1, 2, 2), (3, 2, 2),
(1, None, None), (2, None, None), (3, None, None),
(1, -1, 1), (2, -1, 1), (3, -1, 2), (4, -2, 2)]
for cores_available, processes_req, expected in combos:
with self.subTest(cpu_count_available=cores_available,
processes_requested=processes_req):
mock_cpu_count.return_value = cores_available
with multicore.Pool(augseq,
processes=processes_req) as _pool:
pass
if expected is None:
assert mock_Pool.call_args[0][0] is None
else:
assert mock_Pool.call_args[0][0] == expected
def test_cpu_count_does_not_exist(self, mock_pool):
def _side_effect():
raise NotImplementedError
old_method = multicore._get_context().cpu_count
mock_cpu_count = mock.Mock()
mock_cpu_count.side_effect = _side_effect
multicore._get_context().cpu_count = mock_cpu_count
augseq = iaa.Identity()
with warnings.catch_warnings(record=True) as caught_warnings:
warnings.simplefilter("always")
with multicore.Pool(augseq, processes=-1):
pass
assert mock_cpu_count.call_count == 1
assert mock_pool.call_count == 1
# 'processes' arg to Pool was expected to be set to None as cpu_count
# produced an error
assert mock_pool.call_args_list[0][0][0] is None
assert len(caught_warnings) == 1
assert ("Could not find method multiprocessing.cpu_count(). "
in str(caught_warnings[-1].message))
multicore._get_context().cpu_count = old_method
def test_property_pool(self):
mock_Pool = mock.MagicMock()
mock_Pool.return_value = mock_Pool
mock_Pool.close.return_value = None
mock_Pool.join.return_value = None
# We cannot just mock multiprocessing.Pool here, because of using
# a custom context. We would have to mock each possible context's
# Pool() method or overwrite here the Pool() method of the
# actually used context.
with mock.patch("multiprocessing.pool.Pool", mock_Pool):
augseq = iaa.Identity()
pool_config = multicore.Pool(augseq,
processes=1,
maxtasksperchild=4,
seed=123)
with pool_config as pool:
assert pool.processes == 1
assert pool._pool is None
assert mock_Pool.call_count == 1
assert mock_Pool.close.call_count == 1
assert mock_Pool.join.call_count == 1
# see
# https://github.com/
# python/cpython/blob/master/Lib/multiprocessing/context.py
# L119 (method Pool()) for an example of how Pool() is called
# internally.
assert mock_Pool.call_args[0][0] == 1 # processes
assert mock_Pool.call_args[0][1] is multicore._Pool_initialize_worker
assert mock_Pool.call_args[0][2] == (augseq, 123)
assert mock_Pool.call_args[0][3] == 4
def _test_map_batches_both(self, call_async):
augseq = iaa.Noop()
mock_Pool = mock.MagicMock()
mock_Pool.return_value = mock_Pool
mock_Pool.map.return_value = "X"
mock_Pool.map_async.return_value = "X"
with mock.patch("multiprocessing.Pool", mock_Pool):
batches = [
ia.Batch(images=[ia.quokka()]),
ia.Batch(images=[ia.quokka() + 1])
]
with multicore.Pool(augseq, processes=1) as pool:
if call_async:
_ = pool.map_batches_async(batches)
else:
_ = pool.map_batches(batches)
if call_async:
to_check = mock_Pool.map_async
else:
to_check = mock_Pool.map
assert to_check.call_count == 1
# args, arg 0
assert to_check.call_args[0][0] == multicore._Pool_starworker
# args, arg 1 (batches with ids), tuple 0, entry 0 in tuple (=> batch id)
assert to_check.call_args[0][1][0][0] == 0
# args, arg 1 (batches with ids), tuple 0, entry 1 in tuple (=> batch)
assert np.array_equal(to_check.call_args[0][1][0][1].images_unaug,
batches[0].images_unaug)
# args, arg 1 (batches with ids), tuple 1, entry 0 in tuple (=> batch id)
assert to_check.call_args[0][1][1][0] == 1
# args, arg 1 (batches with ids), tuple 1, entry 1 in tuple (=> batch)
assert np.array_equal(to_check.call_args[0][1][1][1].images_unaug,
batches[1].images_unaug)
def test_fast(processes, chunksize):
augseq = iaa.Dropout(0.1)
with multicore.Pool(augseq, processes=processes) as pool:
batches = list(load_images(n_batches=10000, draw_text=False))
time_start = time.time()
batches_aug = pool.map_batches(batches, chunksize=chunksize)
assert len(batches_aug) == 10000
print("chunksize=%d, worker=%s, time=%.4fs" %
(chunksize, processes, time.time() - time_start))
def test_heavy(processes, chunksize):
augseq_heavy = iaa.PiecewiseAffine(scale=0.2, nb_cols=8, nb_rows=8)
with multicore.Pool(augseq_heavy, processes=processes) as pool:
batches = list(load_images(n_batches=500, draw_text=False))
time_start = time.time()
batches_aug = pool.map_batches(batches, chunksize=chunksize)
assert len(batches_aug) == 500
print("chunksize=%d, worker=%s, time=%.4fs" %
(chunksize, processes, time.time() - time_start))
def test_augmentations_with_seed_match(self):
nb_batches = 60
augseq = iaa.AddElementwise((0, 255))
image = np.zeros((10, 10, 1), dtype=np.uint8)
batch = ia.Batch(images=np.uint8([image, image]))
batches = [batch.deepcopy() for _ in sm.xrange(nb_batches)]
# seed=1
with multicore.Pool(augseq, processes=2, maxtasksperchild=30,
seed=1) as pool:
batches_aug1 = pool.map_batches(batches, chunksize=2)
# seed=1
with multicore.Pool(augseq, processes=2, seed=1) as pool:
batches_aug2 = pool.map_batches(batches, chunksize=1)
# seed=2
with multicore.Pool(augseq, processes=2, seed=2) as pool:
batches_aug3 = pool.map_batches(batches, chunksize=1)
assert len(batches_aug1) == nb_batches
assert len(batches_aug2) == nb_batches
assert len(batches_aug3) == nb_batches
for b1, b2, b3 in zip(batches_aug1, batches_aug2, batches_aug3):
# images were augmented
assert not np.array_equal(b1.images_unaug, b1.images_aug)
assert not np.array_equal(b2.images_unaug, b2.images_aug)
assert not np.array_equal(b3.images_unaug, b3.images_aug)
# original images still the same
assert np.array_equal(b1.images_unaug, batch.images_unaug)
assert np.array_equal(b2.images_unaug, batch.images_unaug)
assert np.array_equal(b3.images_unaug, batch.images_unaug)
# augmentations for same seed are the same
assert np.array_equal(b1.images_aug, b2.images_aug)
# augmentations for different seeds are different
assert not np.array_equal(b1.images_aug, b3.images_aug)
# make sure that batches for the two pools with same seed did not
# repeat within results (only between the results of the two pools)
for batches_aug in [batches_aug1, batches_aug2, batches_aug3]:
self._assert_each_augmentation_not_more_than_once(batches_aug)
def test_augmentations_without_seed_differ_for_images_and_keypoints(self):
augseq = iaa.AddElementwise((0, 255))
image = np.zeros((10, 10, 1), dtype=np.uint8)
# keypoints here will not be changed by augseq, but they will induce deterministic mode to start in
# augment_batches() as each batch contains images AND keypoints
kps = ia.KeypointsOnImage([ia.Keypoint(x=2, y=0)], shape=(10, 10, 1))
batch = ia.Batch(images=np.uint8([image, image]), keypoints=[kps, kps])
batches = [batch.deepcopy() for _ in sm.xrange(20)]
with multicore.Pool(augseq, processes=2, maxtasksperchild=5) as pool:
batches_aug = pool.map_batches(batches, chunksize=2)
with multicore.Pool(augseq, processes=2) as pool:
batches_aug.extend(pool.map_batches(batches, chunksize=1))
assert len(batches_aug) == 2 * 20
for batch in batches_aug:
for keypoints_aug in batch.keypoints_aug:
assert keypoints_aug.keypoints[0].x == 2
assert keypoints_aug.keypoints[0].y == 0
self._assert_each_augmentation_not_more_than_once(batches_aug)
def _test_imap_batches_both(cls, call_unordered):
for clazz in [Batch, UnnormalizedBatch]:
batches = [
clazz(images=[ia.quokka()]),
clazz(images=[ia.quokka() + 1])
]
def _generate_batches():
for batch in batches:
yield batch
augseq = iaa.Identity()
mock_Pool = mock.MagicMock()
mock_Pool.return_value = mock_Pool
mock_Pool.imap.return_value = batches
mock_Pool.imap_unordered.return_value = batches
with mock.patch("multiprocessing.pool.Pool", mock_Pool):
with multicore.Pool(augseq, processes=1) as pool:
gen = _generate_batches()
if call_unordered:
_ = list(pool.imap_batches_unordered(gen))
else:
_ = list(pool.imap_batches(gen))
if call_unordered:
to_check = mock_Pool.imap_unordered
else:
to_check = mock_Pool.imap
assert to_check.call_count == 1
assert to_check.call_args[0][0] == multicore._Pool_starworker
# convert generator to list, make it subscriptable
arg_batches = list(to_check.call_args[0][1])
# args, arg 1 (batches with ids), tuple 0,
# entry 0 in tuple (=> batch id)
assert arg_batches[0][0] == 0
# tuple 0, entry 1 in tuple (=> batch)
assert np.array_equal(arg_batches[0][1].images_unaug,
batches[0].images_unaug)
# tuple 1, entry 0 in tuple (=> batch id)
assert arg_batches[1][0] == 1
# tuple 1, entry 1 in tuple (=> batch)
assert np.array_equal(arg_batches[1][1].images_unaug,
batches[1].images_unaug)
def test_processes(self):
augseq = iaa.Identity()
mock_Pool = mock.MagicMock()
mock_cpu_count = mock.Mock()
# We cannot just mock multiprocessing.Pool here, because of using
# a custom context. We would have to mock each possible context's
# Pool() method or overwrite here the Pool() method of the
# actually used context.
patch_pool = mock.patch("multiprocessing.pool.Pool", mock_Pool)
# Multiprocessing seems to always access os.cpu_count to get the
# current count of cpu cores.
# See
# https://github.com/
# python/cpython/blob/master/Lib/multiprocessing/context.py
# L41.
fname = ("os.cpu_count" if IS_SUPPORTING_CONTEXTS
else "multiprocessing.cpu_count")
patch_cpu_count = mock.patch(fname, mock_cpu_count)
with patch_pool, patch_cpu_count:
# (cpu cores available, processes requested, processes started)
combos = [
(1, 1, 1),
(2, 1, 1),
(3, 1, 1),
(1, 2, 2),
(3, 2, 2),
(1, None, None),
(2, None, None),
(3, None, None),
(1, -1, 1),
(2, -1, 1),
(3, -1, 2),
(4, -2, 2)
]
for cores_available, processes_req, expected in combos:
with self.subTest(cpu_count_available=cores_available,
processes_requested=processes_req):
mock_cpu_count.return_value = cores_available
with multicore.Pool(augseq,
processes=processes_req) as _pool:
pass
if expected is None:
assert mock_Pool.call_args[0][0] is None
else:
assert mock_Pool.call_args[0][0] == expected
def test_join_via_mock(self, mock_pool):
# According to codecov, the join() does not get beyond its initial
# if statement in the test_join() test, even though it should be.
# Might be a simple travis multicore problem?
# It is tested here again via some mocking.
mock_pool.return_value = mock_pool
mock_pool.join.return_value = True
with multicore.Pool(iaa.Identity(), processes=2) as pool:
pool.join()
# Make sure that __exit__ does not call close(), which would then
# call join() again and we would get a call_count of 2
pool._pool = None
assert mock_pool.join.call_count == 1
def test_property_pool(self):
mock_Pool = mock.MagicMock()
mock_Pool.return_value = mock_Pool
mock_Pool.close.return_value = None
mock_Pool.join.return_value = None
with mock.patch("multiprocessing.Pool", mock_Pool):
augseq = iaa.Noop()
with multicore.Pool(augseq,
processes=1,
maxtasksperchild=4,
seed=123) as pool:
assert pool.processes == 1
assert pool._pool is None
assert mock_Pool.call_count == 1
assert mock_Pool.close.call_count == 1
assert mock_Pool.join.call_count == 1
assert mock_Pool.call_args[0][0] == 1 # processes
assert mock_Pool.call_args[1]["initargs"] == (augseq, 123)
assert mock_Pool.call_args[1]["maxtasksperchild"] == 4
def test_processes(self):
augseq = iaa.Noop()
mock_Pool = mock.MagicMock()
mock_cpu_count = mock.Mock()
with mock.patch("multiprocessing.Pool",
mock_Pool), mock.patch("multiprocessing.cpu_count",
mock_cpu_count):
combos = [(1, 1, 1), (2, 1, 1), (3, 1, 1), (1, 2, 2), (3, 2, 2),
(1, None, None), (2, None, None), (3, None, None),
(1, -1, 1), (2, -1, 1), (3, -1, 2), (4, -2, 2)]
for ret_val, inputs, expected in combos:
mock_cpu_count.return_value = ret_val
with multicore.Pool(augseq, processes=inputs) as _pool:
pass
if expected is None:
assert mock_Pool.call_args[0][0] is None
else:
assert mock_Pool.call_args[0][0] == expected
def main():
augseq = iaa.Sequential(
[iaa.Fliplr(0.5),
iaa.CoarseDropout(p=0.1, size_percent=0.1)])
def func_images(images, random_state, parents, hooks):
time.sleep(0.2)
return images
def func_heatmaps(heatmaps, random_state, parents, hooks):
return heatmaps
def func_keypoints(keypoints_on_images, random_state, parents, hooks):
return keypoints_on_images
augseq_slow = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Lambda(func_images=func_images,
func_heatmaps=func_heatmaps,
func_keypoints=func_keypoints)
])
print("------------------")
print(".pool()")
print("------------------")
with augseq.pool() as pool:
time_start = time.time()
batches = list(load_images())
batches_aug = pool.map_batches(batches)
images_aug = []
keypoints_aug = []
for batch_aug in batches_aug:
images_aug.append(batch_aug.images_aug)
keypoints_aug.append(batch_aug.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
# ia.imshow(draw_grid(images_aug, keypoints_aug))
print("------------------")
print("Pool.map_batches(batches)")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches = list(load_images())
batches_aug = pool.map_batches(batches)
images_aug = []
keypoints_aug = []
for batch_aug in batches_aug:
images_aug.append(batch_aug.images_aug)
keypoints_aug.append(batch_aug.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
# ia.imshow(draw_grid(images_aug, keypoints_aug))
print("------------------")
print("Pool.imap_batches(batches)")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images())
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
# ia.imshow(draw_grid(images_aug, keypoints_aug))
print("------------------")
print("Pool.imap_batches(batches, chunksize=32)")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=1000),
chunksize=32)
count = 0
for batch in batches_aug:
count += 1
assert count == 1000
print("Done in %.4fs" % (time.time() - time_start, ))
print("------------------")
print("Pool.imap_batches(batches, chunksize=2)")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=1000),
chunksize=2)
count = 0
for batch in batches_aug:
count += 1
assert count == 1000
print("Done in %.4fs" % (time.time() - time_start, ))
print("------------------")
print("Pool.imap_batches(batches, chunksize=1)")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=1000),
chunksize=1)
count = 0
for batch in batches_aug:
count += 1
assert count == 1000
print("Done in %.4fs" % (time.time() - time_start, ))
print("------------------")
print("Pool.map_batches(batches, chunksize=32)")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches_aug = pool.map_batches(list(load_images(n_batches=1000)),
chunksize=32)
assert len(batches_aug) == 1000
print("Done in %.4fs" % (time.time() - time_start, ))
print("------------------")
print("Pool.map_batches chunksize with fast aug")
print("------------------")
def test_fast(processes, chunksize):
augseq = iaa.Dropout(0.1)
with multicore.Pool(augseq, processes=processes) as pool:
batches = list(load_images(n_batches=10000, draw_text=False))
time_start = time.time()
batches_aug = pool.map_batches(batches, chunksize=chunksize)
assert len(batches_aug) == 10000
print("chunksize=%d, worker=%s, time=%.4fs" %
(chunksize, processes, time.time() - time_start))
test_fast(-4, 1)
test_fast(1, 1)
test_fast(None, 1)
test_fast(1, 4)
test_fast(None, 4)
test_fast(1, 32)
test_fast(None, 32)
print("------------------")
print("Pool.imap_batches chunksize with fast aug")
print("------------------")
def test_fast_imap(processes, chunksize):
augseq = iaa.Dropout(0.1)
with multicore.Pool(augseq, processes=processes) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=10000,
draw_text=False),
chunksize=chunksize)
batches_aug = list(batches_aug)
assert len(batches_aug) == 10000
print("chunksize=%d, worker=%s, time=%.4fs" %
(chunksize, processes, time.time() - time_start))
test_fast_imap(-4, 1)
test_fast_imap(1, 1)
test_fast_imap(None, 1)
test_fast_imap(1, 4)
test_fast_imap(None, 4)
test_fast_imap(1, 32)
test_fast_imap(None, 32)
print("------------------")
print("Pool.map_batches with computationally expensive aug")
print("------------------")
def test_heavy(processes, chunksize):
augseq_heavy = iaa.PiecewiseAffine(scale=0.2, nb_cols=8, nb_rows=8)
with multicore.Pool(augseq_heavy, processes=processes) as pool:
batches = list(load_images(n_batches=500, draw_text=False))
time_start = time.time()
batches_aug = pool.map_batches(batches, chunksize=chunksize)
assert len(batches_aug) == 500
print("chunksize=%d, worker=%s, time=%.4fs" %
(chunksize, processes, time.time() - time_start))
test_heavy(-4, 1)
test_heavy(1, 1)
test_heavy(None, 1)
test_heavy(1, 4)
test_heavy(None, 4)
test_heavy(1, 32)
test_heavy(None, 32)
print("------------------")
print("Pool.imap_batches(batches), slow loading")
print("------------------")
with multicore.Pool(augseq) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=100, sleep=0.2))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
print("------------------")
print("Pool.imap_batches(batches), maxtasksperchild=4")
print("------------------")
with multicore.Pool(augseq, maxtasksperchild=4) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=100))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
ia.imshow(draw_grid(images_aug, keypoints_aug))
print("------------------")
print("Pool.imap_batches(batches), seed=1")
print("------------------")
# we color here the images of the first worker to see in the grids which images belong to one worker
with PoolWithMarkedWorker(augseq, seed=1) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=4))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
grid_a = draw_grid(images_aug, keypoints_aug)
with multicore.Pool(augseq, seed=1) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=4))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
grid_b = draw_grid(images_aug, keypoints_aug)
grid_b[:, 0:2, 0] = 0
grid_b[:, 0:2, 1] = 255
grid_b[:, 0:2, 2] = 0
ia.imshow(np.hstack([grid_a, grid_b]))
print("------------------")
print("Pool.imap_batches(batches), seed=None")
print("------------------")
with multicore.Pool(augseq, seed=None) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=4))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
grid_a = draw_grid(images_aug, keypoints_aug)
with multicore.Pool(augseq, seed=None) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=4))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
grid_b = draw_grid(images_aug, keypoints_aug)
ia.imshow(np.hstack([grid_a, grid_b]))
print("------------------")
print("Pool.imap_batches(batches), maxtasksperchild=4, seed=1")
print("------------------")
with multicore.Pool(augseq, maxtasksperchild=4, seed=1) as pool:
time_start = time.time()
batches_aug = pool.imap_batches(load_images(n_batches=100))
images_aug = []
keypoints_aug = []
for batch in batches_aug:
images_aug.append(batch.images_aug)
keypoints_aug.append(batch.keypoints_aug)
print("Done in %.4fs" % (time.time() - time_start, ))
ia.imshow(draw_grid(images_aug, keypoints_aug))
for augseq_i in [augseq, augseq_slow]:
print("------------------")
print("Many very small runs (batches=1)")
print("------------------")
with multicore.Pool(augseq_i) as pool:
time_start = time.time()
for i in range(100):
_ = pool.map_batches(list(load_images(n_batches=1)))
print("Done in %.4fs" % (time.time() - time_start, ))
print("------------------")
print("Many very small runs (batches=2)")
print("------------------")
with multicore.Pool(augseq_i) as pool:
time_start = time.time()
for i in range(100):
_ = pool.map_batches(list(load_images(n_batches=2)))
print("Done in %.4fs" % (time.time() - time_start, ))
def test_close(self):
augseq = iaa.Noop()
with multicore.Pool(augseq, processes=2) as pool:
pool.close()
def test_join(self):
augseq = iaa.Identity()
with multicore.Pool(augseq, processes=2) as pool:
pool.close()
pool.join()
def test_terminate(self):
augseq = iaa.Identity()
with multicore.Pool(augseq, processes=2) as pool:
pool.terminate()
def test___init___seed_out_of_bounds(self):
augseq = iaa.Identity()
with self.assertRaises(AssertionError) as context:
_ = multicore.Pool(augseq, seed=iarandom.SEED_MAX_VALUE + 100)
assert "Expected `seed` to be" in str(context.exception)
def _test_imap_batches_both_output_buffer_size(cls,
call_unordered,
timeout=0.075):
batches = [
ia.Batch(images=[np.full((1, 1), i, dtype=np.uint8)])
for i in range(8)
]
def _generate_batches(times):
for batch in batches:
yield batch
times.append(time.time())
def callfunc(pool, gen, output_buffer_size):
func = (pool.imap_batches_unordered
if call_unordered else pool.imap_batches)
for v in func(gen, output_buffer_size=output_buffer_size):
yield v
def contains_all_ids(inputs):
arrs = np.uint8([batch.images_aug for batch in inputs])
ids_uq = np.unique(arrs)
return (len(ids_uq) == len(batches) and np.all(0 <= ids_uq)
and np.all(ids_uq < len(batches)))
augseq = iaa.Identity()
with multicore.Pool(augseq, processes=1) as pool:
# no output buffer limit, there should be no noteworthy lag
# for any batch requested from _generate_batches()
times = []
gen = callfunc(pool, _generate_batches(times), None)
result = next(gen)
time.sleep(timeout)
result = [result] + list(gen)
times = np.float64(times)
times_diffs = times[1:] - times[0:-1]
assert np.all(times_diffs < timeout * 1.01)
assert contains_all_ids(result)
# with output buffer limit, but set to the number of batches,
# i.e. should again not lead to any lag
times = []
gen = callfunc(pool, _generate_batches(times), len(batches))
result = next(gen)
time.sleep(timeout)
result = [result] + list(gen)
times = np.float64(times)
times_diffs = times[1:] - times[0:-1]
assert np.all(times_diffs < timeout * 1.01)
assert contains_all_ids(result)
# With output buffer limit of #batches/2 (=4), followed by a
# timeout after starting the loading process. This should quickly
# load batches until the buffer is full, then wait until the
# batches are requested from the buffer (i.e. after the timeout
# ended) and then proceed to produce batches at the speed at which
# they are requested. This should lead to a measureable lag between
# batch 4 and 5 (matching the timeout).
times = []
gen = callfunc(pool, _generate_batches(times), 4)
result = next(gen)
time.sleep(timeout)
result = [result] + list(gen)
times = np.float64(times)
times_diffs = times[1:] - times[0:-1]
# use -1 here because we have N-1 times for N batches as
# diffs denote diffs between Nth and N+1th batch
assert np.all(times_diffs[0:4 - 1] < timeout * 1.01)
assert np.all(times_diffs[4 - 1:4 - 1 + 1] >= timeout * 0.99)
assert np.all(times_diffs[4 - 1 + 1:] < timeout * 1.01)
assert contains_all_ids(result)
