def _load_batches(cls, load_batch_func, queue_internal, join_signal,
seedval):
if seedval is not None:
random.seed(seedval)
np.random.seed(seedval)
iarandom.seed(seedval)
try:
gen = (load_batch_func() if not ia.is_generator(load_batch_func)
else load_batch_func)
for batch in gen:
ia.do_assert(
isinstance(batch, Batch),
"Expected batch returned by load_batch_func to "
"be of class imgaug.Batch, got %s." % (type(batch), ))
batch_pickled = pickle.dumps(batch, protocol=-1)
while not join_signal.is_set():
try:
queue_internal.put(batch_pickled, timeout=0.005)
break
except QueueFull:
pass
if join_signal.is_set():
break
except Exception:
traceback.print_exc()
finally:
queue_internal.put("")
time.sleep(0.01)
def _load_batches(self, load_batch_func, queue_internal, join_signal, seedval):
if seedval is not None:
random.seed(seedval)
np.random.seed(seedval)
ia.seed(seedval)
try:
gen = load_batch_func() if not ia.is_generator(load_batch_func) else load_batch_func
for batch in gen:
ia.do_assert(isinstance(batch, Batch),
"Expected batch returned by load_batch_func to be of class imgaug.Batch, got %s." % (
type(batch),))
batch_pickled = pickle.dumps(batch, protocol=-1)
while not join_signal.is_set():
try:
queue_internal.put(batch_pickled, timeout=0.005)
break
except QueueFull:
pass
if join_signal.is_set():
break
except Exception:
traceback.print_exc()
finally:
queue_internal.put("")
time.sleep(0.01)
def imap_batches_unordered(self, batches, chunksize=1):
"""
Augment batches from a generator in a way that does not guarantee to preserve order.
Parameters
----------
batches : generator of imgaug.augmentables.batches.Batch
The batches to augment, provided as a generator. Each call to the generator should yield exactly one
batch.
chunksize : None or int, optional
Rough indicator of how many tasks should be sent to each worker. Increasing this number can improve
performance.
Yields
------
imgaug.augmentables.batches.Batch
Augmented batch.
"""
assert ia.is_generator(batches), (
"Expected to get a generator as 'batches', got type %s. " +
"Call map_batches() if you use lists.") % (type(batches), )
# TODO change this to 'yield from' once switched to 3.3+
gen = self.pool.imap_unordered(_Pool_starworker,
self._handle_batch_ids_gen(batches),
chunksize=chunksize)
for batch in gen:
yield batch
def imap_batches_unordered(self, batches, chunksize=1,
output_buffer_size=None):
"""
Augment batches from a generator in a way that does not guarantee to preserve order.
Pattern for output buffer constraint is from
https://stackoverflow.com/a/47058399.
Parameters
----------
batches : generator of imgaug.augmentables.batches.Batch
The batches to augment, provided as a generator. Each call to the generator should yield exactly one
batch.
chunksize : None or int, optional
Rough indicator of how many tasks should be sent to each worker. Increasing this number can improve
performance.
output_buffer_size : None or int, optional
Max number of batches to handle *at the same time* in the *whole*
pipeline (including already augmented batches that are waiting to
be requested). If the buffer size is reached, no new batches will
be loaded from `batches` until a produced (i.e. augmented) batch is
consumed (i.e. requested from this method).
The buffer is unlimited if this is set to ``None``. For large
datasets, this should be set to an integer value to avoid filling
the whole RAM if loading+augmentation happens faster than training.
*New in version 0.3.0.*
Yields
------
imgaug.augmentables.batches.Batch
Augmented batch.
"""
assert ia.is_generator(batches), ("Expected to get a generator as 'batches', got type %s. "
+ "Call map_batches() if you use lists.") % (type(batches),)
# buffer is either None or a Semaphore
output_buffer_left = _create_output_buffer_left(output_buffer_size)
# TODO change this to 'yield from' once switched to 3.3+
gen = self.pool.imap_unordered(
_Pool_starworker,
self._ibuffer_batch_loading(
self._handle_batch_ids_gen(batches),
output_buffer_left
),
chunksize=chunksize
)
for batch in gen:
yield batch
if output_buffer_left is not None:
output_buffer_left.release()
def _assert_batches_is_generator(cls, batches):
assert ia.is_generator(
batches), ("Expected `batches` to be generator, got type %s. Call "
"map_batches() if you use lists.") % (type(batches), )
