def istarmap(self, func, iterable, chunksize=1):
"""starmap-version of imap
From https://stackoverflow.com/a/57364423/2340703.
"""
if self._state != mpp.RUN:
raise ValueError("Pool not running")
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
try:
result = mpp.IMapIterator(self._cache)
except AttributeError:
# python >=3.8 should pass `pool` instead of `cache`
result = mpp.IMapIterator(self)
self._taskqueue.put(
(
self._guarded_task_generation(
result._job, mpp.starmapstar, task_batches
),
result._set_length,
)
)
return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1):
"""starmap-version of imap."""
if self._state != mpp.RUN:
raise ValueError("Pool not running")
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
if sys.version_info < (3, 8):
result = mpp.IMapIterator(self._cache)
else:
result = mpp.IMapIterator(self)
self._taskqueue.put(
(self._guarded_task_generation(result._job, mpp.starmapstar,
task_batches), result._set_length))
return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1):
"""starmap-version of imap
"""
self._check_running()
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
result = mpp.IMapIterator(self)
self._taskqueue.put(
(self._guarded_task_generation(result._job, mpp.starmapstar,
task_batches), result._set_length))
return (item for chunk in result for item in chunk)
def _istarmap_38(self, func, iterable, chunksize=1):
"""starmap-version of imap
Ref: https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm
"""
self._check_running()
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
result = mpp.IMapIterator(self)
self._taskqueue.put(
(self._guarded_task_generation(result._job, mpp.starmapstar,
task_batches), result._set_length))
return (item for chunk in result for item in chunk)
def _istarmap(self, func, iterable, chunksize=1):
"""
Starmap-version of imap, see: https://stackoverflow.com/a/57364423/16031961
"""
if self._state != mpp.RUN:
raise ValueError("Pool not running")
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
result = mpp.IMapIterator(self._cache)
self._taskqueue.put(
(self._guarded_task_generation(result._job, mpp.starmapstar,
task_batches), result._set_length))
return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1):
""" Starmap-version of imap. The code is adapted from
https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm
"""
if self._state != mpp.RUN:
raise ValueError("Pool not running")
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
result = mpp.IMapIterator(self._cache)
self._taskqueue.put(
(self._guarded_task_generation(result._job, mpp.starmapstar,
task_batches), result._set_length))
return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1):
"""
This is a hack to get tqdm working with starmap.
"""
if self._state != mp.RUN:
raise ValueError("Pool not running...")
if chunksize < 1:
raise ValueError(
f"Expected chunksize to be equal or more than 1. Got {chunksize}."
)
task_batches = mp.Pool._get_tasks(func, iterable, chunksize)
result = mp.IMapIterator(self._cache)
self._taskqueue.put(
(self._guarded_task_generation(
result._job,
mp.starmapstar,
task_batches),
result._set_length,
))
return (item for chunk in result for item in chunk)
def multiprocessing_istarmap(self, func, iterable, chunksize=1):
"""
starmap-version of imap
This is only for possibility of displaying progress bar in jupyter notebook during
multiprocessing of images to superpixels graphs.
Source: https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm/57364423#57364423
"""
self._check_running()
if chunksize < 1:
raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize))
task_batches = mpp.Pool._get_tasks(func, iterable, chunksize)
result = mpp.IMapIterator(self)
self._taskqueue.put(
(
self._guarded_task_generation(result._job, mpp.starmapstar, task_batches),
result._set_length,
)
)
return (item for chunk in result for item in chunk)
