def istarmap(self, func, iterable, chunksize=1): """starmap-version of imap From https://stackoverflow.com/a/57364423/2340703. """ if self._state != mpp.RUN: raise ValueError("Pool not running") if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) try: result = mpp.IMapIterator(self._cache) except AttributeError: # python >=3.8 should pass `pool` instead of `cache` result = mpp.IMapIterator(self) self._taskqueue.put( ( self._guarded_task_generation( result._job, mpp.starmapstar, task_batches ), result._set_length, ) ) return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1): """starmap-version of imap.""" if self._state != mpp.RUN: raise ValueError("Pool not running") if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) if sys.version_info < (3, 8): result = mpp.IMapIterator(self._cache) else: result = mpp.IMapIterator(self) self._taskqueue.put( (self._guarded_task_generation(result._job, mpp.starmapstar, task_batches), result._set_length)) return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1): """starmap-version of imap """ self._check_running() if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) result = mpp.IMapIterator(self) self._taskqueue.put( (self._guarded_task_generation(result._job, mpp.starmapstar, task_batches), result._set_length)) return (item for chunk in result for item in chunk)
def _istarmap_38(self, func, iterable, chunksize=1): """starmap-version of imap Ref: https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm """ self._check_running() if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) result = mpp.IMapIterator(self) self._taskqueue.put( (self._guarded_task_generation(result._job, mpp.starmapstar, task_batches), result._set_length)) return (item for chunk in result for item in chunk)
def _istarmap(self, func, iterable, chunksize=1): """ Starmap-version of imap, see: https://stackoverflow.com/a/57364423/16031961 """ if self._state != mpp.RUN: raise ValueError("Pool not running") if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) result = mpp.IMapIterator(self._cache) self._taskqueue.put( (self._guarded_task_generation(result._job, mpp.starmapstar, task_batches), result._set_length)) return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1): """ Starmap-version of imap. The code is adapted from https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm """ if self._state != mpp.RUN: raise ValueError("Pool not running") if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) result = mpp.IMapIterator(self._cache) self._taskqueue.put( (self._guarded_task_generation(result._job, mpp.starmapstar, task_batches), result._set_length)) return (item for chunk in result for item in chunk)
def istarmap(self, func, iterable, chunksize=1): """ This is a hack to get tqdm working with starmap. """ if self._state != mp.RUN: raise ValueError("Pool not running...") if chunksize < 1: raise ValueError( f"Expected chunksize to be equal or more than 1. Got {chunksize}." ) task_batches = mp.Pool._get_tasks(func, iterable, chunksize) result = mp.IMapIterator(self._cache) self._taskqueue.put( (self._guarded_task_generation( result._job, mp.starmapstar, task_batches), result._set_length, )) return (item for chunk in result for item in chunk)
def multiprocessing_istarmap(self, func, iterable, chunksize=1): """ starmap-version of imap This is only for possibility of displaying progress bar in jupyter notebook during multiprocessing of images to superpixels graphs. Source: https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm/57364423#57364423 """ self._check_running() if chunksize < 1: raise ValueError("Chunksize must be 1+, not {0:n}".format(chunksize)) task_batches = mpp.Pool._get_tasks(func, iterable, chunksize) result = mpp.IMapIterator(self) self._taskqueue.put( ( self._guarded_task_generation(result._job, mpp.starmapstar, task_batches), result._set_length, ) ) return (item for chunk in result for item in chunk)