def print_progress(self, index):
"""Display the process of the parallel execution only a fraction
of time, controled by self.verbose.
"""
if not self.verbose:
return
elapsed_time = time.time() - self._start_time
# This is heuristic code to print only 'verbose' times a messages
# The challenge is that we may not know the queue length
if self._iterable:
if _verbosity_filter(index, self.verbose):
return
self._print("Done %3i jobs | elapsed: %s", (index + 1, short_format_time(elapsed_time)))
else:
# We are finished dispatching
queue_length = self.n_dispatched
# We always display the first loop
if not index == 0:
# Display depending on the number of remaining items
# A message as soon as we finish dispatching, cursor is 0
cursor = queue_length - index + 1 - self._pre_dispatch_amount
frequency = (queue_length // self.verbose) + 1
is_last_item = index + 1 == queue_length
if is_last_item or cursor % frequency:
return
remaining_time = elapsed_time / (index + 1) * (self.n_dispatched - index - 1.0)
self._print(
"Done %3i out of %3i | elapsed: %s remaining: %s",
(index + 1, queue_length, short_format_time(elapsed_time), short_format_time(remaining_time)),
)
def dispatch(self, func, args, kwargs):
""" Queue the function for computing, with or without multiprocessing
"""
if self._pool is None:
job = ImmediateApply(func, args, kwargs)
index = len(self._jobs)
if not _verbosity_filter(index, self.verbose):
self._print('Done %3i jobs | elapsed: %s',
(index + 1,
short_format_time(time.time() - self._start_time)
))
self._jobs.append(job)
self.n_dispatched += 1
else:
self._lock.acquire()
# If job.get() catches an exception, it closes the queue:
try:
try:
job = self._pool.apply_async(SafeFunction(func), args,
kwargs, callback=CallBack(self.n_dispatched, self))
self._jobs.append(job)
self.n_dispatched += 1
except AssertionError:
print '[Parallel] Pool seems closed'
finally:
self._lock.release()
def __call__(self, iterable):
if self._jobs:
raise ValueError("This Parallel instance is already running")
n_jobs = self.n_jobs
if n_jobs < 0 and multiprocessing is not None:
n_jobs = max(multiprocessing.cpu_count() + 1 + n_jobs, 1)
# The list of exceptions that we will capture
self.exceptions = [TransportableException]
if n_jobs is None or multiprocessing is None or n_jobs == 1:
n_jobs = 1
self._pool = None
else:
if multiprocessing.current_process()._daemonic:
# Daemonic processes cannot have children
n_jobs = 1
self._pool = None
warnings.warn("Parallel loops cannot be nested, setting n_jobs=1", stacklevel=2)
else:
self._pool = multiprocessing.Pool(n_jobs)
self._lock = threading.Lock()
# We are using multiprocessing, we also want to capture
# KeyboardInterrupts
self.exceptions.extend([KeyboardInterrupt, WorkerInterrupt])
if self.pre_dispatch == "all" or n_jobs == 1:
self._iterable = None
self._pre_dispatch_amount = 0
else:
self._iterable = iterable
self._dispatch_amount = 0
pre_dispatch = self.pre_dispatch
if hasattr(pre_dispatch, "endswith"):
pre_dispatch = eval(pre_dispatch)
self._pre_dispatch_amount = pre_dispatch = int(pre_dispatch)
iterable = itertools.islice(iterable, pre_dispatch)
self._start_time = time.time()
self.n_dispatched = 0
try:
for function, args, kwargs in iterable:
self.dispatch(function, args, kwargs)
self.retrieve()
# Make sure that we get a last message telling us we are done
elapsed_time = time.time() - self._start_time
self._print(
"Done %3i out of %3i | elapsed: %s finished",
(len(self._output), len(self._output), short_format_time(elapsed_time)),
)
finally:
if n_jobs > 1:
self._pool.close()
self._pool.join()
self._jobs = list()
output = self._output
self._output = None
return output
def print_progress(self, index):
"""Display the process of the parallel execution only a fraction
of time, controled by self.verbose.
"""
if not self.verbose:
return
elapsed_time = time.time() - self._start_time
# This is heuristic code to print only 'verbose' times a messages
# The challenge is that we may not know the queue length
if self._iterable:
if _verbosity_filter(index, self.verbose):
return
self._print('Done %3i jobs | elapsed: %s',
(index + 1,
short_format_time(elapsed_time),
))
else:
# We are finished dispatching
queue_length = self.n_dispatched
# We always display the first loop
if not index == 0:
# Display depending on the number of remaining items
# A message as soon as we finish dispatching, cursor is 0
cursor = (queue_length - index + 1
- self._pre_dispatch_amount)
frequency = (queue_length // self.verbose) + 1
is_last_item = (index + 1 == queue_length)
if (is_last_item or cursor % frequency):
return
remaining_time = (elapsed_time / (index + 1) *
(self.n_dispatched - index - 1.))
self._print('Done %3i out of %3i | elapsed: %s remaining: %s',
(index + 1,
queue_length,
short_format_time(elapsed_time),
short_format_time(remaining_time),
))
def __call__(self, iterable):
if self._jobs:
raise ValueError('This Parallel instance is already running')
n_jobs = self.n_jobs
if n_jobs < 0 and multiprocessing is not None:
n_jobs = max(multiprocessing.cpu_count() + 1 + n_jobs, 1)
# The list of exceptions that we will capture
self.exceptions = [TransportableException]
if n_jobs is None or multiprocessing is None or n_jobs == 1:
n_jobs = 1
self._pool = None
else:
if multiprocessing.current_process()._daemonic:
# Daemonic processes cannot have children
n_jobs = 1
self._pool = None
warnings.warn(
'Parallel loops cannot be nested, setting n_jobs=1',
stacklevel=2)
else:
self._pool = multiprocessing.Pool(n_jobs)
self._lock = threading.Lock()
# We are using multiprocessing, we also want to capture
# KeyboardInterrupts
self.exceptions.extend([KeyboardInterrupt, WorkerInterrupt])
if self.pre_dispatch == 'all' or n_jobs == 1:
self._iterable = None
self._pre_dispatch_amount = 0
else:
self._iterable = iterable
self._dispatch_amount = 0
pre_dispatch = self.pre_dispatch
if hasattr(pre_dispatch, 'endswith'):
pre_dispatch = eval(pre_dispatch)
self._pre_dispatch_amount = pre_dispatch = int(pre_dispatch)
iterable = itertools.islice(iterable, pre_dispatch)
self._start_time = time.time()
self.n_dispatched = 0
try:
for function, args, kwargs in iterable:
self.dispatch(function, args, kwargs)
self.retrieve()
# Make sure that we get a last message telling us we are done
elapsed_time = time.time() - self._start_time
self._print('Done %3i out of %3i | elapsed: %s finished',
(len(self._output),
len(self._output),
short_format_time(elapsed_time)
))
finally:
if n_jobs > 1:
self._pool.close()
self._pool.join()
self._jobs = list()
output = self._output
self._output = None
return output
