def plot_rt_point(key, y, plot_ind=1, x=None):
global _n_plots
if _suppress_plots:
return
fig = plt.gcf()
if _n_plots == 0:
_init_plot()
if key not in _rt_plot_state:
_n_plots = max(plot_ind, _n_plots)
_rt_plot_state[key] = RTPlotState(
[], [], [], [],
_get_ax(plot_ind).plot([], [], marker='.', label=key)[0],
[0])
plot_state = _rt_plot_state[key]
x = x if x is not None else time.time() - _plot_start
plot_state.xbuf.append(x)
plot_state.ybuf.append(y)
plot_state.n_so_far[0] += 1
if timing._register_exec(__name__, 'plot', 1):
_clear_bufs()
def pmap(fn, args_kwargs, verbose=True, super_verbose=False, n_procs=None):
""" Multiprocessing abstraction that includes a progress meter.
Usage:
def foo(x): return x * 2
pmap(foo, [([1], {}), ([2], {})]) => [2, 4]
"""
global _pool_state
try:
args_kwargs = process_args_kwargs(args_kwargs)
n_procs = n_procs or mp.cpu_count() - 1
if verbose:
print 'Using', n_procs, 'processes.'
mp.freeze_support()
p = mp.Pool(n_procs)
ret = [None] * len(args_kwargs)
processed_args = [(fn, (i, a_k)) for i, a_k in enumerate(args_kwargs)]
start_time = time.time()
for job_ind, (arg_ind, result) in \
enumerate(p.imap_unordered(wrapper_fcn, processed_args, chunksize=1)):
n_completed = job_ind + 1
avg_time = (time.time() - start_time) / n_completed
num_remaining = len(args_kwargs) - n_completed
eta = avg_time * num_remaining
if verbose and timing._register_exec(__file__, 'pmap', 20):
print '\n>>>>>>>>>>> PROGRESS:', job_ind+1, '/', len(args_kwargs), 'done. ETA:', eta
ret[arg_ind] = result
if super_verbose:
print 'Args remaining:', [arg for i, arg in enumerate(args_kwargs) if ret[i] == None]
p.terminate()
p.join()
del p
if verbose:
print '\n>>>>>>>>>>> DONE.', job_ind+1, '/', len(args_kwargs)
return ret
except Exception, e:
# If there was an exception thrown above, then re-run it single-threaded until we hit
# the exception so we can raise it with the right stack trace.
print 'Hit exception in parallel run:', str(e)
print 'Re-running single-threaded.'
ret = []
for i, (args, kwargs) in enumerate(args_kwargs):
if verbose:
print '>>>>>>>>>>> SINGLE-THREAD PROGRESS:', i+1, '/', len(args_kwargs), 'done'
ret.append(fn(*args, **kwargs))
# Throw the original if we didn't hit any exception in the single-threaded execution
raise
return ret