def plot_rt_point(key, y, plot_ind=1, x=None): global _n_plots if _suppress_plots: return fig = plt.gcf() if _n_plots == 0: _init_plot() if key not in _rt_plot_state: _n_plots = max(plot_ind, _n_plots) _rt_plot_state[key] = RTPlotState( [], [], [], [], _get_ax(plot_ind).plot([], [], marker='.', label=key)[0], [0]) plot_state = _rt_plot_state[key] x = x if x is not None else time.time() - _plot_start plot_state.xbuf.append(x) plot_state.ybuf.append(y) plot_state.n_so_far[0] += 1 if timing._register_exec(__name__, 'plot', 1): _clear_bufs()
def pmap(fn, args_kwargs, verbose=True, super_verbose=False, n_procs=None): """ Multiprocessing abstraction that includes a progress meter. Usage: def foo(x): return x * 2 pmap(foo, [([1], {}), ([2], {})]) => [2, 4] """ global _pool_state try: args_kwargs = process_args_kwargs(args_kwargs) n_procs = n_procs or mp.cpu_count() - 1 if verbose: print 'Using', n_procs, 'processes.' mp.freeze_support() p = mp.Pool(n_procs) ret = [None] * len(args_kwargs) processed_args = [(fn, (i, a_k)) for i, a_k in enumerate(args_kwargs)] start_time = time.time() for job_ind, (arg_ind, result) in \ enumerate(p.imap_unordered(wrapper_fcn, processed_args, chunksize=1)): n_completed = job_ind + 1 avg_time = (time.time() - start_time) / n_completed num_remaining = len(args_kwargs) - n_completed eta = avg_time * num_remaining if verbose and timing._register_exec(__file__, 'pmap', 20): print '\n>>>>>>>>>>> PROGRESS:', job_ind+1, '/', len(args_kwargs), 'done. ETA:', eta ret[arg_ind] = result if super_verbose: print 'Args remaining:', [arg for i, arg in enumerate(args_kwargs) if ret[i] == None] p.terminate() p.join() del p if verbose: print '\n>>>>>>>>>>> DONE.', job_ind+1, '/', len(args_kwargs) return ret except Exception, e: # If there was an exception thrown above, then re-run it single-threaded until we hit # the exception so we can raise it with the right stack trace. print 'Hit exception in parallel run:', str(e) print 'Re-running single-threaded.' ret = [] for i, (args, kwargs) in enumerate(args_kwargs): if verbose: print '>>>>>>>>>>> SINGLE-THREAD PROGRESS:', i+1, '/', len(args_kwargs), 'done' ret.append(fn(*args, **kwargs)) # Throw the original if we didn't hit any exception in the single-threaded execution raise return ret