def run_parallel(graph, nprocs=None, sleep=0.2, raise_errors=False):
nprocs = nprocs or mp.cpu_count() - 1
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
with mp.Pool(nprocs) as pool:
exception_q = mp.Queue(10)
def error_callback(exception):
exception_q.put_nowait(exception)
pool.terminate()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph, reverse=False):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: assigning task {}'.format(
os.getpid(), task))
pool.apply_async(run_task,
args=(graph, task, raise_errors),
error_callback=error_callback)
time.sleep(sleep)
if not exception_q.empty():
raise exception_q.get()
return tgraph.recover_values_from_manager(graph)
def run_parallel_async(graph, nprocs=None, sleep=0.2):
if nprocs == 1:
return run_async(graph)
nprocs = nprocs or mp.cpu_count() // 2
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
ioq = mp.Queue(len(graph.funcs.keys()))
cpuq = mp.Queue(len(graph.funcs.keys()))
for _ in range(nprocs):
proc = mp.Process(target=run_scheduler,
args=(graph, sleep, ioq, cpuq))
proc.start()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: queueing task {}'.format(
os.getpid(), task))
if task in graph.io_bound:
ioq.put(task)
else:
cpuq.put(task)
time.sleep(sleep)
return tgraph.recover_values_from_manager(graph)
def run_parallel_async(graph, nprocs=None, sleep=0.2, raise_errors=False):
if nprocs == 1:
return run_async(graph, sleep=sleep, raise_errors=raise_errors)
nprocs = nprocs or mp.cpu_count() // 2
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
ioq = mp.Queue(len(graph.funcs.keys()))
cpuq = mp.Queue(len(graph.funcs.keys()))
procs = [mp.Process(target=run_scheduler,
args=(graph, sleep, ioq, cpuq, raise_errors))
for _ in range(nprocs)]
for proc in procs:
proc.start()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info(
'pid {}: queueing task {}'.format(os.getpid(), task))
if task in graph.io_bound:
ioq.put(task)
else:
cpuq.put(task)
time.sleep(sleep)
if raise_errors and sum(not p.exitcode for p in procs):
raise RuntimeError('An async task has failed. Please check your logs')
return tgraph.recover_values_from_manager(graph)
async def run_task_async(graph, task):
try:
graph = tgraph.mark_as_in_progress(graph, task)
args = get_task_args(graph, task)
log(graph).info('pid {}: starting task {}'.format(os.getpid(), task))
result = await graph.funcs[task](*args)
return task_success(graph, task, result)
except Exception as error:
return task_error(graph, task, error)
async def queue_loader(graph, ioq, cpuq, sleep):
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
logger.info('pid {}: queueing task {}'.format(os.getpid(), task))
if task in graph.io_bound:
await ioq.put(task)
else:
await cpuq.put(task)
await asyncio.sleep(sleep)
def run_parallel(graph, nprocs=None, sleep=0.2):
nprocs = nprocs or mp.cpu_count() - 1
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
with mp.Pool(nprocs) as pool:
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph, reverse=False):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: assigning task {}'.format(
os.getpid(), task))
pool.apply_async(run_task, args=(graph, task))
time.sleep(sleep)
return tgraph.recover_values_from_manager(graph)
async def run_task_async(graph, task, raise_errors=False):
graph = tgraph.mark_as_in_progress(graph, task)
args = get_task_args(graph, task)
log(graph).debug('pid {}: starting task {}'.format(os.getpid(), task))
try:
result = await asyncio.coroutine(graph.funcs[task])(*args)
return task_success(graph, task, result)
except Exception as error:
graph = task_error(graph, task, error)
if raise_errors:
raise
return graph