def _process_worker(call_queue, result_queue, const_args=[], shared_arrays=[]):
"""Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
shared_arrays_np = [
np.ctypeslib.as_array(arr).view(dtype).reshape(shape)
for arr, dtype, shape in shared_arrays
]
while True:
call_item = call_queue.get(block=True)
if call_item is None:
result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args,
const_args=const_args,
shared_arrays=shared_arrays_np,
**call_item.kwargs)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue):
"""
Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
prctl.set_pdeathsig(signal.SIGKILL)
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(None)
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException:
e = sys.exc_info()[1]
result_queue.put(_ResultItem(call_item.work_id, exception=e))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_actor_eventloop(_call_queue, _result_queue, _ActorClass, *args,
**kwargs):
"""
actor event loop run in a separate process.
Creates the instance of the actor (passing in the required *args, and
**kwargs). Then the eventloop starts and feeds the actor messages from the
_call_queue. Results are placed in the _result_queue, which are then placed
in Future objects.
"""
actor = _ActorClass(*args, **kwargs)
while True:
call_item = _call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
_result_queue.put(os.getpid())
return
try:
r = actor.handle(call_item.message)
except BaseException as e:
exc = process._ExceptionWithTraceback(e, e.__traceback__)
_result_queue.put(
process._ResultItem(call_item.work_id, exception=exc))
else:
_result_queue.put(process._ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue):
"""Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
logger.exception(e) # 主要是直接显示错误。
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue):
"""
Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
prctl.set_pdeathsig(signal.SIGKILL)
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(None)
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException:
e = sys.exc_info()[1]
result_queue.put(_ResultItem(call_item.work_id,
exception=e))
else:
result_queue.put(_ResultItem(call_item.work_id,
result=r))
def _process_worker(call_queue, result_queue, progress_queue, initializer,
initargs):
if initializer is not None:
try:
initializer(*initargs)
except BaseException:
_base.LOGGER.critical('Exception in initializer:', exc_info=True)
return
while True:
call_item = call_queue.get(block=True)
if call_item is None:
result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
if isinstance(call_item.fn, ProgressedTask):
for step in r:
progress_queue.put(step)
r = None
else:
progress_queue.put(1)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def process_worker(call_queue, result_queue):
"""
A copy of Python's process_worker function in :class:`concurrent.futures.ProcessPoolExecutor`.
This copy was changed to not die on KeyboardInterrupt, but to exit gracefully.
Also, no traceback is printed upon :class:`NoKnownPathwaysError` or :class:`CancelledError`.
Note
----
Copyright © 2001-2018 Python Software Foundation; All Rights Reserved
"""
while True:
try:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(os.getpid())
return
except KeyboardInterrupt as e:
sys.exit()
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException as e:
from FEV_KEGG.KEGG.Database import NoKnownPathwaysError
if isinstance(e, NoKnownPathwaysError) or isinstance(e, concurrent.futures._base.CancelledError):
pass
elif isinstance(e, Exception):
traceback.print_exc()
result_queue.put(_ResultItem(call_item.work_id,
exception=e))
else:
result_queue.put(_ResultItem(call_item.work_id,
result=r))
def _return_result(call_item, result_queue, future):
try:
r = future.result()
except BaseException as e:
if _ExceptionWithTraceback is None:
result_queue.put(_ResultItem(call_item.work_id, exception=e))
else:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue, connection_args, logger):
"""Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
vmem_process = None
while True:
try:
call_item = call_queue.get(block=True, timeout=0.5)
except Queue.Empty:
continue
if call_item is None:
# Wake up queue management thread
if vmem_process:
vmem_process.connection.logout()
result_queue.put(None)
return
if vmem_process is None:
vmem_process = ViolinAPI(connection_args, logger)
try:
function = getattr(vmem_process, call_item.fn)
r = function(*call_item.args, **call_item.kwargs)
except BaseException:
e = sys.exc_info()[1]
result_queue.put(_ResultItem(call_item.work_id,
exception=e))
else:
result_queue.put(_ResultItem(call_item.work_id,
result=r))