def _process_worker(call_queue, result_queue, const_args=[], shared_arrays=[]):
"""Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
shared_arrays_np = [
np.ctypeslib.as_array(arr).view(dtype).reshape(shape)
for arr, dtype, shape in shared_arrays
]
while True:
call_item = call_queue.get(block=True)
if call_item is None:
result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args,
const_args=const_args,
shared_arrays=shared_arrays_np,
**call_item.kwargs)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue):
"""Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
logger.exception(e) # 主要是直接显示错误。
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_actor_eventloop(_call_queue, _result_queue, _ActorClass, *args,
**kwargs):
"""
actor event loop run in a separate process.
Creates the instance of the actor (passing in the required *args, and
**kwargs). Then the eventloop starts and feeds the actor messages from the
_call_queue. Results are placed in the _result_queue, which are then placed
in Future objects.
"""
actor = _ActorClass(*args, **kwargs)
while True:
call_item = _call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
_result_queue.put(os.getpid())
return
try:
r = actor.handle(call_item.message)
except BaseException as e:
exc = process._ExceptionWithTraceback(e, e.__traceback__)
_result_queue.put(
process._ResultItem(call_item.work_id, exception=exc))
else:
_result_queue.put(process._ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue, progress_queue, initializer,
initargs):
if initializer is not None:
try:
initializer(*initargs)
except BaseException:
_base.LOGGER.critical('Exception in initializer:', exc_info=True)
return
while True:
call_item = call_queue.get(block=True)
if call_item is None:
result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
if isinstance(call_item.fn, ProgressedTask):
for step in r:
progress_queue.put(step)
r = None
else:
progress_queue.put(1)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def _return_result(call_item, result_queue, future):
try:
r = future.result()
except BaseException as e:
if _ExceptionWithTraceback is None:
result_queue.put(_ResultItem(call_item.work_id, exception=e))
else:
exc = _ExceptionWithTraceback(e, e.__traceback__)
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))