def _process_worker(call_queue, result_queue, const_args=[], shared_arrays=[]): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. Args: call_queue: A multiprocessing.Queue of _CallItems that will be read and evaluated by the worker. result_queue: A multiprocessing.Queue of _ResultItems that will written to by the worker. shutdown: A multiprocessing.Event that will be set as a signal to the worker that it should exit when call_queue is empty. """ shared_arrays_np = [ np.ctypeslib.as_array(arr).view(dtype).reshape(shape) for arr, dtype, shape in shared_arrays ] while True: call_item = call_queue.get(block=True) if call_item is None: result_queue.put(os.getpid()) return try: r = call_item.fn(*call_item.args, const_args=const_args, shared_arrays=shared_arrays_np, **call_item.kwargs) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) result_queue.put(_ResultItem(call_item.work_id, exception=exc)) else: result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue): """ Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. Args: call_queue: A multiprocessing.Queue of _CallItems that will be read and evaluated by the worker. result_queue: A multiprocessing.Queue of _ResultItems that will written to by the worker. shutdown: A multiprocessing.Event that will be set as a signal to the worker that it should exit when call_queue is empty. """ prctl.set_pdeathsig(signal.SIGKILL) while True: call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread result_queue.put(None) return try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException: e = sys.exc_info()[1] result_queue.put(_ResultItem(call_item.work_id, exception=e)) else: result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_actor_eventloop(_call_queue, _result_queue, _ActorClass, *args, **kwargs): """ actor event loop run in a separate process. Creates the instance of the actor (passing in the required *args, and **kwargs). Then the eventloop starts and feeds the actor messages from the _call_queue. Results are placed in the _result_queue, which are then placed in Future objects. """ actor = _ActorClass(*args, **kwargs) while True: call_item = _call_queue.get(block=True) if call_item is None: # Wake up queue management thread _result_queue.put(os.getpid()) return try: r = actor.handle(call_item.message) except BaseException as e: exc = process._ExceptionWithTraceback(e, e.__traceback__) _result_queue.put( process._ResultItem(call_item.work_id, exception=exc)) else: _result_queue.put(process._ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. Args: call_queue: A multiprocessing.Queue of _CallItems that will be read and evaluated by the worker. result_queue: A multiprocessing.Queue of _ResultItems that will written to by the worker. shutdown: A multiprocessing.Event that will be set as a signal to the worker that it should exit when call_queue is empty. """ while True: call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread result_queue.put(os.getpid()) return try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) result_queue.put(_ResultItem(call_item.work_id, exception=exc)) logger.exception(e) # 主要是直接显示错误。 else: result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue, progress_queue, initializer, initargs): if initializer is not None: try: initializer(*initargs) except BaseException: _base.LOGGER.critical('Exception in initializer:', exc_info=True) return while True: call_item = call_queue.get(block=True) if call_item is None: result_queue.put(os.getpid()) return try: r = call_item.fn(*call_item.args, **call_item.kwargs) if isinstance(call_item.fn, ProgressedTask): for step in r: progress_queue.put(step) r = None else: progress_queue.put(1) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) result_queue.put(_ResultItem(call_item.work_id, exception=exc)) else: result_queue.put(_ResultItem(call_item.work_id, result=r))
def process_worker(call_queue, result_queue): """ A copy of Python's process_worker function in :class:`concurrent.futures.ProcessPoolExecutor`. This copy was changed to not die on KeyboardInterrupt, but to exit gracefully. Also, no traceback is printed upon :class:`NoKnownPathwaysError` or :class:`CancelledError`. Note ---- Copyright © 2001-2018 Python Software Foundation; All Rights Reserved """ while True: try: call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread result_queue.put(os.getpid()) return except KeyboardInterrupt as e: sys.exit() try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: from FEV_KEGG.KEGG.Database import NoKnownPathwaysError if isinstance(e, NoKnownPathwaysError) or isinstance(e, concurrent.futures._base.CancelledError): pass elif isinstance(e, Exception): traceback.print_exc() result_queue.put(_ResultItem(call_item.work_id, exception=e)) else: result_queue.put(_ResultItem(call_item.work_id, result=r))
def _return_result(call_item, result_queue, future): try: r = future.result() except BaseException as e: if _ExceptionWithTraceback is None: result_queue.put(_ResultItem(call_item.work_id, exception=e)) else: exc = _ExceptionWithTraceback(e, e.__traceback__) result_queue.put(_ResultItem(call_item.work_id, exception=exc)) else: result_queue.put(_ResultItem(call_item.work_id, result=r))
def _process_worker(call_queue, result_queue, connection_args, logger): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. Args: call_queue: A multiprocessing.Queue of _CallItems that will be read and evaluated by the worker. result_queue: A multiprocessing.Queue of _ResultItems that will written to by the worker. shutdown: A multiprocessing.Event that will be set as a signal to the worker that it should exit when call_queue is empty. """ vmem_process = None while True: try: call_item = call_queue.get(block=True, timeout=0.5) except Queue.Empty: continue if call_item is None: # Wake up queue management thread if vmem_process: vmem_process.connection.logout() result_queue.put(None) return if vmem_process is None: vmem_process = ViolinAPI(connection_args, logger) try: function = getattr(vmem_process, call_item.fn) r = function(*call_item.args, **call_item.kwargs) except BaseException: e = sys.exc_info()[1] result_queue.put(_ResultItem(call_item.work_id, exception=e)) else: result_queue.put(_ResultItem(call_item.work_id, result=r))