def _queue_management_worker(self): """ TODO: docstring """ logger.debug("[MTHREAD] queue management worker starting") while not self.bad_state_is_set: task_id, buf = self.incoming_q.get() # TODO: why does this hang? msg = deserialize(buf)[0] # TODO: handle exceptions task_fut = self.tasks[task_id] logger.debug("Got response for task id {}".format(task_id)) if "result" in msg: task_fut.set_result(msg["result"]) elif "exception" in msg: # TODO: handle exception pass elif 'exception' in msg: logger.warning("Task: {} has returned with an exception") try: s = deserialize(msg['exception']) exception = ValueError( "Remote exception description: {}".format(s)) task_fut.set_exception(exception) except Exception as e: # TODO could be a proper wrapped exception? task_fut.set_exception( DeserializationError( "Received exception, but handling also threw an exception: {}" .format(e))) else: raise BadMessage( "Message received is neither result nor exception") if not self.is_alive: break logger.info("[MTHREAD] queue management worker finished")
def _queue_management_worker(self): """Listen to the queue for task status messages and handle them. Depending on the message, tasks will be updated with results, exceptions, or updates. It expects the following messages: .. code:: python { "task_id" : <task_id> "result" : serialized result object, if task succeeded ... more tags could be added later } { "task_id" : <task_id> "exception" : serialized exception object, on failure } We do not support these yet, but they could be added easily. .. code:: python { "task_id" : <task_id> "cpu_stat" : <> "mem_stat" : <> "io_stat" : <> "started" : tstamp } The `None` message is a die request. """ logger.debug("[MTHREAD] queue management worker starting") while not self.bad_state_is_set: try: msgs = self.incoming_q.get(timeout=1) except queue.Empty: logger.debug("[MTHREAD] queue empty") # Timed out. pass except IOError as e: logger.exception( "[MTHREAD] Caught broken queue with exception code {}: {}". format(e.errno, e)) return except Exception as e: logger.exception( "[MTHREAD] Caught unknown exception: {}".format(e)) return else: if msgs is None: logger.debug("[MTHREAD] Got None, exiting") return else: for serialized_msg in msgs: try: msg = pickle.loads(serialized_msg) tid = msg['task_id'] except pickle.UnpicklingError: raise BadMessage( "Message received could not be unpickled") except Exception: raise BadMessage( "Message received does not contain 'task_id' field" ) if tid == -1 and 'exception' in msg: logger.warning( "Executor shutting down due to exception from interchange" ) exception = deserialize(msg['exception']) self.set_bad_state_and_fail_all(exception) break task_fut = self.tasks.pop(tid) if 'result' in msg: result = deserialize(msg['result']) task_fut.set_result(result) elif 'exception' in msg: try: s = deserialize(msg['exception']) # s should be a RemoteExceptionWrapper... so we can reraise it if isinstance(s, RemoteExceptionWrapper): try: s.reraise() except Exception as e: task_fut.set_exception(e) elif isinstance(s, Exception): task_fut.set_exception(s) else: raise ValueError( "Unknown exception-like type received: {}" .format(type(s))) except Exception as e: # TODO could be a proper wrapped exception? task_fut.set_exception( DeserializationError( "Received exception, but handling also threw an exception: {}" .format(e))) else: raise BadMessage( "Message received is neither result or exception" ) if not self.is_alive: break logger.info("[MTHREAD] queue management worker finished")
def _queue_management_worker(self): """Listen to the queue for task status messages and handle them. Depending on the message, tasks will be updated with results, exceptions, or updates. It expects the following messages: .. code:: python { "task_id" : <task_id> "result" : serialized result object, if task succeeded ... more tags could be added later } { "task_id" : <task_id> "exception" : serialized exception object, on failure } We do not support these yet, but they could be added easily. .. code:: python { "task_id" : <task_id> "cpu_stat" : <> "mem_stat" : <> "io_stat" : <> "started" : tstamp } The `None` message is a die request. """ log.debug("[MTHREAD] queue management worker starting") while not self._executor_bad_state.is_set(): try: msgs = self.incoming_q.get(timeout=1) self.last_response_time = time.time() except queue.Empty: log.debug("[MTHREAD] queue empty") # Timed out. pass except OSError as e: log.exception( "[MTHREAD] Caught broken queue with exception code {}: {}". format(e.errno, e)) return except Exception as e: log.exception(f"[MTHREAD] Caught unknown exception: {e}") return else: if msgs is None: log.debug("[MTHREAD] Got None, exiting") return elif isinstance(msgs, EPStatusReport): log.debug(f"[MTHREAD] Received EPStatusReport {msgs}") if self.passthrough: self.results_passthrough.put({ "task_id": None, "message": pickle.dumps(msgs) }) else: log.debug("[MTHREAD] Unpacking results") for serialized_msg in msgs: try: msg = pickle.loads(serialized_msg) tid = msg["task_id"] except pickle.UnpicklingError: raise BadMessage( "Message received could not be unpickled") except Exception: raise BadMessage( "Message received does not contain 'task_id' field" ) if tid == -2 and "info" in msg: log.warning( "[MTHREAD[ Received info response : {}".format( msg["info"])) if tid == -1 and "exception" in msg: # TODO: This could be handled better we are # essentially shutting down the client with little # indication to the user. log.warning( "[MTHREAD] Executor shutting down due to fatal " "exception from interchange") self._executor_exception = fx_serializer.deserialize( msg["exception"]) log.exception("[MTHREAD] Exception: {}".format( self._executor_exception)) # Set bad state to prevent new tasks from being submitted self._executor_bad_state.set() # We set all current tasks to this exception to make sure # that this is raised in the main context. for task_id in self.tasks: try: self.tasks[task_id].set_exception( self._executor_exception) except concurrent.futures.InvalidStateError: # Task was already cancelled, the exception can be # ignored log.debug( f"Task:{task_id} result couldn't be set. " "Already in terminal state") break if self.passthrough is True: log.debug( f"[MTHREAD] Pushing results for task:{tid}") # we are only interested in actual task ids here, not # identifiers for other message types sent_task_id = tid if isinstance(tid, str) else None x = self.results_passthrough.put({ "task_id": sent_task_id, "message": serialized_msg }) log.debug(f"[MTHREAD] task:{tid} ret value: {x}") log.debug( "[MTHREAD] task:%s items in queue: %s", tid, self.results_passthrough.qsize(), ) continue try: task_fut = self.tasks.pop(tid) except KeyError: # This is triggered when the result of a cancelled task is # returned # We should log, and proceed. log.warning( f"[MTHREAD] Task:{tid} not found in tasks table\n" "Task likely was cancelled and removed.") continue if "result" in msg: result = fx_serializer.deserialize(msg["result"]) try: task_fut.set_result(result) except concurrent.futures.InvalidStateError: log.debug( f"Task:{tid} result couldn't be set. " "Already in terminal state") elif "exception" in msg: exception = fx_serializer.deserialize( msg["exception"]) try: task_fut.set_result(exception) except concurrent.futures.InvalidStateError: log.debug( f"Task:{tid} result couldn't be set. " "Already in terminal state") else: raise BadMessage( "[MTHREAD] Message received is neither result or " "exception") if not self.is_alive: break log.info("[MTHREAD] queue management worker finished")