def _queue_management_worker(self): """ TODO: docstring """ logger.debug("[MTHREAD] queue management worker starting") while not self.bad_state_is_set: task_id, buf = self.incoming_q.get() # TODO: why does this hang? msg = deserialize(buf)[0] # TODO: handle exceptions task_fut = self.tasks[task_id] logger.debug("Got response for task id {}".format(task_id)) if "result" in msg: task_fut.set_result(msg["result"]) elif "exception" in msg: # TODO: handle exception pass elif 'exception' in msg: logger.warning("Task: {} has returned with an exception") try: s = deserialize(msg['exception']) exception = ValueError( "Remote exception description: {}".format(s)) task_fut.set_exception(exception) except Exception as e: # TODO could be a proper wrapped exception? task_fut.set_exception( DeserializationError( "Received exception, but handling also threw an exception: {}" .format(e))) else: raise BadMessage( "Message received is neither result nor exception") if not self.is_alive: break logger.info("[MTHREAD] queue management worker finished")
def _queue_management_worker(self): """Listen to the queue for task status messages and handle them. Depending on the message, tasks will be updated with results, exceptions, or updates. It expects the following messages: .. code:: python { "task_id" : <task_id> "result" : serialized result object, if task succeeded ... more tags could be added later } { "task_id" : <task_id> "exception" : serialized exception object, on failure } We do not support these yet, but they could be added easily. .. code:: python { "task_id" : <task_id> "cpu_stat" : <> "mem_stat" : <> "io_stat" : <> "started" : tstamp } The `None` message is a die request. """ logger.debug("[MTHREAD] queue management worker starting") while not self.bad_state_is_set: try: msgs = self.incoming_q.get(timeout=1) except queue.Empty: logger.debug("[MTHREAD] queue empty") # Timed out. pass except IOError as e: logger.exception( "[MTHREAD] Caught broken queue with exception code {}: {}". format(e.errno, e)) return except Exception as e: logger.exception( "[MTHREAD] Caught unknown exception: {}".format(e)) return else: if msgs is None: logger.debug("[MTHREAD] Got None, exiting") return else: for serialized_msg in msgs: try: msg = pickle.loads(serialized_msg) tid = msg['task_id'] except pickle.UnpicklingError: raise BadMessage( "Message received could not be unpickled") except Exception: raise BadMessage( "Message received does not contain 'task_id' field" ) if tid == -1 and 'exception' in msg: logger.warning( "Executor shutting down due to exception from interchange" ) exception = deserialize(msg['exception']) self.set_bad_state_and_fail_all(exception) break task_fut = self.tasks.pop(tid) if 'result' in msg: result = deserialize(msg['result']) task_fut.set_result(result) elif 'exception' in msg: try: s = deserialize(msg['exception']) # s should be a RemoteExceptionWrapper... so we can reraise it if isinstance(s, RemoteExceptionWrapper): try: s.reraise() except Exception as e: task_fut.set_exception(e) elif isinstance(s, Exception): task_fut.set_exception(s) else: raise ValueError( "Unknown exception-like type received: {}" .format(type(s))) except Exception as e: # TODO could be a proper wrapped exception? task_fut.set_exception( DeserializationError( "Received exception, but handling also threw an exception: {}" .format(e))) else: raise BadMessage( "Message received is neither result or exception" ) if not self.is_alive: break logger.info("[MTHREAD] queue management worker finished")