def _queue_management_worker(self):
""" TODO: docstring """
logger.debug("[MTHREAD] queue management worker starting")
while not self.bad_state_is_set:
task_id, buf = self.incoming_q.get() # TODO: why does this hang?
msg = deserialize(buf)[0]
# TODO: handle exceptions
task_fut = self.tasks[task_id]
logger.debug("Got response for task id {}".format(task_id))
if "result" in msg:
task_fut.set_result(msg["result"])
elif "exception" in msg:
# TODO: handle exception
pass
elif 'exception' in msg:
logger.warning("Task: {} has returned with an exception")
try:
s = deserialize(msg['exception'])
exception = ValueError(
"Remote exception description: {}".format(s))
task_fut.set_exception(exception)
except Exception as e:
# TODO could be a proper wrapped exception?
task_fut.set_exception(
DeserializationError(
"Received exception, but handling also threw an exception: {}"
.format(e)))
else:
raise BadMessage(
"Message received is neither result nor exception")
if not self.is_alive:
break
logger.info("[MTHREAD] queue management worker finished")
def _queue_management_worker(self):
"""Listen to the queue for task status messages and handle them.
Depending on the message, tasks will be updated with results, exceptions,
or updates. It expects the following messages:
.. code:: python
{
"task_id" : <task_id>
"result" : serialized result object, if task succeeded
... more tags could be added later
}
{
"task_id" : <task_id>
"exception" : serialized exception object, on failure
}
We do not support these yet, but they could be added easily.
.. code:: python
{
"task_id" : <task_id>
"cpu_stat" : <>
"mem_stat" : <>
"io_stat" : <>
"started" : tstamp
}
The `None` message is a die request.
"""
logger.debug("[MTHREAD] queue management worker starting")
while not self.bad_state_is_set:
try:
msgs = self.incoming_q.get(timeout=1)
except queue.Empty:
logger.debug("[MTHREAD] queue empty")
# Timed out.
pass
except IOError as e:
logger.exception(
"[MTHREAD] Caught broken queue with exception code {}: {}".
format(e.errno, e))
return
except Exception as e:
logger.exception(
"[MTHREAD] Caught unknown exception: {}".format(e))
return
else:
if msgs is None:
logger.debug("[MTHREAD] Got None, exiting")
return
else:
for serialized_msg in msgs:
try:
msg = pickle.loads(serialized_msg)
tid = msg['task_id']
except pickle.UnpicklingError:
raise BadMessage(
"Message received could not be unpickled")
except Exception:
raise BadMessage(
"Message received does not contain 'task_id' field"
)
if tid == -1 and 'exception' in msg:
logger.warning(
"Executor shutting down due to exception from interchange"
)
exception = deserialize(msg['exception'])
self.set_bad_state_and_fail_all(exception)
break
task_fut = self.tasks.pop(tid)
if 'result' in msg:
result = deserialize(msg['result'])
task_fut.set_result(result)
elif 'exception' in msg:
try:
s = deserialize(msg['exception'])
# s should be a RemoteExceptionWrapper... so we can reraise it
if isinstance(s, RemoteExceptionWrapper):
try:
s.reraise()
except Exception as e:
task_fut.set_exception(e)
elif isinstance(s, Exception):
task_fut.set_exception(s)
else:
raise ValueError(
"Unknown exception-like type received: {}"
.format(type(s)))
except Exception as e:
# TODO could be a proper wrapped exception?
task_fut.set_exception(
DeserializationError(
"Received exception, but handling also threw an exception: {}"
.format(e)))
else:
raise BadMessage(
"Message received is neither result or exception"
)
if not self.is_alive:
break
logger.info("[MTHREAD] queue management worker finished")
