def _queue_management_worker(self):
""" TODO: docstring """
logger.debug("[MTHREAD] queue management worker starting")
while not self.bad_state_is_set:
task_id, buf = self.incoming_q.get() # TODO: why does this hang?
msg = deserialize(buf)[0]
# TODO: handle exceptions
task_fut = self.tasks[task_id]
logger.debug("Got response for task id {}".format(task_id))
if "result" in msg:
task_fut.set_result(msg["result"])
elif "exception" in msg:
# TODO: handle exception
pass
elif 'exception' in msg:
logger.warning("Task: {} has returned with an exception")
try:
s = deserialize(msg['exception'])
exception = ValueError(
"Remote exception description: {}".format(s))
task_fut.set_exception(exception)
except Exception as e:
# TODO could be a proper wrapped exception?
task_fut.set_exception(
DeserializationError(
"Received exception, but handling also threw an exception: {}"
.format(e)))
else:
raise BadMessage(
"Message received is neither result nor exception")
if not self.is_alive:
break
logger.info("[MTHREAD] queue management worker finished")
def _queue_management_worker(self):
"""Listen to the queue for task status messages and handle them.
Depending on the message, tasks will be updated with results, exceptions,
or updates. It expects the following messages:
.. code:: python
{
"task_id" : <task_id>
"result" : serialized result object, if task succeeded
... more tags could be added later
}
{
"task_id" : <task_id>
"exception" : serialized exception object, on failure
}
We do not support these yet, but they could be added easily.
.. code:: python
{
"task_id" : <task_id>
"cpu_stat" : <>
"mem_stat" : <>
"io_stat" : <>
"started" : tstamp
}
The `None` message is a die request.
"""
logger.debug("[MTHREAD] queue management worker starting")
while not self.bad_state_is_set:
try:
msgs = self.incoming_q.get(timeout=1)
except queue.Empty:
logger.debug("[MTHREAD] queue empty")
# Timed out.
pass
except IOError as e:
logger.exception(
"[MTHREAD] Caught broken queue with exception code {}: {}".
format(e.errno, e))
return
except Exception as e:
logger.exception(
"[MTHREAD] Caught unknown exception: {}".format(e))
return
else:
if msgs is None:
logger.debug("[MTHREAD] Got None, exiting")
return
else:
for serialized_msg in msgs:
try:
msg = pickle.loads(serialized_msg)
tid = msg['task_id']
except pickle.UnpicklingError:
raise BadMessage(
"Message received could not be unpickled")
except Exception:
raise BadMessage(
"Message received does not contain 'task_id' field"
)
if tid == -1 and 'exception' in msg:
logger.warning(
"Executor shutting down due to exception from interchange"
)
exception = deserialize(msg['exception'])
self.set_bad_state_and_fail_all(exception)
break
task_fut = self.tasks.pop(tid)
if 'result' in msg:
result = deserialize(msg['result'])
task_fut.set_result(result)
elif 'exception' in msg:
try:
s = deserialize(msg['exception'])
# s should be a RemoteExceptionWrapper... so we can reraise it
if isinstance(s, RemoteExceptionWrapper):
try:
s.reraise()
except Exception as e:
task_fut.set_exception(e)
elif isinstance(s, Exception):
task_fut.set_exception(s)
else:
raise ValueError(
"Unknown exception-like type received: {}"
.format(type(s)))
except Exception as e:
# TODO could be a proper wrapped exception?
task_fut.set_exception(
DeserializationError(
"Received exception, but handling also threw an exception: {}"
.format(e)))
else:
raise BadMessage(
"Message received is neither result or exception"
)
if not self.is_alive:
break
logger.info("[MTHREAD] queue management worker finished")
def _queue_management_worker(self):
"""Listen to the queue for task status messages and handle them.
Depending on the message, tasks will be updated with results, exceptions,
or updates. It expects the following messages:
.. code:: python
{
"task_id" : <task_id>
"result" : serialized result object, if task succeeded
... more tags could be added later
}
{
"task_id" : <task_id>
"exception" : serialized exception object, on failure
}
We do not support these yet, but they could be added easily.
.. code:: python
{
"task_id" : <task_id>
"cpu_stat" : <>
"mem_stat" : <>
"io_stat" : <>
"started" : tstamp
}
The `None` message is a die request.
"""
log.debug("[MTHREAD] queue management worker starting")
while not self._executor_bad_state.is_set():
try:
msgs = self.incoming_q.get(timeout=1)
self.last_response_time = time.time()
except queue.Empty:
log.debug("[MTHREAD] queue empty")
# Timed out.
pass
except OSError as e:
log.exception(
"[MTHREAD] Caught broken queue with exception code {}: {}".
format(e.errno, e))
return
except Exception as e:
log.exception(f"[MTHREAD] Caught unknown exception: {e}")
return
else:
if msgs is None:
log.debug("[MTHREAD] Got None, exiting")
return
elif isinstance(msgs, EPStatusReport):
log.debug(f"[MTHREAD] Received EPStatusReport {msgs}")
if self.passthrough:
self.results_passthrough.put({
"task_id":
None,
"message":
pickle.dumps(msgs)
})
else:
log.debug("[MTHREAD] Unpacking results")
for serialized_msg in msgs:
try:
msg = pickle.loads(serialized_msg)
tid = msg["task_id"]
except pickle.UnpicklingError:
raise BadMessage(
"Message received could not be unpickled")
except Exception:
raise BadMessage(
"Message received does not contain 'task_id' field"
)
if tid == -2 and "info" in msg:
log.warning(
"[MTHREAD[ Received info response : {}".format(
msg["info"]))
if tid == -1 and "exception" in msg:
# TODO: This could be handled better we are
# essentially shutting down the client with little
# indication to the user.
log.warning(
"[MTHREAD] Executor shutting down due to fatal "
"exception from interchange")
self._executor_exception = fx_serializer.deserialize(
msg["exception"])
log.exception("[MTHREAD] Exception: {}".format(
self._executor_exception))
# Set bad state to prevent new tasks from being submitted
self._executor_bad_state.set()
# We set all current tasks to this exception to make sure
# that this is raised in the main context.
for task_id in self.tasks:
try:
self.tasks[task_id].set_exception(
self._executor_exception)
except concurrent.futures.InvalidStateError:
# Task was already cancelled, the exception can be
# ignored
log.debug(
f"Task:{task_id} result couldn't be set. "
"Already in terminal state")
break
if self.passthrough is True:
log.debug(
f"[MTHREAD] Pushing results for task:{tid}")
# we are only interested in actual task ids here, not
# identifiers for other message types
sent_task_id = tid if isinstance(tid,
str) else None
x = self.results_passthrough.put({
"task_id":
sent_task_id,
"message":
serialized_msg
})
log.debug(f"[MTHREAD] task:{tid} ret value: {x}")
log.debug(
"[MTHREAD] task:%s items in queue: %s",
tid,
self.results_passthrough.qsize(),
)
continue
try:
task_fut = self.tasks.pop(tid)
except KeyError:
# This is triggered when the result of a cancelled task is
# returned
# We should log, and proceed.
log.warning(
f"[MTHREAD] Task:{tid} not found in tasks table\n"
"Task likely was cancelled and removed.")
continue
if "result" in msg:
result = fx_serializer.deserialize(msg["result"])
try:
task_fut.set_result(result)
except concurrent.futures.InvalidStateError:
log.debug(
f"Task:{tid} result couldn't be set. "
"Already in terminal state")
elif "exception" in msg:
exception = fx_serializer.deserialize(
msg["exception"])
try:
task_fut.set_result(exception)
except concurrent.futures.InvalidStateError:
log.debug(
f"Task:{tid} result couldn't be set. "
"Already in terminal state")
else:
raise BadMessage(
"[MTHREAD] Message received is neither result or "
"exception")
if not self.is_alive:
break
log.info("[MTHREAD] queue management worker finished")
