def process_flush_queues(self, proc):
"""Flushes all queues, including the outbound buffer, so that
all tasks that have not been started will be discarded.
In Celery this is called whenever the transport connection is lost
(consumer restart).
"""
resq = proc.outq._reader
on_state_change = self._result_handler.on_state_change
while not resq.closed and resq.poll(0) and self._state != TERMINATE:
setblocking(resq, 1)
try:
task = resq.recv()
except (IOError, EOFError) as exc:
debug('got %r while flushing process %r',
exc, proc, exc_info=1)
break
else:
if task is not None:
on_state_change(task)
else:
debug('got sentinel while flushing process %r', proc)
finally:
setblocking(resq, 0)
assert not isblocking(resq)
def _flush_outqueue(self, fd, remove, process_index, on_state_change):
try:
proc = process_index[fd]
except KeyError:
# process already found terminated
# this means its outqueue has already been processed
# by the worker lost handler.
return remove(fd)
reader = proc.outq._reader
try:
setblocking(reader, 1)
except (OSError, IOError):
return remove(fd)
try:
if reader.poll(0):
task = reader.recv()
else:
task = None
sleep(0.5)
except (IOError, EOFError):
return remove(fd)
else:
if task:
on_state_change(task)
finally:
try:
setblocking(reader, 0)
except (OSError, IOError):
return remove(fd)
def _stop_task_handler(task_handler):
"""Called at shutdown to tell processes that we are shutting down."""
for proc in task_handler.pool:
setblocking(proc.inq._writer, 1)
try:
proc.inq.put(None)
except OSError as exc:
if get_errno(exc) != errno.EBADF:
raise
def _stop_task_handler(task_handler):
"""Called at shutdown to tell processes that we're shutting down."""
for proc in task_handler.pool:
try:
setblocking(proc.inq._writer, 1)
except (OSError, IOError):
pass
else:
try:
proc.inq.put(None)
except OSError as exc:
if exc.errno != errno.EBADF:
raise
def _stop_task_handler(task_handler):
"""Called at shutdown to tell processes that we're shutting down."""
for proc in task_handler.pool:
try:
setblocking(proc.inq._writer, 1)
except (OSError, IOError):
pass
else:
try:
proc.inq.put(None)
except OSError as exc:
if exc.errno != errno.EBADF:
raise