def process_flush_queues(self, proc):
"""Flushes all queues, including the outbound buffer, so that
all tasks that have not been started will be discarded.
In Celery this is called whenever the transport connection is lost
(consumer restart).
"""
resq = proc.outq._reader
on_state_change = self._result_handler.on_state_change
while not resq.closed and resq.poll(0) and self._state != TERMINATE:
setblocking(resq, 1)
try:
task = resq.recv()
except (IOError, EOFError) as exc:
debug('got %r while flushing process %r',
exc, proc, exc_info=1)
break
else:
if task is not None:
on_state_change(task)
else:
debug('got sentinel while flushing process %r', proc)
finally:
setblocking(resq, 0)
assert not isblocking(resq)
def process_flush_queues(self, proc):
"""Flushes all queues, including the outbound buffer, so that
all tasks that have not been started will be discarded.
In Celery this is called whenever the transport connection is lost
(consumer restart).
"""
resq = proc.outq._reader
on_state_change = self._result_handler.on_state_change
while not resq.closed and resq.poll(0) and self._state != TERMINATE:
setblocking(resq, 1)
try:
task = resq.recv()
except (IOError, EOFError) as exc:
debug('got %r while flushing process %r',
exc,
proc,
exc_info=1)
break
else:
if task is not None:
on_state_change(task)
else:
debug('got sentinel while flushing process %r', proc)
finally:
setblocking(resq, 0)
assert not isblocking(resq)
def on_process_up(proc):
"""Called when a process has started."""
# If we got the same fd as a previous process then we will also
# receive jobs in the old buffer, so we need to reset the
# job._write_to and job._scheduled_for attributes used to recover
# message boundaries when processes exit.
infd = proc.inqW_fd
for job in values(cache):
if job._write_to and job._write_to.inqW_fd == infd:
job._write_to = proc
if job._scheduled_for and job._scheduled_for.inqW_fd == infd:
job._scheduled_for = proc
fileno_to_outq[proc.outqR_fd] = proc
# maintain_pool is called whenever a process exits.
add_reader(
proc.sentinel, event_process_exit, hub, proc.sentinel,
)
assert not isblocking(proc.outq._reader)
# handle_result_event is called when the processes outqueue is
# readable.
add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd)
waiting_to_start.add(proc)
hub.call_later(
self._proc_alive_timeout, verify_process_alive, proc,
)
def on_process_up(proc):
"""Called when a process has started."""
# If we got the same fd as a previous process then we'll also
# receive jobs in the old buffer, so we need to reset the
# job._write_to and job._scheduled_for attributes used to recover
# message boundaries when processes exit.
infd = proc.inqW_fd
for job in cache.values():
if job._write_to and job._write_to.inqW_fd == infd:
job._write_to = proc
if job._scheduled_for and job._scheduled_for.inqW_fd == infd:
job._scheduled_for = proc
fileno_to_outq[proc.outqR_fd] = proc
# maintain_pool is called whenever a process exits.
self._track_child_process(proc, hub)
assert not isblocking(proc.outq._reader)
# handle_result_event is called when the processes outqueue is
# readable.
add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd)
waiting_to_start.add(proc)
hub.call_later(
self._proc_alive_timeout, verify_process_alive, ref(proc),
)
def _recv_message(
self,
add_reader,
fd,
callback,
__read__=__read__,
readcanbuf=readcanbuf,
BytesIO=BytesIO,
unpack_from=unpack_from,
load=_pickle.load,
):
Hr = Br = 0
if readcanbuf:
buf = bytearray(4)
bufv = memoryview(buf)
else:
buf = bufv = BytesIO()
# header
assert not isblocking(fd)
while Hr < 4:
try:
n = __read__(fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
if n == 0:
raise (OSError("End of file during message") if Hr else EOFError())
Hr += n
body_size, = unpack_from(">i", bufv)
if readcanbuf:
buf = bytearray(body_size)
bufv = memoryview(buf)
else:
buf = bufv = BytesIO()
while Br < body_size:
try:
n = __read__(fd, bufv[Br:] if readcanbuf else bufv, body_size - Br)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
if n == 0:
raise (OSError("End of file during message") if Br else EOFError())
Br += n
add_reader(fd, self.handle_event, fd)
if readcanbuf:
message = load(BytesIO(bufv))
else:
bufv.seek(0)
message = load(bufv)
if message:
callback(message)
def _recv_message(self, add_reader, fd, callback,
__read__=__read__, readcanbuf=readcanbuf,
BytesIO=BytesIO, unpack_from=unpack_from,
load=_pickle.load):
Hr = Br = 0
if readcanbuf:
buf = bytearray(4)
bufv = memoryview(buf)
else:
buf = bufv = BytesIO()
# header
assert not isblocking(fd)
while Hr < 4:
try:
n = __read__(
fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr,
)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
if n == 0:
raise (OSError('End of file during message') if Hr
else EOFError())
Hr += n
body_size, = unpack_from('>i', bufv)
if readcanbuf:
buf = bytearray(body_size)
bufv = memoryview(buf)
else:
buf = bufv = BytesIO()
while Br < body_size:
try:
n = __read__(
fd, bufv[Br:] if readcanbuf else bufv, body_size - Br,
)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
if n == 0:
raise (OSError('End of file during message') if Br
else EOFError())
Br += n
add_reader(fd, self.handle_event, fd)
if readcanbuf:
message = load(BytesIO(bufv))
else:
bufv.seek(0)
message = load(bufv)
if message:
callback(message)
def _recv_message(self,
add_reader,
fd,
callback,
read=os.read,
unpack=struct.unpack,
loads=_pickle.loads,
BytesIO=BytesIO):
buf = BytesIO()
# header
remaining = 4
bsize = None
assert not isblocking(fd)
while remaining > 0:
try:
bsize = read(fd, remaining)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
n = len(bsize)
if n == 0:
if remaining == 4:
raise EOFError()
else:
raise OSError("Got end of file during message")
remaining -= n
remaining, = size, = unpack('>i', bsize)
while remaining > 0:
try:
chunk = read(fd, remaining)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
n = len(chunk)
if n == 0:
if remaining == size:
raise EOFError()
else:
raise IOError('Got end of file during message')
buf.write(chunk)
remaining -= n
add_reader(fd, self.handle_event, fd)
message = loads(buf.getvalue())
if message:
callback(message)
def _recv_message(self, add_reader, fd, callback,
read=os.read, unpack=struct.unpack,
loads=_pickle.loads, BytesIO=BytesIO):
buf = BytesIO()
# header
remaining = 4
bsize = None
assert not isblocking(fd)
while remaining > 0:
try:
bsize = read(fd, remaining)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
n = len(bsize)
if n == 0:
if remaining == 4:
raise EOFError()
else:
raise OSError("Got end of file during message")
remaining -= n
remaining, = size, = unpack('>i', bsize)
while remaining > 0:
try:
chunk = read(fd, remaining)
except OSError as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
else:
n = len(chunk)
if n == 0:
if remaining == size:
raise EOFError()
else:
raise IOError('Got end of file during message')
buf.write(chunk)
remaining -= n
add_reader(fd, self.handle_event, fd)
message = loads(buf.getvalue())
if message:
callback(message)
def create_process_queues(self):
"""Create new in, out, etc. queues, returned as a tuple."""
# NOTE: Pipes must be set O_NONBLOCK at creation time (the original
# fd), otherwise it won't be possible to change the flags until
# there's an actual reader/writer on the other side.
inq = _SimpleQueue(wnonblock=True)
outq = _SimpleQueue(rnonblock=True)
synq = None
assert isblocking(inq._reader)
assert not isblocking(inq._writer)
assert not isblocking(outq._reader)
assert isblocking(outq._writer)
if self.synack:
synq = _SimpleQueue(wnonblock=True)
assert isblocking(synq._reader)
assert not isblocking(synq._writer)
return inq, outq, synq
def create_process_queues(self):
"""Create new in, out, etc. queues, returned as a tuple."""
# NOTE: Pipes must be set O_NONBLOCK at creation time (the original
# fd), otherwise it won't be possible to change the flags until
# there's an actual reader/writer on the other side.
inq = _SimpleQueue(wnonblock=True)
outq = _SimpleQueue(rnonblock=True)
synq = None
assert isblocking(inq._reader)
assert not isblocking(inq._writer)
assert not isblocking(outq._reader)
assert isblocking(outq._writer)
if self.synack:
synq = _SimpleQueue(wnonblock=True)
assert isblocking(synq._reader)
assert not isblocking(synq._writer)
return inq, outq, synq
