def process_flush_queues(self, proc): """Flushes all queues, including the outbound buffer, so that all tasks that have not been started will be discarded. In Celery this is called whenever the transport connection is lost (consumer restart). """ resq = proc.outq._reader on_state_change = self._result_handler.on_state_change while not resq.closed and resq.poll(0) and self._state != TERMINATE: setblocking(resq, 1) try: task = resq.recv() except (IOError, EOFError) as exc: debug('got %r while flushing process %r', exc, proc, exc_info=1) break else: if task is not None: on_state_change(task) else: debug('got sentinel while flushing process %r', proc) finally: setblocking(resq, 0) assert not isblocking(resq)
def on_process_up(proc): """Called when a process has started.""" # If we got the same fd as a previous process then we will also # receive jobs in the old buffer, so we need to reset the # job._write_to and job._scheduled_for attributes used to recover # message boundaries when processes exit. infd = proc.inqW_fd for job in values(cache): if job._write_to and job._write_to.inqW_fd == infd: job._write_to = proc if job._scheduled_for and job._scheduled_for.inqW_fd == infd: job._scheduled_for = proc fileno_to_outq[proc.outqR_fd] = proc # maintain_pool is called whenever a process exits. add_reader( proc.sentinel, event_process_exit, hub, proc.sentinel, ) assert not isblocking(proc.outq._reader) # handle_result_event is called when the processes outqueue is # readable. add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd) waiting_to_start.add(proc) hub.call_later( self._proc_alive_timeout, verify_process_alive, proc, )
def on_process_up(proc): """Called when a process has started.""" # If we got the same fd as a previous process then we'll also # receive jobs in the old buffer, so we need to reset the # job._write_to and job._scheduled_for attributes used to recover # message boundaries when processes exit. infd = proc.inqW_fd for job in cache.values(): if job._write_to and job._write_to.inqW_fd == infd: job._write_to = proc if job._scheduled_for and job._scheduled_for.inqW_fd == infd: job._scheduled_for = proc fileno_to_outq[proc.outqR_fd] = proc # maintain_pool is called whenever a process exits. self._track_child_process(proc, hub) assert not isblocking(proc.outq._reader) # handle_result_event is called when the processes outqueue is # readable. add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd) waiting_to_start.add(proc) hub.call_later( self._proc_alive_timeout, verify_process_alive, ref(proc), )
def _recv_message( self, add_reader, fd, callback, __read__=__read__, readcanbuf=readcanbuf, BytesIO=BytesIO, unpack_from=unpack_from, load=_pickle.load, ): Hr = Br = 0 if readcanbuf: buf = bytearray(4) bufv = memoryview(buf) else: buf = bufv = BytesIO() # header assert not isblocking(fd) while Hr < 4: try: n = __read__(fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: if n == 0: raise (OSError("End of file during message") if Hr else EOFError()) Hr += n body_size, = unpack_from(">i", bufv) if readcanbuf: buf = bytearray(body_size) bufv = memoryview(buf) else: buf = bufv = BytesIO() while Br < body_size: try: n = __read__(fd, bufv[Br:] if readcanbuf else bufv, body_size - Br) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: if n == 0: raise (OSError("End of file during message") if Br else EOFError()) Br += n add_reader(fd, self.handle_event, fd) if readcanbuf: message = load(BytesIO(bufv)) else: bufv.seek(0) message = load(bufv) if message: callback(message)
def _recv_message(self, add_reader, fd, callback, __read__=__read__, readcanbuf=readcanbuf, BytesIO=BytesIO, unpack_from=unpack_from, load=_pickle.load): Hr = Br = 0 if readcanbuf: buf = bytearray(4) bufv = memoryview(buf) else: buf = bufv = BytesIO() # header assert not isblocking(fd) while Hr < 4: try: n = __read__( fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr, ) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: if n == 0: raise (OSError('End of file during message') if Hr else EOFError()) Hr += n body_size, = unpack_from('>i', bufv) if readcanbuf: buf = bytearray(body_size) bufv = memoryview(buf) else: buf = bufv = BytesIO() while Br < body_size: try: n = __read__( fd, bufv[Br:] if readcanbuf else bufv, body_size - Br, ) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: if n == 0: raise (OSError('End of file during message') if Br else EOFError()) Br += n add_reader(fd, self.handle_event, fd) if readcanbuf: message = load(BytesIO(bufv)) else: bufv.seek(0) message = load(bufv) if message: callback(message)
def _recv_message(self, add_reader, fd, callback, read=os.read, unpack=struct.unpack, loads=_pickle.loads, BytesIO=BytesIO): buf = BytesIO() # header remaining = 4 bsize = None assert not isblocking(fd) while remaining > 0: try: bsize = read(fd, remaining) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: n = len(bsize) if n == 0: if remaining == 4: raise EOFError() else: raise OSError("Got end of file during message") remaining -= n remaining, = size, = unpack('>i', bsize) while remaining > 0: try: chunk = read(fd, remaining) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield n = len(chunk) if n == 0: if remaining == size: raise EOFError() else: raise IOError('Got end of file during message') buf.write(chunk) remaining -= n add_reader(fd, self.handle_event, fd) message = loads(buf.getvalue()) if message: callback(message)
def _recv_message(self, add_reader, fd, callback, read=os.read, unpack=struct.unpack, loads=_pickle.loads, BytesIO=BytesIO): buf = BytesIO() # header remaining = 4 bsize = None assert not isblocking(fd) while remaining > 0: try: bsize = read(fd, remaining) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: n = len(bsize) if n == 0: if remaining == 4: raise EOFError() else: raise OSError("Got end of file during message") remaining -= n remaining, = size, = unpack('>i', bsize) while remaining > 0: try: chunk = read(fd, remaining) except OSError as exc: if get_errno(exc) not in UNAVAIL: raise yield else: n = len(chunk) if n == 0: if remaining == size: raise EOFError() else: raise IOError('Got end of file during message') buf.write(chunk) remaining -= n add_reader(fd, self.handle_event, fd) message = loads(buf.getvalue()) if message: callback(message)
def create_process_queues(self): """Create new in, out, etc. queues, returned as a tuple.""" # NOTE: Pipes must be set O_NONBLOCK at creation time (the original # fd), otherwise it won't be possible to change the flags until # there's an actual reader/writer on the other side. inq = _SimpleQueue(wnonblock=True) outq = _SimpleQueue(rnonblock=True) synq = None assert isblocking(inq._reader) assert not isblocking(inq._writer) assert not isblocking(outq._reader) assert isblocking(outq._writer) if self.synack: synq = _SimpleQueue(wnonblock=True) assert isblocking(synq._reader) assert not isblocking(synq._writer) return inq, outq, synq