def iter_read(fds, deadline=None): fds = list(fds) bits = [] timeout = None while fds: if deadline is not None: timeout = max(0, deadline - time.time()) if timeout == 0: break rfds, _, _ = select.select(fds, [], [], timeout) if not rfds: continue for fd in rfds: s, disconnected = mitogen.core.io_op(os.read, fd, 4096) if disconnected or not s: IOLOG.debug('iter_read(%r) -> disconnected', fd) fds.remove(fd) else: IOLOG.debug('iter_read(%r) -> %r', fd, s) bits.append(s) yield s if not fds: raise mitogen.core.StreamError( 'EOF on stream; last 300 bytes received: %r' % (''.join(bits)[-300:], )) raise mitogen.core.TimeoutError('read timed out')
def on_receive(self, broker): s = self.receive_side.read() IOLOG.debug('%r.on_receive() -> len %r', self, len(s)) if s: mitogen.core.fire(self, 'receive', s) else: self.on_disconnect(broker)
def discard_until(fd, s, deadline): for buf in iter_read([fd], deadline): if IOLOG.level == logging.DEBUG: for line in buf.splitlines(): IOLOG.debug('discard_until: discarding %r', line) if buf.endswith(s): return
def _on_stdin(self, msg): if msg.is_dead: IOLOG.debug('%r._on_stdin() -> %r', self, data) self.pump.close() return data = msg.unpickle() IOLOG.debug('%r._on_stdin() -> len %d', self, len(data)) self.pump.write(data)
def _forward_module(self, context, fullname): IOLOG.debug('%r._forward_module(%r, %r)', self, context, fullname) path = [] while fullname: path.append(fullname) fullname, _, _ = fullname.rpartition('.') for fullname in reversed(path): stream = self._router.stream_by_id(context.context_id) self._send_module_and_related(stream, fullname) self._send_forward_module(stream, context, fullname)
def _on_stdin(self, msg): if msg == mitogen.core._DEAD: return data = msg.unpickle() if data == mitogen.core._DEAD: IOLOG.debug('%r._on_stdin() -> %r', self, data) self.pump.close() return IOLOG.debug('%r._on_stdin() -> len %d', self, len(data)) self.pump.write(data)
def on_transmit(self, broker): written = self.transmit_side.write(self._output_buf) IOLOG.debug('%r.on_transmit() -> len %r', self, written) if written is None: self.on_disconnect(broker) else: self._output_buf = self._output_buf[written:] if not self._output_buf: broker._stop_transmit(self) if self._closed: self.on_disconnect(broker)
def on_transmit(self, broker): written = self.transmit_side.write(self._output_buf) IOLOG.debug('%r.on_transmit() -> len %r', self, written) if written is None: self.on_disconnect(broker) else: self._output_buf = self._output_buf[written:] if not self._output_buf: broker.stop_transmit(self) if self._closed: self.on_disconnect(broker)
def poll(self, timeout=None): the_timeout = -1 if timeout is not None: the_timeout = timeout events, _ = mitogen.core.io_op(self._epoll.poll, the_timeout, 32) for fd, event in events: if event & self._inmask and fd in self._rfds: # Events can still be read for an already-discarded fd. mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd) yield self._rfds[fd] if event & select.EPOLLOUT and fd in self._wfds: mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd) yield self._wfds[fd]
def poll(self, timeout=None): changelist = self._changelist self._changelist = [] events, _ = mitogen.core.io_op(self._kqueue.control, changelist, 32, timeout) for event in events: fd = event.ident if event.filter == select.KQ_FILTER_READ and fd in self._rfds: # Events can still be read for an already-discarded fd. mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd) yield self._rfds[fd] elif event.filter == select.KQ_FILTER_WRITE and fd in self._wfds: mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd) yield self._wfds[fd]
def find(self, fullname): """ Find `fullname` using :func:`pkgutil.find_loader`. """ try: # Pre-'import spec' this returned None, in Python3.6 it raises # ImportError. loader = pkgutil.find_loader(fullname) except ImportError: e = sys.exc_info()[1] LOG.debug('%r._get_module_via_pkgutil(%r): %s', self, fullname, e) return None IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r', self, fullname, loader) if not loader: return try: path, is_special = _py_filename(loader.get_filename(fullname)) source = loader.get_source(fullname) is_pkg = loader.is_package(fullname) # workaround for special python modules that might only exist in memory if is_special and is_pkg and not source: source = '\n' except (AttributeError, ImportError): # - Per PEP-302, get_source() and is_package() are optional, # calling them may throw AttributeError. # - get_filename() may throw ImportError if pkgutil.find_loader() # picks a "parent" package's loader for some crap that's been # stuffed in sys.modules, for example in the case of urllib3: # "loader for urllib3.contrib.pyopenssl cannot handle # requests.packages.urllib3.contrib.pyopenssl" e = sys.exc_info()[1] LOG.debug('%r: loading %r using %r failed: %s', self, fullname, loader, e) return if path is None or source is None: return if isinstance(source, mitogen.core.UnicodeType): # get_source() returns "string" according to PEP-302, which was # reinterpreted for Python 3 to mean a Unicode string. source = source.encode('utf-8') return path, source, is_pkg
def poll(self, timeout=None): changelist = self._changelist self._changelist = [] events, _ = mitogen.core.io_op(self._kqueue.control, changelist, 32, timeout) for event in events: fd = event.ident if event.flags & select.KQ_EV_ERROR: LOG.debug('ignoring stale event for fd %r: errno=%d: %s', fd, event.data, errno.errorcode.get(event.data)) elif event.filter == select.KQ_FILTER_READ and fd in self._rfds: # Events can still be read for an already-discarded fd. mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd) yield self._rfds[fd] elif event.filter == select.KQ_FILTER_WRITE and fd in self._wfds: mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd) yield self._wfds[fd]
def _get_module_via_pkgutil(self, fullname): """ Attempt to fetch source code via pkgutil. In an ideal world, this would be the only required implementation of get_module(). """ try: # Pre-'import spec' this returned None, in Python3.6 it raises # ImportError. loader = pkgutil.find_loader(fullname) except ImportError: e = sys.exc_info()[1] LOG.debug('%r._get_module_via_pkgutil(%r): %s', self, fullname, e) return None IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r', self, fullname, loader) if not loader: return try: path = self._py_filename(loader.get_filename(fullname)) source = loader.get_source(fullname) is_pkg = loader.is_package(fullname) except (AttributeError, ImportError): # - Per PEP-302, get_source() and is_package() are optional, # calling them may throw AttributeError. # - get_filename() may throw ImportError if pkgutil.find_loader() # picks a "parent" package's loader for some crap that's been # stuffed in sys.modules, for example in the case of urllib3: # "loader for urllib3.contrib.pyopenssl cannot handle # requests.packages.urllib3.contrib.pyopenssl" e = sys.exc_info()[1] LOG.debug('%r: loading %r using %r failed: %s', self, fullname, loader, e) return if path is None or source is None: return if isinstance(source, mitogen.core.UnicodeType): # get_source() returns "string" according to PEP-302, which was # reinterpreted for Python 3 to mean a Unicode string. source = source.encode('utf-8') return path, source, is_pkg
def _control(self, fd, filters, flags): mitogen.core._vv and IOLOG.debug('%r._control(%r, %r, %r)', self, fd, filters, flags) # TODO: at shutdown it is currently possible for KQ_EV_ADD/KQ_EV_DEL # pairs to be pending after the associated file descriptor has already # been closed. Fixing this requires maintaining extra state, or perhaps # making fd closure the poller's responsibility. In the meantime, # simply apply changes immediately. # self._changelist.append(select.kevent(fd, filters, flags)) changelist = [select.kevent(fd, filters, flags)] events, _ = mitogen.core.io_op(self._kqueue.control, changelist, 0, 0) assert not events
def _control(self, fd): mitogen.core._vv and IOLOG.debug('%r._control(%r)', self, fd) mask = (((fd in self._rfds) and select.EPOLLIN) | ((fd in self._wfds) and select.EPOLLOUT)) if mask: if fd in self._registered_fds: self._epoll.modify(fd, mask) else: self._epoll.register(fd, mask) self._registered_fds.add(fd) elif fd in self._registered_fds: self._epoll.unregister(fd) self._registered_fds.remove(fd)
def stop_transmit(self, fd): mitogen.core._vv and IOLOG.debug('%r.stop_transmit(%r)', self, fd) self._wfds.pop(fd, None) self._control(fd)
def start_transmit(self, fd, data=None): mitogen.core._vv and IOLOG.debug('%r.start_transmit(%r, %r)', self, fd, data) self._wfds[fd] = data or fd self._control(fd)
def stop_receive(self, fd): mitogen.core._vv and IOLOG.debug('%r.stop_receive(%r)', self, fd) self._rfds.pop(fd, None) self._control(fd)
def stop_transmit(self, fd): mitogen.core._vv and IOLOG.debug('%r.stop_transmit(%r)', self, fd) if fd in self._wfds: self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) del self._wfds[fd]
def start_transmit(self, fd, data=None): mitogen.core._vv and IOLOG.debug('%r.start_transmit(%r, %r)', self, fd, data) if fd not in self._wfds: self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) self._wfds[fd] = data or fd
def stop_receive(self, fd): mitogen.core._vv and IOLOG.debug('%r.stop_receive(%r)', self, fd) if fd in self._rfds: self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE) del self._rfds[fd]
def _on_pump_receive(self, s): IOLOG.info('%r._on_pump_receive(len %d)', self, len(s)) self.stdin.put(s)
def _control(self, fd, filters, flags): mitogen.core._vv and IOLOG.debug( '%r._control(%r, %r, %r)', self, fd, filters, flags) self._changelist.append(select.kevent(fd, filters, flags))
def _forward_modules(self, context, fullnames): IOLOG.debug('%r._forward_modules(%r, %r)', self, context, fullnames) for fullname in fullnames: self._forward_one_module(context, fullname)
def start_receive(self, fd, data=None): mitogen.core._vv and IOLOG.debug('%r.start_receive(%r, %r)', self, fd, data) if fd not in self._rfds: self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD) self._rfds[fd] = data or fd