def _scan(self): with closing(select.epoll()) as epoll: with _monitoring_socket(self._queue, self._groups, epoll) as sock: with _pipetrick(epoll) as self._pipetrick: self._scanning_started.set() while True: if self._timeout: timeout = self._end_time - monotonic_time() # timeout expired if timeout <= 0: self._scanning_stopped.set() self._queue.put(_TIMEOUT_FLAG) break else: timeout = -1 events = NoIntrPoll(epoll.poll, timeout=timeout) # poll timeouted if len(events) == 0: self._scanning_stopped.set() self._queue.put(_TIMEOUT_FLAG) break # stopped by pipetrick elif (self._pipetrick[0], select.POLLIN) in events: uninterruptible(os.read, self._pipetrick[0], 1) self._queue.put(_STOP_FLAG) break _nl_recvmsgs_default(sock)
def test_echo(self, concurrency): msg = b"ping" sockets = [] try: for i in range(concurrency): sock1, sock2 = socket.socketpair() self.loop.create_dispatcher(Echo, sock2) sockets.append(sock1) t = concurrent.thread(self.loop.run_forever) t.start() try: start = time.time() for sock in sockets: osutils.uninterruptible(sock.send, msg) for sock in sockets: data = osutils.uninterruptible(sock.recv, len(msg)) self.assertEqual(data, msg) elapsed = time.time() - start print("%7d echos: %f seconds" % (concurrency, elapsed)) finally: self.loop.call_soon_threadsafe(self.loop.stop) t.join() finally: for sock in sockets: sock.close()
def _scan(self): with closing(select.epoll()) as epoll: with _monitoring_socket(self._queue, self._groups, epoll) as sock: with _pipetrick(epoll) as self._pipetrick: self._scanning_started.set() while True: if self._timeout: timeout = self._end_time - monotonic_time() # timeout expired if timeout <= 0: self._scanning_stopped.set() self._queue.put(_TIMEOUT_FLAG) break else: timeout = -1 events = uninterruptible_poll(epoll.poll, timeout=timeout) # poll timeouted if len(events) == 0: self._scanning_stopped.set() self._queue.put(_TIMEOUT_FLAG) break # stopped by pipetrick elif (self._pipetrick[0], select.POLLIN) in events: uninterruptible(os.read, self._pipetrick[0], 1) self._queue.put(_STOP_FLAG) break libnl.nl_recvmsgs_default(sock)
def acquire(self, hostId, lease): if lease != self._lease: raise MultipleLeasesNotSupported("acquire", lease) with self._globalLockMapSync: self.log.info("Acquiring local lock for domain %s (id: %s)", self._sdUUID, hostId) hostId, lockFile = self._getLease() if lockFile: try: osutils.uninterruptible(fcntl.fcntl, lockFile, fcntl.F_GETFD) except IOError as e: # We found a stale file descriptor, removing. del self._globalLockMap[self._sdUUID] # Raise any other unkown error. if e.errno != errno.EBADF: raise else: self.log.debug("Local lock already acquired for domain " "%s (id: %s)", self._sdUUID, hostId) return # success, the lock was already acquired lockFile = osutils.uninterruptible(os.open, self._idsPath, os.O_RDONLY) try: osutils.uninterruptible(fcntl.flock, lockFile, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: osutils.close_fd(lockFile) if e.errno in (errno.EACCES, errno.EAGAIN): raise se.AcquireLockFailure(self._sdUUID, e.errno, "Cannot acquire local lock", str(e)) raise else: self._globalLockMap[self._sdUUID] = (hostId, lockFile) self.log.debug("Local lock for domain %s successfully acquired " "(id: %s)", self._sdUUID, hostId)
def pread(self, offset, buf): """ Read len(buf) bytes from storage at offset into mmap buf. Returns: The number bytes read (int). """ self._file.seek(offset, os.SEEK_SET) pos = 0 if six.PY2: # There is no way to create a writable memoryview on mmap object in # python 2, so we must read into a temporary buffer and copy into # the given buffer. rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED) with utils.closing(rbuf, log=log.name): while pos < len(buf): # TODO: Handle EOF nread = uninterruptible(self._file.readinto, rbuf) if nread == 0: break # EOF buf.write(rbuf[:nread]) pos += nread else: # In python 3 we can read directly into the underlying buffer # without any copies using a memoryview. while pos < len(buf): rbuf = memoryview(buf)[pos:] # TODO: Handle EOF nread = uninterruptible(self._file.readinto, rbuf) if nread == 0: break # EOF pos += nread return pos
def wakeup(self): try: osutils.uninterruptible(os.write, self._wfd, b"\0") except OSError as e: if self.closing: # Another thread tried to wake up after loop was closed. return if e.errno == errno.EAGAIN: # The pipe is full, no need to write. return raise
def test_args_kwargs(self): def func(*args, **kwargs): return args, kwargs self.assertEqual(osutils.uninterruptible(func, "a", "b", c=3), (("a", "b"), {"c": 3}))
def test_args_kwargs(self): def func(*args, **kwargs): return args, kwargs self.assertEqual(osutils.uninterruptible(func, "a", "b", c=3), (("a", "b"), { "c": 3 }))
def write(self, buf): pos = 0 while pos < len(buf): if six.PY2: wbuf = buffer(buf, pos) else: wbuf = memoryview(buf)[pos:] pos += uninterruptible(self._file.write, wbuf)
def acquire(self, hostId, lease): if lease != self._lease: raise MultipleLeasesNotSupported("acquire", lease) with self._globalLockMapSync: self.log.info("Acquiring local lock for domain %s (id: %s)", self._sdUUID, hostId) hostId, lockFile = self._getLease() if lockFile: try: osutils.uninterruptible(fcntl.fcntl, lockFile, fcntl.F_GETFD) except IOError as e: # We found a stale file descriptor, removing. del self._globalLockMap[self._sdUUID] # Raise any other unkown error. if e.errno != errno.EBADF: raise else: self.log.debug( "Local lock already acquired for domain " "%s (id: %s)", self._sdUUID, hostId) return # success, the lock was already acquired lockFile = osutils.uninterruptible(os.open, self._idsPath, os.O_RDONLY) try: osutils.uninterruptible(fcntl.flock, lockFile, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: osutils.close_fd(lockFile) if e.errno in (errno.EACCES, errno.EAGAIN): raise se.AcquireLockFailure(self._sdUUID, e.errno, "Cannot acquire local lock", str(e)) raise else: self._globalLockMap[self._sdUUID] = (hostId, lockFile) self.log.debug( "Local lock for domain %s successfully acquired " "(id: %s)", self._sdUUID, hostId)
def test_retry_on_eintr(self): count = [0] def fail(n): count[0] += 1 if count[0] == n: return n raise OSError(errno.EINTR, "Fake error") self.assertEqual(osutils.uninterruptible(fail, 3), 3)
def readinto(self, buf): pos = 0 if six.PY2: # There is no way to create a writable memoryview on mmap object in # python 2, so we must read into a temporary buffer and copy into # the given buffer. rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED) with utils.closing(rbuf, log=log.name): while pos < len(buf): nread = uninterruptible(self._file.readinto, rbuf) buf.write(rbuf[:nread]) pos += nread else: # In python 3 we can read directly into the underlying buffer # without any copies using a memoryview. while pos < len(buf): rbuf = memoryview(buf)[pos:] pos += uninterruptible(self._file.readinto, rbuf) return pos
def _scan(self): try: epoll = select.epoll() with closing(epoll): with _monitoring_socket( self._queue, self._groups, epoll, self._c_callback_function ) as sock: with _pipetrick(epoll) as self._pipetrick: self._scanning_started.set() while True: if self._timeout: timeout = self._end_time - monotonic_time() # timeout expired if timeout <= 0: self._scanning_stopped.set() self._queue.put(Event(EventType.TIMEOUT)) break else: timeout = -1 events = uninterruptible_poll( epoll.poll, timeout=timeout ) # poll timeouted if len(events) == 0: self._scanning_stopped.set() self._queue.put(Event(EventType.TIMEOUT)) break # stopped by pipetrick elif (self._pipetrick[0], select.POLLIN) in events: uninterruptible(os.read, self._pipetrick[0], 1) self._queue.put(Event(EventType.STOP)) break libnl.nl_recvmsgs_default(sock) except: event = Event(EventType.EXCEPTION, sys.exc_info()) self._queue.put(event) raise
def readinto(self, buf): pos = 0 if six.PY2: # There is no way to create a writable memoryview on mmap object in # python 2, so we must read into a temporary buffer and copy into # the given buffer. rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED) with utils.closing(rbuf, log=log.name): while pos < len(buf): # TODO: Handle EOF nread = uninterruptible(self._file.readinto, rbuf) buf.write(rbuf[:nread]) pos += nread else: # In python 3 we can read directly into the underlying buffer # without any copies using a memoryview. while pos < len(buf): rbuf = memoryview(buf)[pos:] # TODO: Handle EOF nread = uninterruptible(self._file.readinto, rbuf) pos += nread return pos
def pwrite(self, offset, buf): """ Write mmap buf to storage at offset, and wait until the device reports that the transfer has completed. """ self._file.seek(offset, os.SEEK_SET) pos = 0 while pos < len(buf): if six.PY2: wbuf = buffer(buf, pos) else: wbuf = memoryview(buf)[pos:] pos += uninterruptible(self._file.write, wbuf) os.fsync(self._file.fileno())
def _scan(self): try: epoll = select.epoll() with closing(epoll): with _monitoring_socket( self._queue, self._groups, epoll) as sock: with _pipetrick(epoll) as self._pipetrick: self._scanning_started.set() while True: if self._timeout: timeout = self._end_time - monotonic_time() # timeout expired if timeout <= 0: self._scanning_stopped.set() self._queue.put(Event(EventType.TIMEOUT)) break else: timeout = -1 events = uninterruptible_poll(epoll.poll, timeout=timeout) # poll timeouted if len(events) == 0: self._scanning_stopped.set() self._queue.put(Event(EventType.TIMEOUT)) break # stopped by pipetrick elif (self._pipetrick[0], select.POLLIN) in events: uninterruptible(os.read, self._pipetrick[0], 1) self._queue.put(Event(EventType.STOP)) break libnl.nl_recvmsgs_default(sock) except: event = Event(EventType.EXCEPTION, sys.exc_info()) self._queue.put(event) raise
def wrapper(*a, **kw): r, w = os.pipe() try: pid = os.fork() if pid == 0: try: f(*a, **kw) os._exit(0) except Exception as e: os.write(w, pickle.dumps(e)) os._exit(1) else: _, status = osutils.uninterruptible(os.waitpid, pid, 0) if status != 0: e = pickle.loads(os.read(r, 4006)) raise e finally: osutils.close_fd(r) osutils.close_fd(w)
def pinger(sock): msg = b"ping" osutils.uninterruptible(sock.send, msg) osutils.uninterruptible(sock.recv, len(msg)) sock.close() self.loop.call_soon_threadsafe(self.loop.stop)
def receive(p, timeout=None, bufsize=io.DEFAULT_BUFFER_SIZE): """ Receive data from a process, yielding data read from stdout and stderr until proccess terminates or timeout expires. Unlike Popen.communicate(), this supports a timeout, and allows reading both stdout and stderr with a single thread. Example usage:: # Reading data from both stdout and stderr until process # terminates: for src, data in cmdutils.receive(p): if src == cmdutils.OUT: # handle output elif src == cmdutils.ERR: # handler errors # Receiving data with a timeout: try: received = list(cmdutils.receive(p, timeout=10)) except cmdutils.TimeoutExpired: # handle timeout Arguments: p (`subprocess.Popen`): A subprocess created with subprocess.Popen or subprocess32.Popen or cpopen.CPopen. timeout (float): Number of seconds to wait for process. Timeout resolution is limited by the resolution of `common.time.monotonic_time`, typically 10 milliseconds. bufsize (int): Number of bytes to read from the process in each iteration. Returns: Generator of tuples (SRC, bytes). SRC may be either `cmdutils.OUT` or `cmdutils.ERR`, and bytes is a bytes object read from process stdout or stderr. Raises: `cmdutils.TimeoutExpired` if process did not terminate within the specified timeout. """ if timeout is not None: deadline = monotonic_time() + timeout remaining = timeout else: deadline = None remaining = None fds = {} if p.stdout: fds[p.stdout.fileno()] = OUT if p.stderr: fds[p.stderr.fileno()] = ERR if fds: poller = select.poll() for fd in fds: poller.register(fd, select.POLLIN) def discard(fd): if fd in fds: del fds[fd] poller.unregister(fd) while fds: log.debug("Waiting for process (pid=%d, remaining=%s)", p.pid, remaining) # Unlike all other time apis, poll is using milliseconds remaining_msec = remaining * 1000 if deadline else None try: ready = poller.poll(remaining_msec) except select.error as e: if e.args[0] != errno.EINTR: raise log.debug("Polling process (pid=%d) interrupted", p.pid) else: for fd, mode in ready: if mode & select.POLLIN: data = osutils.uninterruptible(os.read, fd, bufsize) if not data: log.debug("Fd %d closed, unregistering", fd) discard(fd) continue yield fds[fd], data else: log.debug("Fd %d hangup/error, unregistering", fd) discard(fd) if deadline: remaining = deadline - monotonic_time() if remaining <= 0: raise TimeoutExpired(p.pid) _wait(p, deadline)
def handle_read(self): osutils.uninterruptible(self.socket.read, 1024)
def handle_read(self): data = osutils.uninterruptible(self.socket.recv, 4096) osutils.uninterruptible(self.socket.send, data)
def receive(p, timeout=None, bufsize=io.DEFAULT_BUFFER_SIZE): """ Receive data from a process, yielding data read from stdout and stderr until proccess terminates or timeout expires. Unlike Popen.communicate(), this supports a timeout, and allows reading both stdout and stderr with a single thread. Example usage:: # Reading data from both stdout and stderr until process # terminates: for src, data in cmdutils.receive(p): if src == cmdutils.OUT: # handle output elif src == cmdutils.ERR: # handler errors # Receiving data with a timeout: try: received = list(cmdutils.receive(p, timeout=10)) except cmdutils.TimeoutExpired: # handle timeout Arguments: p (`subprocess.Popen`): A subprocess created with subprocess.Popen or subprocess32.Popen or cpopen.CPopen. timeout (float): Number of seconds to wait for process. Timeout resolution is limited by the resolution of `common.time.monotonic_time`, typically 10 milliseconds. bufsize (int): Number of bytes to read from the process in each iteration. Returns: Generator of tuples (SRC, bytes). SRC may be either `cmdutils.OUT` or `cmdutils.ERR`, and bytes is a bytes object read from process stdout or stderr. Raises: `cmdutils.TimeoutExpired` if process did not terminate within the specified timeout. """ if timeout is not None: deadline = monotonic_time() + timeout remaining = timeout else: deadline = None remaining = None fds = {} if p.stdout: fds[p.stdout.fileno()] = OUT if p.stderr: fds[p.stderr.fileno()] = ERR if fds: poller = select.poll() for fd in fds: poller.register(fd, select.POLLIN) def discard(fd): if fd in fds: del fds[fd] poller.unregister(fd) while fds: log.debug("Waiting for process (pid=%d, remaining=%s)", p.pid, remaining) # Unlike all other time apis, poll is using milliseconds remaining_msec = remaining * 1000 if deadline else None try: ready = poller.poll(remaining_msec) except select.error as e: if e[0] != errno.EINTR: raise log.debug("Polling process (pid=%d) interrupted", p.pid) else: for fd, mode in ready: if mode & select.POLLIN: data = osutils.uninterruptible(os.read, fd, bufsize) if not data: log.debug("Fd %d closed, unregistering", fd) discard(fd) continue yield fds[fd], data else: log.debug("Fd %d hangup/error, unregistering", fd) discard(fd) if deadline: remaining = deadline - monotonic_time() if remaining <= 0: raise TimeoutExpired(p.pid) _wait(p, deadline)