def _scan(self):
with closing(select.epoll()) as epoll:
with _monitoring_socket(self._queue, self._groups, epoll) as sock:
with _pipetrick(epoll) as self._pipetrick:
self._scanning_started.set()
while True:
if self._timeout:
timeout = self._end_time - monotonic_time()
# timeout expired
if timeout <= 0:
self._scanning_stopped.set()
self._queue.put(_TIMEOUT_FLAG)
break
else:
timeout = -1
events = NoIntrPoll(epoll.poll, timeout=timeout)
# poll timeouted
if len(events) == 0:
self._scanning_stopped.set()
self._queue.put(_TIMEOUT_FLAG)
break
# stopped by pipetrick
elif (self._pipetrick[0], select.POLLIN) in events:
uninterruptible(os.read, self._pipetrick[0], 1)
self._queue.put(_STOP_FLAG)
break
_nl_recvmsgs_default(sock)
def test_echo(self, concurrency):
msg = b"ping"
sockets = []
try:
for i in range(concurrency):
sock1, sock2 = socket.socketpair()
self.loop.create_dispatcher(Echo, sock2)
sockets.append(sock1)
t = concurrent.thread(self.loop.run_forever)
t.start()
try:
start = time.time()
for sock in sockets:
osutils.uninterruptible(sock.send, msg)
for sock in sockets:
data = osutils.uninterruptible(sock.recv, len(msg))
self.assertEqual(data, msg)
elapsed = time.time() - start
print("%7d echos: %f seconds" % (concurrency, elapsed))
finally:
self.loop.call_soon_threadsafe(self.loop.stop)
t.join()
finally:
for sock in sockets:
sock.close()
def _scan(self):
with closing(select.epoll()) as epoll:
with _monitoring_socket(self._queue, self._groups, epoll) as sock:
with _pipetrick(epoll) as self._pipetrick:
self._scanning_started.set()
while True:
if self._timeout:
timeout = self._end_time - monotonic_time()
# timeout expired
if timeout <= 0:
self._scanning_stopped.set()
self._queue.put(_TIMEOUT_FLAG)
break
else:
timeout = -1
events = uninterruptible_poll(epoll.poll,
timeout=timeout)
# poll timeouted
if len(events) == 0:
self._scanning_stopped.set()
self._queue.put(_TIMEOUT_FLAG)
break
# stopped by pipetrick
elif (self._pipetrick[0], select.POLLIN) in events:
uninterruptible(os.read, self._pipetrick[0], 1)
self._queue.put(_STOP_FLAG)
break
libnl.nl_recvmsgs_default(sock)
def acquire(self, hostId, lease):
if lease != self._lease:
raise MultipleLeasesNotSupported("acquire", lease)
with self._globalLockMapSync:
self.log.info("Acquiring local lock for domain %s (id: %s)", self._sdUUID, hostId)
hostId, lockFile = self._getLease()
if lockFile:
try:
osutils.uninterruptible(fcntl.fcntl, lockFile, fcntl.F_GETFD)
except IOError as e:
# We found a stale file descriptor, removing.
del self._globalLockMap[self._sdUUID]
# Raise any other unkown error.
if e.errno != errno.EBADF:
raise
else:
self.log.debug("Local lock already acquired for domain " "%s (id: %s)", self._sdUUID, hostId)
return # success, the lock was already acquired
lockFile = osutils.uninterruptible(os.open, self._idsPath, os.O_RDONLY)
try:
osutils.uninterruptible(fcntl.flock, lockFile, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
osutils.close_fd(lockFile)
if e.errno in (errno.EACCES, errno.EAGAIN):
raise se.AcquireLockFailure(self._sdUUID, e.errno, "Cannot acquire local lock", str(e))
raise
else:
self._globalLockMap[self._sdUUID] = (hostId, lockFile)
self.log.debug("Local lock for domain %s successfully acquired " "(id: %s)", self._sdUUID, hostId)
def test_echo(self, concurrency):
msg = b"ping"
sockets = []
try:
for i in range(concurrency):
sock1, sock2 = socket.socketpair()
self.loop.create_dispatcher(Echo, sock2)
sockets.append(sock1)
t = concurrent.thread(self.loop.run_forever)
t.start()
try:
start = time.time()
for sock in sockets:
osutils.uninterruptible(sock.send, msg)
for sock in sockets:
data = osutils.uninterruptible(sock.recv, len(msg))
self.assertEqual(data, msg)
elapsed = time.time() - start
print("%7d echos: %f seconds" % (concurrency, elapsed))
finally:
self.loop.call_soon_threadsafe(self.loop.stop)
t.join()
finally:
for sock in sockets:
sock.close()
def pread(self, offset, buf):
"""
Read len(buf) bytes from storage at offset into mmap buf.
Returns:
The number bytes read (int).
"""
self._file.seek(offset, os.SEEK_SET)
pos = 0
if six.PY2:
# There is no way to create a writable memoryview on mmap object in
# python 2, so we must read into a temporary buffer and copy into
# the given buffer.
rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED)
with utils.closing(rbuf, log=log.name):
while pos < len(buf):
# TODO: Handle EOF
nread = uninterruptible(self._file.readinto, rbuf)
if nread == 0:
break # EOF
buf.write(rbuf[:nread])
pos += nread
else:
# In python 3 we can read directly into the underlying buffer
# without any copies using a memoryview.
while pos < len(buf):
rbuf = memoryview(buf)[pos:]
# TODO: Handle EOF
nread = uninterruptible(self._file.readinto, rbuf)
if nread == 0:
break # EOF
pos += nread
return pos
def pread(self, offset, buf):
"""
Read len(buf) bytes from storage at offset into mmap buf.
Returns:
The number bytes read (int).
"""
self._file.seek(offset, os.SEEK_SET)
pos = 0
if six.PY2:
# There is no way to create a writable memoryview on mmap object in
# python 2, so we must read into a temporary buffer and copy into
# the given buffer.
rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED)
with utils.closing(rbuf, log=log.name):
while pos < len(buf):
# TODO: Handle EOF
nread = uninterruptible(self._file.readinto, rbuf)
if nread == 0:
break # EOF
buf.write(rbuf[:nread])
pos += nread
else:
# In python 3 we can read directly into the underlying buffer
# without any copies using a memoryview.
while pos < len(buf):
rbuf = memoryview(buf)[pos:]
# TODO: Handle EOF
nread = uninterruptible(self._file.readinto, rbuf)
if nread == 0:
break # EOF
pos += nread
return pos
def wakeup(self):
try:
osutils.uninterruptible(os.write, self._wfd, b"\0")
except OSError as e:
if self.closing:
# Another thread tried to wake up after loop was closed.
return
if e.errno == errno.EAGAIN:
# The pipe is full, no need to write.
return
raise
def wakeup(self):
try:
osutils.uninterruptible(os.write, self._wfd, b"\0")
except OSError as e:
if self.closing:
# Another thread tried to wake up after loop was closed.
return
if e.errno == errno.EAGAIN:
# The pipe is full, no need to write.
return
raise
def test_args_kwargs(self):
def func(*args, **kwargs):
return args, kwargs
self.assertEqual(osutils.uninterruptible(func, "a", "b", c=3),
(("a", "b"), {"c": 3}))
def test_args_kwargs(self):
def func(*args, **kwargs):
return args, kwargs
self.assertEqual(osutils.uninterruptible(func, "a", "b", c=3),
(("a", "b"), {
"c": 3
}))
def write(self, buf):
pos = 0
while pos < len(buf):
if six.PY2:
wbuf = buffer(buf, pos)
else:
wbuf = memoryview(buf)[pos:]
pos += uninterruptible(self._file.write, wbuf)
def acquire(self, hostId, lease):
if lease != self._lease:
raise MultipleLeasesNotSupported("acquire", lease)
with self._globalLockMapSync:
self.log.info("Acquiring local lock for domain %s (id: %s)",
self._sdUUID, hostId)
hostId, lockFile = self._getLease()
if lockFile:
try:
osutils.uninterruptible(fcntl.fcntl, lockFile,
fcntl.F_GETFD)
except IOError as e:
# We found a stale file descriptor, removing.
del self._globalLockMap[self._sdUUID]
# Raise any other unkown error.
if e.errno != errno.EBADF:
raise
else:
self.log.debug(
"Local lock already acquired for domain "
"%s (id: %s)", self._sdUUID, hostId)
return # success, the lock was already acquired
lockFile = osutils.uninterruptible(os.open, self._idsPath,
os.O_RDONLY)
try:
osutils.uninterruptible(fcntl.flock, lockFile,
fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
osutils.close_fd(lockFile)
if e.errno in (errno.EACCES, errno.EAGAIN):
raise se.AcquireLockFailure(self._sdUUID, e.errno,
"Cannot acquire local lock",
str(e))
raise
else:
self._globalLockMap[self._sdUUID] = (hostId, lockFile)
self.log.debug(
"Local lock for domain %s successfully acquired "
"(id: %s)", self._sdUUID, hostId)
def test_retry_on_eintr(self):
count = [0]
def fail(n):
count[0] += 1
if count[0] == n:
return n
raise OSError(errno.EINTR, "Fake error")
self.assertEqual(osutils.uninterruptible(fail, 3), 3)
def test_retry_on_eintr(self):
count = [0]
def fail(n):
count[0] += 1
if count[0] == n:
return n
raise OSError(errno.EINTR, "Fake error")
self.assertEqual(osutils.uninterruptible(fail, 3), 3)
def readinto(self, buf):
pos = 0
if six.PY2:
# There is no way to create a writable memoryview on mmap object in
# python 2, so we must read into a temporary buffer and copy into
# the given buffer.
rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED)
with utils.closing(rbuf, log=log.name):
while pos < len(buf):
nread = uninterruptible(self._file.readinto, rbuf)
buf.write(rbuf[:nread])
pos += nread
else:
# In python 3 we can read directly into the underlying buffer
# without any copies using a memoryview.
while pos < len(buf):
rbuf = memoryview(buf)[pos:]
pos += uninterruptible(self._file.readinto, rbuf)
return pos
def _scan(self):
try:
epoll = select.epoll()
with closing(epoll):
with _monitoring_socket(
self._queue, self._groups, epoll, self._c_callback_function
) as sock:
with _pipetrick(epoll) as self._pipetrick:
self._scanning_started.set()
while True:
if self._timeout:
timeout = self._end_time - monotonic_time()
# timeout expired
if timeout <= 0:
self._scanning_stopped.set()
self._queue.put(Event(EventType.TIMEOUT))
break
else:
timeout = -1
events = uninterruptible_poll(
epoll.poll, timeout=timeout
)
# poll timeouted
if len(events) == 0:
self._scanning_stopped.set()
self._queue.put(Event(EventType.TIMEOUT))
break
# stopped by pipetrick
elif (self._pipetrick[0], select.POLLIN) in events:
uninterruptible(os.read, self._pipetrick[0], 1)
self._queue.put(Event(EventType.STOP))
break
libnl.nl_recvmsgs_default(sock)
except:
event = Event(EventType.EXCEPTION, sys.exc_info())
self._queue.put(event)
raise
def readinto(self, buf):
pos = 0
if six.PY2:
# There is no way to create a writable memoryview on mmap object in
# python 2, so we must read into a temporary buffer and copy into
# the given buffer.
rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED)
with utils.closing(rbuf, log=log.name):
while pos < len(buf):
# TODO: Handle EOF
nread = uninterruptible(self._file.readinto, rbuf)
buf.write(rbuf[:nread])
pos += nread
else:
# In python 3 we can read directly into the underlying buffer
# without any copies using a memoryview.
while pos < len(buf):
rbuf = memoryview(buf)[pos:]
# TODO: Handle EOF
nread = uninterruptible(self._file.readinto, rbuf)
pos += nread
return pos
def pwrite(self, offset, buf):
"""
Write mmap buf to storage at offset, and wait until the device reports
that the transfer has completed.
"""
self._file.seek(offset, os.SEEK_SET)
pos = 0
while pos < len(buf):
if six.PY2:
wbuf = buffer(buf, pos)
else:
wbuf = memoryview(buf)[pos:]
pos += uninterruptible(self._file.write, wbuf)
os.fsync(self._file.fileno())
def pwrite(self, offset, buf):
"""
Write mmap buf to storage at offset, and wait until the device reports
that the transfer has completed.
"""
self._file.seek(offset, os.SEEK_SET)
pos = 0
while pos < len(buf):
if six.PY2:
wbuf = buffer(buf, pos)
else:
wbuf = memoryview(buf)[pos:]
pos += uninterruptible(self._file.write, wbuf)
os.fsync(self._file.fileno())
def _scan(self):
try:
epoll = select.epoll()
with closing(epoll):
with _monitoring_socket(
self._queue, self._groups, epoll) as sock:
with _pipetrick(epoll) as self._pipetrick:
self._scanning_started.set()
while True:
if self._timeout:
timeout = self._end_time - monotonic_time()
# timeout expired
if timeout <= 0:
self._scanning_stopped.set()
self._queue.put(Event(EventType.TIMEOUT))
break
else:
timeout = -1
events = uninterruptible_poll(epoll.poll,
timeout=timeout)
# poll timeouted
if len(events) == 0:
self._scanning_stopped.set()
self._queue.put(Event(EventType.TIMEOUT))
break
# stopped by pipetrick
elif (self._pipetrick[0], select.POLLIN) in events:
uninterruptible(os.read, self._pipetrick[0], 1)
self._queue.put(Event(EventType.STOP))
break
libnl.nl_recvmsgs_default(sock)
except:
event = Event(EventType.EXCEPTION, sys.exc_info())
self._queue.put(event)
raise
def wrapper(*a, **kw):
r, w = os.pipe()
try:
pid = os.fork()
if pid == 0:
try:
f(*a, **kw)
os._exit(0)
except Exception as e:
os.write(w, pickle.dumps(e))
os._exit(1)
else:
_, status = osutils.uninterruptible(os.waitpid, pid, 0)
if status != 0:
e = pickle.loads(os.read(r, 4006))
raise e
finally:
osutils.close_fd(r)
osutils.close_fd(w)
def wrapper(*a, **kw):
r, w = os.pipe()
try:
pid = os.fork()
if pid == 0:
try:
f(*a, **kw)
os._exit(0)
except Exception as e:
os.write(w, pickle.dumps(e))
os._exit(1)
else:
_, status = osutils.uninterruptible(os.waitpid, pid, 0)
if status != 0:
e = pickle.loads(os.read(r, 4006))
raise e
finally:
osutils.close_fd(r)
osutils.close_fd(w)
def pinger(sock):
msg = b"ping"
osutils.uninterruptible(sock.send, msg)
osutils.uninterruptible(sock.recv, len(msg))
sock.close()
self.loop.call_soon_threadsafe(self.loop.stop)
def receive(p, timeout=None, bufsize=io.DEFAULT_BUFFER_SIZE):
"""
Receive data from a process, yielding data read from stdout and stderr
until proccess terminates or timeout expires.
Unlike Popen.communicate(), this supports a timeout, and allows
reading both stdout and stderr with a single thread.
Example usage::
# Reading data from both stdout and stderr until process
# terminates:
for src, data in cmdutils.receive(p):
if src == cmdutils.OUT:
# handle output
elif src == cmdutils.ERR:
# handler errors
# Receiving data with a timeout:
try:
received = list(cmdutils.receive(p, timeout=10))
except cmdutils.TimeoutExpired:
# handle timeout
Arguments:
p (`subprocess.Popen`): A subprocess created with
subprocess.Popen or subprocess32.Popen or cpopen.CPopen.
timeout (float): Number of seconds to wait for process. Timeout
resolution is limited by the resolution of
`common.time.monotonic_time`, typically 10 milliseconds.
bufsize (int): Number of bytes to read from the process in each
iteration.
Returns:
Generator of tuples (SRC, bytes). SRC may be either
`cmdutils.OUT` or `cmdutils.ERR`, and bytes is a bytes object
read from process stdout or stderr.
Raises:
`cmdutils.TimeoutExpired` if process did not terminate within
the specified timeout.
"""
if timeout is not None:
deadline = monotonic_time() + timeout
remaining = timeout
else:
deadline = None
remaining = None
fds = {}
if p.stdout:
fds[p.stdout.fileno()] = OUT
if p.stderr:
fds[p.stderr.fileno()] = ERR
if fds:
poller = select.poll()
for fd in fds:
poller.register(fd, select.POLLIN)
def discard(fd):
if fd in fds:
del fds[fd]
poller.unregister(fd)
while fds:
log.debug("Waiting for process (pid=%d, remaining=%s)",
p.pid, remaining)
# Unlike all other time apis, poll is using milliseconds
remaining_msec = remaining * 1000 if deadline else None
try:
ready = poller.poll(remaining_msec)
except select.error as e:
if e.args[0] != errno.EINTR:
raise
log.debug("Polling process (pid=%d) interrupted", p.pid)
else:
for fd, mode in ready:
if mode & select.POLLIN:
data = osutils.uninterruptible(os.read, fd, bufsize)
if not data:
log.debug("Fd %d closed, unregistering", fd)
discard(fd)
continue
yield fds[fd], data
else:
log.debug("Fd %d hangup/error, unregistering", fd)
discard(fd)
if deadline:
remaining = deadline - monotonic_time()
if remaining <= 0:
raise TimeoutExpired(p.pid)
_wait(p, deadline)
def pinger(sock):
msg = b"ping"
osutils.uninterruptible(sock.send, msg)
osutils.uninterruptible(sock.recv, len(msg))
sock.close()
self.loop.call_soon_threadsafe(self.loop.stop)
def handle_read(self):
osutils.uninterruptible(self.socket.read, 1024)
def handle_read(self):
osutils.uninterruptible(self.socket.read, 1024)
def handle_read(self):
data = osutils.uninterruptible(self.socket.recv, 4096)
osutils.uninterruptible(self.socket.send, data)
def handle_read(self):
data = osutils.uninterruptible(self.socket.recv, 4096)
osutils.uninterruptible(self.socket.send, data)
def receive(p, timeout=None, bufsize=io.DEFAULT_BUFFER_SIZE):
"""
Receive data from a process, yielding data read from stdout and stderr
until proccess terminates or timeout expires.
Unlike Popen.communicate(), this supports a timeout, and allows
reading both stdout and stderr with a single thread.
Example usage::
# Reading data from both stdout and stderr until process
# terminates:
for src, data in cmdutils.receive(p):
if src == cmdutils.OUT:
# handle output
elif src == cmdutils.ERR:
# handler errors
# Receiving data with a timeout:
try:
received = list(cmdutils.receive(p, timeout=10))
except cmdutils.TimeoutExpired:
# handle timeout
Arguments:
p (`subprocess.Popen`): A subprocess created with
subprocess.Popen or subprocess32.Popen or cpopen.CPopen.
timeout (float): Number of seconds to wait for process. Timeout
resolution is limited by the resolution of
`common.time.monotonic_time`, typically 10 milliseconds.
bufsize (int): Number of bytes to read from the process in each
iteration.
Returns:
Generator of tuples (SRC, bytes). SRC may be either
`cmdutils.OUT` or `cmdutils.ERR`, and bytes is a bytes object
read from process stdout or stderr.
Raises:
`cmdutils.TimeoutExpired` if process did not terminate within
the specified timeout.
"""
if timeout is not None:
deadline = monotonic_time() + timeout
remaining = timeout
else:
deadline = None
remaining = None
fds = {}
if p.stdout:
fds[p.stdout.fileno()] = OUT
if p.stderr:
fds[p.stderr.fileno()] = ERR
if fds:
poller = select.poll()
for fd in fds:
poller.register(fd, select.POLLIN)
def discard(fd):
if fd in fds:
del fds[fd]
poller.unregister(fd)
while fds:
log.debug("Waiting for process (pid=%d, remaining=%s)", p.pid,
remaining)
# Unlike all other time apis, poll is using milliseconds
remaining_msec = remaining * 1000 if deadline else None
try:
ready = poller.poll(remaining_msec)
except select.error as e:
if e[0] != errno.EINTR:
raise
log.debug("Polling process (pid=%d) interrupted", p.pid)
else:
for fd, mode in ready:
if mode & select.POLLIN:
data = osutils.uninterruptible(os.read, fd, bufsize)
if not data:
log.debug("Fd %d closed, unregistering", fd)
discard(fd)
continue
yield fds[fd], data
else:
log.debug("Fd %d hangup/error, unregistering", fd)
discard(fd)
if deadline:
remaining = deadline - monotonic_time()
if remaining <= 0:
raise TimeoutExpired(p.pid)
_wait(p, deadline)
