def _launch(self, process_obj):
self.sentinel = None
try:
super()._launch(process_obj)
finally:
if self.sentinel is not None:
make_nonblocking(self.sentinel)
def __init__(self, fileno, mode='r', closefd=True):
RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called
self._closefd = closefd
self._fileno = fileno
make_nonblocking(fileno)
readable = 'r' in mode
writable = 'w' in mode
self.hub = get_hub()
io_watcher = self.hub.loop.io
try:
if readable:
self._read_event = io_watcher(fileno, 1)
if writable:
self._write_event = io_watcher(fileno, 2)
except:
# If anything goes wrong, it's important to go ahead and
# close these watchers *now*, especially under libuv, so
# that they don't get eventually reclaimed by the garbage
# collector at some random time, thanks to the C level
# slot (even though we don't seem to have any actual references
# at the Python level). Previously, if we didn't close now,
# that random close in the future would cause issues if we had duplicated
# the fileno (if a wrapping with statement had closed an open fileobject,
# for example)
# test__fileobject can show a failure if this doesn't happen
# TRAVIS=true GEVENT_LOOP=libuv python -m gevent.tests.test__fileobject \
# TestFileObjectPosix.test_seek TestFileObjectThread.test_bufsize_0
self.close()
raise
def __init__(self, fileno, open_descriptor, closefd=True):
RawIOBase.__init__(self)
self._closefd = closefd
self._fileno = fileno
self.mode = open_descriptor.fileio_mode
make_nonblocking(fileno)
readable = open_descriptor.can_read
writable = open_descriptor.can_write
self.hub = get_hub()
io_watcher = self.hub.loop.io
try:
if readable:
self._read_watcher = io_watcher(fileno, 1)
if writable:
self._write_watcher = io_watcher(fileno, 2)
except:
# If anything goes wrong, it's important to go ahead and
# close these watchers *now*, especially under libuv, so
# that they don't get eventually reclaimed by the garbage
# collector at some random time, thanks to the C level
# slot (even though we don't seem to have any actual references
# at the Python level). Previously, if we didn't close now,
# that random close in the future would cause issues if we had duplicated
# the fileno (if a wrapping with statement had closed an open fileobject,
# for example)
# test__fileobject can show a failure if this doesn't happen
# TRAVIS=true GEVENT_LOOP=libuv python -m gevent.tests.test__fileobject \
# TestFileObjectPosix.test_seek TestFileObjectThread.test_bufsize_0
self.close()
raise
def test(self): # pylint:disable=too-many-locals
# If this test is broken, there are a few failure modes.
# - In the original examples, the parent process just hangs, because the
# child has raced ahead, spawned the greenlet and read the data. When the
# greenlet goes to read in the parent, it blocks, and the hub and loop
# wait for it.
# - Here, our child detects the greenlet ran when it shouldn't and
# raises an error, which translates to a non-zero exit status,
# which the parent checks for and fails by raising an exception before
# returning control to the hub. We can replicate the hang by removing the
# assertion in the child.
from time import sleep as hang
from gevent import get_hub
from gevent import spawn
from gevent.socket import wait_read
from gevent.os import nb_read
from gevent.os import nb_write
from gevent.os import make_nonblocking
from gevent.os import fork
from gevent.os import waitpid
pipe_read_fd, pipe_write_fd = os.pipe()
make_nonblocking(pipe_read_fd)
make_nonblocking(pipe_write_fd)
run = []
def reader():
run.append(1)
return nb_read(pipe_read_fd, 4096)
# Put data in the pipe
DATA = b'test'
nb_write(pipe_write_fd, DATA)
# Make sure we're ready to read it
wait_read(pipe_read_fd)
# Schedule a greenlet to start
reader = spawn(reader)
hub = get_hub()
pid = fork()
if pid == 0:
# Child destroys the hub. The reader should not have run.
hub.destroy(destroy_loop=True)
self.assertFalse(run)
os._exit(0)
return
# The parent.
# Briefly prevent us from spinning our event loop.
hang(0.5)
wait_child_result = waitpid(pid, 0)
self.assertEqual(wait_child_result, (pid, 0))
# We should get the data; the greenlet only runs in the parent.
data = reader.get()
self.assertEqual(run, [1])
self.assertEqual(data, DATA)
def preHook(self):
if self.kwargs.timestamp:
self.getTimestamp = self.returnTimestamp
else:
self.getTimestamp = self.returnNoTimestamp
self.file = open(self.kwargs.location, "a")
make_nonblocking(self.file)
def preHook(self):
if self.kwargs.timestamp:
self.getTimestamp = self.returnTimestamp
else:
self.getTimestamp = self.returnNoTimestamp
self.file = open(self.kwargs.location, "a")
make_nonblocking(self.file)
def readFile(self, filename):
try:
if filename.endswith("ready") and self.loop():
with open(filename, "rb") as output_file:
make_nonblocking(output_file)
self.logging.info("Read file %s" % filename)
for e in self.__pickleReader(output_file):
self.submit(e, self.pool.queue.outbox)
os.remove(filename)
except Exception as err:
self.logging.error("Failed to read file %s. Reason: %s" % (filename, str(err)))
def __init__(self, fileno, mode=None, close=True):
if not isinstance(fileno, integer_types):
raise TypeError('fileno must be int: %r' % fileno)
self._fileno = fileno
self._mode = mode or 'rb'
self._close = close
self._translate = 'U' in self._mode
make_nonblocking(fileno)
self._eat_newline = False
self.hub = get_hub()
io = self.hub.loop.io
self._read_event = io(fileno, 1)
self._write_event = io(fileno, 2)
def __init__(self, fileno, mode=None, close=True):
if not isinstance(fileno, integer_types):
raise TypeError('fileno must be int: %r' % fileno)
self._fileno = fileno
self._mode = mode or 'rb'
self._close = close
self._translate = 'U' in self._mode
make_nonblocking(fileno)
self._eat_newline = False
self.hub = get_hub()
io = self.hub.loop.io
self._read_event = io(fileno, 1)
self._write_event = io(fileno, 2)
def preHook(self):
if self.kwargs.timestamp:
self.getTimestamp = self.returnTimestamp
else:
self.getTimestamp = self.returnNoTimestamp
if self.kwargs.complete:
self.getData = self.returnComplete
else:
self.getData = self.returnDataOnly
self.file = open(self.kwargs.location, "a")
make_nonblocking(self.file)
def __init__(self, fileno, mode='r', closefd=True):
RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called
self._closefd = closefd
self._fileno = fileno
make_nonblocking(fileno)
readable = 'r' in mode
writable = 'w' in mode
self.hub = get_hub()
io_watcher = self.hub.loop.io
if readable:
self._read_event = io_watcher(fileno, 1)
if writable:
self._write_event = io_watcher(fileno, 2)
def __init__(self, fileno, mode=None, close=True):
if not isinstance(fileno, integer_types):
raise TypeError("fileno must be int: %r" % fileno)
self._fileno = fileno
self._mode = mode or "rb"
self._close = close
self._translate = not PY3 and "U" in self._mode
make_nonblocking(fileno)
self._eat_newline = False
self.hub = get_hub()
io = self.hub.loop.io
self._read_event = io(fileno, 1)
self._write_event = io(fileno, 2)
self._io_refs = 0 # for Python 3
self._closed = False
def drain(self):
'''Reads the named pipe.'''
self.logging.info('Started.')
fd = os.open(self.path, os.O_RDWR | os.O_NONBLOCK)
gevent_os.make_nonblocking(fd)
while self.loop():
try:
lines = gevent_os.nb_read(fd, 4096).splitlines()
if len(lines) == 0:
sleep(0.5)
else:
self.consume(lines)
except OSError:
pass
def drain(self, p):
'''Reads the named pipe.'''
self.logging.info('Started.')
fd = os.open(p, os.O_RDWR | os.O_NONBLOCK)
gevent_os.make_nonblocking(fd)
while self.loop():
try:
lines = gevent_os.nb_read(fd, 4096).splitlines()
if len(lines) == 0:
sleep(0.5)
else:
self.consume(lines)
except OSError:
pass
def __init__(self, fileno, mode='r', closefd=True):
RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called
self._closed = False
self._closefd = closefd
self._fileno = fileno
make_nonblocking(fileno)
self._readable = 'r' in mode
self._writable = 'w' in mode
self.hub = get_hub()
io_watcher = self.hub.loop.io
if self._readable:
self._read_event = io_watcher(fileno, 1)
if self._writable:
self._write_event = io_watcher(fileno, 2)
self._seekable = None
def __init__(self, fileno, mode='r', closefd=True):
RawIOBase.__init__(self)
self._closed = False
self._closefd = closefd
self._fileno = fileno
make_nonblocking(fileno)
self._readable = 'r' in mode
self._writable = 'w' in mode
self.hub = get_hub()
io = self.hub.loop.io
if self._readable:
self._read_event = io(fileno, 1)
else:
self._read_event = None
if self._writable:
self._write_event = io(fileno, 2)
else:
self._write_event = None
def flushDisk(self):
if self.pool.queue.disk.size() > 0 and not self.__flush_lock:
self.__flush_lock = True
i = str(uuid4())
try:
with open(r"%s/%s.%s.writing" % (self.kwargs.directory, self.name, i), "wb") as output_file:
make_nonblocking(output_file)
size = self.pool.queue.disk.size()
for event in self.pool.queue.disk.dump():
pickle.dump(event, output_file)
except Exception as err:
self.logging.error("Failed to write file '%s' to '%s'. Reason: '%s'." % (self.name, self.kwargs.directory, err))
try:
os.remove("%s/%s.%s.writing" % (self.kwargs.directory, self.name, i))
except Exception as err:
self.logging.debug("No file %s/%s.%s.writing to remove" % (self.kwargs.directory, self.name, i))
self.submit(event, self.pool.queue.disk)
else:
os.rename("%s/%s.%s.writing" % (self.kwargs.directory, self.name, i), "%s/%s.%s.ready" % (self.kwargs.directory, self.name, i))
self.logging.info("Wrote %s events to file %s/%s.%s.ready" % (size, self.kwargs.directory, self.name, i))
self.__flush_lock = False
def pipe(self):
r, w = pipe()
os.make_nonblocking(r)
os.make_nonblocking(w)
return r, w
def open(file, flags, mode=0o777):
fd = _open(file, flags, mode)
# this is (almost) pointless since local files may block even if we set non-blocking on Linux
# it's still interesting if the file is a named pipe, UNIX-domain socket, etc.
make_nonblocking(fd)
return fd
def pipe(self):
r, w = pipe()
os.make_nonblocking(r)
os.make_nonblocking(w)
return r, w
def read_events(self, event_buffer_size=DEFAULT_EVENT_BUFFER_SIZE):
"""
Reads events from inotify and yields them.
"""
from gevent import os as gos
gos.make_nonblocking(self._inotify_fd)
while True:
try:
event_buffer = gos.nb_read(self._inotify_fd, event_buffer_size)
except OSError as e:
if e.errno == errno.EINTR:
continue
break
with self._lock:
event_list = []
for wd, mask, cookie, name in Inotify._parse_event_buffer(event_buffer):
if wd == -1:
continue
wd_path = self._path_for_wd[wd]
src_path = absolute_path(os.path.join(wd_path, name))
inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)
if inotify_event.is_moved_from:
self.remember_move_from_event(inotify_event)
elif inotify_event.is_moved_to:
move_src_path = self.source_for_move(inotify_event)
if move_src_path in self._wd_for_path:
# update old path -> new path
moved_wd = self._wd_for_path[move_src_path]
del self._wd_for_path[move_src_path]
self._wd_for_path[inotify_event.src_path] = moved_wd
self._path_for_wd[moved_wd] = inotify_event.src_path
src_path = absolute_path(os.path.join(wd_path, name))
inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)
if inotify_event.is_ignored:
# Clean up book-keeping for deleted watches.
self._remove_watch_bookkeeping(src_path)
continue
event_list.append(inotify_event)
if self.is_recursive and inotify_event.is_directory and inotify_event.is_create:
# HACK: We need to traverse the directory path
# recursively and simulate events for newly
# created subdirectories/files. This will handle
# mkdir -p foobar/blah/bar; touch foobar/afile
# TODO: When a directory from another part of the
# filesystem is moved into a watched directory, this
# will not generate events for the directory tree.
# We need to coalesce IN_MOVED_TO events and those
# IN_MOVED_TO events which don't pair up with
# IN_MOVED_FROM events should be marked IN_CREATE
# instead relative to this directory.
try:
self._add_watch(src_path, self._event_mask)
except OSError:
continue
for root, dirnames, filenames in os.walk(src_path):
for dirname in dirnames:
try:
full_path = absolute_path(os.path.join(root, dirname))
wd_dir = self._add_watch(full_path, self._event_mask)
event_list.append(InotifyEvent(wd_dir,
InotifyConstants.IN_CREATE | InotifyConstants.IN_ISDIR, 0, dirname, full_path))
except OSError:
pass
for filename in filenames:
full_path = absolute_path(os.path.join(root, filename))
wd_parent_dir = self._wd_for_path[absolute_path(os.path.dirname(full_path))]
event_list.append(InotifyEvent(wd_parent_dir,
InotifyConstants.IN_CREATE, 0, filename, full_path))
return event_list
def ensure_running(self):
super().ensure_running()
make_nonblocking(self._forkserver_alive_fd)
def connect_to_new_process(self, fds):
parent_r, parent_w = super().connect_to_new_process(fds)
make_nonblocking(parent_r)
make_nonblocking(parent_w)
return parent_r, parent_w
def ensure_running(self):
super().ensure_running()
make_nonblocking(self._fd)
def pipe(self):
r, w = super(TestOS_nb, self).pipe()
os.make_nonblocking(r)
os.make_nonblocking(w)
return r, w
def pipe(self):
r, w = super(TestOS_nb, self).pipe()
os.make_nonblocking(r)
os.make_nonblocking(w)
return r, w
def dhcp_worker(self):
make_nonblocking(self.dhcp_server.bpf.fd)
self.dhcp_server.serve()
