def _launch(self, process_obj): self.sentinel = None try: super()._launch(process_obj) finally: if self.sentinel is not None: make_nonblocking(self.sentinel)
def __init__(self, fileno, mode='r', closefd=True): RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called self._closefd = closefd self._fileno = fileno make_nonblocking(fileno) readable = 'r' in mode writable = 'w' in mode self.hub = get_hub() io_watcher = self.hub.loop.io try: if readable: self._read_event = io_watcher(fileno, 1) if writable: self._write_event = io_watcher(fileno, 2) except: # If anything goes wrong, it's important to go ahead and # close these watchers *now*, especially under libuv, so # that they don't get eventually reclaimed by the garbage # collector at some random time, thanks to the C level # slot (even though we don't seem to have any actual references # at the Python level). Previously, if we didn't close now, # that random close in the future would cause issues if we had duplicated # the fileno (if a wrapping with statement had closed an open fileobject, # for example) # test__fileobject can show a failure if this doesn't happen # TRAVIS=true GEVENT_LOOP=libuv python -m gevent.tests.test__fileobject \ # TestFileObjectPosix.test_seek TestFileObjectThread.test_bufsize_0 self.close() raise
def __init__(self, fileno, open_descriptor, closefd=True): RawIOBase.__init__(self) self._closefd = closefd self._fileno = fileno self.mode = open_descriptor.fileio_mode make_nonblocking(fileno) readable = open_descriptor.can_read writable = open_descriptor.can_write self.hub = get_hub() io_watcher = self.hub.loop.io try: if readable: self._read_watcher = io_watcher(fileno, 1) if writable: self._write_watcher = io_watcher(fileno, 2) except: # If anything goes wrong, it's important to go ahead and # close these watchers *now*, especially under libuv, so # that they don't get eventually reclaimed by the garbage # collector at some random time, thanks to the C level # slot (even though we don't seem to have any actual references # at the Python level). Previously, if we didn't close now, # that random close in the future would cause issues if we had duplicated # the fileno (if a wrapping with statement had closed an open fileobject, # for example) # test__fileobject can show a failure if this doesn't happen # TRAVIS=true GEVENT_LOOP=libuv python -m gevent.tests.test__fileobject \ # TestFileObjectPosix.test_seek TestFileObjectThread.test_bufsize_0 self.close() raise
def test(self): # pylint:disable=too-many-locals # If this test is broken, there are a few failure modes. # - In the original examples, the parent process just hangs, because the # child has raced ahead, spawned the greenlet and read the data. When the # greenlet goes to read in the parent, it blocks, and the hub and loop # wait for it. # - Here, our child detects the greenlet ran when it shouldn't and # raises an error, which translates to a non-zero exit status, # which the parent checks for and fails by raising an exception before # returning control to the hub. We can replicate the hang by removing the # assertion in the child. from time import sleep as hang from gevent import get_hub from gevent import spawn from gevent.socket import wait_read from gevent.os import nb_read from gevent.os import nb_write from gevent.os import make_nonblocking from gevent.os import fork from gevent.os import waitpid pipe_read_fd, pipe_write_fd = os.pipe() make_nonblocking(pipe_read_fd) make_nonblocking(pipe_write_fd) run = [] def reader(): run.append(1) return nb_read(pipe_read_fd, 4096) # Put data in the pipe DATA = b'test' nb_write(pipe_write_fd, DATA) # Make sure we're ready to read it wait_read(pipe_read_fd) # Schedule a greenlet to start reader = spawn(reader) hub = get_hub() pid = fork() if pid == 0: # Child destroys the hub. The reader should not have run. hub.destroy(destroy_loop=True) self.assertFalse(run) os._exit(0) return # The parent. # Briefly prevent us from spinning our event loop. hang(0.5) wait_child_result = waitpid(pid, 0) self.assertEqual(wait_child_result, (pid, 0)) # We should get the data; the greenlet only runs in the parent. data = reader.get() self.assertEqual(run, [1]) self.assertEqual(data, DATA)
def preHook(self): if self.kwargs.timestamp: self.getTimestamp = self.returnTimestamp else: self.getTimestamp = self.returnNoTimestamp self.file = open(self.kwargs.location, "a") make_nonblocking(self.file)
def readFile(self, filename): try: if filename.endswith("ready") and self.loop(): with open(filename, "rb") as output_file: make_nonblocking(output_file) self.logging.info("Read file %s" % filename) for e in self.__pickleReader(output_file): self.submit(e, self.pool.queue.outbox) os.remove(filename) except Exception as err: self.logging.error("Failed to read file %s. Reason: %s" % (filename, str(err)))
def __init__(self, fileno, mode=None, close=True): if not isinstance(fileno, integer_types): raise TypeError('fileno must be int: %r' % fileno) self._fileno = fileno self._mode = mode or 'rb' self._close = close self._translate = 'U' in self._mode make_nonblocking(fileno) self._eat_newline = False self.hub = get_hub() io = self.hub.loop.io self._read_event = io(fileno, 1) self._write_event = io(fileno, 2)
def preHook(self): if self.kwargs.timestamp: self.getTimestamp = self.returnTimestamp else: self.getTimestamp = self.returnNoTimestamp if self.kwargs.complete: self.getData = self.returnComplete else: self.getData = self.returnDataOnly self.file = open(self.kwargs.location, "a") make_nonblocking(self.file)
def __init__(self, fileno, mode='r', closefd=True): RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called self._closefd = closefd self._fileno = fileno make_nonblocking(fileno) readable = 'r' in mode writable = 'w' in mode self.hub = get_hub() io_watcher = self.hub.loop.io if readable: self._read_event = io_watcher(fileno, 1) if writable: self._write_event = io_watcher(fileno, 2)
def __init__(self, fileno, mode=None, close=True): if not isinstance(fileno, integer_types): raise TypeError("fileno must be int: %r" % fileno) self._fileno = fileno self._mode = mode or "rb" self._close = close self._translate = not PY3 and "U" in self._mode make_nonblocking(fileno) self._eat_newline = False self.hub = get_hub() io = self.hub.loop.io self._read_event = io(fileno, 1) self._write_event = io(fileno, 2) self._io_refs = 0 # for Python 3 self._closed = False
def drain(self): '''Reads the named pipe.''' self.logging.info('Started.') fd = os.open(self.path, os.O_RDWR | os.O_NONBLOCK) gevent_os.make_nonblocking(fd) while self.loop(): try: lines = gevent_os.nb_read(fd, 4096).splitlines() if len(lines) == 0: sleep(0.5) else: self.consume(lines) except OSError: pass
def drain(self, p): '''Reads the named pipe.''' self.logging.info('Started.') fd = os.open(p, os.O_RDWR | os.O_NONBLOCK) gevent_os.make_nonblocking(fd) while self.loop(): try: lines = gevent_os.nb_read(fd, 4096).splitlines() if len(lines) == 0: sleep(0.5) else: self.consume(lines) except OSError: pass
def __init__(self, fileno, mode='r', closefd=True): RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called self._closed = False self._closefd = closefd self._fileno = fileno make_nonblocking(fileno) self._readable = 'r' in mode self._writable = 'w' in mode self.hub = get_hub() io_watcher = self.hub.loop.io if self._readable: self._read_event = io_watcher(fileno, 1) if self._writable: self._write_event = io_watcher(fileno, 2) self._seekable = None
def __init__(self, fileno, mode='r', closefd=True): RawIOBase.__init__(self) self._closed = False self._closefd = closefd self._fileno = fileno make_nonblocking(fileno) self._readable = 'r' in mode self._writable = 'w' in mode self.hub = get_hub() io = self.hub.loop.io if self._readable: self._read_event = io(fileno, 1) else: self._read_event = None if self._writable: self._write_event = io(fileno, 2) else: self._write_event = None
def flushDisk(self): if self.pool.queue.disk.size() > 0 and not self.__flush_lock: self.__flush_lock = True i = str(uuid4()) try: with open(r"%s/%s.%s.writing" % (self.kwargs.directory, self.name, i), "wb") as output_file: make_nonblocking(output_file) size = self.pool.queue.disk.size() for event in self.pool.queue.disk.dump(): pickle.dump(event, output_file) except Exception as err: self.logging.error("Failed to write file '%s' to '%s'. Reason: '%s'." % (self.name, self.kwargs.directory, err)) try: os.remove("%s/%s.%s.writing" % (self.kwargs.directory, self.name, i)) except Exception as err: self.logging.debug("No file %s/%s.%s.writing to remove" % (self.kwargs.directory, self.name, i)) self.submit(event, self.pool.queue.disk) else: os.rename("%s/%s.%s.writing" % (self.kwargs.directory, self.name, i), "%s/%s.%s.ready" % (self.kwargs.directory, self.name, i)) self.logging.info("Wrote %s events to file %s/%s.%s.ready" % (size, self.kwargs.directory, self.name, i)) self.__flush_lock = False
def pipe(self): r, w = pipe() os.make_nonblocking(r) os.make_nonblocking(w) return r, w
def open(file, flags, mode=0o777): fd = _open(file, flags, mode) # this is (almost) pointless since local files may block even if we set non-blocking on Linux # it's still interesting if the file is a named pipe, UNIX-domain socket, etc. make_nonblocking(fd) return fd
def read_events(self, event_buffer_size=DEFAULT_EVENT_BUFFER_SIZE): """ Reads events from inotify and yields them. """ from gevent import os as gos gos.make_nonblocking(self._inotify_fd) while True: try: event_buffer = gos.nb_read(self._inotify_fd, event_buffer_size) except OSError as e: if e.errno == errno.EINTR: continue break with self._lock: event_list = [] for wd, mask, cookie, name in Inotify._parse_event_buffer(event_buffer): if wd == -1: continue wd_path = self._path_for_wd[wd] src_path = absolute_path(os.path.join(wd_path, name)) inotify_event = InotifyEvent(wd, mask, cookie, name, src_path) if inotify_event.is_moved_from: self.remember_move_from_event(inotify_event) elif inotify_event.is_moved_to: move_src_path = self.source_for_move(inotify_event) if move_src_path in self._wd_for_path: # update old path -> new path moved_wd = self._wd_for_path[move_src_path] del self._wd_for_path[move_src_path] self._wd_for_path[inotify_event.src_path] = moved_wd self._path_for_wd[moved_wd] = inotify_event.src_path src_path = absolute_path(os.path.join(wd_path, name)) inotify_event = InotifyEvent(wd, mask, cookie, name, src_path) if inotify_event.is_ignored: # Clean up book-keeping for deleted watches. self._remove_watch_bookkeeping(src_path) continue event_list.append(inotify_event) if self.is_recursive and inotify_event.is_directory and inotify_event.is_create: # HACK: We need to traverse the directory path # recursively and simulate events for newly # created subdirectories/files. This will handle # mkdir -p foobar/blah/bar; touch foobar/afile # TODO: When a directory from another part of the # filesystem is moved into a watched directory, this # will not generate events for the directory tree. # We need to coalesce IN_MOVED_TO events and those # IN_MOVED_TO events which don't pair up with # IN_MOVED_FROM events should be marked IN_CREATE # instead relative to this directory. try: self._add_watch(src_path, self._event_mask) except OSError: continue for root, dirnames, filenames in os.walk(src_path): for dirname in dirnames: try: full_path = absolute_path(os.path.join(root, dirname)) wd_dir = self._add_watch(full_path, self._event_mask) event_list.append(InotifyEvent(wd_dir, InotifyConstants.IN_CREATE | InotifyConstants.IN_ISDIR, 0, dirname, full_path)) except OSError: pass for filename in filenames: full_path = absolute_path(os.path.join(root, filename)) wd_parent_dir = self._wd_for_path[absolute_path(os.path.dirname(full_path))] event_list.append(InotifyEvent(wd_parent_dir, InotifyConstants.IN_CREATE, 0, filename, full_path)) return event_list
def ensure_running(self): super().ensure_running() make_nonblocking(self._forkserver_alive_fd)
def connect_to_new_process(self, fds): parent_r, parent_w = super().connect_to_new_process(fds) make_nonblocking(parent_r) make_nonblocking(parent_w) return parent_r, parent_w
def ensure_running(self): super().ensure_running() make_nonblocking(self._fd)
def pipe(self): r, w = super(TestOS_nb, self).pipe() os.make_nonblocking(r) os.make_nonblocking(w) return r, w
def dhcp_worker(self): make_nonblocking(self.dhcp_server.bpf.fd) self.dhcp_server.serve()