def test_set_child_watcher(self): p = objc_asyncio.PyObjCEventLoopPolicy() watcher = asyncio.SafeChildWatcher() p.set_child_watcher(watcher) self.assertIs(p.get_child_watcher(), watcher)
def test_read_all_from_pipe_reader(self): # See Tulip issue 168. This test is derived from the example # subprocess_attach_read_pipe.py, but we configure the # StreamReader's limit so that twice it is less than the size # of the data writter. Also we must explicitly attach a child # watcher to the event loop. code = """\ import os, sys fd = int(sys.argv[1]) os.write(fd, b'data') os.close(fd) """ rfd, wfd = os.pipe() args = [sys.executable, '-c', code, str(wfd)] pipe = open(rfd, 'rb', 0) reader = asyncio.StreamReader(loop=self.loop, limit=1) protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) transport, _ = self.loop.run_until_complete( self.loop.connect_read_pipe(lambda: protocol, pipe)) watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) try: asyncio.set_child_watcher(watcher) proc = self.loop.run_until_complete( asyncio.create_subprocess_exec(*args, pass_fds={wfd}, loop=self.loop)) self.loop.run_until_complete(proc.wait()) finally: asyncio.set_child_watcher(None) os.close(wfd) data = self.loop.run_until_complete(reader.read(-1)) self.assertEqual(data, b'data')
def _setup_eventloop(self): """Sets up a new eventloop as the current one according to the OS.""" if os.name == "nt": self.eventloop = asyncio.ProactorEventLoop() else: try: self.eventloop = asyncio.get_event_loop_policy( ).get_event_loop() except RuntimeError: if threading.current_thread() != threading.main_thread(): # Ran not in main thread, make a new eventloop self.eventloop = asyncio.new_event_loop() asyncio.set_event_loop(self.eventloop) else: raise if os.name == "posix" and isinstance(threading.current_thread(), threading._MainThread): # The default child watchers (ThreadedChildWatcher) attach_loop method is empty! # While using pyshark with ThreadedChildWatcher, asyncio could raise a ChildProcessError # "Unknown child process pid %d, will report returncode 255" # This led to a TSharkCrashException in _cleanup_subprocess. # Using the SafeChildWatcher fixes this issue, but it is slower. # SafeChildWatcher O(n) -> large numbers of processes are slow # ThreadedChildWatcher O(1) -> independent of process number # asyncio.get_child_watcher().attach_loop(self.eventloop) asyncio.set_child_watcher(asyncio.SafeChildWatcher()) asyncio.get_child_watcher().attach_loop(self.eventloop)
def get_child_watcher(self): if self._watcher is None: self._watcher = asyncio.SafeChildWatcher() if self._loop: self._watcher.attach_loop(self._loop) return self._watcher
def start(self): """Start the event loop and call `start_service`. Pyhap will be stopped gracefully on a KeyBoardInterrupt. """ try: logger.info('Starting the event loop') if threading.current_thread() is threading.main_thread(): logger.debug('Setting child watcher') watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) asyncio.set_child_watcher(watcher) else: logger.debug( 'Not setting a child watcher. Set one if ' 'subprocesses will be started outside the main thread.') self.add_job(self.start_service) self.loop.run_forever() except KeyboardInterrupt: logger.debug('Got a KeyboardInterrupt, stopping driver') self.loop.call_soon_threadsafe(self.loop.create_task, self.async_stop()) self.loop.run_forever() finally: self.loop.close() logger.info('Closed the event loop')
def setup_test_loop( loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, ) -> asyncio.AbstractEventLoop: """Create and return an asyncio.BaseEventLoop instance. The caller should also call teardown_test_loop, once they are done with the loop. """ loop = loop_factory() try: module = loop.__class__.__module__ skip_watcher = "uvloop" in module except AttributeError: # pragma: no cover # Just in case skip_watcher = True asyncio.set_event_loop(loop) if sys.platform != "win32" and not skip_watcher: policy = asyncio.get_event_loop_policy() watcher: asyncio.AbstractChildWatcher try: # Python >= 3.8 # Refs: # * https://github.com/pytest-dev/pytest-xdist/issues/620 # * https://stackoverflow.com/a/58614689/595220 # * https://bugs.python.org/issue35621 # * https://github.com/python/cpython/pull/14344 watcher = asyncio.ThreadedChildWatcher() except AttributeError: # Python < 3.8 watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) with contextlib.suppress(NotImplementedError): policy.set_child_watcher(watcher) return loop
def test_read_all_from_pipe_reader(self): code = ( "import os, sys\nfd = int(sys.argv[1])\nos.write(fd, b'data')\nos.close(fd)\n" ) rfd, wfd = os.pipe() args = [sys.executable, '-c', code, str(wfd)] pipe = open(rfd, 'rb', 0) reader = asyncio.StreamReader(loop=self.loop, limit=1) protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) transport, _ = self.loop.run_until_complete( self.loop.connect_read_pipe(lambda: protocol, pipe)) watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) try: asyncio.set_child_watcher(watcher) create = asyncio.create_subprocess_exec(*args, pass_fds={wfd}, loop=self.loop) proc = self.loop.run_until_complete(create) self.loop.run_until_complete(proc.wait()) finally: asyncio.set_child_watcher(None) os.close(wfd) data = self.loop.run_until_complete(reader.read(-1)) self.assertEqual(data, b'data')
def run(self, queues, casd_process_manager): # Hold on to the queues to process self.queues = queues # NOTE: Enforce use of `SafeChildWatcher` as we generally don't want # background threads. # In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and # not `SafeChildWatcher`. asyncio.set_child_watcher(asyncio.SafeChildWatcher()) # Ensure that we have a fresh new event loop, in case we want # to run another test in this thread. self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) # Add timeouts self.loop.call_later(1, self._tick) # Add exception handler self.loop.set_exception_handler(self._handle_exception) # Handle unix signals while running self._connect_signals() # Watch casd while running to ensure it doesn't die self._casd_process = casd_process_manager.process _watcher = asyncio.get_child_watcher() def abort_casd(pid, returncode): asyncio.get_event_loop().call_soon(self._abort_on_casd_failure, pid, returncode) _watcher.add_child_handler(self._casd_process.pid, abort_casd) # Start the profiler with PROFILER.profile(Topics.SCHEDULER, "_".join(queue.action_name for queue in self.queues)): # Run the queues self._sched() self.loop.run_forever() self.loop.close() # Stop watching casd _watcher.remove_child_handler(self._casd_process.pid) self._casd_process = None # Stop handling unix signals self._disconnect_signals() failed = any(queue.any_failed_elements() for queue in self.queues) self.loop = None if failed: status = SchedStatus.ERROR elif self.terminated: status = SchedStatus.TERMINATED else: status = SchedStatus.SUCCESS return status
def get_child_watcher(self): self._check_unix() if self.loop: if self.watcher is None: self.watcher = asyncio.SafeChildWatcher() self.watcher.attach_loop(self.loop) return self.watcher else: return self.original_policy.get_child_watcher()
def test_process_str(): cmd = 'python -c "for i in range(4): print(i)"' s = Source.from_process(cmd) if sys.platform != "win32": # don't know why - something with pytest and new processes policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() policy.set_child_watcher(watcher) watcher.attach_loop(s.loop.asyncio_loop) out = s.sink_to_list() s.start() yield await_for(lambda: out == [b'0\n', b'1\n', b'2\n', b'3\n'], timeout=5) s.stop()
def test_process(): cmd = ["python", "-c", "for i in range(4): print(i, end='')"] s = Source.from_process(cmd, with_end=True) if sys.platform != "win32": # don't know why - something with pytest and new processes policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() policy.set_child_watcher(watcher) watcher.attach_loop(s.loop.asyncio_loop) out = s.sink_to_list() s.start() yield await_for(lambda: out == [b'0123'], timeout=5) s.stop()
def setup_test_loop(loop_factory=asyncio.new_event_loop): """Create and return an asyncio.BaseEventLoop instance. The caller should also call teardown_test_loop, once they are done with the loop. """ loop = loop_factory() asyncio.set_event_loop(loop) if sys.platform != "win32": policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) policy.set_child_watcher(watcher) return loop
def start_sniffer_handler(self): # Pass the function to execute policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() loop = asyncio.new_event_loop() watcher.attach_loop(loop) policy.set_child_watcher(watcher) fn = partial(self.run_sniffer, loop=loop) self.worker = Worker( fn) # Any other args, kwargs are passed to the run function # worker.signals.result.connect(self.print_output) # worker.signals.finished.connect(self.thread_complete) # worker.signals.progress.connect(self.progress_fn) # Execute self.threadpool.start(self.worker)
def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]: if sys.platform != "win32": asyncio.set_event_loop( None ) # type: ignore # see https://github.com/pytest-dev/pytest-asyncio/issues/73 loop = get_event_loop() if sys.platform != "win32": # on UNIX we also need to attach the loop to the child watcher for asyncio.subprocess policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() # type: ignore # undocumented? watcher.attach_loop(loop) policy.set_child_watcher(watcher) try: yield loop finally: loop.close()
def event_loop(): """pytest-asyncio customization""" if sys.platform != "win32": asyncio.set_event_loop( None) # see https://github.com/pytest-dev/pytest-asyncio/issues/73 loop = get_event_loop() if sys.platform != "win32": # on UNIX we also need to attach the loop to the child watcher for asyncio.subprocess policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) policy.set_child_watcher(watcher) try: yield loop finally: loop.close()
def test_set_child_watcher_on_thread(self): p = objc_asyncio.PyObjCEventLoopPolicy() watcher = asyncio.SafeChildWatcher() def thread_main(): thread_loop = objc_asyncio.PyObjCEventLoop() p.set_event_loop(thread_loop) p.set_child_watcher(watcher) t = threading.Thread(target=thread_main) t.start() t.join() self.assertIs(p.get_child_watcher(), watcher) self.assertIs(watcher._loop, None)
def setup_test_loop( loop_factory=asyncio.new_event_loop) -> asyncio.AbstractEventLoop: loop = loop_factory() try: module = loop.__class__.__module__ skip_watcher = 'uvloop' in module except AttributeError: # pragma: no cover # Just in case skip_watcher = True asyncio.set_event_loop(loop) if sys.platform != 'win32' and not skip_watcher: policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() # type: ignore watcher.attach_loop(loop) with contextlib.suppress(NotImplementedError): policy.set_child_watcher(watcher) return loop
def test_write_config(self): loop = asyncio.new_event_loop() policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) policy.set_child_watcher(watcher) async def t(): os.environ['SUBIQUITY_REPLAY_TIMESCALE'] = '100' with tempfile.TemporaryDirectory() as tmpdir: new_setting = KeyboardSetting('fr', 'azerty') await set_keyboard(tmpdir, new_setting, True) read_setting = from_config_file( os.path.join(tmpdir, 'etc', 'default', 'keyboard')) self.assertEqual(new_setting, read_setting) loop.run_until_complete(t()) loop.close()
def test_resetting_child_watcher(self): p = objc_asyncio.PyObjCEventLoopPolicy() p.get_event_loop() watcher = p.get_child_watcher() self.assertIsInstance(watcher, objc_asyncio.KQueueChildWatcher) self.assertIs(watcher._loop, p.get_event_loop()) new_watcher = asyncio.SafeChildWatcher() p.set_child_watcher(new_watcher) self.assertIs(watcher._loop, None) self.assertIs(new_watcher._loop, None) loop = objc_asyncio.PyObjCEventLoop() p.set_event_loop(loop) self.assertIs(watcher._loop, None) self.assertIs(new_watcher._loop, loop) self.assertIs(p.get_event_loop(), loop)
def setup_test_loop(loop_factory=asyncio.new_event_loop): """Create and return an asyncio.BaseEventLoop instance. The caller should also call teardown_test_loop, once they are done with the loop. """ loop = loop_factory() try: module = loop.__class__.__module skip_watcher = 'uvloop' in module except AttributeError: # Just in case skip_watcher = True asyncio.set_event_loop(loop) if sys.platform != 'win32' and not skip_watcher: policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) with contextlib.suppress(NotImplementedError): policy.set_child_watcher(watcher) return loop
def test_write_config(self): loop = asyncio.new_event_loop() policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) policy.set_child_watcher(watcher) async def t(): os.environ['SUBIQUITY_REPLAY_TIMESCALE'] = '100' with tempfile.TemporaryDirectory() as tmpdir: new_setting = KeyboardSetting('fr', 'azerty') model = KeyboardModel(tmpdir) model.setting = new_setting c = object.__new__(KeyboardController) c.opts = opts c.model = model await c.set_keyboard() read_setting = KeyboardModel(tmpdir).setting self.assertEqual(new_setting, read_setting) loop.run_until_complete(t()) loop.close()
def safe_watcher(): if sys.platform == "win32": yield return from asyncio import SafeChildWatcher policy = asyncio.get_event_loop_policy() old_watcher = policy.get_child_watcher() if isinstance(old_watcher, SafeChildWatcher): yield return loop = asyncio.get_event_loop() try: watcher = asyncio.SafeChildWatcher() watcher.attach_loop(loop) policy.set_child_watcher(watcher) yield finally: watcher.close() policy.set_child_watcher(old_watcher)
def setup_test_loop( loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, ) -> asyncio.AbstractEventLoop: """Create and return an asyncio.BaseEventLoop instance. The caller should also call teardown_test_loop, once they are done with the loop. """ loop = loop_factory() try: module = loop.__class__.__module__ skip_watcher = "uvloop" in module except AttributeError: # pragma: no cover # Just in case skip_watcher = True asyncio.set_event_loop(loop) if sys.platform != "win32" and not skip_watcher: policy = asyncio.get_event_loop_policy() watcher = asyncio.SafeChildWatcher() # type: ignore watcher.attach_loop(loop) with contextlib.suppress(NotImplementedError): policy.set_child_watcher(watcher) return loop
def setUp(self): super().setUp() watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) asyncio.set_child_watcher(watcher)
SECRET_KEY = "CHANGEME" BANNED_TIMEOUT = 45 * 60 # 45 minutes SOLVE_DURATION = 3 * 60 # 3 minutes proxies = ProxyDB(last_banned_timeout=BANNED_TIMEOUT) proxy_source = None # Can be URL or file location proxy_username, proxy_password = (None, None) if sys.platform == "win32": parent_loop = asyncio.ProactorEventLoop() asyncio.set_event_loop(parent_loop) else: parent_loop = asyncio.get_event_loop() asyncio.set_child_watcher(asyncio.SafeChildWatcher()) asyncio.get_child_watcher().attach_loop(parent_loop) app = web.Application() # Clear Chrome temporary profiles dir = f"{Path.home()}/.pyppeteer/.dev_profile" shutil.rmtree(dir, ignore_errors=True) # Bugs are to be expected, despite my efforts. Apparently, event loops paired # with threads is nothing short of a hassle. class TaskRerun(object): def __init__(self, coro, duration): self.coro = coro self.duration = duration
def run(self, queues, casd_process_manager): # Hold on to the queues to process self.queues = queues # NOTE: Enforce use of `SafeChildWatcher` as we generally don't want # background threads. # In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and # not `SafeChildWatcher`. asyncio.set_child_watcher(asyncio.SafeChildWatcher()) # Ensure that we have a fresh new event loop, in case we want # to run another test in this thread. self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) # Add timeouts self.loop.call_later(1, self._tick) # Add exception handler self.loop.set_exception_handler(self._handle_exception) # Handle unix signals while running self._connect_signals() # Watch casd while running to ensure it doesn't die self._casd_process = casd_process_manager.process _watcher = asyncio.get_child_watcher() def abort_casd(pid, returncode): asyncio.get_event_loop().call_soon(self._abort_on_casd_failure, pid, returncode) _watcher.add_child_handler(self._casd_process.pid, abort_casd) # Start the profiler with PROFILER.profile( Topics.SCHEDULER, "_".join(queue.action_name for queue in self.queues)): # This is not a no-op. Since it is the first signal registration # that is set, it allows then other threads to register signal # handling routines, which would not be possible if the main thread # hadn't set it before. # FIXME: this should be done in a cleaner way with _signals.suspendable( lambda: None, lambda: None), _signals.terminator(lambda: None): with ThreadPoolExecutor(max_workers=sum( self.resources._max_resources.values())) as pool: self.loop.set_default_executor(pool) # Run the queues self._sched() self.loop.run_forever() self.loop.close() # Invoke the ticker callback a final time to render pending messages self._ticker_callback() # Stop watching casd _watcher.remove_child_handler(self._casd_process.pid) self._casd_process = None # Stop handling unix signals self._disconnect_signals() failed = any(queue.any_failed_elements() for queue in self.queues) self.loop = None if failed: status = SchedStatus.ERROR elif self.terminated: status = SchedStatus.TERMINATED else: status = SchedStatus.SUCCESS return status