def test_set_child_watcher(self):
p = objc_asyncio.PyObjCEventLoopPolicy()
watcher = asyncio.SafeChildWatcher()
p.set_child_watcher(watcher)
self.assertIs(p.get_child_watcher(), watcher)
def test_read_all_from_pipe_reader(self):
# See Tulip issue 168. This test is derived from the example
# subprocess_attach_read_pipe.py, but we configure the
# StreamReader's limit so that twice it is less than the size
# of the data writter. Also we must explicitly attach a child
# watcher to the event loop.
code = """\
import os, sys
fd = int(sys.argv[1])
os.write(fd, b'data')
os.close(fd)
"""
rfd, wfd = os.pipe()
args = [sys.executable, '-c', code, str(wfd)]
pipe = open(rfd, 'rb', 0)
reader = asyncio.StreamReader(loop=self.loop, limit=1)
protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
transport, _ = self.loop.run_until_complete(
self.loop.connect_read_pipe(lambda: protocol, pipe))
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
try:
asyncio.set_child_watcher(watcher)
proc = self.loop.run_until_complete(
asyncio.create_subprocess_exec(*args, pass_fds={wfd}, loop=self.loop))
self.loop.run_until_complete(proc.wait())
finally:
asyncio.set_child_watcher(None)
os.close(wfd)
data = self.loop.run_until_complete(reader.read(-1))
self.assertEqual(data, b'data')
def _setup_eventloop(self):
"""Sets up a new eventloop as the current one according to the OS."""
if os.name == "nt":
self.eventloop = asyncio.ProactorEventLoop()
else:
try:
self.eventloop = asyncio.get_event_loop_policy(
).get_event_loop()
except RuntimeError:
if threading.current_thread() != threading.main_thread():
# Ran not in main thread, make a new eventloop
self.eventloop = asyncio.new_event_loop()
asyncio.set_event_loop(self.eventloop)
else:
raise
if os.name == "posix" and isinstance(threading.current_thread(),
threading._MainThread):
# The default child watchers (ThreadedChildWatcher) attach_loop method is empty!
# While using pyshark with ThreadedChildWatcher, asyncio could raise a ChildProcessError
# "Unknown child process pid %d, will report returncode 255"
# This led to a TSharkCrashException in _cleanup_subprocess.
# Using the SafeChildWatcher fixes this issue, but it is slower.
# SafeChildWatcher O(n) -> large numbers of processes are slow
# ThreadedChildWatcher O(1) -> independent of process number
# asyncio.get_child_watcher().attach_loop(self.eventloop)
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
asyncio.get_child_watcher().attach_loop(self.eventloop)
def get_child_watcher(self):
if self._watcher is None:
self._watcher = asyncio.SafeChildWatcher()
if self._loop:
self._watcher.attach_loop(self._loop)
return self._watcher
def start(self):
"""Start the event loop and call `start_service`.
Pyhap will be stopped gracefully on a KeyBoardInterrupt.
"""
try:
logger.info('Starting the event loop')
if threading.current_thread() is threading.main_thread():
logger.debug('Setting child watcher')
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
asyncio.set_child_watcher(watcher)
else:
logger.debug(
'Not setting a child watcher. Set one if '
'subprocesses will be started outside the main thread.')
self.add_job(self.start_service)
self.loop.run_forever()
except KeyboardInterrupt:
logger.debug('Got a KeyboardInterrupt, stopping driver')
self.loop.call_soon_threadsafe(self.loop.create_task,
self.async_stop())
self.loop.run_forever()
finally:
self.loop.close()
logger.info('Closed the event loop')
def setup_test_loop(
loop_factory: _LOOP_FACTORY = asyncio.new_event_loop,
) -> asyncio.AbstractEventLoop:
"""Create and return an asyncio.BaseEventLoop instance.
The caller should also call teardown_test_loop,
once they are done with the loop.
"""
loop = loop_factory()
try:
module = loop.__class__.__module__
skip_watcher = "uvloop" in module
except AttributeError: # pragma: no cover
# Just in case
skip_watcher = True
asyncio.set_event_loop(loop)
if sys.platform != "win32" and not skip_watcher:
policy = asyncio.get_event_loop_policy()
watcher: asyncio.AbstractChildWatcher
try: # Python >= 3.8
# Refs:
# * https://github.com/pytest-dev/pytest-xdist/issues/620
# * https://stackoverflow.com/a/58614689/595220
# * https://bugs.python.org/issue35621
# * https://github.com/python/cpython/pull/14344
watcher = asyncio.ThreadedChildWatcher()
except AttributeError: # Python < 3.8
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
with contextlib.suppress(NotImplementedError):
policy.set_child_watcher(watcher)
return loop
def test_read_all_from_pipe_reader(self):
code = (
"import os, sys\nfd = int(sys.argv[1])\nos.write(fd, b'data')\nos.close(fd)\n"
)
rfd, wfd = os.pipe()
args = [sys.executable, '-c', code, str(wfd)]
pipe = open(rfd, 'rb', 0)
reader = asyncio.StreamReader(loop=self.loop, limit=1)
protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
transport, _ = self.loop.run_until_complete(
self.loop.connect_read_pipe(lambda: protocol, pipe))
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
try:
asyncio.set_child_watcher(watcher)
create = asyncio.create_subprocess_exec(*args,
pass_fds={wfd},
loop=self.loop)
proc = self.loop.run_until_complete(create)
self.loop.run_until_complete(proc.wait())
finally:
asyncio.set_child_watcher(None)
os.close(wfd)
data = self.loop.run_until_complete(reader.read(-1))
self.assertEqual(data, b'data')
def run(self, queues, casd_process_manager):
# Hold on to the queues to process
self.queues = queues
# NOTE: Enforce use of `SafeChildWatcher` as we generally don't want
# background threads.
# In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and
# not `SafeChildWatcher`.
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
# Ensure that we have a fresh new event loop, in case we want
# to run another test in this thread.
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
# Add timeouts
self.loop.call_later(1, self._tick)
# Add exception handler
self.loop.set_exception_handler(self._handle_exception)
# Handle unix signals while running
self._connect_signals()
# Watch casd while running to ensure it doesn't die
self._casd_process = casd_process_manager.process
_watcher = asyncio.get_child_watcher()
def abort_casd(pid, returncode):
asyncio.get_event_loop().call_soon(self._abort_on_casd_failure, pid, returncode)
_watcher.add_child_handler(self._casd_process.pid, abort_casd)
# Start the profiler
with PROFILER.profile(Topics.SCHEDULER, "_".join(queue.action_name for queue in self.queues)):
# Run the queues
self._sched()
self.loop.run_forever()
self.loop.close()
# Stop watching casd
_watcher.remove_child_handler(self._casd_process.pid)
self._casd_process = None
# Stop handling unix signals
self._disconnect_signals()
failed = any(queue.any_failed_elements() for queue in self.queues)
self.loop = None
if failed:
status = SchedStatus.ERROR
elif self.terminated:
status = SchedStatus.TERMINATED
else:
status = SchedStatus.SUCCESS
return status
def get_child_watcher(self):
self._check_unix()
if self.loop:
if self.watcher is None:
self.watcher = asyncio.SafeChildWatcher()
self.watcher.attach_loop(self.loop)
return self.watcher
else:
return self.original_policy.get_child_watcher()
def test_process_str():
cmd = 'python -c "for i in range(4): print(i)"'
s = Source.from_process(cmd)
if sys.platform != "win32":
# don't know why - something with pytest and new processes
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
policy.set_child_watcher(watcher)
watcher.attach_loop(s.loop.asyncio_loop)
out = s.sink_to_list()
s.start()
yield await_for(lambda: out == [b'0\n', b'1\n', b'2\n', b'3\n'], timeout=5)
s.stop()
def test_process():
cmd = ["python", "-c", "for i in range(4): print(i, end='')"]
s = Source.from_process(cmd, with_end=True)
if sys.platform != "win32":
# don't know why - something with pytest and new processes
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
policy.set_child_watcher(watcher)
watcher.attach_loop(s.loop.asyncio_loop)
out = s.sink_to_list()
s.start()
yield await_for(lambda: out == [b'0123'], timeout=5)
s.stop()
def setup_test_loop(loop_factory=asyncio.new_event_loop):
"""Create and return an asyncio.BaseEventLoop instance.
The caller should also call teardown_test_loop, once they are done
with the loop.
"""
loop = loop_factory()
asyncio.set_event_loop(loop)
if sys.platform != "win32":
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
return loop
def start_sniffer_handler(self):
# Pass the function to execute
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
loop = asyncio.new_event_loop()
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
fn = partial(self.run_sniffer, loop=loop)
self.worker = Worker(
fn) # Any other args, kwargs are passed to the run function
# worker.signals.result.connect(self.print_output)
# worker.signals.finished.connect(self.thread_complete)
# worker.signals.progress.connect(self.progress_fn)
# Execute
self.threadpool.start(self.worker)
def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]:
if sys.platform != "win32":
asyncio.set_event_loop(
None
) # type: ignore # see https://github.com/pytest-dev/pytest-asyncio/issues/73
loop = get_event_loop()
if sys.platform != "win32":
# on UNIX we also need to attach the loop to the child watcher for asyncio.subprocess
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher() # type: ignore # undocumented?
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
try:
yield loop
finally:
loop.close()
def event_loop():
"""pytest-asyncio customization"""
if sys.platform != "win32":
asyncio.set_event_loop(
None) # see https://github.com/pytest-dev/pytest-asyncio/issues/73
loop = get_event_loop()
if sys.platform != "win32":
# on UNIX we also need to attach the loop to the child watcher for asyncio.subprocess
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
try:
yield loop
finally:
loop.close()
def test_set_child_watcher_on_thread(self):
p = objc_asyncio.PyObjCEventLoopPolicy()
watcher = asyncio.SafeChildWatcher()
def thread_main():
thread_loop = objc_asyncio.PyObjCEventLoop()
p.set_event_loop(thread_loop)
p.set_child_watcher(watcher)
t = threading.Thread(target=thread_main)
t.start()
t.join()
self.assertIs(p.get_child_watcher(), watcher)
self.assertIs(watcher._loop, None)
def setup_test_loop(
loop_factory=asyncio.new_event_loop) -> asyncio.AbstractEventLoop:
loop = loop_factory()
try:
module = loop.__class__.__module__
skip_watcher = 'uvloop' in module
except AttributeError: # pragma: no cover
# Just in case
skip_watcher = True
asyncio.set_event_loop(loop)
if sys.platform != 'win32' and not skip_watcher:
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher() # type: ignore
watcher.attach_loop(loop)
with contextlib.suppress(NotImplementedError):
policy.set_child_watcher(watcher)
return loop
def test_write_config(self):
loop = asyncio.new_event_loop()
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
async def t():
os.environ['SUBIQUITY_REPLAY_TIMESCALE'] = '100'
with tempfile.TemporaryDirectory() as tmpdir:
new_setting = KeyboardSetting('fr', 'azerty')
await set_keyboard(tmpdir, new_setting, True)
read_setting = from_config_file(
os.path.join(tmpdir, 'etc', 'default', 'keyboard'))
self.assertEqual(new_setting, read_setting)
loop.run_until_complete(t())
loop.close()
def test_resetting_child_watcher(self):
p = objc_asyncio.PyObjCEventLoopPolicy()
p.get_event_loop()
watcher = p.get_child_watcher()
self.assertIsInstance(watcher, objc_asyncio.KQueueChildWatcher)
self.assertIs(watcher._loop, p.get_event_loop())
new_watcher = asyncio.SafeChildWatcher()
p.set_child_watcher(new_watcher)
self.assertIs(watcher._loop, None)
self.assertIs(new_watcher._loop, None)
loop = objc_asyncio.PyObjCEventLoop()
p.set_event_loop(loop)
self.assertIs(watcher._loop, None)
self.assertIs(new_watcher._loop, loop)
self.assertIs(p.get_event_loop(), loop)
def setup_test_loop(loop_factory=asyncio.new_event_loop):
"""Create and return an asyncio.BaseEventLoop instance.
The caller should also call teardown_test_loop, once they are done
with the loop.
"""
loop = loop_factory()
try:
module = loop.__class__.__module
skip_watcher = 'uvloop' in module
except AttributeError:
# Just in case
skip_watcher = True
asyncio.set_event_loop(loop)
if sys.platform != 'win32' and not skip_watcher:
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
with contextlib.suppress(NotImplementedError):
policy.set_child_watcher(watcher)
return loop
def test_write_config(self):
loop = asyncio.new_event_loop()
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
async def t():
os.environ['SUBIQUITY_REPLAY_TIMESCALE'] = '100'
with tempfile.TemporaryDirectory() as tmpdir:
new_setting = KeyboardSetting('fr', 'azerty')
model = KeyboardModel(tmpdir)
model.setting = new_setting
c = object.__new__(KeyboardController)
c.opts = opts
c.model = model
await c.set_keyboard()
read_setting = KeyboardModel(tmpdir).setting
self.assertEqual(new_setting, read_setting)
loop.run_until_complete(t())
loop.close()
def safe_watcher():
if sys.platform == "win32":
yield
return
from asyncio import SafeChildWatcher
policy = asyncio.get_event_loop_policy()
old_watcher = policy.get_child_watcher()
if isinstance(old_watcher, SafeChildWatcher):
yield
return
loop = asyncio.get_event_loop()
try:
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(loop)
policy.set_child_watcher(watcher)
yield
finally:
watcher.close()
policy.set_child_watcher(old_watcher)
def setup_test_loop(
loop_factory: _LOOP_FACTORY = asyncio.new_event_loop,
) -> asyncio.AbstractEventLoop:
"""Create and return an asyncio.BaseEventLoop
instance.
The caller should also call teardown_test_loop,
once they are done with the loop.
"""
loop = loop_factory()
try:
module = loop.__class__.__module__
skip_watcher = "uvloop" in module
except AttributeError: # pragma: no cover
# Just in case
skip_watcher = True
asyncio.set_event_loop(loop)
if sys.platform != "win32" and not skip_watcher:
policy = asyncio.get_event_loop_policy()
watcher = asyncio.SafeChildWatcher() # type: ignore
watcher.attach_loop(loop)
with contextlib.suppress(NotImplementedError):
policy.set_child_watcher(watcher)
return loop
def setUp(self):
super().setUp()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
asyncio.set_child_watcher(watcher)
SECRET_KEY = "CHANGEME"
BANNED_TIMEOUT = 45 * 60 # 45 minutes
SOLVE_DURATION = 3 * 60 # 3 minutes
proxies = ProxyDB(last_banned_timeout=BANNED_TIMEOUT)
proxy_source = None # Can be URL or file location
proxy_username, proxy_password = (None, None)
if sys.platform == "win32":
parent_loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(parent_loop)
else:
parent_loop = asyncio.get_event_loop()
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
asyncio.get_child_watcher().attach_loop(parent_loop)
app = web.Application()
# Clear Chrome temporary profiles
dir = f"{Path.home()}/.pyppeteer/.dev_profile"
shutil.rmtree(dir, ignore_errors=True)
# Bugs are to be expected, despite my efforts. Apparently, event loops paired
# with threads is nothing short of a hassle.
class TaskRerun(object):
def __init__(self, coro, duration):
self.coro = coro
self.duration = duration
def run(self, queues, casd_process_manager):
# Hold on to the queues to process
self.queues = queues
# NOTE: Enforce use of `SafeChildWatcher` as we generally don't want
# background threads.
# In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and
# not `SafeChildWatcher`.
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
# Ensure that we have a fresh new event loop, in case we want
# to run another test in this thread.
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
# Add timeouts
self.loop.call_later(1, self._tick)
# Add exception handler
self.loop.set_exception_handler(self._handle_exception)
# Handle unix signals while running
self._connect_signals()
# Watch casd while running to ensure it doesn't die
self._casd_process = casd_process_manager.process
_watcher = asyncio.get_child_watcher()
def abort_casd(pid, returncode):
asyncio.get_event_loop().call_soon(self._abort_on_casd_failure,
pid, returncode)
_watcher.add_child_handler(self._casd_process.pid, abort_casd)
# Start the profiler
with PROFILER.profile(
Topics.SCHEDULER,
"_".join(queue.action_name for queue in self.queues)):
# This is not a no-op. Since it is the first signal registration
# that is set, it allows then other threads to register signal
# handling routines, which would not be possible if the main thread
# hadn't set it before.
# FIXME: this should be done in a cleaner way
with _signals.suspendable(
lambda: None,
lambda: None), _signals.terminator(lambda: None):
with ThreadPoolExecutor(max_workers=sum(
self.resources._max_resources.values())) as pool:
self.loop.set_default_executor(pool)
# Run the queues
self._sched()
self.loop.run_forever()
self.loop.close()
# Invoke the ticker callback a final time to render pending messages
self._ticker_callback()
# Stop watching casd
_watcher.remove_child_handler(self._casd_process.pid)
self._casd_process = None
# Stop handling unix signals
self._disconnect_signals()
failed = any(queue.any_failed_elements() for queue in self.queues)
self.loop = None
if failed:
status = SchedStatus.ERROR
elif self.terminated:
status = SchedStatus.TERMINATED
else:
status = SchedStatus.SUCCESS
return status
