def test_read_all_from_pipe_reader(self):
# See asyncio issue 168. This test is derived from the example
# subprocess_attach_read_pipe.py, but we configure the
# StreamReader's limit so that twice it is less than the size
# of the data writter. Also we must explicitly attach a child
# watcher to the event loop.
code = """\
import os, sys
fd = int(sys.argv[1])
os.write(fd, b'data')
os.close(fd)
"""
rfd, wfd = os.pipe()
args = [sys.executable, "-c", code, str(wfd)]
pipe = open(rfd, "rb", 0)
reader = asyncio.StreamReader(loop=self.loop, limit=1)
protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
transport, _ = self.loop.run_until_complete(self.loop.connect_read_pipe(lambda: protocol, pipe))
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
try:
asyncio.set_child_watcher(watcher)
create = asyncio.create_subprocess_exec(*args, pass_fds={wfd}, loop=self.loop)
proc = self.loop.run_until_complete(create)
self.loop.run_until_complete(proc.wait())
finally:
asyncio.set_child_watcher(None)
os.close(wfd)
data = self.loop.run_until_complete(reader.read(-1))
self.assertEqual(data, b"data")
def test_read_all_from_pipe_reader(self):
# See Tulip issue 168. This test is derived from the example
# subprocess_attach_read_pipe.py, but we configure the
# StreamReader's limit so that twice it is less than the size
# of the data writter. Also we must explicitly attach a child
# watcher to the event loop.
code = """\
import os, sys
try:
import faulthandler
except ImportError:
pass
else:
faulthandler.dump_traceback_later(60, exit=True)
fd = int(sys.argv[1])
os.write(fd, b'data')
os.close(fd)
"""
rfd, wfd = os.pipe()
args = [sys.executable, '-c', code, str(wfd)]
# FIXME: Debug code for issue #21645
import logging
self.loop.set_debug(True)
logger = logging.getLogger('asyncio')
log_level = logger.level
try:
log_handler = logging.StreamHandler(sys.__stderr__)
logger.addHandler(log_handler)
logger.setLevel(logging.DEBUG)
# FIXME: Debug code for issue #21645 ---
pipe = open(rfd, 'rb', 0)
reader = asyncio.StreamReader(loop=self.loop, limit=1)
protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
transport, _ = self.loop.run_until_complete(
self.loop.connect_read_pipe(lambda: protocol, pipe))
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
try:
asyncio.set_child_watcher(watcher)
proc = self.loop.run_until_complete(
asyncio.create_subprocess_exec(*args, pass_fds={wfd}, loop=self.loop))
self.loop.run_until_complete(proc.wait())
finally:
asyncio.set_child_watcher(None)
os.close(wfd)
data = self.loop.run_until_complete(reader.read(-1))
self.assertEqual(data, b'data')
finally:
# FIXME: Debug code for issue #21645
logger.removeHandler(log_handler)
logger.setLevel(log_level)
def test_read_all_from_pipe_reader(self):
# See asyncio issue 168. This test is derived from the example
# subprocess_attach_read_pipe.py, but we configure the
# StreamReader's limit so that twice it is less than the size
# of the data writter. Also we must explicitly attach a child
# watcher to the event loop.
code = """\
import os, sys
fd = int(sys.argv[1])
os.write(fd, b'data')
os.close(fd)
"""
rfd, wfd = os.pipe()
args = [sys.executable, '-c', code, str(wfd)]
pipe = open(rfd, 'rb', 0)
reader = asyncio.StreamReader(loop=self.loop,
limit=1,
_asyncio_internal=True)
protocol = asyncio.StreamReaderProtocol(reader,
loop=self.loop,
_asyncio_internal=True)
transport, _ = self.loop.run_until_complete(
self.loop.connect_read_pipe(lambda: protocol, pipe))
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
try:
asyncio.set_child_watcher(watcher)
create = asyncio.create_subprocess_exec(*args,
pass_fds={wfd},
loop=self.loop)
proc = self.loop.run_until_complete(create)
self.loop.run_until_complete(proc.wait())
finally:
asyncio.set_child_watcher(None)
os.close(wfd)
data = self.loop.run_until_complete(reader.read(-1))
self.assertEqual(data, b'data')
def get_event_loop():
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# On Windows the default SelectorEventLoop is not available:
# https://docs.python.org/3.5/library/asyncio-subprocess.html#windows-event-loop
if sys.platform == 'win32' and \
not isinstance(loop, asyncio.ProactorEventLoop):
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
# Avoid spammy BlockingIOError warnings with older python versions
if sys.platform != 'win32' and \
sys.version_info < (3, 8, 0):
asyncio.set_child_watcher(asyncio.FastChildWatcher())
asyncio.get_child_watcher().attach_loop(loop)
return loop
def main(cls, loop=None, argv=sys.argv):
"""
Runs cli commands in asyncio loop and outputs in appropriate format
"""
if loop is None:
loop = asyncio.get_event_loop()
# In Python 3.8 ThreadedChildWatcher becomes the default which
# should work fine for us. However, in Python 3.7 SafeChildWatcher
# is the default and may cause BlockingIOErrors when many
# subprocesses are created
# https://docs.python.org/3/library/asyncio-policy.html#asyncio.FastChildWatcher
if sys.version_info.major == 3 and sys.version_info.minor == 7:
watcher = asyncio.FastChildWatcher()
asyncio.set_child_watcher(watcher)
watcher.attach_loop(loop)
result = None
try:
result = loop.run_until_complete(cls._main(*argv[1:]))
except KeyboardInterrupt: # pragma: no cover
pass # pragma: no cover
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def setUp(self):
super().setUp()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
asyncio.set_child_watcher(watcher)
from pathlib import Path
from nonocaptcha import util
from nonocaptcha.proxy import ProxyDB
from nonocaptcha.solver import Solver
SECRET_KEY = "CHANGEME"
proxies = ProxyDB(last_banned_timeout=45 * 60) # This is 45 minutes
proxy_source = None # Can be URL or file location
proxy_username, proxy_password = (None, None)
parent_loop = asyncio.get_event_loop()
# I'm not sure exactly if FastChildWatcher() is really any faster, requires
# further research.
asyncio.set_child_watcher(asyncio.FastChildWatcher())
asyncio.get_child_watcher().attach_loop(parent_loop)
app = web.Application()
# Clear Chrome temporary profiles
dir = f"{Path.home()}/.pyppeteer/.dev_profile"
shutil.rmtree(dir, ignore_errors=True)
# Should be less crash prone since we use the main loop, only spawning the
# task in a future within an executor. Maybe.
class TaskRerun(object):
def __init__(self, coro, duration):
self.coro = coro
self.duration = duration
def update_event(self, inp=-1):
self.set_output_val(0, asyncio.set_child_watcher(self.input(0)))
"\033[0m", # End of color
"\033[36m", # Cyan
"\033[91m", # Red
"\033[35m", # Magenta
)
async def makerandom(idx: int, threshold: int = 6) -> int:
print(c[idx + 1] + f"Initiated makerandom({idx}).")
i = random.randint(0, 10)
while i <= threshold:
print(c[idx + 1] + f"makerandom({idx}) == {i} too low; retrying.")
await asyncio.sleep(idx + 1)
i = random.randint(0, 10)
print(c[idx + 1] + f"---> Finished: makerandom({idx}) == {i}" + c[0])
return i
async def main():
res = await asyncio.gather(*(makerandom(i, 10 - i - 1) for i in range(3)))
return res
if __name__ == "__main__":
random.seed(444)
watcher = asyncio.PidfdChildWatcher()
asyncio.set_child_watcher(watcher)
r1, r2, r3 = asyncio.run(main())
print()
print(f"r1: {r1}, r2: {r2}, r3: {r3}")
def attach_loop(loop):
watcher = asyncio.SafeChildWatcher()
# FIXME: remove
watcher.attach_loop(loop)
asyncio.set_child_watcher(watcher)
def run(self, queues, casd_process_manager):
# Hold on to the queues to process
self.queues = queues
# NOTE: Enforce use of `SafeChildWatcher` as we generally don't want
# background threads.
# In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and
# not `SafeChildWatcher`.
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
# Ensure that we have a fresh new event loop, in case we want
# to run another test in this thread.
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
# Add timeouts
self.loop.call_later(1, self._tick)
# Add exception handler
self.loop.set_exception_handler(self._handle_exception)
# Handle unix signals while running
self._connect_signals()
# Watch casd while running to ensure it doesn't die
self._casd_process = casd_process_manager.process
_watcher = asyncio.get_child_watcher()
def abort_casd(pid, returncode):
asyncio.get_event_loop().call_soon(self._abort_on_casd_failure,
pid, returncode)
_watcher.add_child_handler(self._casd_process.pid, abort_casd)
# Start the profiler
with PROFILER.profile(
Topics.SCHEDULER,
"_".join(queue.action_name for queue in self.queues)):
# This is not a no-op. Since it is the first signal registration
# that is set, it allows then other threads to register signal
# handling routines, which would not be possible if the main thread
# hadn't set it before.
# FIXME: this should be done in a cleaner way
with _signals.suspendable(
lambda: None,
lambda: None), _signals.terminator(lambda: None):
with ThreadPoolExecutor(max_workers=sum(
self.resources._max_resources.values())) as pool:
self.loop.set_default_executor(pool)
# Run the queues
self._sched()
self.loop.run_forever()
self.loop.close()
# Invoke the ticker callback a final time to render pending messages
self._ticker_callback()
# Stop watching casd
_watcher.remove_child_handler(self._casd_process.pid)
self._casd_process = None
# Stop handling unix signals
self._disconnect_signals()
failed = any(queue.any_failed_elements() for queue in self.queues)
self.loop = None
if failed:
status = SchedStatus.ERROR
elif self.terminated:
status = SchedStatus.TERMINATED
else:
status = SchedStatus.SUCCESS
return status
def tearDown(self):
asyncio.set_child_watcher(None)
super().tearDown()
def setUp(self):
super().setUp()
watcher = asyncio.SafeChildWatcher()
watcher.attach_loop(self.loop)
asyncio.set_child_watcher(watcher)
SECRET_KEY = "CHANGEME"
BANNED_TIMEOUT = 45 * 60 # 45 minutes
SOLVE_DURATION = 3 * 60 # 3 minutes
proxies = ProxyDB(last_banned_timeout=BANNED_TIMEOUT)
proxy_source = None # Can be URL or file location
proxy_username, proxy_password = (None, None)
if sys.platform == "win32":
parent_loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(parent_loop)
else:
parent_loop = asyncio.get_event_loop()
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
asyncio.get_child_watcher().attach_loop(parent_loop)
app = web.Application()
# Clear Chrome temporary profiles
dir = f"{Path.home()}/.pyppeteer/.dev_profile"
shutil.rmtree(dir, ignore_errors=True)
# Bugs are to be expected, despite my efforts. Apparently, event loops paired
# with threads is nothing short of a hassle.
class TaskRerun(object):
def __init__(self, coro, duration):
self.coro = coro
self.duration = duration
def tearDown(self):
asyncio.set_child_watcher(None)
super().tearDown()
def run(self, queues, casd_process_manager):
# Hold on to the queues to process
self.queues = queues
# NOTE: Enforce use of `SafeChildWatcher` as we generally don't want
# background threads.
# In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and
# not `SafeChildWatcher`.
asyncio.set_child_watcher(asyncio.SafeChildWatcher())
# Ensure that we have a fresh new event loop, in case we want
# to run another test in this thread.
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
# Add timeouts
self.loop.call_later(1, self._tick)
# Add exception handler
self.loop.set_exception_handler(self._handle_exception)
# Handle unix signals while running
self._connect_signals()
# Watch casd while running to ensure it doesn't die
self._casd_process = casd_process_manager.process
_watcher = asyncio.get_child_watcher()
def abort_casd(pid, returncode):
asyncio.get_event_loop().call_soon(self._abort_on_casd_failure,
pid, returncode)
_watcher.add_child_handler(self._casd_process.pid, abort_casd)
# Start the profiler
with PROFILER.profile(
Topics.SCHEDULER,
"_".join(queue.action_name for queue in self.queues)):
# Run the queues
self._sched()
self.loop.run_forever()
self.loop.close()
# Stop watching casd
_watcher.remove_child_handler(self._casd_process.pid)
self._casd_process = None
# Stop handling unix signals
self._disconnect_signals()
failed = any(queue.any_failed_elements() for queue in self.queues)
self.loop = None
if failed:
status = SchedStatus.ERROR
elif self.terminated:
status = SchedStatus.TERMINATED
else:
status = SchedStatus.SUCCESS
return status
# Python 3.6 has no WindowsProactorEventLoopPolicy class
from asyncio import events
class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy
):
_loop_factory = asyncio.ProactorEventLoop
else:
WindowsProactorEventLoopPolicy = asyncio.WindowsProactorEventLoopPolicy
asyncio.set_event_loop_policy(WindowsProactorEventLoopPolicy())
else:
if sys.version_info < (3, 8):
from .asyncio_utils import ThreadedChildWatcher
asyncio.set_child_watcher(ThreadedChildWatcher())
log = logging.getLogger(__name__)
def setup_logging(verbosity: int, color: bool) -> None:
root_logger = logging.getLogger()
handler = ConsoleHandler()
root_logger.addHandler(handler)
root_logger.setLevel(logging.DEBUG)
if color:
format_class: Type[logging.Formatter] = ConsoleWarningFormatter
else:
format_class = logging.Formatter