async def test_create_app_server_port_conflict(
self, client: ClientSession) -> None:
code = AuthCode()
app = create_auth_code_app(code)
outer_port = unused_port()
inner_port = unused_port()
async with create_app_server(app, ports=[outer_port,
inner_port]) as url:
assert url == URL(f"http://127.0.0.1:{outer_port}")
async with create_app_server(app, ports=[outer_port,
inner_port]) as url:
assert url == URL(f"http://127.0.0.1:{inner_port}")
await self.assert_code_callback_success(code, client, url)
def test_tcp_connector_uses_provided_local_addr(self):
@asyncio.coroutine
def handler(request):
return web.HTTPOk()
app, srv, url = self.loop.run_until_complete(
self.create_server('get', '/', handler)
)
port = unused_port()
conn = aiohttp.TCPConnector(loop=self.loop,
local_addr=('127.0.0.1', port))
session = aiohttp.ClientSession(connector=conn)
r = self.loop.run_until_complete(
session.request('get', url)
)
r.release()
first_conn = next(iter(conn._conns.values()))[0][0]
self.assertEqual(
first_conn.transport._sock.getsockname(), ('127.0.0.1', port))
r.close()
session.close()
conn.close()
async def test_probe_stop_no_session(loop):
"""Assert the probe stops without an exception being thrown."""
probe = Probes(loop=loop, port=unused_port())
await probe.stop()
# no exception was thrown
assert True
async def start(self):
port = unused_port()
self.handler = self.app.make_handler()
self.server = await self.loop.create_server(self.handler,
'127.0.0.1', port,
ssl=self.ssl_context)
return {'graph.facebook.com': port}
def test_runs_webserver(self):
module = InterfaceReader()
data = helpers.create_data(self.stream.layout, length=10)
port = unused_port()
args = argparse.Namespace(pipes="unset",
module_config="unset",
socket="unset",
port=port,
host="127.0.0.1",
url='http://localhost:8080',
node="",
api_socket="",
mock_data=data)
def get_page():
time.sleep(0.5)
resp = requests.get('http://localhost:%d' % port)
self.assertEqual(resp.content.decode('utf8'), 'Hello World')
getter = threading.Thread(target=get_page)
getter.start()
f = io.StringIO()
with redirect_stdout(f):
module.start(args)
getter.join()
async def start_server(self) -> None:
"""Start the web server asynchronously."""
app = web.Application()
app.add_routes(
[web.get("/", self.handle_home), web.get("/ws", self.handle_websocket)]
)
port = self._args.port or unused_port()
host = self._args.host
self.html = load_site()
self._runner = web.AppRunner(app)
if not self._runner:
raise AustinWebError("Cannot create web app runner.")
await self._runner.setup()
site = web.TCPSite(self._runner, host, port)
await site.start()
print(Figlet(font="speed", width=240).renderText("* Austin Web *"))
print(
f"⏲️ Sampling process with PID {self.get_process().pid} "
f"({self.get_command_line()})"
)
print(f"🏃 Austin Web is running on http://{host}:{port}. Press Ctrl+C to stop.")
def test_server():
app = web.Application()
app.add_routes(router)
port = test_utils.unused_port()
print(f'listen on {port}')
def run_server(loop: asyncio.AbstractEventLoop):
# https://stackoverflow.com/questions/51610074/how-to-run-an-aiohttp-server-in-a-thread
runner = web.AppRunner(app)
loop.run_until_complete(runner.setup())
site = web.TCPSite(runner, '127.0.0.1', port)
loop.run_until_complete(site.start())
loop.run_forever()
loop = asyncio.new_event_loop()
srv_thread = threading.Thread(target=run_server, args=(loop, ))
srv_thread.start()
yield {'port': port}
# https://stackoverflow.com/questions/46093238/python-asyncio-event-loop-does-not-seem-to-stop-when-stop-method-is-called
loop.call_soon_threadsafe(loop.stop)
def test_tcp_connector_uses_provided_local_addr(self):
@asyncio.coroutine
def handler(request):
return web.HTTPOk()
app, srv, url = self.loop.run_until_complete(
self.create_server('get', '/', handler)
)
port = unused_port()
conn = aiohttp.TCPConnector(loop=self.loop,
local_addr=('127.0.0.1', port))
r = self.loop.run_until_complete(
aiohttp.request(
'get', url,
connector=conn
))
self.loop.run_until_complete(r.release())
first_conn = next(iter(conn._conns.values()))[0][0]
self.assertEqual(first_conn._sock.getsockname(), ('127.0.0.1', port))
r.close()
conn.close()
def test_tcp_connector_do_not_raise_connector_ssl_error(self):
async def handler(request):
return web.HTTPOk()
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
keyfile = os.path.join(here, 'sample.key')
certfile = os.path.join(here, 'sample.crt')
sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
sslcontext.load_cert_chain(certfile, keyfile)
app, srv, url = self.loop.run_until_complete(
self.create_server('get', '/', handler, ssl_context=sslcontext))
port = unused_port()
conn = aiohttp.TCPConnector(loop=self.loop,
local_addr=('127.0.0.1', port))
session = aiohttp.ClientSession(connector=conn)
r = self.loop.run_until_complete(
session.request('get', url, ssl_context=sslcontext))
r.release()
first_conn = next(iter(conn._conns.values()))[0][0]
try:
_sslcontext = first_conn.transport._ssl_protocol._sslcontext
except AttributeError:
_sslcontext = first_conn.transport._sslcontext
self.assertIs(_sslcontext, sslcontext)
r.close()
session.close()
conn.close()
async def test_scan_port_open():
port = unused_port()
loop = asyncio.get_event_loop()
webserver = RawTestServer(lambda request: web.Response(), host='127.1', port=port)
await webserver.start_server(loop=loop)
result = await scan_port(loop, '127.1', port)
assert result == '{"port": %s, "state": "open"}' % port
def go():
server = yield from self.loop.create_server(
Proto, '127.0.0.1', unused_port())
addr = server.sockets[0].getsockname()
connector = aiohttp.TCPConnector(loop=self.loop, capacity=1)
url = 'http://{}:{}/'.format(*addr)
r = yield from client.request('GET',
url,
connector=connector,
loop=self.loop)
yield from r.read()
self.assertEqual(1, len(connector._conns))
with self.assertRaises(aiohttp.ClientError):
yield from client.request('GET',
url,
connector=connector,
loop=self.loop)
self.assertEqual(0, len(connector._conns))
connector.close()
server.close()
yield from server.wait_closed()
def create_server(self,
method,
path,
handler=None,
ssl_ctx=None,
logger=log.server_logger,
handler_kwargs=None):
app = web.Application(loop=self.loop)
if handler:
app.router.add_route(method, path, handler)
port = unused_port()
self.handler = app.make_handler(keep_alive_on=False,
access_log=log.access_logger,
logger=logger,
**(handler_kwargs or {}))
srv = yield from self.loop.create_server(self.handler,
'127.0.0.1',
port,
ssl=ssl_ctx)
protocol = "https" if ssl_ctx else "http"
url = "{}://127.0.0.1:{}".format(protocol, port) + path
self.addCleanup(srv.close)
return app, srv, url
app.router.add_static = self.patch_sendfile(app.router.add_static)
return app, srv, url
async def start(self):
port = unused_port()
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, "127.0.0.1", port)
await site.start()
return {"localhost": port}
def test_tcp_connector_raise_connector_ssl_error(self):
@asyncio.coroutine
def handler(request):
return web.HTTPOk()
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
keyfile = os.path.join(here, 'sample.key')
certfile = os.path.join(here, 'sample.crt')
sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
sslcontext.load_cert_chain(certfile, keyfile)
app, srv, url = self.loop.run_until_complete(
self.create_server('get', '/', handler, ssl_context=sslcontext))
port = unused_port()
conn = aiohttp.TCPConnector(loop=self.loop,
local_addr=('127.0.0.1', port))
session = aiohttp.ClientSession(connector=conn)
with pytest.raises(aiohttp.ClientConnectorSSLError) as ctx:
self.loop.run_until_complete(session.request('get', url))
self.assertIsInstance(ctx.value.os_error, ssl.SSLError)
self.assertTrue(ctx.value, aiohttp.ClientSSLError)
session.close()
conn.close()
async def test_port_forward(nmrc_path: Path, nginx_job_async: Tuple[str, str]) -> None:
loop_sleep = 1
service_wait_time = 10 * 60
async def get_(url: str) -> int:
status = 999
start_time = time()
async with aiohttp.ClientSession() as session:
while status != 200 and (int(time() - start_time) < service_wait_time):
try:
async with session.get(url) as resp:
status = resp.status
text = await resp.text()
assert text == nginx_job_async[1], (
f"Secret not found "
f"via {url}. Like as it's not our test server."
)
except aiohttp.ClientConnectionError:
status = 599
if status != 200:
await asyncio.sleep(loop_sleep)
return status
async with api_get(path=nmrc_path, timeout=CLIENT_TIMEOUT) as client:
port = unused_port()
# We test client instead of run_cli as asyncio subprocesses do
# not work if run from thread other than main.
async with client.jobs.port_forward(
nginx_job_async[0], port, 80, no_key_check=True
):
await asyncio.sleep(loop_sleep)
url = f"http://127.0.0.1:{port}/secret.txt"
probe = await get_(url)
assert probe == 200
def get_admin_server(self,
settings: dict = None,
context: InjectionContext = None) -> AdminServer:
if not context:
context = InjectionContext()
if settings:
context.update_settings(settings)
# middleware is task queue xor collector: cover both over test suite
task_queue = (settings or {}).pop("task_queue", None)
plugin_registry = async_mock.MagicMock(test_module.PluginRegistry,
autospec=True)
plugin_registry.post_process_routes = async_mock.MagicMock()
context.injector.bind_instance(test_module.PluginRegistry,
plugin_registry)
collector = Collector()
context.injector.bind_instance(test_module.Collector, collector)
profile = InMemoryProfile.test_profile()
self.port = unused_port()
return AdminServer(
"0.0.0.0",
self.port,
context,
profile,
self.outbound_message_router,
self.webhook_router,
conductor_stop=async_mock.CoroutineMock(),
task_queue=TaskQueue(max_active=4) if task_queue else None,
conductor_stats=(None if task_queue else async_mock.CoroutineMock(
return_value={"a": 1})),
)
async def start(self) -> Dict[str, int]:
port = unused_port()
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, "127.0.0.1", port, ssl_context=self.ssl_context)
await site.start()
return {"pypi.org": port}
async def test_run_with_port_forward(helper: Helper) -> None:
port = unused_port()
job_id = None
secret = uuid4()
command = (
f"bash -c \"echo -n '{secret}' > /usr/share/nginx/html/secret.txt; "
f"timeout 15m /usr/sbin/nginx -g 'daemon off;'\""
)
proc = await helper.acli(
["run", "--port-forward", f"{port}:80", "nginx:latest", command]
)
try:
await asyncio.sleep(1)
url = f"http://127.0.0.1:{port}/secret.txt"
probe = await fetch_http(url, str(secret))
assert probe == 200
assert proc.returncode is None
assert proc.stdout is not None
out = await proc.stdout.read(64 * 1024)
job_id = helper.find_job_id(out.decode("utf-8", "replace"))
assert job_id is not None
finally:
proc.terminate()
await proc.wait()
if job_id is not None:
await helper.akill_job(job_id)
async def start(self):
port = unused_port()
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, '127.0.0.1', port)
await site.start()
return {FAKE_BASE_URL: port}
async def invest_api_server(
loop: asyncio.AbstractEventLoop,
invest_api_session: orm.Session,
) -> AsyncIterator:
assert loop.is_running()
db_url = str(invest_api_session.bind.url)
config = {
"db": {
"pool": {
"dsn": db_url,
"min_size": 1,
"max_size": 5,
},
"logger": {
"name": "db",
},
},
}
host = LOCALHOST
port = unused_port()
async with invest_api.create_tcp_server(host, port, config) as url:
yield url
async def start(self):
port = unused_port()
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, "127.0.0.1", port, ssl_context=self.ssl_context)
await site.start()
return {"graph.facebook.com": port}
def get_admin_server(self) -> AdminServer:
context = InjectionContext()
context.injector.bind_provider(
BaseOutboundMessageQueue, ClassProvider(BasicOutboundMessageQueue))
server = AdminServer("localhost", unused_port(), context,
self.outbound_message_router)
return server
async def start(self):
port = unused_port()
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, '127.0.0.1', port)
await site.start()
return {'smog1.ampio.pl': port}
def start_server(self, server):
key = master.make_key()
self.proc = psutil.Process()
port = unused_port()
self.msgs = multiprocessing.Queue()
self.outbound_msgs = multiprocessing.Queue()
self.server_proc = multiprocessing.Process(target=server.start,
args=(port, self.msgs,
self.outbound_msgs,
key))
self.server_proc.start()
ready = False
time.sleep(0.01)
self.conf_dir = tempfile.TemporaryDirectory()
while not ready:
for conn in psutil.net_connections():
if conn.laddr.port == port:
ready = True
break
else:
time.sleep(0.1)
# create a .joule config directory with key info
with open(os.path.join(self.conf_dir.name, "nodes.json"), 'w') as f:
f.write(
json.dumps([{
"name": "fake_joule",
"key": key,
"url": "http://127.0.0.1:%d" % port
}]))
with open(os.path.join(self.conf_dir.name, "default_node.txt"),
'w') as f:
f.write("fake_joule")
os.environ["JOULE_USER_CONFIG_DIR"] = self.conf_dir.name
def go():
server = yield from self.loop.create_server(
Proto, '127.0.0.1', unused_port())
addr = server.sockets[0].getsockname()
connector = aiohttp.TCPConnector(loop=self.loop)
url = 'http://{}:{}/'.format(*addr)
r = yield from client.request('GET', url,
connector=connector,
loop=self.loop)
yield from r.read()
self.assertEqual(1, len(connector._conns))
with self.assertRaises(aiohttp.ClientError):
yield from client.request('GET', url,
connector=connector,
loop=self.loop)
self.assertEqual(0, len(connector._conns))
connector.close()
server.close()
yield from server.wait_closed()
def get_admin_server(self) -> AdminServer:
context = InjectionContext()
context.injector.bind_provider(
BaseOutboundMessageQueue, ClassProvider(BasicOutboundMessageQueue))
context.settings["admin.admin_insecure_mode"] = True
server = AdminServer("0.0.0.0", unused_port(), context,
self.outbound_message_router)
return server
def get_admin_server(self) -> AdminServer:
context = InjectionContext()
context.injector.bind_provider(
BaseOutboundMessageQueue, ClassProvider(BasicOutboundMessageQueue))
context.settings["admin.admin_api_key"] = self.TEST_API_KEY
self.server = AdminServer("0.0.0.0", unused_port(), context,
self.outbound_message_router)
return self.server
async def start(self):
port = unused_port()
self.handler = self.app.make_handler()
self.server = await self.loop.create_server(self.handler,
'127.0.0.1',
port,
ssl=self.ssl_context)
return {'graph.facebook.com': port}
async def start(self):
port = unused_port()
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, '127.0.0.1', port,
ssl_context=self.ssl_context)
await site.start()
return {'graph.facebook.com': port}
def get_admin_server(self, settings: dict = None) -> AdminServer:
context = InjectionContext()
context.injector.bind_provider(
BaseOutboundMessageQueue, ClassProvider(BasicOutboundMessageQueue))
if settings:
context.update_settings(settings)
return AdminServer("0.0.0.0", unused_port(), context,
self.outbound_message_router)
def setUp(self):
self.message_results = []
self.port = unused_port()
self.session = None
self.transport = WsTransport("0.0.0.0", self.port, self.create_session)
self.transport.wire_format = JsonWireFormat()
self.result_event = None
super().setUp()
async def test_probe_stop_with_session(loop):
"""Assert that call to probe stop doesn't throw an exception."""
probe = Probes(loop=loop, port=unused_port())
info = await probe.start()
print(info)
await probe.stop()
assert True
async def test_create_app_server_no_ports(self) -> None:
code = AuthCode()
app = create_auth_code_app(code)
port = unused_port()
async with create_app_server_once(app, host="127.0.0.1", port=port):
with pytest.raises(RuntimeError, match="No free ports."):
async with create_app_server(app, ports=[port]):
pass
async def test_create_app_server(self, client: ClientSession) -> None:
code = AuthCode()
app = create_auth_code_app(code)
port = unused_port()
async with create_app_server(app, host="127.0.0.1",
ports=[port]) as url:
assert url == URL(f"http://127.0.0.1:{port}")
await self.assert_code_callback_success(code, client, url)
def create_server(self, method, path, handler):
app = web.Application(loop=self.loop)
app.router.add_route(method, path, handler)
port = unused_port()
srv = yield from self.loop.create_server(
app.make_handler(), '127.0.0.1', port)
url = "http://127.0.0.1:{}".format(port) + path
self.addCleanup(srv.close)
return app, srv, url
def create(*, debug=False, ssl_ctx=None, proto='http'):
nonlocal app, handler, srv
app = web.Application(loop=loop)
port = unused_port()
handler = app.make_handler(debug=debug, keep_alive_on=False)
srv = yield from loop.create_server(handler, '127.0.0.1', port,
ssl=ssl_ctx)
if ssl_ctx:
proto += 's'
url = "{}://127.0.0.1:{}".format(proto, port)
return app, url
def __init__(self, handler, loop, protocol="http"):
self._loop = loop
self.port = unused_port()
self._handler = handler
self._server = loop.run_until_complete(loop.create_server(
lambda: handler, '127.0.0.1', self.port
))
self._session = ClientSession(loop=self._loop)
self._root = "{}://127.0.0.1:{}".format(
protocol, self.port
)
self._closed = False
def test_resolver_not_called_with_address_is_ip(self):
resolver = unittest.mock.MagicMock()
connector = aiohttp.TCPConnector(resolver=resolver, loop=self.loop)
req = ClientRequest('GET', 'http://127.0.0.1:{}'.format(unused_port()),
loop=self.loop,
response_class=unittest.mock.Mock())
with self.assertRaises(OSError):
self.loop.run_until_complete(connector.connect(req))
resolver.resolve.assert_not_called()
def go():
server = yield from self.loop.create_server(Proto, "127.0.0.1", unused_port())
addr = server.sockets[0].getsockname()
connector = aiohttp.TCPConnector(loop=self.loop)
url = "http://{}:{}/".format(*addr)
for i in range(2):
r = yield from client.request("GET", url, connector=connector, loop=self.loop)
yield from r.read()
self.assertEqual(0, len(connector._conns))
connector.close()
server.close()
yield from server.wait_closed()
def go():
server = yield from self.loop.create_server(
Proto, '127.0.0.1', unused_port())
addr = server.sockets[0].getsockname()
connector = aiohttp.TCPConnector(loop=self.loop, limit=1)
session = client.ClientSession(loop=self.loop, connector=connector)
url = 'http://{}:{}/'.format(*addr)
for i in range(2):
r = yield from session.request('GET', url)
yield from r.read()
self.assertEqual(0, len(connector._conns))
session.close()
connector.close()
server.close()
yield from server.wait_closed()
def create_server(self, method, path, handler=None, ssl_ctx=None,
logger=log.server_logger, handler_kwargs=None):
app = web.Application(
loop=self.loop)
if handler:
app.router.add_route(method, path, handler)
port = unused_port()
self.handler = app.make_handler(
keep_alive_on=False,
access_log=log.access_logger,
logger=logger,
**(handler_kwargs or {}))
srv = yield from self.loop.create_server(
self.handler, '127.0.0.1', port, ssl=ssl_ctx)
protocol = "https" if ssl_ctx else "http"
url = "{}://127.0.0.1:{}".format(protocol, port) + path
self.addCleanup(srv.close)
return app, srv, url
def __init__(self, loop):
self._loop = loop
self._transports = []
self._futures = []
self._srv = None
self.port = unused_port()
def __init__(self, loop, write_buff):
self._loop = loop
self._write_buff = write_buff
self._transports = []
self._srv = None
self.port = unused_port()
