def main(argv):
if len(argv) < 3:
print('usage: %s {client|server} url ...' % argv[0], file=sys.stderr)
return 1
if argv[1] == 'client' and len(argv) < 4:
print('usage: %s client url request' % argv[0], file=sys.stderr)
return 1
async def serve():
with nng.asyncs.Socket(nng.Protocols.REP0) as socket:
socket.listen(argv[2])
while True:
data = await socket.recv()
print('serve: recv: %r' % data)
await socket.send(data)
async def request():
with nng.asyncs.Socket(nng.Protocols.REQ0) as socket:
socket.dial(argv[2])
await socket.send(argv[3].encode('utf-8'))
print((await socket.recv()).decode('utf-8'))
target = request if argv[1] == 'client' else serve
kernels.run(target)
return 0
def test_run_application_sendfile(self):
mock_file = unittest.mock.Mock()
mock_app = unittest.mock.AsyncMock()
mock_app.return_value = wsgi.FileWrapper(mock_file)
session = wsgi.HttpSession(None, mock_app, {})
context = wsgi._ApplicationContext()
run_task = tasks.spawn(session._run_application(context, {}))
get_task = tasks.spawn(self.get_body_chunks(context))
kernels.run(timeout=0.01)
self.assertTrue(context._chunks.is_closed())
self.assertTrue(run_task.is_completed())
run_task.get_result_nonblocking()
self.assertTrue(get_task.is_completed())
self.assertEqual(get_task.get_result_nonblocking(), [])
self.assertIs(context.file, mock_file)
# `close` is not called because ownership is transferred to
# context.
mock_file.close.assert_not_called()
def test_put_short_response_not_keep_alive(self):
session = wsgi.HttpSession(self.mock_sock, None, {})
kernels.run(
session._put_short_response(http.HTTPStatus.OK, False),
timeout=0.01,
)
self.assert_send(b'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n')
def test_shutdown(self):
async def catch(caughts):
try:
await timers.sleep(None)
except BaseException as exc:
caughts.append(exc)
raise
async def shutdown(bg):
await timers.sleep(0.01)
bg.shutdown()
caughts = []
bg = g1.backgrounds.tasks.BackgroundTasks()
catch_tasks = [bg.queue.spawn(catch(caughts)) for _ in range(3)]
shutdown_tasks = [tasks.spawn(shutdown(bg)) for _ in range(10)]
supervisor_task = tasks.spawn(bg.supervise())
with self.assertLogs(g1.backgrounds.tasks.__name__,
level='DEBUG') as cm:
kernels.run(timeout=0.1)
self.assert_completed(supervisor_task)
for catch_task in catch_tasks:
self.assert_cancelled(catch_task)
for shutdown_task in shutdown_tasks:
self.assert_completed(shutdown_task)
self.assertEqual(len(caughts), len(catch_tasks))
for caught in caughts:
self.assertIsInstance(caught,
g1.asyncs.kernels.errors.TaskCancellation)
self.assertEqual(len(cm.output), 1)
self.assertIn('cancel 3 background tasks on exit', cm.output[0])
def test_handle_request_100_continue(self):
session = wsgi.HttpSession(self.mock_sock, None, {})
for conn_header, keep_alive, expect_keep_alive in [
('Keep-Alive', True, True),
('Keep-Alive', False, True),
('close', True, False),
('close', False, False),
(None, True, True),
(None, False, False),
]:
with self.subTest((conn_header, keep_alive, expect_keep_alive)):
environ = {'HTTP_EXPECT': '100-Continue'}
if conn_header is not None:
environ['HTTP_CONNECTION'] = conn_header
if expect_keep_alive:
kernels.run(
session._handle_request(environ, keep_alive),
timeout=0.01,
)
self.assert_send(b'HTTP/1.1 100 Continue\r\n'
b'Connection: keep-alive\r\n'
b'\r\n')
else:
with self.assertRaises(wsgi._SessionExit):
kernels.run(
session._handle_request(environ, keep_alive),
timeout=0.01,
)
self.assert_send(b'HTTP/1.1 100 Continue\r\n'
b'Connection: close\r\n'
b'\r\n')
def run_handler(self, headers):
self.response = wsgi_apps._Response(None, False)
self.response.headers.update(headers)
kernels.run(
self.handler(None, wsgi_apps.Response(self.response)),
timeout=0.01,
)
def test_timeout_other(self):
t = tasks.spawn(timers.sleep(100))
timers.timeout_after(0, task=t)
kernels.run(timeout=1)
self.assertTrue(t.is_completed())
with self.assertRaises(timers.Timeout):
t.get_result_nonblocking()
def test_token_bucket_raise_when_empty(self):
self.monotonic_mock.return_value = 99
tb = policies.TokenBucket(1, 1, True)
with self.assertRaisesRegex(policies.Unavailable,
r'rate limit exceeded'):
kernels.run(tb)
self.assertEqual(tb._num_tokens, 0)
def test_cancel(self):
exc = BaseException()
async def handler_uncommitted(request, response):
del request, response
raise exc
async def handler_committed(request, response):
del request
response.commit()
raise exc
kernels.run(self.run_handler(handler_uncommitted), timeout=0.01)
self.start_response_mock.assert_called_once_with(
'503 Service Unavailable',
[('Retry-After', '60')],
)
self.start_response_mock.reset_mock()
kernels.run(self.run_handler(handler_committed), timeout=0.01)
self.start_response_mock.assert_has_calls([
unittest.mock.call('200 OK', []),
unittest.mock.call(
'503 Service Unavailable',
[('Retry-After', '60')],
(exc.__class__, exc, unittest.mock.ANY),
),
])
def run_handler(self, path):
self.set_request(PATH_INFO=path)
self.response = wsgi_apps._Response(None, False)
self.calls.clear()
kernels.run(
self.handler(self.request, wsgi_apps.Response(self.response)),
timeout=0.01,
)
def run_handler(self, method):
self.set_request(REQUEST_METHOD=method)
self.response = wsgi_apps._Response(None, False)
self.calls.clear()
kernels.run(
self.handler(self.request, wsgi_apps.Response(self.response)),
timeout=0.01,
)
def test_already_closed_output(self):
q1 = queues.Queue()
q_out = queues.Queue()
q_out.close()
multiplexer_task = tasks.spawn(more_queues.multiplex([q1], q_out))
kernels.run(timeout=0.01)
self.assertTrue(multiplexer_task.is_completed())
self.assertIsNone(multiplexer_task.get_result_nonblocking())
def main(_):
logging.basicConfig(level=logging.INFO)
start = time.perf_counter()
task = tasks.spawn(to_be_cancelled)
kernels.run(cancel_task(task))
elapsed = time.perf_counter() - start
print('total elapsed time: %.3f seconds' % elapsed)
return 0
def run_handler(self):
self.response = wsgi_apps._Response(unittest.mock.Mock(), True)
kernels.run(
self.handler(self.request, wsgi_apps.Response(self.response)),
timeout=0.01,
)
self.response.close()
if self.response.file is not None:
self.response.file.close()
def test_context_green(self):
breaker = self.make(2, 1, 1, 2)
self.assert_breaker(breaker, policies._States.GREEN, [], 0)
self.assertIs(kernels.run(breaker.__aenter__()), breaker)
self.assert_breaker(breaker, policies._States.GREEN, [], 1)
self.assertIsNone(kernels.run(breaker.__aexit__(None, None, None)))
self.assert_breaker(breaker, policies._States.GREEN, [], 0)
def test_error(self):
session = clients.Session(
executor=self.executor,
retry=policies.ExponentialBackoff(2, 0.001),
)
self.mock_session.get.side_effect = Exception('some error')
with self.assertRaisesRegex(Exception, r'some error'):
kernels.run(session.send(self.REQUEST))
self.mock_session.get.assert_called_once_with(self.URL)
def test_http_error_no_retry_4xx(self):
session = clients.Session(
executor=self.executor,
retry=policies.ExponentialBackoff(2, 0.001),
)
self.set_mock_response(404)
with self.assertRaises(requests.RequestException):
kernels.run(session.send(self.REQUEST))
self.mock_session.get.assert_called_once_with(self.URL)
def main(
queue: g1.backgrounds.tasks.LABELS.queue,
supervise_agents: g1.asyncs.agents.parts.LABELS.supervise_agents,
graceful_exit: g1.asyncs.agents.parts.LABELS.graceful_exit,
):
queue.spawn(sleep_forever)
queue.spawn(raises)
queue.spawn(do_exit(graceful_exit))
kernels.run(supervise_agents)
return 0
def test_sticky_key(self):
session = clients.Session(executor=self.executor)
self.set_mock_response(200)
response = kernels.run(session.send(self.REQUEST, sticky_key='y'))
for _ in range(3):
self.assertIs(
kernels.run(session.send(self.REQUEST, sticky_key='y')),
response,
)
self.mock_session.get.assert_called_once_with(self.URL)
def test_close_not_graceful(self):
queue = queues.Queue()
self.assertFalse(queue.is_closed())
for x in (42, 43, 44):
kernels.run(queue.put(x))
self.assertEqual(queue.close(False), [42, 43, 44])
self.assertTrue(queue.is_closed())
self.assertFalse(queue)
def do_test(s0):
m0 = nng.Message(b'hello world')
t0 = tasks.spawn(s0.sendmsg(m0))
t0.cancel()
kernels.run(timeout=1)
self.assertTrue(t0.is_completed())
self.assertIsInstance(t0.get_exception_nonblocking(),
tasks.Cancelled)
# Ownership was not transferred.
self.assertIsNotNone(m0._msg_p)
def main(argv):
if len(argv) < 2:
print('usage: %s duration' % argv[0], file=sys.stderr)
return 1
duration = float(argv[1])
print('expect to time out after %.3f seconds' % duration)
start = time.perf_counter()
kernels.run(do_timeout_after(duration))
actual_duration = time.perf_counter() - start
print('actually time out after %.3f seconds' % actual_duration)
return 0
def do_test(s0, s1):
m0 = nng.Message(b'hello world')
t0 = tasks.spawn(s0.sendmsg(m0))
t1 = tasks.spawn(s1.recv())
kernels.run(timeout=1)
self.assertTrue(t0.is_completed())
self.assertTrue(t1.is_completed())
self.assertEqual(t1.get_result_nonblocking(), b'hello world')
# Ownership is transferred on success.
self.assertIsNone(m0._msg_p)
def test_context_red(self, mock_time):
mock_monotonic = mock_time.monotonic
breaker = self.make(2, 1, 1, 2)
mock_monotonic.side_effect = [100]
breaker._change_state_red(99)
self.assert_breaker(breaker, policies._States.RED, [99], 0)
with self.assertRaisesRegex(policies.Unavailable,
r'circuit breaker disconnected: test'):
kernels.run(breaker.__aenter__())
self.assert_breaker(breaker, policies._States.RED, [99], 0)
def test_router(self):
self.handler = composers.PathPatternRouter([
(r'/a/p', self.make_noop_handler('/a/p')),
(r'/a/q', self.make_noop_handler('/a/q')),
(r'/a', self.make_noop_handler('/a')),
])
self.set_request()
self.assertEqual(self.request.context.asdict(), {})
self.assertIsNone(composers.group(self.request))
self.assertIsNone(self.request.context.get(composers.PATH_MATCH))
self.assertEqual(composers.get_path_str(self.request), '/foo/bar')
self.run_handler('/a/p/x')
self.assertEqual(self.calls, ['/a/p'])
self.assert_context_keys([composers.PATH_MATCH])
self.assertEqual(composers.get_path_str(self.request), '/x')
self.assert_response(consts.Statuses.OK, {})
self.run_handler('/a/q')
self.assertEqual(self.calls, ['/a/q'])
self.assert_context_keys([composers.PATH_MATCH])
self.assertEqual(composers.get_path_str(self.request), '')
self.assert_response(consts.Statuses.OK, {})
self.run_handler('/a/q/')
self.assertEqual(self.calls, ['/a/q'])
self.assert_context_keys([composers.PATH_MATCH])
self.assertEqual(composers.get_path_str(self.request), '/')
self.assert_response(consts.Statuses.OK, {})
self.run_handler('/a/r/foo/bar')
self.assertEqual(self.calls, ['/a'])
self.assert_context_keys([composers.PATH_MATCH])
self.assertEqual(composers.get_path_str(self.request), '/r/foo/bar')
self.assert_response(consts.Statuses.OK, {})
with self.assertRaisesRegex(
wsgi_apps.HttpError,
r'path does not match any pattern: /foo/bar',
) as cm:
self.run_handler('/foo/bar')
self.assertEqual(self.calls, [])
self.assert_http_error(cm.exception, consts.Statuses.NOT_FOUND, {})
# You cannot override a PATH_MATCH entry in context.
self.run_handler('/a/p/x')
self.assertIn(composers.PATH_MATCH, self.request.context.asdict())
with self.assertRaisesRegex(AssertionError, r'expect.*not in'):
kernels.run(
self.handler(self.request, wsgi_apps.Response(self.response)),
timeout=0.01,
)
def test_closed_output(self):
q1 = queues.Queue()
q_out = queues.Queue()
multiplexer_task = tasks.spawn(more_queues.multiplex([q1], q_out))
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
q1.put_nonblocking('x')
q_out.close()
kernels.run(timeout=0.01)
self.assertEqual(get_many(q1), ['x'])
self.assertEqual(get_many(q_out), [])
self.assertTrue(multiplexer_task.is_completed())
self.assertIsNone(multiplexer_task.get_result_nonblocking())
def main(argv):
if len(argv) < 3:
print('usage: %s {client|server} url ...' % argv[0], file=sys.stderr)
return 1
if argv[1] == 'client' and len(argv) < 6:
print('usage: %s client url op x y' % argv[0], file=sys.stderr)
return 1
if argv[1] == 'server':
func = run_server
else:
func = run_client
kernels.run(func(*argv[2:]))
return 0
def test_rate_limiter(self, mock_monotonic):
mock_monotonic.return_value = 0
handler = requests.RateLimiter(
token_rate=2,
bucket_size=5,
get_bucket_key=lambda _: 'key',
)
kernels.run(handler(None, None))
self.assertEqual(handler._buckets['key']._num_tokens, 4)
mock_monotonic.return_value = 0.5
kernels.run(handler(None, None))
self.assertEqual(handler._buckets['key']._num_tokens, 4)
for expect in [3, 2, 1, 0]:
kernels.run(handler(None, None))
self.assertEqual(handler._buckets['key']._num_tokens, expect)
with self.assertRaisesRegex(
wsgi_apps.HttpError,
r'rate limit exceeded: bucket_key=\'key\' retry_after=2.5',
) as cm:
kernels.run(handler(None, None))
self.assertIs(cm.exception.status, consts.Statuses.TOO_MANY_REQUESTS)
self.assertEqual(cm.exception.headers,
{consts.HEADER_RETRY_AFTER: '3'})
def test_serve(self):
server = servers.Server(
TestApplication(),
Request,
Response,
WIRE_DATA,
invalid_request_error=InvalidRequestError(),
internal_server_error=InternalServerError(),
)
wire_request = WIRE_DATA.to_lower(
Request(args=Request.m.greet(name='world')))
self.assertEqual(
WIRE_DATA.to_upper(
Response,
kernels.run(server._serve(wire_request)),
),
Response(result=Response.Result(greet='Hello, world')),
)
with self.assertLogs(servers.__name__, level='DEBUG') as cm:
self.assertEqual(
kernels.run(server._serve(b'')),
server._invalid_request_error_wire,
)
self.assertRegex('\n'.join(cm.output), r'to_upper error: ')
wire_request = WIRE_DATA.to_lower(Request(args=Request.m.f()))
with self.assertLogs(servers.__name__, level='DEBUG') as cm:
self.assertEqual(
kernels.run(server._serve(wire_request)),
server._invalid_request_error_wire,
)
self.assertRegex('\n'.join(cm.output), r'unknown method: f: ')
wire_request = WIRE_DATA.to_lower(Request(args=Request.m.g()))
with self.assertLogs(servers.__name__, level='DEBUG') as cm:
self.assertEqual(
kernels.run(server._serve(wire_request)),
server._internal_server_error_wire,
)
self.assertRegex('\n'.join(cm.output), r'to_lower error: ')
wire_request = WIRE_DATA.to_lower(Request(args=Request.m.h()))
with self.assertLogs(servers.__name__, level='DEBUG') as cm:
self.assertEqual(
kernels.run(server._serve(wire_request)),
server._internal_server_error_wire,
)
self.assertRegex('\n'.join(cm.output), r'server error: ')
def main(
args: asyncs.LABELS.args,
stub: LABELS.stub.stub,
supervise_agents: g1.asyncs.agents.parts.LABELS.supervise_agents,
graceful_exit: g1.asyncs.agents.parts.LABELS.graceful_exit,
):
supervisor_task = tasks.spawn(supervise_agents)
request = clients.Request('GET', args.url)
response = kernels.run(stub.send(request, priority=0))
for link in response.html().xpath('//a'):
print(link.get('href'))
graceful_exit.set()
kernels.run(supervisor_task.get_result())
return 0
