def test_already_closed_output(self):
q1 = queues.Queue()
q_out = queues.Queue()
q_out.close()
multiplexer_task = tasks.spawn(more_queues.multiplex([q1], q_out))
kernels.run(timeout=0.01)
self.assertTrue(multiplexer_task.is_completed())
self.assertIsNone(multiplexer_task.get_result_nonblocking())
def test_closed_output(self):
q1 = queues.Queue()
q_out = queues.Queue()
multiplexer_task = tasks.spawn(more_queues.multiplex([q1], q_out))
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
q1.put_nonblocking('x')
q_out.close()
kernels.run(timeout=0.01)
self.assertEqual(get_many(q1), ['x'])
self.assertEqual(get_many(q_out), [])
self.assertTrue(multiplexer_task.is_completed())
self.assertIsNone(multiplexer_task.get_result_nonblocking())
def test_close(self):
queue = queues.Queue()
self.assertFalse(queue.is_closed())
for x in (42, 43, 44):
kernels.run(queue.put(x))
self.assertEqual(queue.close(), [])
self.assertTrue(queue.is_closed())
self.assertTrue(queue)
self.assertEqual(queue.close(), [])
with self.assertRaises(queues.Closed):
kernels.run(queue.put(45))
with self.assertRaises(queues.Closed):
queue.put_nonblocking(45)
actual = []
while queue:
actual.append(kernels.run(queue.get()))
self.assertEqual(actual, [42, 43, 44])
with self.assertRaises(queues.Closed):
kernels.run(queue.get())
with self.assertRaises(queues.Closed):
queue.get_nonblocking()
def test_capacity(self):
queue = queues.Queue(3)
self.assertFalse(queue.is_closed())
for x in (42, 43, 44):
self.assertFalse(queue.is_full())
kernels.run(queue.put(x))
self.assertTrue(queue.is_full())
with self.assertRaises(queues.Full):
queue.put_nonblocking(45)
self.assertTrue(queue.is_full())
with self.assertRaises(kernels.KernelTimeout):
kernels.run(queue.put(45), timeout=0)
self.assertTrue(queue.is_full())
self.assertEqual(kernels.run(queue.get()), 42)
self.assertFalse(queue.is_full())
kernels.run()
actual = []
while queue:
actual.append(kernels.run(queue.get()))
self.assertEqual(actual, [43, 44, 45])
def __init__(self):
self._is_committed = False
self._status = None
self._headers = []
self._send_mechanism = _SendMechanisms.UNDECIDED
# Set capacity to 1 to prevent excessive buffering.
self._chunks = queues.Queue(capacity=1)
self.file = None
def make_context(status, headers, *body_chunks):
context = wsgi._ApplicationContext()
context._status = status
context._headers = headers
context._chunks = queues.Queue() # Unset capacity for test.
for chunk in body_chunks:
context._chunks.put_nonblocking(chunk)
context.end_body_chunks()
return context
def test_close_not_graceful(self):
queue = queues.Queue()
self.assertFalse(queue.is_closed())
for x in (42, 43, 44):
kernels.run(queue.put(x))
self.assertEqual(queue.close(False), [42, 43, 44])
self.assertTrue(queue.is_closed())
self.assertFalse(queue)
def test_pubsub(self):
with contextlib.ExitStack() as stack:
wiredata = jsons.JsonWireData()
p_queue = queues.Queue()
s1_queue = queues.Queue()
s2_queue = queues.Queue()
publisher = stack.enter_context(
publishers.Publisher(p_queue, wiredata))
subscriber1 = stack.enter_context(
subscribers.Subscriber(Message, s1_queue, wiredata))
subscriber2 = stack.enter_context(
subscribers.Subscriber(Message, s2_queue, wiredata))
publisher.socket.listen('inproc://test_pubsub')
subscriber1.socket.dial('inproc://test_pubsub')
subscriber2.socket.dial('inproc://test_pubsub')
p_task = tasks.spawn(publisher.serve())
s1_task = tasks.spawn(subscriber1.serve())
s2_task = tasks.spawn(subscriber2.serve())
with self.assertRaises(kernels.KernelTimeout):
# Unfortunately this test is a somehow timing sensitive.
# If we remove this kernels.run call, the subscribers
# might sometimes not receive all messages.
kernels.run(timeout=0.01)
expect = (Message(content='hello'), Message(content='world'))
for message in expect:
publisher.publish_nonblocking(message)
self.assertFalse(s1_queue)
self.assertFalse(s2_queue)
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
self.assertEqual(len(s1_queue), 2)
self.assertEqual(len(s2_queue), 2)
for message in expect:
self.assertEqual(s1_queue.get_nonblocking(), message)
self.assertEqual(s2_queue.get_nonblocking(), message)
publisher.shutdown()
subscriber1.shutdown()
subscriber2.shutdown()
kernels.run(timeout=0.01)
self.assertIsNone(p_task.get_result_nonblocking())
self.assertIsNone(s1_task.get_result_nonblocking())
self.assertIsNone(s2_task.get_result_nonblocking())
def setUp(self):
super().setUp()
self.main_task = None
self.agent_queue = tasks.CompletionQueue()
self.graceful_exit = locks.Event()
self.signal_queue = queues.Queue()
mock = unittest.mock.patch(agents.__name__ + '.signals').start()
mock.SignalSource().__enter__().get = self.signal_queue.get
self._assert_logs = self.assertLogs(agents.__name__, level='DEBUG')
self.cm = self._assert_logs.__enter__()
def test_multiplex(self):
q1 = queues.Queue()
q2 = queues.Queue()
q_out = queues.Queue()
multiplexer_task = tasks.spawn(more_queues.multiplex([q1, q2], q_out))
q1.put_nonblocking('x')
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
self.assertEqual(get_many(q_out), ['x'])
q2.put_nonblocking('a')
q1.put_nonblocking('y')
q2.put_nonblocking('b')
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
# Hmmm... The order of q_out items depends on the internal
# implementation of multiplex. I am not sure whether this is a
# good test case.
self.assertEqual(get_many(q_out), ['a', 'y', 'b'])
q2.put_nonblocking('c')
q2.put_nonblocking('d')
q2.put_nonblocking('e')
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
self.assertEqual(get_many(q_out), ['c', 'd', 'e'])
q2.close()
q1.put_nonblocking('z')
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
self.assertEqual(get_many(q_out), ['z'])
self.assertFalse(multiplexer_task.is_completed())
q1.put_nonblocking('w')
q1.close()
kernels.run(timeout=0.01)
self.assertEqual(get_many(q_out), ['w'])
self.assertTrue(q_out.is_closed())
self.assertTrue(multiplexer_task.is_completed())
self.assertIsNone(multiplexer_task.get_result_nonblocking())
def test_select(self):
q1 = queues.Queue()
q1.put_nonblocking('x')
q1.put_nonblocking('y')
q1.close()
q2 = queues.Queue()
q2.put_nonblocking('a')
q2.put_nonblocking('b')
q2.put_nonblocking('c')
q2.put_nonblocking('d')
q2.close()
generator = more_queues.select([q1, q2])
async def anext():
return await generator.__anext__() # pylint: disable=no-member
for item in 'xaybcd':
self.assertEqual(kernels.run(anext(), timeout=0.01), item)
with self.assertRaises(StopAsyncIteration):
kernels.run(anext(), timeout=0.01)
def test_cancel(self):
q1 = queues.Queue()
q2 = queues.Queue()
q_out = queues.Queue(1)
multiplexer_task = tasks.spawn(more_queues.multiplex([q1, q2], q_out))
q1.put_nonblocking('x')
q1.put_nonblocking('y')
q2.put_nonblocking('a')
q2.put_nonblocking('b')
with self.assertRaises(kernels.KernelTimeout):
kernels.run(timeout=0.01)
multiplexer_task.cancel()
kernels.run(timeout=0.01)
self.assertTrue(multiplexer_task.is_completed())
self.assertIsInstance(multiplexer_task.get_exception_nonblocking(),
tasks.Cancelled)
self.assertTrue(q_out.is_closed())
# No input item is lost.
self.assertEqual(get_many(q1), ['y'])
self.assertEqual(get_many(q2), ['a', 'b'])
self.assertEqual(get_many(q_out), ['x'])
def test_round_robin(self):
q1 = queues.Queue()
q1.put_nonblocking('x')
q1.put_nonblocking('y')
q1.put_nonblocking('z')
q1.close()
q2 = queues.Queue()
q2.put_nonblocking('a')
q2.put_nonblocking('b')
q2.put_nonblocking('c')
q2.put_nonblocking('d')
q2.put_nonblocking('e')
q2.put_nonblocking('f')
q2.close()
q_out = queues.Queue()
kernels.run(more_queues.multiplex([q1, q2], q_out), timeout=0.01)
self.assertTrue(q_out.is_closed())
self.assertEqual(
q_out.close(graceful=False),
['x', 'a', 'y', 'b', 'z', 'c', 'd', 'e', 'f'],
)
def __init__(self, start_response, is_sendfile_supported):
self._start_response = start_response
self._status = consts.Statuses.OK
self.headers = self.Headers(self.is_uncommitted)
self._precommit = self._make_precommit()
# Set capacity to 1 to prevent excessive buffering.
self._body = queues.Queue(capacity=1)
self.file = None
self._send_mechanism = _SendMechanisms.UNDECIDED
self._send_mechanism_decided = locks.Event()
if not is_sendfile_supported:
self._set_send_mechanism(_SendMechanisms.SEND)
'graceful_exit',
'grace_period',
# shutdown_agents.
'shutdown_queue',
)
PARAMS = parameters.define(
agents.__name__,
parameters.Namespace(grace_period=parameters.Parameter(4,
type=(int, float),
unit='seconds'), ),
)
startup.set(LABELS.agent_queue, tasks.CompletionQueue())
startup.set(LABELS.graceful_exit, locks.Event())
startup.set(LABELS.shutdown_queue, queues.Queue())
utils.depend_parameter_for(LABELS.grace_period, PARAMS.grace_period)
utils.define_binder(
agents.supervise_agents,
LABELS.supervise_agents,
{
'agent_queue': LABELS.agent_queue,
'graceful_exit': LABELS.graceful_exit,
'grace_period': LABELS.grace_period,
},
)
@startup
def make_queue(shutdown_queue: g1.asyncs.agents.parts.LABELS.shutdown_queue):
queue = queues.Queue(capacity=32)
shutdown_queue.put_nonblocking(queue.close)
return queue
def make_publisher():
return publishers.Publisher(queues.Queue(capacity=32), capnps.WIRE_DATA)