async def __chat_internal(self, chat_input, topic, history, entities,
chat_reload_training):
if self.last_chat_time is None:
await self.controller.start_chat_for_ai(self)
if self.__chat_process_pool is None:
process_name = 'Chat_{0}'.format(self.ai_id)
chat_pool = a_pool.AsyncProcessPool(
self.controller.multiprocessing_manager,
process_name,
num_processes=1)
chat_process_worker, process_kwargs = \
self.create_chat_process_worker()
if process_kwargs is None:
process_kwargs = {}
await chat_pool.initialize_processes(chat_process_worker,
**process_kwargs)
self.__chat_process_pool = chat_pool
msg = self.create_wake_chat_message(str(self.ai_data_directory),
self.ai_id)
await self.__chat_process_pool.do_work(msg)
self.last_chat_time = datetime.datetime.utcnow()
msg = ait_c.ChatRequestMessage(chat_input, topic, history,
chat_reload_training, entities)
result = await self.__chat_process_pool.do_work(msg)
return result
async def test_pool1_jobfail1(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest")
await pool.initialize_processes(ProcessWorker)
msg = DoWork(3, fail_me=True)
# use the higher level messaging functions
with pytest.raises(a_pool.FailedJobError):
await pool.do_work(msg)
await pool.shutdown()
async def test_pool8_ping4(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=8)
await pool.initialize_processes(ProcessWorker)
msgs = [Ping() for x in range(4)]
# use the higher level do_worklist function
resp = await pool.do_worklist(msgs)
logger = _get_logger()
logger.debug('received {}'.format(resp))
await pool.shutdown()
async def test_pool1_job1(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest")
await pool.initialize_processes(ProcessWorker)
msg = DoWork(3)
# use the higher level messaging functions
resp = await pool.do_work(msg)
assert isinstance(resp, a_pool.Response)
logger = _get_logger()
logger.debug('received {}'.format(resp))
await pool.shutdown()
async def create_training_process_pool(self, training_processes: int,
training_queue_size: int,
worker_type: type):
"""Create a training pool"""
training_pool = a_pool.AsyncProcessPool(
self.__multiprocessing_manager, 'Training_pool',
training_processes, training_queue_size, training_queue_size)
await training_pool.initialize_processes(
worker_type, save_controller=self.__save_controller)
return training_pool
async def test_pool4_ping3(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=4)
await pool.initialize_processes(ProcessWorker)
msg = Ping()
# use the higher level do_work function
resp = await pool.do_work(msg)
assert isinstance(resp, a_pool.Response)
logger = _get_logger()
logger.debug('received {}'.format(resp))
await pool.shutdown()
async def test_get_message_out_timeout(manager):
pool = a_pool.AsyncProcessPool(manager,
"SimpleTest",
q_in_size=1,
q_out_size=2)
await pool.initialize_processes(ProcessWorker)
# Try and read message with 0.05s timeout
with pytest.raises(queue.Empty):
await pool.get_message_out(timeout=0.05)
async def test_pool4_fail(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=4)
await pool.initialize_processes(ProcessWorker)
msg = FailMe()
# use the low level messaging functions
await pool.send_message_in(msg)
resp = await pool.get_message_out()
assert isinstance(resp, a_pool.ErrorResponse)
logger = _get_logger()
logger.debug('received {}'.format(resp))
await pool.shutdown()
async def test_cancel_2(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest")
await pool.initialize_processes(ProcessWorker)
msg = DoWork(4, wait_for_cancel=True)
# use the low level messaging functions
await pool.send_message_in(msg)
await asyncio.sleep(0.1)
await pool.send_cancel(msg)
with pytest.raises(a_pool.JobCancelledError):
await pool.get_message_out_with_id(msg.msg_id)
await pool.shutdown()
async def test_cancel_1(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest")
await pool.initialize_processes(ProcessWorker)
msg = DoWork(4, wait_for_cancel=True)
# use the low level messaging functions
await pool.send_message_in(msg)
await asyncio.sleep(0.1)
await pool.send_cancel(msg)
resp = await pool.get_message_out()
assert isinstance(resp, a_pool.CancelResponse)
await pool.shutdown()
async def test_pool8_ping4_fail1(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=8)
await pool.initialize_processes(ProcessWorker)
msgs = [Ping() for x in range(4)]
msgs.append(FailMe())
# use the higher level do_worklist function
with pytest.raises(a_pool.FailedJobError):
resp = await pool.do_worklist(msgs)
logger = _get_logger()
logger.debug('received {}'.format(resp))
await pool.shutdown()
async def test_get_message_out_with_id_timeout(manager):
pool = a_pool.AsyncProcessPool(manager,
"SimpleTest",
q_in_size=1,
q_out_size=2)
await pool.initialize_processes(ProcessWorker)
# Create message but never send it
msg = DoWork(4, wait_for_cancel=True)
# Therefore there can be no reply within 0.05s
with pytest.raises(queue.Empty):
await pool.get_message_out_with_id(msg, timeout=0.05)
async def test_pool_unhealthy_on_dead_worker_2(manager, mocker):
pool = a_pool.AsyncProcessPool(manager,
"SimpleTest",
q_in_size=1,
q_out_size=2)
await pool.initialize_processes(ProcessWorker)
# Patch is_alive to indicate our worker process died
mock_call = mocker.patch('multiprocessing.Process.is_alive')
mock_call.return_value = False
with pytest.raises(a_pool.PoolUnhealthyError):
await pool.get_message_out()
async def test_cancel_3(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=1)
await pool.initialize_processes(ProcessWorker)
msgs = [DoWork(4, wait_for_cancel=True) for x in range(4)]
future1 = asyncio.ensure_future(send_cancel_after_100ms(pool, msgs[0]))
future2 = asyncio.ensure_future(pool.do_worklist(msgs))
tasks = [future1, future2]
combined_future = asyncio.gather(*tasks)
with pytest.raises(a_pool.JobCancelledError):
await combined_future
await pool.shutdown()
async def test_pool_raises_if_overflow(manager):
pool = a_pool.AsyncProcessPool(manager,
"SimpleTest",
q_in_size=1,
q_out_size=2)
await pool.initialize_processes(ProcessWorker)
msg = DoWork(4, wait_for_cancel=True)
# This one will be run
await pool.send_message_in(msg)
# This one will be queued
await pool.send_message_in(msg)
# This one will overflow the queue
with pytest.raises(queue.Full):
await pool.send_message_in(msg, timeout=0.05)
async def test_pool8_fail1_ping4(manager):
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=8)
await pool.initialize_processes(ProcessWorker)
msgs = [FailMe()]
msgs += [Ping() for x in range(4)]
# use the higher level do_worklist function
with pytest.raises(a_pool.FailedJobError):
resp = await pool.do_worklist(msgs)
logger = _get_logger()
logger.debug('received {}'.format(resp))
# check that the messages that got posted after the failure are not
# in the queue as they were ignored later as part of the single Ping()
await asyncio.sleep(0.05)
await pool.do_work(Ping())
assert pool.get_queue_status() == (0, 0, 0)
await pool.shutdown()
async def test_cancel_from_q_in(manager):
"""This test can only pass if the cancelled message is handled while it is still queued.
This is because every task will block,
we have only one execution process
and we cancel the SECOND message that was queued."""
pool = a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=1)
await pool.initialize_processes(ProcessWorker)
msgs = [DoWork(4, wait_for_cancel=True) for x in range(4)]
tasks = []
cancel_future = asyncio.ensure_future(
send_cancel_after_100ms(pool, msgs[1]))
tasks.append(cancel_future)
tasks.append(asyncio.ensure_future(pool.do_worklist(msgs)))
combined_future = asyncio.gather(*tasks)
with pytest.raises(a_pool.JobCancelledError):
await combined_future
await pool.shutdown()
async def test_pool_unhealthy_on_dead_worker_3(manager, mocker):
pool = a_pool.AsyncProcessPool(manager,
"SimpleTest",
q_in_size=1,
q_out_size=2)
await pool.initialize_processes(ProcessWorker)
with pytest.raises(a_pool.PoolUnhealthyError):
msg = DoWork("Hi", wait_for_cancel=True)
work_task = asyncio.ensure_future(pool.do_work(msg))
await asyncio.sleep(1)
# check that the work_task is still "running"
assert not work_task.done()
# NOW - once the message was sent and we waited a bit, now
# patch is_alive to indicate our worker process died
mock_call = mocker.patch('multiprocessing.Process.is_alive')
mock_call.return_value = False
# This should now cause the do_work operation to bomb
await work_task
# Shutdown pool to cleanup the hanging background process
await pool.shutdown()
async def test_create_fail_0_size(manager):
with pytest.raises(a_pool.ProcessPoolConfigurationError):
a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=0)
async def test_create_ok(manager):
a_pool.AsyncProcessPool(manager, "SimpleTest", num_processes=1)
