async def on_post_ai(self, req: aiohttp.web.Request):
"""POST handler - upload training data"""
dev_id, ai_id, training_data = await self._get_training_data_from_request(req)
self.logger.info("Add training data for AI {}/{}".format(
dev_id, ai_id))
item = self.training_lookup.lookup_item(dev_id, ai_id)
if item is None:
self.logger.info(
"AI {}/{} is not found - creating "
"training data".format(dev_id, ai_id))
item = self.training_lookup.create(dev_id, ai_id)
else:
if item.status.is_training:
self.logger.info(
"AI {}/{} is training - stopping before changing "
"training data".format(dev_id, ai_id))
await item.stop_training()
item.controller = self
hash_value = ait.training_file.write_training_data_to_disk_v1(
item.ai_data_directory, training_data)
status = ait.AiTrainingStatusWithProgress(
ait.AiTrainingState.ai_ready_to_train,
training_file_hash=hash_value)
# we have an explicit request to change state, so we will
# always save this
item.reset_status(status, always_save=True)
await item.notify_status_update()
url = "{}/{}/{}".format(req.url, dev_id, ai_id)
data = {'status': item.status.state.name, 'url': url}
resp = web.json_response(data)
return resp
async def test_internal_error_500(cli, mock_training):
mock_training.add_training_with_state(
'test_error',
'test_error',
ait.AiTrainingStatusWithProgress(ait.AiTrainingState.ai_undefined),
exception_on_access=True)
resp = await cli.get('/ai/test_error/test_error/chat?q=hi')
assert resp.status == 500
text_data = await resp.text()
text_data_lower = text_data.lower()
assert "internal server error" in text_data_lower
assert "Traceback (" in text_data
assert "ConfTestException" in text_data
async def process_training_message(self, msg: TrainingMessage):
"""Train the model"""
self.logger.info("Training started for {}".format(msg.ai_id))
self.pool.check_for_cancel(msg)
resp = StatusResponse(msg, msg.ai_id, ait.AiTrainingState.ai_training,
0.0, None)
await self.pool.send_message_out(resp)
self.callback_object = StatusCallback(msg, self.pool,
self.save_controller, msg.ai_id)
train_start = datetime.datetime.utcnow()
try:
# check for blank training
training_file = Path(
msg.ai_path) / ait.AI_TRAINING_STANDARD_FILE_NAME
topic = ait.file_load_training_data_v1(training_file)
if topic.is_empty():
self.logger.warning(
"Training data empty for AI: {}, marking as complete".
format(msg.ai_id))
final_status = ait.AiTrainingState.ai_training_complete
training_data_hash = None
empty_state = ait.AiTrainingStatusWithProgress(final_status)
# wait to have permission to write the save status
await self.callback_object.wait_to_save()
empty_state.save(
Path(msg.ai_path) / AI_TRAINING_STATUS_FILENAME)
else:
final_status, training_data_hash = await self.train(
msg, topic, self.callback_object)
finally:
self.callback_object = None
train_duration = datetime.datetime.utcnow() - train_start
self.logger.info(
"Training ended for {}, status:{}, duration:{}, hash:{}".format(
msg.ai_id, final_status, train_duration, training_data_hash),
extra={
"ai_id": msg.ai_id,
"status": str(final_status),
"duration": str(train_duration),
"hash": training_data_hash
})
resp = StatusResponse(msg,
msg.ai_id,
final_status,
training_data_hash=training_data_hash)
return resp
def __init__(self):
self.__controller = None
self.max_chat_lock_seconds = DEFAULT_MAX_CHAT_LOCK_SECONDS
self.__status = ait.AiTrainingStatusWithProgress(
ait.AiTrainingState.ai_undefined)
self.dev_id = None
self.ai_id = None
self.__status_file = None
self.last_chat_time = None
self.training_msg = None
self.__asyncio_condition_variable = None
self.__training_stopped_condition_variable = None
self.__start_training_mutex = None
self.__shutting_down = False
self.__update_pending = False
self.__chat_process_pool = None
self.__watch_training_future = None
def __init__(self,
mock_training_provider,
training_data_root,
dev_id,
ai_id,
status=ait.AiTrainingStatusWithProgress(
ait.AiTrainingState.ai_undefined)):
super().__init__()
self.dev_id = dev_id
self.ai_id = ai_id
self.mock_training_provider = mock_training_provider
self.__ai_data_directory = pathlib.Path(
str(training_data_root)) / dev_id / ai_id
self.__ai_data_directory.mkdir(exist_ok=True, parents=True)
self.__training_data = None
self.exception_on_access = False
self.reset_status(status)