def train_model_for_bot(bot: str):
"""
loads bot data from mongo into individual files for training
:param bot: bot id
:return: model path
"""
processor = MongoProcessor()
nlu = processor.load_nlu(bot)
if not nlu.training_examples:
raise AppException("Training data does not exists!")
domain = processor.load_domain(bot)
stories = processor.load_stories(bot)
config = processor.load_config(bot)
rules = processor.get_rules_for_training(bot)
directory = Utility.write_training_data(nlu, domain, config, stories,
rules)
output = os.path.join(DEFAULT_MODELS_PATH, bot)
if not os.path.exists(output):
os.mkdir(output)
model = train(domain=os.path.join(directory, DEFAULT_DOMAIN_PATH),
config=os.path.join(directory, DEFAULT_CONFIG_PATH),
training_files=os.path.join(directory, DEFAULT_DATA_PATH),
output=output,
core_additional_arguments={
"augmentation_factor": 100
},
force_training=True).model
Utility.delete_directory(directory)
del processor
del nlu
del domain
del stories
del config
Utility.move_old_models(output, model)
return model
async def trigger_data_importer(bot: Text,
user: Text,
save_data: bool,
overwrite: bool = True):
"""
Triggers data importer event which validates and imports data into kairon.
@param bot: bot id.
@param user: kairon username.
@param save_data: Flag to import data into kairon. If set to false, then only validation is run.
Otherwise, both validation and import is done.
@param overwrite: Overwrite existing data(if set to true) or append (if set to false).
@return:
"""
validation_status = 'Failure'
path = None
event_url = Utility.get_event_url("DATA_IMPORTER")
try:
if not Utility.check_empty_string(event_url):
import_flag = '--import-data' if save_data else ''
overwrite_flag = '--overwrite' if overwrite else ''
env_var = {
'BOT': bot,
'USER': user,
"IMPORT_DATA": import_flag,
"OVERWRITE": overwrite_flag
}
event_request = Utility.build_event_request(env_var)
Utility.http_request(
"POST", Utility.environment['model']['data_importer'].get(
'event_url'), None, user, event_request)
DataImporterLogProcessor.add_log(
bot, user, event_status=EVENT_STATUS.TASKSPAWNED.value)
else:
path = Utility.get_latest_file(
os.path.join('training_data', bot))
files_received = DataImporterLogProcessor.get_files_received_for_latest_event(
bot)
DataImporterLogProcessor.add_log(
bot, user, event_status=EVENT_STATUS.PARSE.value)
data_importer = DataImporter(path, bot, user, files_received,
save_data, overwrite)
DataImporterLogProcessor.add_log(
bot, user, event_status=EVENT_STATUS.VALIDATING.value)
summary, component_count = await data_importer.validate()
initiate_import = Utility.is_data_import_allowed(
summary, bot, user)
status = 'Success' if initiate_import else 'Failure'
DataImporterLogProcessor.update_summary(
bot,
user,
component_count,
summary,
status=status,
event_status=EVENT_STATUS.SAVE.value)
if initiate_import:
data_importer.import_data()
DataImporterLogProcessor.add_log(
bot, user, event_status=EVENT_STATUS.COMPLETED.value)
except exceptions.ConnectionError as e:
logger.error(str(e))
DataImporterLogProcessor.add_log(
bot,
user,
exception=f'Failed to trigger the event. {e}',
status=validation_status,
event_status=EVENT_STATUS.FAIL.value)
except Exception as e:
logger.error(str(e))
DataImporterLogProcessor.add_log(
bot,
user,
exception=str(e),
status=validation_status,
event_status=EVENT_STATUS.FAIL.value)
if path:
Utility.delete_directory(path)