def train_model_for_bot(bot: str): """ loads bot data from mongo into individual files for training :param bot: bot id :return: model path """ processor = MongoProcessor() nlu = processor.load_nlu(bot) if not nlu.training_examples: raise AppException("Training data does not exists!") domain = processor.load_domain(bot) stories = processor.load_stories(bot) config = processor.load_config(bot) rules = processor.get_rules_for_training(bot) directory = Utility.write_training_data(nlu, domain, config, stories, rules) output = os.path.join(DEFAULT_MODELS_PATH, bot) model = train( domain=os.path.join(directory, DEFAULT_DOMAIN_PATH), config=os.path.join(directory, DEFAULT_CONFIG_PATH), training_files=os.path.join(directory, DEFAULT_DATA_PATH), output=output, ) Utility.delete_directory(directory) del processor del nlu del domain del stories del config return model
def train_model_for_bot(bot: str): """ loads bot data from mongo into individual files for training :param bot: bot id :return: model path """ processor = MongoProcessor() nlu = processor.load_nlu(bot) if not nlu.training_examples: raise AppException("Training data does not exists!") domain = processor.load_domain(bot) stories = processor.load_stories(bot) config = processor.load_config(bot) directory = Utility.save_files( nlu.nlu_as_markdown().encode(), domain.as_yaml().encode(), stories.as_story_string().encode(), yaml.dump(config).encode(), ) output = os.path.join(DEFAULT_MODELS_PATH, bot) model = train( domain=os.path.join(directory, DEFAULT_DOMAIN_PATH), config=os.path.join(directory, DEFAULT_CONFIG_PATH), training_files=os.path.join(directory, DEFAULT_DATA_PATH), output=output, ) Utility.delete_directory(directory) del processor del nlu del domain del stories del config return model
async def trigger_data_importer(bot: Text, user: Text, save_data: bool, overwrite: bool = True): """ Triggers data importer event which validates and imports data into kairon. @param bot: bot id. @param user: kairon username. @param save_data: Flag to import data into kairon. If set to false, then only validation is run. Otherwise, both validation and import is done. @param overwrite: Overwrite existing data(if set to true) or append (if set to false). @return: """ validation_status = 'Failure' path = None try: if Utility.get_event_url("DATA_IMPORTER"): import_flag = '--import-data' if save_data else '' overwrite_flag = '--overwrite' if overwrite else '' env_var = { 'BOT': bot, 'USER': user, "IMPORT_DATA": import_flag, "OVERWRITE": overwrite_flag } event_request = Utility.build_event_request(env_var) Utility.http_request( "POST", Utility.environment['model']['data_importer'].get( 'event_url'), None, user, event_request) DataImporterLogProcessor.add_log( bot, user, event_status=EVENT_STATUS.TASKSPAWNED.value) else: path = Utility.get_latest_file( os.path.join('training_data', bot)) files_received = DataImporterLogProcessor.get_files_received_for_latest_event( bot) DataImporterLogProcessor.add_log( bot, user, event_status=EVENT_STATUS.PARSE.value) data_importer = DataImporter(path, bot, user, files_received, save_data, overwrite) DataImporterLogProcessor.add_log( bot, user, event_status=EVENT_STATUS.VALIDATING.value) summary, component_count = await data_importer.validate() is_data_valid = all( [not summary[key] for key in summary.keys()]) validation_status = 'Success' if is_data_valid else 'Failure' DataImporterLogProcessor.update_summary( bot, user, component_count, summary, status=validation_status, event_status=EVENT_STATUS.SAVE.value) if is_data_valid: data_importer.import_data() DataImporterLogProcessor.add_log( bot, user, event_status=EVENT_STATUS.COMPLETED.value) except exceptions.ConnectionError as e: logger.error(str(e)) DataImporterLogProcessor.add_log( bot, user, exception='Failed to trigger the event.', status=validation_status, event_status=EVENT_STATUS.FAIL.value) except Exception as e: logger.error(str(e)) DataImporterLogProcessor.add_log( bot, user, exception=str(e), status=validation_status, event_status=EVENT_STATUS.FAIL.value) if path: Utility.delete_directory(path)