def create(bot: str, use_test_stories: bool = False):
from kairon import Utility
from itertools import chain
from rasa.shared.nlu.training_data.training_data import TrainingData
bot_home = os.path.join('testing_data', bot)
Utility.make_dirs(bot_home)
processor = MongoProcessor()
intents_and_training_examples = processor.get_intents_and_training_examples(bot)
aug_training_examples = map(lambda training_data: TestDataGenerator.__prepare_nlu(training_data[0], training_data[1]), intents_and_training_examples.items())
messages = list(chain.from_iterable(aug_training_examples))
nlu_data = TrainingData(training_examples=messages)
stories = processor.load_stories(bot)
rules = processor.get_rules_for_training(bot)
stories = stories.merge(rules)
if stories.is_empty() or nlu_data.is_empty():
raise AppException('Not enough training data exists. Please add some training data.')
nlu_as_str = nlu_data.nlu_as_yaml().encode()
nlu_path = os.path.join(bot_home, "nlu.yml")
Utility.write_to_file(nlu_path, nlu_as_str)
if use_test_stories:
stories_path = os.path.join(bot_home, "test_stories.yml")
else:
stories_path = os.path.join(bot_home, "stories.yml")
YAMLStoryWriter().dump(stories_path, stories.story_steps, is_test_story=use_test_stories)
return nlu_path, stories_path
async def test_trigger_data_importer_validate_existing_data(self, monkeypatch, get_training_data):
bot = 'test_trigger_data_importer_domain_only'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(uuid.uuid4()))
Utility.make_dirs(test_data_path)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot, user)
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 2
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert [action.get('data') for action in logs[0].get('actions') if action.get('type') == 'http_actions'] == [[]]
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
mongo_processor = MongoProcessor()
assert len(mongo_processor.fetch_stories(bot)) == 2
assert len(list(mongo_processor.fetch_training_examples(bot))) == 7
assert len(list(mongo_processor.fetch_responses(bot))) == 3
assert len(mongo_processor.fetch_actions(bot)) == 2
assert len(mongo_processor.fetch_rule_block_names(bot)) == 3
async def test_trigger_data_importer_rules_only(self, monkeypatch,
get_training_data):
bot = 'test_trigger_data_importer_rules_only'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
data_path = os.path.join(test_data_path, 'data')
Utility.make_dirs(data_path)
shutil.copy2('tests/testing_data/validator/valid/data/rules.yml',
data_path)
nlu, story_graph, domain, config, http_actions = await get_training_data(
'tests/testing_data/validator/valid')
mongo_processor = MongoProcessor()
mongo_processor.save_domain(domain, bot, user)
mongo_processor.save_nlu(nlu, bot, user)
config["bot"] = bot
config["user"] = user
config_obj = Configs._from_son(config)
config_obj.save()
mongo_processor.save_stories(story_graph.story_steps, bot, user)
mongo_processor.save_http_action(http_actions, bot, user)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot, user, files_received=["rules"])
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 1
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert not logs[0].get('http_actions').get('data')
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
assert len(mongo_processor.fetch_stories(bot)) == 2
assert len(list(mongo_processor.fetch_training_examples(bot))) == 7
assert len(list(mongo_processor.fetch_responses(bot))) == 2
assert len(mongo_processor.fetch_actions(bot)) == 2
assert len(mongo_processor.fetch_rule_block_names(bot)) == 3