async def test_import_data_dont_save(self):
path = 'tests/testing_data/validator/common_training_examples'
bot = 'test_data_import'
bot_2 = 'test_data_import_bot'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
shutil.copytree(path, test_data_path)
importer = DataImporter(test_data_path, bot, user, set(), False)
await importer.validate()
importer.import_data()
processor = MongoProcessor()
intents = processor.fetch_intents(bot)
assert 'greet' in intents
assert 'deny' in intents
assert 'location' in intents
assert 'affirm' in intents
assert len(processor.fetch_stories(bot)) == 4
assert len(list(processor.fetch_training_examples(bot))) == 13
assert len(list(processor.fetch_responses(bot))) == 6
assert len(processor.fetch_actions(bot)) == 4
assert len(processor.fetch_rule_block_names(bot)) == 4
assert len(processor.fetch_intents(bot_2)) == 0
assert len(processor.fetch_stories(bot_2)) == 0
assert len(list(processor.fetch_training_examples(bot_2))) == 0
assert len(list(processor.fetch_responses(bot_2))) == 0
assert len(processor.fetch_actions(bot_2)) == 0
assert len(processor.fetch_rule_block_names(bot_2)) == 0
async def test_import_data_validation_failed(self):
path = 'tests/testing_data/validator/common_training_examples'
bot = 'test_data_import_bot'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
shutil.copytree(path, test_data_path)
importer = DataImporter(test_data_path, bot, user, REQUIREMENTS.copy(),
True)
summary, component_count = await importer.validate()
assert not summary.get('intents')
assert not summary.get('stories')
assert not summary.get('utterances')
assert not summary.get('http_actions')
assert summary.get('training_examples')
assert not summary.get('domain')
assert not summary.get('config')
assert not summary.get('exception')
importer.validator.intents = []
importer.import_data()
processor = MongoProcessor()
assert len(processor.fetch_intents(bot)) == 0
assert len(processor.fetch_stories(bot)) == 0
assert len(list(processor.fetch_training_examples(bot))) == 0
assert len(list(processor.fetch_responses(bot))) == 0
assert len(processor.fetch_actions(bot)) == 0
async def test_trigger_data_importer_rules_only(self, monkeypatch,
get_training_data):
bot = 'test_trigger_data_importer_rules_only'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
data_path = os.path.join(test_data_path, 'data')
Utility.make_dirs(data_path)
shutil.copy2('tests/testing_data/validator/valid/data/rules.yml',
data_path)
nlu, story_graph, domain, config, http_actions = await get_training_data(
'tests/testing_data/validator/valid')
mongo_processor = MongoProcessor()
mongo_processor.save_domain(domain, bot, user)
mongo_processor.save_nlu(nlu, bot, user)
config["bot"] = bot
config["user"] = user
config_obj = Configs._from_son(config)
config_obj.save()
mongo_processor.save_stories(story_graph.story_steps, bot, user)
mongo_processor.save_http_action(http_actions, bot, user)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot, user, files_received=["rules"])
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 1
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert not logs[0].get('http_actions').get('data')
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
assert len(mongo_processor.fetch_stories(bot)) == 2
assert len(list(mongo_processor.fetch_training_examples(bot))) == 7
assert len(list(mongo_processor.fetch_responses(bot))) == 2
assert len(mongo_processor.fetch_actions(bot)) == 2
assert len(mongo_processor.fetch_rule_block_names(bot)) == 3
async def test_trigger_data_importer_validate_and_save_append(
self, monkeypatch):
bot = 'test_events'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
shutil.copytree('tests/testing_data/validator/append', test_data_path)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot,
user,
files_received=REQUIREMENTS -
{"http_actions", "rules"})
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 6
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert not logs[0].get('http_actions').get('data')
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
processor = MongoProcessor()
assert 'greet' in processor.fetch_intents(bot)
assert 'deny' in processor.fetch_intents(bot)
assert 'location' in processor.fetch_intents(bot)
assert 'affirm' in processor.fetch_intents(bot)
assert len(processor.fetch_stories(bot)) == 4
assert len(list(processor.fetch_training_examples(bot))) == 13
assert len(list(processor.fetch_responses(bot))) == 6
assert len(processor.fetch_actions(bot)) == 4
assert len(processor.fetch_rule_block_names(bot)) == 4
async def test_import_data(self):
path = 'tests/testing_data/validator/valid'
bot = 'test_data_import'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
shutil.copytree(path, test_data_path)
importer = DataImporter(test_data_path, bot, user,
REQUIREMENTS - {"http_actions"}, True, True)
await importer.validate()
importer.import_data()
processor = MongoProcessor()
assert 'greet' in processor.fetch_intents(bot)
assert 'deny' in processor.fetch_intents(bot)
assert len(processor.fetch_stories(bot)) == 2
assert len(list(processor.fetch_training_examples(bot))) == 7
assert len(list(processor.fetch_responses(bot))) == 4
assert len(processor.fetch_actions(bot)) == 2
assert len(processor.fetch_rule_block_names(bot)) == 4
async def test_trigger_data_importer_import_with_intent_issues(
self, monkeypatch):
bot = 'test_trigger_data_importer_import_with_intent_issues'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
shutil.copytree('tests/testing_data/validator/intent_name_mismatch',
test_data_path)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
BotSettings(ignore_utterances=True, bot=bot, user=user).save()
DataImporterLogProcessor.add_log(
bot, user, files_received=['nlu', 'stories', 'domain', 'config'])
await EventsTrigger.trigger_data_importer(bot, user, True, True)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 1
assert logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert not logs[0].get('http_actions').get('data')
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Failure'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
mongo_processor = MongoProcessor()
assert len(mongo_processor.fetch_stories(bot)) == 0
assert len(list(mongo_processor.fetch_training_examples(bot))) == 0
assert len(list(mongo_processor.fetch_responses(bot))) == 0
assert len(mongo_processor.fetch_actions(bot)) == 0
assert len(mongo_processor.fetch_rule_block_names(bot)) == 0
async def test_trigger_data_importer_validate_existing_data(
self, monkeypatch, get_training_data):
bot = 'test_trigger_data_importer_domain_only'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
Utility.make_dirs(test_data_path)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot, user)
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 2
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert not logs[0].get('http_actions').get('data')
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
mongo_processor = MongoProcessor()
assert len(mongo_processor.fetch_stories(bot)) == 2
assert len(list(mongo_processor.fetch_training_examples(bot))) == 7
assert len(list(mongo_processor.fetch_responses(bot))) == 2
assert len(mongo_processor.fetch_actions(bot)) == 2
assert len(mongo_processor.fetch_rule_block_names(bot)) == 3