async def test_import_data_dont_save(self): path = 'tests/testing_data/validator/common_training_examples' bot = 'test_data_import' bot_2 = 'test_data_import_bot' user = '******' test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow())) shutil.copytree(path, test_data_path) importer = DataImporter(test_data_path, bot, user, set(), False) await importer.validate() importer.import_data() processor = MongoProcessor() intents = processor.fetch_intents(bot) assert 'greet' in intents assert 'deny' in intents assert 'location' in intents assert 'affirm' in intents assert len(processor.fetch_stories(bot)) == 4 assert len(list(processor.fetch_training_examples(bot))) == 13 assert len(list(processor.fetch_responses(bot))) == 6 assert len(processor.fetch_actions(bot)) == 4 assert len(processor.fetch_rule_block_names(bot)) == 4 assert len(processor.fetch_intents(bot_2)) == 0 assert len(processor.fetch_stories(bot_2)) == 0 assert len(list(processor.fetch_training_examples(bot_2))) == 0 assert len(list(processor.fetch_responses(bot_2))) == 0 assert len(processor.fetch_actions(bot_2)) == 0 assert len(processor.fetch_rule_block_names(bot_2)) == 0
async def test_trigger_data_importer_rules_only(self, monkeypatch, get_training_data): bot = 'test_trigger_data_importer_rules_only' user = '******' test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow())) data_path = os.path.join(test_data_path, 'data') Utility.make_dirs(data_path) shutil.copy2('tests/testing_data/validator/valid/data/rules.yml', data_path) nlu, story_graph, domain, config, http_actions = await get_training_data( 'tests/testing_data/validator/valid') mongo_processor = MongoProcessor() mongo_processor.save_domain(domain, bot, user) mongo_processor.save_nlu(nlu, bot, user) config["bot"] = bot config["user"] = user config_obj = Configs._from_son(config) config_obj.save() mongo_processor.save_stories(story_graph.story_steps, bot, user) mongo_processor.save_http_action(http_actions, bot, user) def _path(*args, **kwargs): return test_data_path monkeypatch.setattr(Utility, "get_latest_file", _path) DataImporterLogProcessor.add_log(bot, user, files_received=["rules"]) await EventsTrigger.trigger_data_importer(bot, user, True, False) logs = list(DataImporterLogProcessor.get_logs(bot)) assert len(logs) == 1 assert not logs[0].get('intents').get('data') assert not logs[0].get('stories').get('data') assert not logs[0].get('utterances').get('data') assert not logs[0].get('http_actions').get('data') assert not logs[0].get('training_examples').get('data') assert not logs[0].get('domain').get('data') assert not logs[0].get('config').get('data') assert not logs[0].get('exception') assert logs[0]['is_data_uploaded'] assert logs[0]['start_timestamp'] assert logs[0]['end_timestamp'] assert logs[0]['status'] == 'Success' assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value assert len(mongo_processor.fetch_stories(bot)) == 2 assert len(list(mongo_processor.fetch_training_examples(bot))) == 7 assert len(list(mongo_processor.fetch_responses(bot))) == 2 assert len(mongo_processor.fetch_actions(bot)) == 2 assert len(mongo_processor.fetch_rule_block_names(bot)) == 3
async def test_trigger_data_importer_validate_and_save_append( self, monkeypatch): bot = 'test_events' user = '******' test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow())) shutil.copytree('tests/testing_data/validator/append', test_data_path) def _path(*args, **kwargs): return test_data_path monkeypatch.setattr(Utility, "get_latest_file", _path) DataImporterLogProcessor.add_log(bot, user, files_received=REQUIREMENTS - {"http_actions", "rules"}) await EventsTrigger.trigger_data_importer(bot, user, True, False) logs = list(DataImporterLogProcessor.get_logs(bot)) assert len(logs) == 6 assert not logs[0].get('intents').get('data') assert not logs[0].get('stories').get('data') assert not logs[0].get('utterances').get('data') assert not logs[0].get('http_actions').get('data') assert not logs[0].get('training_examples').get('data') assert not logs[0].get('domain').get('data') assert not logs[0].get('config').get('data') assert not logs[0].get('exception') assert logs[0]['is_data_uploaded'] assert logs[0]['start_timestamp'] assert logs[0]['end_timestamp'] assert logs[0]['status'] == 'Success' assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value processor = MongoProcessor() assert 'greet' in processor.fetch_intents(bot) assert 'deny' in processor.fetch_intents(bot) assert 'location' in processor.fetch_intents(bot) assert 'affirm' in processor.fetch_intents(bot) assert len(processor.fetch_stories(bot)) == 4 assert len(list(processor.fetch_training_examples(bot))) == 13 assert len(list(processor.fetch_responses(bot))) == 6 assert len(processor.fetch_actions(bot)) == 4 assert len(processor.fetch_rule_block_names(bot)) == 4
async def test_import_data(self): path = 'tests/testing_data/validator/valid' bot = 'test_data_import' user = '******' test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow())) shutil.copytree(path, test_data_path) importer = DataImporter(test_data_path, bot, user, REQUIREMENTS - {"http_actions"}, True, True) await importer.validate() importer.import_data() processor = MongoProcessor() assert 'greet' in processor.fetch_intents(bot) assert 'deny' in processor.fetch_intents(bot) assert len(processor.fetch_stories(bot)) == 2 assert len(list(processor.fetch_training_examples(bot))) == 7 assert len(list(processor.fetch_responses(bot))) == 4 assert len(processor.fetch_actions(bot)) == 2 assert len(processor.fetch_rule_block_names(bot)) == 4
async def test_trigger_data_importer_import_with_intent_issues( self, monkeypatch): bot = 'test_trigger_data_importer_import_with_intent_issues' user = '******' test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow())) shutil.copytree('tests/testing_data/validator/intent_name_mismatch', test_data_path) def _path(*args, **kwargs): return test_data_path monkeypatch.setattr(Utility, "get_latest_file", _path) BotSettings(ignore_utterances=True, bot=bot, user=user).save() DataImporterLogProcessor.add_log( bot, user, files_received=['nlu', 'stories', 'domain', 'config']) await EventsTrigger.trigger_data_importer(bot, user, True, True) logs = list(DataImporterLogProcessor.get_logs(bot)) assert len(logs) == 1 assert logs[0].get('intents').get('data') assert not logs[0].get('stories').get('data') assert not logs[0].get('utterances').get('data') assert not logs[0].get('http_actions').get('data') assert not logs[0].get('training_examples').get('data') assert not logs[0].get('domain').get('data') assert not logs[0].get('config').get('data') assert not logs[0].get('exception') assert logs[0]['is_data_uploaded'] assert logs[0]['start_timestamp'] assert logs[0]['end_timestamp'] assert logs[0]['status'] == 'Failure' assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value mongo_processor = MongoProcessor() assert len(mongo_processor.fetch_stories(bot)) == 0 assert len(list(mongo_processor.fetch_training_examples(bot))) == 0 assert len(list(mongo_processor.fetch_responses(bot))) == 0 assert len(mongo_processor.fetch_actions(bot)) == 0 assert len(mongo_processor.fetch_rule_block_names(bot)) == 0
async def test_trigger_data_importer_validate_existing_data( self, monkeypatch, get_training_data): bot = 'test_trigger_data_importer_domain_only' user = '******' test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow())) Utility.make_dirs(test_data_path) def _path(*args, **kwargs): return test_data_path monkeypatch.setattr(Utility, "get_latest_file", _path) DataImporterLogProcessor.add_log(bot, user) await EventsTrigger.trigger_data_importer(bot, user, True, False) logs = list(DataImporterLogProcessor.get_logs(bot)) assert len(logs) == 2 assert not logs[0].get('intents').get('data') assert not logs[0].get('stories').get('data') assert not logs[0].get('utterances').get('data') assert not logs[0].get('http_actions').get('data') assert not logs[0].get('training_examples').get('data') assert not logs[0].get('domain').get('data') assert not logs[0].get('config').get('data') assert not logs[0].get('exception') assert logs[0]['is_data_uploaded'] assert logs[0]['start_timestamp'] assert logs[0]['end_timestamp'] assert logs[0]['status'] == 'Success' assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value mongo_processor = MongoProcessor() assert len(mongo_processor.fetch_stories(bot)) == 2 assert len(list(mongo_processor.fetch_training_examples(bot))) == 7 assert len(list(mongo_processor.fetch_responses(bot))) == 2 assert len(mongo_processor.fetch_actions(bot)) == 2 assert len(mongo_processor.fetch_rule_block_names(bot)) == 3