def assert_upload_csv_files_from_directory_with_subdir(self, store): store.empty() store.upload_from_directory(os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "csv", format=Store.CSV_FORMAT) set_collection = SetCollection() store.load(set_collection, 'TESTSET') self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values) set_collection = SetCollection() store.load(set_collection, 'TESTSET2') self.assertTrue(set_collection.contains('TESTSET2')) values = set_collection.set('TESTSET2') self.assertEqual(4, len(values)) self.assertTrue('VAL5' in values) self.assertTrue('VAL6' in values) self.assertTrue('VAL7' in values) self.assertTrue('VAL8' in values)
def __init__(self, bot, configuration: BrainConfiguration): self._bot = bot self._configuration = configuration self._tokenizer = self.load_tokenizer() self._aiml_parser = self.load_aiml_parser() self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = PersonCollection() self._rdf_collection = RDFCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._variables_collection = PropertiesCollection() self._preprocessors = ProcessorLoader() self._postprocessors = ProcessorLoader() self._authentication = None self._authorisation = None self._default_oob = None self._oob = {} self._regex_templates = {} self._dynamics_collection = DynamicsCollection() self.load(self.configuration) self.dump_brain_tree()
def __init__(self, configuration: BrainConfiguration): self._configuration = configuration self._aiml_parser = AIMLParser(self) self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = PersonCollection() self._rdf_collection = RDFCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._preprocessors = ProcessorLoader() self._postprocessors = ProcessorLoader() self._authentication = None self._authorisation = None self._default_oob = None self._oob = {} self._regex_templates = {} self._dynamics_collection = DynamicsCollection() self.load(self._configuration)
def test_load_from_file(self): storage_factory = StorageFactory() file_store_config = FileStorageConfiguration() file_store_config._sets_storage = FileStoreConfiguration(dirs=[ os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets" ]) storage_engine = FileStorageEngine(file_store_config) storage_factory._storage_engines[StorageFactory.SETS] = storage_engine storage_factory._store_to_engine_map[ StorageFactory.SETS] = storage_engine collection = SetCollection() self.assertIsNotNone(collection) self.assertTrue(collection.load(storage_factory) > 0) self.assertIsNotNone(collection._sets) self.assertEqual(len(collection._sets), 1) self.assertIsNotNone(collection._stores) self.assertEqual(len(collection._stores), 1) self.assertTrue("TEST_SET" in collection._sets) self.assertTrue("TEST_SET" in collection._stores) self.assertTrue(collection.contains('TEST_SET')) aset = collection.set('TEST_SET') self.assertIsNotNone(aset) values = aset['AIR'] self.assertIsNotNone(values) self.assertTrue(['Air', 'Force', 'blue'] in values)
def test_collection_duplicate(self): collection = SetCollection() set_dict = { "A": [["A", "A B", "A C"]], "D": [["D"]], "E": [["E", "E F"]] } values = { "A": "A", "A B": "A B", "A C": "A C", "D": "D", "E": "E", "E F": "E F" } collection.add_set("TESTSET", set_dict, "teststore", False, values) with self.assertRaises(Exception): set_dict = { "1": [["1", "1 2", "1 3"]], "4": [["4"]], "5": [["5", "5 6"]] } values = { "1": "1", "1 2": "1 2", "1 3": "1 3", "4": "4", "5": "5", "5 6": "5 6" } collection.add_set("TESTSET", set_dict, "teststore", False, values)
def test_load_from_test_dir_with_subdir(self): config = FileStorageConfiguration() config._sets_storage = FileStoreConfiguration(dirs=[ os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "text" ], extension="txt", subdirs=True, format="text", encoding="utf-8", delete_on_start=False) engine = FileStorageEngine(config) engine.initialise() store = FileSetsStore(engine) set_collection = SetCollection() store.load_all(set_collection) self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set_list('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values) self.assertTrue(set_collection.contains('TESTSET2')) values = set_collection.set_list('TESTSET2') self.assertEqual(4, len(values)) self.assertTrue('VAL5' in values) self.assertTrue('VAL6' in values) self.assertTrue('VAL7' in values) self.assertTrue('VAL8' in values)
def test_reload_from_file_jp(self): storage_factory = StorageFactory() file_store_config = FileStorageConfiguration() file_store_config._sets_storage = FileStoreConfiguration(dirs=[ os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets_jp" ]) storage_engine = FileStorageEngine(file_store_config) storage_factory._storage_engines[StorageFactory.SETS] = storage_engine storage_factory._store_to_engine_map[ StorageFactory.SETS] = storage_engine collection = SetCollection() self.assertIsNotNone(collection) collection.load(storage_factory) self.assertIsNotNone(collection._sets) self.assertEqual(len(collection._sets), 1) self.assertIsNotNone(collection._stores) self.assertEqual(len(collection._stores), 1) self.assertTrue("TEST_SET" in collection._sets) self.assertTrue("TEST_SET" in collection._stores) self.assertTrue("TEST_SET" in collection._is_cjk) self.assertTrue(collection.contains('TEST_SET')) self.assertTrue(collection.is_cjk('TEST_SET')) aset = collection.set_list('TEST_SET') self.assertIsNotNone(aset) values = aset['千'] self.assertIsNotNone(values) self.assertEqual(['千葉', '千葉県', '千葉県下'], values) collection.reload(storage_factory, "TEST_SET") self.assertIsNotNone(collection._sets) self.assertEqual(len(collection._sets), 1) self.assertIsNotNone(collection._stores) self.assertEqual(len(collection._stores), 1) self.assertTrue("TEST_SET" in collection._sets) self.assertTrue("TEST_SET" in collection._stores) self.assertTrue(collection.contains('TEST_SET')) values = aset['千'] self.assertIsNotNone(values) self.assertEqual(['千葉', '千葉県', '千葉県下'], values)
def test_load_with_exception(self): config = FileStorageConfiguration() engine = FileStorageEngine(config) engine.initialise() store = FileSetsStore(engine) set_collection = SetCollection() self.assertFalse( store.load( set_collection, os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "text" + os.sep + "testset.txt"))
def test_reload_no_storage(self): storage_factory = StorageFactory() file_store_config = FileStorageConfiguration() file_store_config._sets_storage = FileStoreConfiguration(dirs=[ os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets" ]) collection = SetCollection() self.assertIsNotNone(collection) self.assertTrue(collection.reload(storage_factory, "TEST_SET") == 0)
def assert_upload_from_text_file(self, store): store.empty() store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "text" + os.sep + "testset.txt") set_collection = SetCollection() store.load(set_collection, 'TESTSET') self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values)
def test_add_set_exists(self): collection = SetCollection() collection.add_set("TESTSET", { "A": [["A", "B", "C"]], "D": [["D"]], "E": [["E", "F"]] }, "teststore") with self.assertRaises(Exception): collection.add_set("TESTSET", { "1": [["1", "2", "3"]], "4": [["4"]], "5": [["5", "6"]] }, "teststore")
def assert_upload_text_files_from_directory_no_subdir(self, store): store.empty () store.upload_from_directory(os.path.dirname(__file__)+os.sep+"data"+os.sep+"sets"+os.sep+"text", subdir=False) set_collection = SetCollection() store.load(set_collection, 'TESTSET') self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values)
def assert_upload_from_csv_file(self, store): store.empty() store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "csv" + os.sep + "testset.csv", fileformat=Store.CSV_FORMAT) set_collection = SetCollection() store.load(set_collection, 'TESTSET') self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values)
def test_collection_operations_jp(self): collection = SetCollection() set_dict = {"千": ["千葉", "千葉県"], "東": ["東京", "東京都"]} values = {"千葉": "千葉", "千葉県": "千葉県", "東京": "東京", "東京都": "東京都"} collection.add_set("TESTSET", set_dict, "teststore", True, values) set_dict = {"神": ["神奈川", "神戸", "神田"], "大": ["大阪", "大分", "大津"]} values = { "神奈川": "神奈川", "神戸": "神戸", "神田": "神田", "大阪": "大阪", "大分": "大分", "大津": "大津" } collection.add_set("TESTSET2", set_dict, "teststore", True, values) self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertTrue(collection.contains("TESTSET")) self.assertTrue(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) self.assertEqual(collection.store_name("TESTSET"), "teststore") self.assertEqual(collection.store_name("TESTSET2"), "teststore") self.assertIsNone(collection.store_name("TESTSET3")) self.assertTrue(collection.is_cjk("TESTSET")) self.assertTrue(collection.is_cjk("TESTSET2")) self.assertIsNone(collection.is_cjk("TESTSET3")) aset = collection.set_list("TESTSET") self.assertIsNotNone(aset) self.assertEqual(10, collection.count_words_in_sets()) collection.remove("TESTSET2") self.assertTrue(collection.contains("TESTSET")) self.assertFalse(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) collection.empty() self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertFalse(collection.contains("TESTSET")) self.assertIsNone(collection.set_list("TESTSET")) self.assertNotEquals(collection.store_name("TESTSET"), "teststore")
def __init__(self, bot, configuration: BrainConfiguration): assert bot is not None assert configuration is not None self._questions = 0 self._bot = bot self._configuration = configuration self._pattern_factory = None self._template_factory = None self._binaries = BinariesManager(configuration.binaries) self._braintree = BraintreeManager(configuration.braintree) self._tokenizer = Tokenizer.load_tokenizer(configuration.tokenizer) self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = Person2Collection() self._rdf_collection = RDFCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._default_variables_collection = DefaultVariablesCollection() self._regex_templates = RegexTemplatesCollection() self._dynamics_collection = DynamicsCollection() self._preprocessors = PreProcessorCollection() self._postprocessors = PostProcessorCollection() self._postquestionprocessors = PostQuestionProcessorCollection() self._services = ServiceHandler() self._oobhandler = OOBHandler() self._security = SecurityManager(configuration.security) self._aiml_parser = self.load_aiml_parser() self.load(self.configuration)
def test_load_with_exception(self): storage_factory = StorageFactory() file_store_config = FileStorageConfiguration() file_store_config._sets_storage = FileStoreConfiguration(dirs=[ os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets" ]) storage_engine = FileStorageEngine(file_store_config) storage_factory._storage_engines[StorageFactory.SETS] = storage_engine storage_factory._store_to_engine_map[ StorageFactory.SETS] = storage_engine collection = SetCollection() self.assertIsNotNone(collection) self.assertTrue(collection.load(storage_factory) == 0)
def test_collection_duplicate(self): collection = SetCollection() set_dict = { "A": [["A", "A B", "A C"]], "D": [["D"]], "E": [["E", "E F"]] } values = { "A": "A", "A B": "A B", "A C": "A C", "D": "D", "E": "E", "E F": "E F" } collection.add_set("TESTSET", set_dict, "teststore", False, values) set_dict = { "1": [["1", "1 2", "1 3"]], "4": [["4"]], "5": [["5", "5 6"]] } values = { "1": "1", "1 2": "1 2", "1 3": "1 3", "4": "4", "5": "5", "5 6": "5 6" } collection.add_set("TESTSET", set_dict, "teststore", False, values) aset = collection.set_list('TESTSET') self.assertIsNotNone(aset) values = aset['A'] self.assertIsNotNone(values) self.assertTrue(["A", "A B", "A C"] in values) self.assertTrue("D" in aset) self.assertTrue("E" in aset) self.assertFalse("1" in aset) self.assertFalse("4" in aset) self.assertFalse("5" in aset)
def assert_upload_from_text(self, store): store.empty () store.upload_from_text('TESTSET', """ VAL1 VAL2 VAL3 VAL4 """) set_collection = SetCollection() store.load(set_collection, 'TESTSET') self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values)
def test_load(self): config = FileStorageConfiguration() engine = FileStorageEngine(config) engine.initialise() store = FileSetsStore(engine) set_collection = SetCollection() self.assertTrue( store.load( set_collection, os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "text" + os.sep + "testset.txt")) self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values)
def __init__(self, configuration: BrainConfiguration): self._configuration = configuration self._aiml_parser = AIMLParser() self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = PersonCollection() self._predicates_collection = PredicatesCollection() self._pronouns_collection = PronounsCollection() self._triples_collection = TriplesCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._preprocessors = ProcessorLoader() self._postprocessors = ProcessorLoader() self.load(self._configuration)
def test_collection_operations(self): collection = SetCollection() collection.add_set("TESTSET", { "A": [["A", "B", "C"]], "D": [["D"]], "E": [["E", "F"]] }, "teststore") collection.add_set("TESTSET2", { "1": [["1", "2", "3"]], "4": [["4"]], "5": [["5", "6"]] }, "teststore") self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertTrue(collection.contains("TESTSET")) self.assertTrue(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) self.assertEqual("teststore", collection.storename("TESTSET")) self.assertIsNone(collection.storename("TESTMAP4")) aset = collection.set("TESTSET") self.assertIsNotNone(aset) self.assertEqual(6, collection.count_words_in_sets()) collection.remove("TESTSET2") self.assertTrue(collection.contains("TESTSET")) self.assertFalse(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) collection.empty() self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertFalse(collection.contains("TESTSET")) self.assertIsNone(collection.set("TESTSET")) self.assertNotEquals(collection.storename("TESTSET"), "teststore")
def test_collection(self): loader = SetLoader() self.assertIsNotNone(loader) collection = SetCollection() self.assertIsNotNone(collection) collection._sets = loader.load_from_text(""" Val1 Val2 Val3 Val4 """) self.assertIsNotNone(collection._sets) self.assertEqual(len(collection._sets), 4) self.assertTrue(collection.contains('VAL1')) self.assertTrue(collection.contains('VAL2')) self.assertTrue(collection.contains('VAL3')) self.assertTrue(collection.contains('VAL4')) self.assertFalse(collection.contains('VAL5'))
def test_load_single_file(self): config = FileStorageConfiguration() config._sets_storage = FileStoreConfiguration(dirs=[ os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "text" + os.sep + "testset.txt" ], extension="txt", format="text", encoding="utf-8", delete_on_start=False) engine = FileStorageEngine(config) engine.initialise() store = FileSetsStore(engine) set_collection = SetCollection() store.load(set_collection) self.assertTrue(set_collection.contains('TESTSET')) values = set_collection.set_list('TESTSET') self.assertTrue('VAL1' in values) self.assertTrue('VAL2' in values) self.assertTrue('VAL3' in values) self.assertTrue('VAL4' in values)
def test_collection_operations(self): collection = SetCollection() set_dict = { "A": [["A", "A B", "A C"]], "D": [["D"]], "E": [["E", "E F"]] } values = { "A": "A", "A B": "A B", "A C": "A C", "D": "D", "E": "E", "E F": "E F" } collection.add_set("TESTSET", set_dict, "teststore", False, values) set_dict = { "1": [["1", "1 2", "1 3"]], "4": [["4"]], "5": [["5", "5 6"]] } values = { "1": "1", "1 2": "1 2", "1 3": "1 3", "4": "4", "5": "5", "5 6": "5 6" } collection.add_set("TESTSET2", set_dict, "teststore", False, values) self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertTrue(collection.contains("TESTSET")) self.assertTrue(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) self.assertEqual(collection.store_name("TESTSET"), "teststore") self.assertEqual(collection.store_name("TESTSET2"), "teststore") self.assertIsNone(collection.store_name("TESTSET3")) self.assertFalse(collection.is_cjk("TESTSET")) self.assertFalse(collection.is_cjk("TESTSET2")) self.assertIsNone(collection.is_cjk("TESTSET3")) aset = collection.set_list("TESTSET") self.assertIsNotNone(aset) self.assertEqual(12, collection.count_words_in_sets()) collection.remove("TESTSET2") self.assertTrue(collection.contains("TESTSET")) self.assertFalse(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) collection.empty() self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertFalse(collection.contains("TESTSET")) self.assertIsNone(collection.set_list("TESTSET")) self.assertNotEquals(collection.store_name("TESTSET"), "teststore")
def test_initialise_collection(self): collection = SetCollection() self.assertIsNotNone(collection) self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertIsNotNone(collection.is_cjk)
def __init__(self, bot, configuration: BrainConfiguration): assert (bot is not None) assert (configuration is not None) self._bot = bot self._configuration = configuration self._binaries = BinariesManager(configuration.binaries) self._braintree = BraintreeManager(configuration.braintree) self._tokenizer = Tokenizer.load_tokenizer(configuration) if configuration.debugfiles.save_errors_collection is True: errors_dict = {} else: errors_dict = None self._denormal_collection = DenormalCollection(errors_dict) self._normal_collection = NormalCollection(errors_dict) self._gender_collection = GenderCollection(errors_dict) self._person_collection = PersonCollection(errors_dict) self._person2_collection = Person2Collection(errors_dict) self._rdf_collection = RDFCollection(errors_dict) self._sets_collection = SetCollection(errors_dict) self._maps_collection = MapCollection(errors_dict) self._properties_collection = PropertiesCollection(errors_dict) self._default_variables_collection = DefaultVariablesCollection( errors_dict) self._botnames_collection = BotNamesCollection(errors_dict) self._preprocessors = PreProcessorCollection(errors_dict) self._postprocessors = PostProcessorCollection(errors_dict) self._pattern_factory = None self._template_factory = None self._security = SecurityManager(configuration.security) self._oobhandler = OOBHandler(configuration.oob) self._regex_templates = RegexTemplatesCollection(errors_dict) self._dynamics_collection = DynamicsCollection() self._aiml_parser = self.load_aiml_parser() self._nlu_collection = NluCollection(bot.client, configuration.nlu, errors_dict) self._nlu = NluRequest.load_nlu(configuration.nlu) self._nlu_utterance = None self.load(self.configuration) if configuration.debugfiles.save_errors_collection is True: storage_factory = self.bot.client.storage_factory if storage_factory.entity_storage_engine_available( StorageFactory.ERRORS_COLLECTION) is True: errors_collection_engine = storage_factory.entity_storage_engine( StorageFactory.ERRORS_COLLECTION) errors_collection_store = errors_collection_engine.errors_collection_store( ) errors_collection_store.save_errors_collection(errors_dict)