def assert_upload_csv_files_from_directory_with_subdir(self, store):
store.empty()
store.upload_from_directory(os.path.dirname(__file__) + os.sep +
"data" + os.sep + "sets" + os.sep + "csv",
format=Store.CSV_FORMAT)
set_collection = SetCollection()
store.load(set_collection, 'TESTSET')
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
set_collection = SetCollection()
store.load(set_collection, 'TESTSET2')
self.assertTrue(set_collection.contains('TESTSET2'))
values = set_collection.set('TESTSET2')
self.assertEqual(4, len(values))
self.assertTrue('VAL5' in values)
self.assertTrue('VAL6' in values)
self.assertTrue('VAL7' in values)
self.assertTrue('VAL8' in values)
def __init__(self, bot, configuration: BrainConfiguration):
self._bot = bot
self._configuration = configuration
self._tokenizer = self.load_tokenizer()
self._aiml_parser = self.load_aiml_parser()
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._variables_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self.configuration)
self.dump_brain_tree()
def __init__(self, configuration: BrainConfiguration):
self._configuration = configuration
self._aiml_parser = AIMLParser(self)
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self._configuration)
def test_load_from_file(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._sets_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets"
])
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[StorageFactory.SETS] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.SETS] = storage_engine
collection = SetCollection()
self.assertIsNotNone(collection)
self.assertTrue(collection.load(storage_factory) > 0)
self.assertIsNotNone(collection._sets)
self.assertEqual(len(collection._sets), 1)
self.assertIsNotNone(collection._stores)
self.assertEqual(len(collection._stores), 1)
self.assertTrue("TEST_SET" in collection._sets)
self.assertTrue("TEST_SET" in collection._stores)
self.assertTrue(collection.contains('TEST_SET'))
aset = collection.set('TEST_SET')
self.assertIsNotNone(aset)
values = aset['AIR']
self.assertIsNotNone(values)
self.assertTrue(['Air', 'Force', 'blue'] in values)
def test_collection_duplicate(self):
collection = SetCollection()
set_dict = {
"A": [["A", "A B", "A C"]],
"D": [["D"]],
"E": [["E", "E F"]]
}
values = {
"A": "A",
"A B": "A B",
"A C": "A C",
"D": "D",
"E": "E",
"E F": "E F"
}
collection.add_set("TESTSET", set_dict, "teststore", False, values)
with self.assertRaises(Exception):
set_dict = {
"1": [["1", "1 2", "1 3"]],
"4": [["4"]],
"5": [["5", "5 6"]]
}
values = {
"1": "1",
"1 2": "1 2",
"1 3": "1 3",
"4": "4",
"5": "5",
"5 6": "5 6"
}
collection.add_set("TESTSET", set_dict, "teststore", False, values)
def test_load_from_test_dir_with_subdir(self):
config = FileStorageConfiguration()
config._sets_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" +
os.sep + "text"
],
extension="txt",
subdirs=True,
format="text",
encoding="utf-8",
delete_on_start=False)
engine = FileStorageEngine(config)
engine.initialise()
store = FileSetsStore(engine)
set_collection = SetCollection()
store.load_all(set_collection)
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set_list('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
self.assertTrue(set_collection.contains('TESTSET2'))
values = set_collection.set_list('TESTSET2')
self.assertEqual(4, len(values))
self.assertTrue('VAL5' in values)
self.assertTrue('VAL6' in values)
self.assertTrue('VAL7' in values)
self.assertTrue('VAL8' in values)
def test_reload_from_file_jp(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._sets_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "test_files" + os.sep +
"sets_jp"
])
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[StorageFactory.SETS] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.SETS] = storage_engine
collection = SetCollection()
self.assertIsNotNone(collection)
collection.load(storage_factory)
self.assertIsNotNone(collection._sets)
self.assertEqual(len(collection._sets), 1)
self.assertIsNotNone(collection._stores)
self.assertEqual(len(collection._stores), 1)
self.assertTrue("TEST_SET" in collection._sets)
self.assertTrue("TEST_SET" in collection._stores)
self.assertTrue("TEST_SET" in collection._is_cjk)
self.assertTrue(collection.contains('TEST_SET'))
self.assertTrue(collection.is_cjk('TEST_SET'))
aset = collection.set_list('TEST_SET')
self.assertIsNotNone(aset)
values = aset['千']
self.assertIsNotNone(values)
self.assertEqual(['千葉', '千葉県', '千葉県下'], values)
collection.reload(storage_factory, "TEST_SET")
self.assertIsNotNone(collection._sets)
self.assertEqual(len(collection._sets), 1)
self.assertIsNotNone(collection._stores)
self.assertEqual(len(collection._stores), 1)
self.assertTrue("TEST_SET" in collection._sets)
self.assertTrue("TEST_SET" in collection._stores)
self.assertTrue(collection.contains('TEST_SET'))
values = aset['千']
self.assertIsNotNone(values)
self.assertEqual(['千葉', '千葉県', '千葉県下'], values)
def test_load_with_exception(self):
config = FileStorageConfiguration()
engine = FileStorageEngine(config)
engine.initialise()
store = FileSetsStore(engine)
set_collection = SetCollection()
self.assertFalse(
store.load(
set_collection,
os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" +
os.sep + "text" + os.sep + "testset.txt"))
def test_reload_no_storage(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._sets_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets"
])
collection = SetCollection()
self.assertIsNotNone(collection)
self.assertTrue(collection.reload(storage_factory, "TEST_SET") == 0)
def assert_upload_from_text_file(self, store):
store.empty()
store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "text" + os.sep + "testset.txt")
set_collection = SetCollection()
store.load(set_collection, 'TESTSET')
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
def test_add_set_exists(self):
collection = SetCollection()
collection.add_set("TESTSET", {
"A": [["A", "B", "C"]],
"D": [["D"]],
"E": [["E", "F"]]
}, "teststore")
with self.assertRaises(Exception):
collection.add_set("TESTSET", {
"1": [["1", "2", "3"]],
"4": [["4"]],
"5": [["5", "6"]]
}, "teststore")
def assert_upload_text_files_from_directory_no_subdir(self, store):
store.empty ()
store.upload_from_directory(os.path.dirname(__file__)+os.sep+"data"+os.sep+"sets"+os.sep+"text", subdir=False)
set_collection = SetCollection()
store.load(set_collection, 'TESTSET')
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
def assert_upload_from_csv_file(self, store):
store.empty()
store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" + os.sep + "csv" + os.sep + "testset.csv", fileformat=Store.CSV_FORMAT)
set_collection = SetCollection()
store.load(set_collection, 'TESTSET')
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
def test_collection_operations_jp(self):
collection = SetCollection()
set_dict = {"千": ["千葉", "千葉県"], "東": ["東京", "東京都"]}
values = {"千葉": "千葉", "千葉県": "千葉県", "東京": "東京", "東京都": "東京都"}
collection.add_set("TESTSET", set_dict, "teststore", True, values)
set_dict = {"神": ["神奈川", "神戸", "神田"], "大": ["大阪", "大分", "大津"]}
values = {
"神奈川": "神奈川",
"神戸": "神戸",
"神田": "神田",
"大阪": "大阪",
"大分": "大分",
"大津": "大津"
}
collection.add_set("TESTSET2", set_dict, "teststore", True, values)
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertTrue(collection.contains("TESTSET"))
self.assertTrue(collection.contains("TESTSET2"))
self.assertFalse(collection.contains("TESTSET3"))
self.assertEqual(collection.store_name("TESTSET"), "teststore")
self.assertEqual(collection.store_name("TESTSET2"), "teststore")
self.assertIsNone(collection.store_name("TESTSET3"))
self.assertTrue(collection.is_cjk("TESTSET"))
self.assertTrue(collection.is_cjk("TESTSET2"))
self.assertIsNone(collection.is_cjk("TESTSET3"))
aset = collection.set_list("TESTSET")
self.assertIsNotNone(aset)
self.assertEqual(10, collection.count_words_in_sets())
collection.remove("TESTSET2")
self.assertTrue(collection.contains("TESTSET"))
self.assertFalse(collection.contains("TESTSET2"))
self.assertFalse(collection.contains("TESTSET3"))
collection.empty()
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertFalse(collection.contains("TESTSET"))
self.assertIsNone(collection.set_list("TESTSET"))
self.assertNotEquals(collection.store_name("TESTSET"), "teststore")
def __init__(self, bot, configuration: BrainConfiguration):
assert bot is not None
assert configuration is not None
self._questions = 0
self._bot = bot
self._configuration = configuration
self._pattern_factory = None
self._template_factory = None
self._binaries = BinariesManager(configuration.binaries)
self._braintree = BraintreeManager(configuration.braintree)
self._tokenizer = Tokenizer.load_tokenizer(configuration.tokenizer)
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = Person2Collection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._default_variables_collection = DefaultVariablesCollection()
self._regex_templates = RegexTemplatesCollection()
self._dynamics_collection = DynamicsCollection()
self._preprocessors = PreProcessorCollection()
self._postprocessors = PostProcessorCollection()
self._postquestionprocessors = PostQuestionProcessorCollection()
self._services = ServiceHandler()
self._oobhandler = OOBHandler()
self._security = SecurityManager(configuration.security)
self._aiml_parser = self.load_aiml_parser()
self.load(self.configuration)
def test_load_with_exception(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._sets_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "sets"
])
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[StorageFactory.SETS] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.SETS] = storage_engine
collection = SetCollection()
self.assertIsNotNone(collection)
self.assertTrue(collection.load(storage_factory) == 0)
def test_collection_duplicate(self):
collection = SetCollection()
set_dict = {
"A": [["A", "A B", "A C"]],
"D": [["D"]],
"E": [["E", "E F"]]
}
values = {
"A": "A",
"A B": "A B",
"A C": "A C",
"D": "D",
"E": "E",
"E F": "E F"
}
collection.add_set("TESTSET", set_dict, "teststore", False, values)
set_dict = {
"1": [["1", "1 2", "1 3"]],
"4": [["4"]],
"5": [["5", "5 6"]]
}
values = {
"1": "1",
"1 2": "1 2",
"1 3": "1 3",
"4": "4",
"5": "5",
"5 6": "5 6"
}
collection.add_set("TESTSET", set_dict, "teststore", False, values)
aset = collection.set_list('TESTSET')
self.assertIsNotNone(aset)
values = aset['A']
self.assertIsNotNone(values)
self.assertTrue(["A", "A B", "A C"] in values)
self.assertTrue("D" in aset)
self.assertTrue("E" in aset)
self.assertFalse("1" in aset)
self.assertFalse("4" in aset)
self.assertFalse("5" in aset)
def assert_upload_from_text(self, store):
store.empty ()
store.upload_from_text('TESTSET', """
VAL1
VAL2
VAL3
VAL4
""")
set_collection = SetCollection()
store.load(set_collection, 'TESTSET')
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
def test_load(self):
config = FileStorageConfiguration()
engine = FileStorageEngine(config)
engine.initialise()
store = FileSetsStore(engine)
set_collection = SetCollection()
self.assertTrue(
store.load(
set_collection,
os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" +
os.sep + "text" + os.sep + "testset.txt"))
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
def __init__(self, configuration: BrainConfiguration):
self._configuration = configuration
self._aiml_parser = AIMLParser()
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._predicates_collection = PredicatesCollection()
self._pronouns_collection = PronounsCollection()
self._triples_collection = TriplesCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self.load(self._configuration)
def test_collection_operations(self):
collection = SetCollection()
collection.add_set("TESTSET", {
"A": [["A", "B", "C"]],
"D": [["D"]],
"E": [["E", "F"]]
}, "teststore")
collection.add_set("TESTSET2", {
"1": [["1", "2", "3"]],
"4": [["4"]],
"5": [["5", "6"]]
}, "teststore")
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertTrue(collection.contains("TESTSET"))
self.assertTrue(collection.contains("TESTSET2"))
self.assertFalse(collection.contains("TESTSET3"))
self.assertEqual("teststore", collection.storename("TESTSET"))
self.assertIsNone(collection.storename("TESTMAP4"))
aset = collection.set("TESTSET")
self.assertIsNotNone(aset)
self.assertEqual(6, collection.count_words_in_sets())
collection.remove("TESTSET2")
self.assertTrue(collection.contains("TESTSET"))
self.assertFalse(collection.contains("TESTSET2"))
self.assertFalse(collection.contains("TESTSET3"))
collection.empty()
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertFalse(collection.contains("TESTSET"))
self.assertIsNone(collection.set("TESTSET"))
self.assertNotEquals(collection.storename("TESTSET"), "teststore")
def test_collection(self):
loader = SetLoader()
self.assertIsNotNone(loader)
collection = SetCollection()
self.assertIsNotNone(collection)
collection._sets = loader.load_from_text("""
Val1
Val2
Val3
Val4
""")
self.assertIsNotNone(collection._sets)
self.assertEqual(len(collection._sets), 4)
self.assertTrue(collection.contains('VAL1'))
self.assertTrue(collection.contains('VAL2'))
self.assertTrue(collection.contains('VAL3'))
self.assertTrue(collection.contains('VAL4'))
self.assertFalse(collection.contains('VAL5'))
def test_load_single_file(self):
config = FileStorageConfiguration()
config._sets_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "data" + os.sep + "sets" +
os.sep + "text" + os.sep + "testset.txt"
],
extension="txt",
format="text",
encoding="utf-8",
delete_on_start=False)
engine = FileStorageEngine(config)
engine.initialise()
store = FileSetsStore(engine)
set_collection = SetCollection()
store.load(set_collection)
self.assertTrue(set_collection.contains('TESTSET'))
values = set_collection.set_list('TESTSET')
self.assertTrue('VAL1' in values)
self.assertTrue('VAL2' in values)
self.assertTrue('VAL3' in values)
self.assertTrue('VAL4' in values)
def test_collection_operations(self):
collection = SetCollection()
set_dict = {
"A": [["A", "A B", "A C"]],
"D": [["D"]],
"E": [["E", "E F"]]
}
values = {
"A": "A",
"A B": "A B",
"A C": "A C",
"D": "D",
"E": "E",
"E F": "E F"
}
collection.add_set("TESTSET", set_dict, "teststore", False, values)
set_dict = {
"1": [["1", "1 2", "1 3"]],
"4": [["4"]],
"5": [["5", "5 6"]]
}
values = {
"1": "1",
"1 2": "1 2",
"1 3": "1 3",
"4": "4",
"5": "5",
"5 6": "5 6"
}
collection.add_set("TESTSET2", set_dict, "teststore", False, values)
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertTrue(collection.contains("TESTSET"))
self.assertTrue(collection.contains("TESTSET2"))
self.assertFalse(collection.contains("TESTSET3"))
self.assertEqual(collection.store_name("TESTSET"), "teststore")
self.assertEqual(collection.store_name("TESTSET2"), "teststore")
self.assertIsNone(collection.store_name("TESTSET3"))
self.assertFalse(collection.is_cjk("TESTSET"))
self.assertFalse(collection.is_cjk("TESTSET2"))
self.assertIsNone(collection.is_cjk("TESTSET3"))
aset = collection.set_list("TESTSET")
self.assertIsNotNone(aset)
self.assertEqual(12, collection.count_words_in_sets())
collection.remove("TESTSET2")
self.assertTrue(collection.contains("TESTSET"))
self.assertFalse(collection.contains("TESTSET2"))
self.assertFalse(collection.contains("TESTSET3"))
collection.empty()
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertFalse(collection.contains("TESTSET"))
self.assertIsNone(collection.set_list("TESTSET"))
self.assertNotEquals(collection.store_name("TESTSET"), "teststore")
def test_initialise_collection(self):
collection = SetCollection()
self.assertIsNotNone(collection)
self.assertIsNotNone(collection.sets)
self.assertIsNotNone(collection.stores)
self.assertIsNotNone(collection.is_cjk)
def __init__(self, bot, configuration: BrainConfiguration):
assert (bot is not None)
assert (configuration is not None)
self._bot = bot
self._configuration = configuration
self._binaries = BinariesManager(configuration.binaries)
self._braintree = BraintreeManager(configuration.braintree)
self._tokenizer = Tokenizer.load_tokenizer(configuration)
if configuration.debugfiles.save_errors_collection is True:
errors_dict = {}
else:
errors_dict = None
self._denormal_collection = DenormalCollection(errors_dict)
self._normal_collection = NormalCollection(errors_dict)
self._gender_collection = GenderCollection(errors_dict)
self._person_collection = PersonCollection(errors_dict)
self._person2_collection = Person2Collection(errors_dict)
self._rdf_collection = RDFCollection(errors_dict)
self._sets_collection = SetCollection(errors_dict)
self._maps_collection = MapCollection(errors_dict)
self._properties_collection = PropertiesCollection(errors_dict)
self._default_variables_collection = DefaultVariablesCollection(
errors_dict)
self._botnames_collection = BotNamesCollection(errors_dict)
self._preprocessors = PreProcessorCollection(errors_dict)
self._postprocessors = PostProcessorCollection(errors_dict)
self._pattern_factory = None
self._template_factory = None
self._security = SecurityManager(configuration.security)
self._oobhandler = OOBHandler(configuration.oob)
self._regex_templates = RegexTemplatesCollection(errors_dict)
self._dynamics_collection = DynamicsCollection()
self._aiml_parser = self.load_aiml_parser()
self._nlu_collection = NluCollection(bot.client, configuration.nlu,
errors_dict)
self._nlu = NluRequest.load_nlu(configuration.nlu)
self._nlu_utterance = None
self.load(self.configuration)
if configuration.debugfiles.save_errors_collection is True:
storage_factory = self.bot.client.storage_factory
if storage_factory.entity_storage_engine_available(
StorageFactory.ERRORS_COLLECTION) is True:
errors_collection_engine = storage_factory.entity_storage_engine(
StorageFactory.ERRORS_COLLECTION)
errors_collection_store = errors_collection_engine.errors_collection_store(
)
errors_collection_store.save_errors_collection(errors_dict)