def test_reload(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._normal_storage = FileStoreConfiguration(
file=os.path.dirname(__file__) + os.sep + "test_files" + os.sep +
"normal.txt",
format="text",
extension="txt",
encoding="utf-8",
delete_on_start=False)
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[
StorageFactory.NORMAL] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.NORMAL] = storage_engine
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.load(storage_factory)
self.assertEqual(
collection.normalise_string(None, "keithsterling.com"),
"keithsterling dot com")
collection.reload(storage_factory)
self.assertEqual(
collection.normalise_string(None, "keithsterling.com"),
"keithsterling dot com")
def __init__(self, bot, configuration: BrainConfiguration):
self._bot = bot
self._configuration = configuration
self._tokenizer = self.load_tokenizer()
self._aiml_parser = self.load_aiml_parser()
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._variables_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self.configuration)
self.dump_brain_tree()
def __init__(self, configuration: BrainConfiguration):
self._configuration = configuration
self._aiml_parser = AIMLParser(self)
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self._configuration)
def assert_upload_from_text(self, store):
store.empty()
store.upload_from_text(
None, """
".uk"," dot uk "
".net"," dot net "
".ca"," dot ca "
".de"," dot de "
".jp"," dot jp "
".fr"," dot fr "
".au"," dot au "
".us"," dot us "
".ru"," dot ru "
".ch"," dot ch "
".it"," dot it "
".nl"," dot nl "
".se"," dot se "
".no"," dot no "
".es"," dot es "
""")
collection = NormalCollection()
store.load(collection)
self.assertEqual(
collection.normalise(".UK"),
[re.compile('(^\\.UK|\\.UK|\\.UK$)', re.IGNORECASE), ' DOT UK '])
self.assertEqual(collection.normalise_string("keiffster.uk"),
"keiffster dot uk")
def test_normalise(self):
collection = NormalCollection ()
self.assertIsNotNone(collection)
count = collection.load_from_text("""
" www. ","www dot "
" www."," www dot "
".com "," dot com "
"%24"," dollars "
"%27","'"
"%2A","*"
"%2D","-"
"%2d","-"
"%2E","."
"%2e","."
" aren t "," are not "
" aren.t "," are not "
" arent "," are not "
" aren't "," are not "
" are'nt "," are not "
" arn t "," are not "
""")
self.assertEqual(count, 16)
self.assertEqual(collection.normalise_string("That will be 24 %24"), "That will be 24 dollars")
self.assertEqual(collection.normalise_string("You aren't him"), "You are not him")
self.assertEqual(collection.normalise_string("www.google.com"), "www dot google dot com")
self.assertIsNone(collection.normalise(" other "))
def test_reload_jp(self):
storage_factory = StorageFactory()
tokenizer = TokenizerJP()
file_store_config = FileStorageConfiguration()
file_store_config._normal_storage = FileStoreConfiguration(
file=os.path.dirname(__file__) + os.sep + "test_files" + os.sep +
"normal_jp.txt",
format="text",
extension="txt",
encoding="utf-8",
delete_on_start=False)
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[
StorageFactory.NORMAL] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.NORMAL] = storage_engine
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.load(storage_factory)
self.assertEqual("丸1の回答",
collection.normalise_string(tokenizer, "①の回答"))
collection.reload(storage_factory)
self.assertEqual("丸1の回答",
collection.normalise_string(tokenizer, "①の回答"))
def test_load(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._normal_storage = FileStoreConfiguration(
file=os.path.dirname(__file__) + os.sep + "test_files" + os.sep +
"normal.txt",
fileformat="text",
extension="txt",
encoding="utf-8",
delete_on_start=False)
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[
StorageFactory.NORMAL] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.NORMAL] = storage_engine
collection = NormalCollection()
self.assertIsNotNone(collection)
self.assertTrue(collection.load(storage_factory))
self.assertEqual(collection.normalise_string("keithsterling.COM"),
"keithsterling dot com")
self.assertEquals([
re.compile('(^\\.COM|\\.COM|\\.COM$)', re.IGNORECASE), ' DOT COM '
], collection.normalise(".COM"))
self.assertEquals(None, collection.normalise(".XXX"))
def test_collection_words_duplicate(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup(" don't", "do not")
collection.add_to_lookup(" don't", "donot")
self.assertEqual("he do not it",
collection.normalise_string(None, "he don't it"))
def test_collection_string_duplicate(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup(".com", 'dot com')
collection.add_to_lookup(".com", "dot co")
self.assertEqual(
"keithsterling dot com",
collection.normalise_string(None, "keithsterling.com"))
def assert_upload_from_file(self, store):
normal_collection = NormalCollection()
store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "lookups" + os.sep + "text" + os.sep + "normal.txt")
store.load(normal_collection)
self.assertEqual(normal_collection.normalise(".COM"),
[re.compile('(^\\.COM|\\.COM|\\.COM$)', re.IGNORECASE), ' DOT COM '])
self.assertEqual(normal_collection.normalise_string("keith.com"), "keith DOT COM")
def test_collection_duplicate_jp(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup("①", '丸1')
collection.add_to_lookup("①", '丸2')
tokenizer = TokenizerJP()
self.assertEqual("丸1の回答",
collection.normalise_string(tokenizer, "①の回答"))
def test_collection_operations(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup(".COM", [re.compile('(^\\.COM|\\.COM|\\.COM$)', re.IGNORECASE), ' DOT COM '])
self.assertTrue(collection.has_key(".COM"))
self.assertEqual([re.compile('(^\\.COM|\\.COM|\\.COM$)', re.IGNORECASE), ' DOT COM '], collection.value(".COM"))
self.assertEqual("keithsterling dot com", collection.normalise_string("keithsterling.COM"))
def test_collection_words_invalid(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup(" do not", "do not")
self.assertFalse(collection.has_keyVal("dont"))
self.assertIsNone(collection.value("dont"))
self.assertIsNone(collection.normalise("dont"))
self.assertEqual("he do nt it",
collection.normalise_string(None, "he do nt it"))
def test_collection_operations_JP(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup("①", '丸１')
tokenizer = TokenizerJP()
self.assertTrue(collection.has_keyVal("①"))
self.assertEqual('丸１', collection.value("①"))
self.assertEqual("丸１の回答",
collection.normalise_string(tokenizer, "①の回答"))
def test_collection_string_invalid(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup(".com", 'dot com')
self.assertFalse(collection.has_replace_key(".cox"))
self.assertIsNone(collection.replace_value(".cox"))
self.assertIsNone(collection.normalise(".cox"))
self.assertEqual(
"keithsterling dot com",
collection.normalise_string(None, "keithsterling.com"))
def test_load_from_file(self):
config = FileStorageConfiguration()
config._normal_storage = FileStoreConfiguration(file=os.path.dirname(__file__) + os.sep + "data" + os.sep + "lookups" + os.sep + "text" + os.sep + "normal.txt", fileformat="text", encoding="utf-8", delete_on_start=False)
engine = FileStorageEngine(config)
engine.initialise()
store = FileNormalStore(engine)
normal_collection = NormalCollection()
store.load(normal_collection)
self.assertEqual(normal_collection.normalise(".COM"), [re.compile('(^\\.COM|\\.COM|\\.COM$)', re.IGNORECASE), ' DOT COM '])
self.assertEqual(normal_collection.normalise_string("keith.com"), "keith dot com")
def test_collection_invalid_jp(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.add_to_lookup("彼岸", 'お彼岸')
self.assertFalse(collection.has_keyVal("彼氏"))
self.assertIsNone(collection.value("彼氏"))
tokenizer = TokenizerJP()
self.assertIsNone(collection.normalise("彼氏"))
self.assertEqual("彼氏の回答",
collection.normalise_string(tokenizer, "彼氏の回答"))
def assert_upload_from_text_file(self, store):
store.empty()
store.upload_from_file(
os.path.dirname(__file__) + os.sep + "data" + os.sep + "lookups" +
os.sep + "text" + os.sep + "normal.txt")
collection = NormalCollection()
store.load(collection)
self.assertEqual(
collection.normalise(".UK"),
[re.compile('(^\\.UK|\\.UK|\\.UK$)', re.IGNORECASE), ' DOT UK '])
self.assertEqual(collection.normalise_string("keiffster.uk"),
"keiffster dot uk")
def __init__(self, bot, configuration: BrainConfiguration):
assert bot is not None
assert configuration is not None
self._questions = 0
self._bot = bot
self._configuration = configuration
self._pattern_factory = None
self._template_factory = None
self._binaries = BinariesManager(configuration.binaries)
self._braintree = BraintreeManager(configuration.braintree)
self._tokenizer = Tokenizer.load_tokenizer(configuration.tokenizer)
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = Person2Collection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._default_variables_collection = DefaultVariablesCollection()
self._regex_templates = RegexTemplatesCollection()
self._dynamics_collection = DynamicsCollection()
self._preprocessors = PreProcessorCollection()
self._postprocessors = PostProcessorCollection()
self._postquestionprocessors = PostQuestionProcessorCollection()
self._services = ServiceHandler()
self._oobhandler = OOBHandler()
self._security = SecurityManager(configuration.security)
self._aiml_parser = self.load_aiml_parser()
self.load(self.configuration)
def __init__(self, configuration: BrainConfiguration):
self._configuration = configuration
self._aiml_parser = AIMLParser()
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._predicates_collection = PredicatesCollection()
self._pronouns_collection = PronounsCollection()
self._triples_collection = TriplesCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self.load(self._configuration)
def test_collection_replace_words(self):
normal_text = """
" couldn't","could not"
" didn't","did not"
" doesn't","does not"
" don't","do not"
" down load","download"
"""
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.load_from_text(normal_text)
self.assertTrue(collection.has_keyVal("down load"))
self.assertEqual('download', collection.value("down load"))
self.assertEqual('download', collection.normalise("down load"))
self.assertEqual(
"gip file download",
collection.normalise_string(None, "gip file down load"))
def test_collection_replace_string(self):
normal_text = """
".ac ","dot ac"
".au","dot au"
".ca","dot ca"
".ch","dot ch"
".com","dot com"
".co","dot co"
"""
collection = NormalCollection()
self.assertIsNotNone(collection)
collection.load_from_text(normal_text)
self.assertTrue(collection.has_replace_key(".com"))
self.assertEqual('dot com', collection.replace_value(".com"))
self.assertEqual(
"keithsterling dot com",
collection.normalise_string(None, "keithsterling.com"))
def test_load_with_exception(self):
storage_factory = StorageFactory()
file_store_config = FileStorageConfiguration()
file_store_config._normal_storage = FileStoreConfiguration(
file=os.path.dirname(__file__) + os.sep + "test_files" + os.sep +
"normal.txt",
fileformat="text",
extension="txt",
encoding="utf-8",
delete_on_start=False)
storage_engine = FileStorageEngine(file_store_config)
storage_factory._storage_engines[
StorageFactory.NORMAL] = storage_engine
storage_factory._store_to_engine_map[
StorageFactory.NORMAL] = storage_engine
collection = NormalCollection()
self.assertIsNotNone(collection)
self.assertFalse(collection.load(storage_factory))
def test_initialise_collection(self):
collection = NormalCollection()
self.assertIsNotNone(collection)
def __init__(self, bot, configuration: BrainConfiguration):
assert (bot is not None)
assert (configuration is not None)
self._bot = bot
self._configuration = configuration
self._binaries = BinariesManager(configuration.binaries)
self._braintree = BraintreeManager(configuration.braintree)
self._tokenizer = Tokenizer.load_tokenizer(configuration)
if configuration.debugfiles.save_errors_collection is True:
errors_dict = {}
else:
errors_dict = None
self._denormal_collection = DenormalCollection(errors_dict)
self._normal_collection = NormalCollection(errors_dict)
self._gender_collection = GenderCollection(errors_dict)
self._person_collection = PersonCollection(errors_dict)
self._person2_collection = Person2Collection(errors_dict)
self._rdf_collection = RDFCollection(errors_dict)
self._sets_collection = SetCollection(errors_dict)
self._maps_collection = MapCollection(errors_dict)
self._properties_collection = PropertiesCollection(errors_dict)
self._default_variables_collection = DefaultVariablesCollection(
errors_dict)
self._botnames_collection = BotNamesCollection(errors_dict)
self._preprocessors = PreProcessorCollection(errors_dict)
self._postprocessors = PostProcessorCollection(errors_dict)
self._pattern_factory = None
self._template_factory = None
self._security = SecurityManager(configuration.security)
self._oobhandler = OOBHandler(configuration.oob)
self._regex_templates = RegexTemplatesCollection(errors_dict)
self._dynamics_collection = DynamicsCollection()
self._aiml_parser = self.load_aiml_parser()
self._nlu_collection = NluCollection(bot.client, configuration.nlu,
errors_dict)
self._nlu = NluRequest.load_nlu(configuration.nlu)
self._nlu_utterance = None
self.load(self.configuration)
if configuration.debugfiles.save_errors_collection is True:
storage_factory = self.bot.client.storage_factory
if storage_factory.entity_storage_engine_available(
StorageFactory.ERRORS_COLLECTION) is True:
errors_collection_engine = storage_factory.entity_storage_engine(
StorageFactory.ERRORS_COLLECTION)
errors_collection_store = errors_collection_engine.errors_collection_store(
)
errors_collection_store.save_errors_collection(errors_dict)
