def test_collection_operations_jp(self): collection = SetCollection() set_dict = {"千": ["千葉", "千葉県"], "東": ["東京", "東京都"]} values = {"千葉": "千葉", "千葉県": "千葉県", "東京": "東京", "東京都": "東京都"} collection.add_set("TESTSET", set_dict, "teststore", True, values) set_dict = {"神": ["神奈川", "神戸", "神田"], "大": ["大阪", "大分", "大津"]} values = { "神奈川": "神奈川", "神戸": "神戸", "神田": "神田", "大阪": "大阪", "大分": "大分", "大津": "大津" } collection.add_set("TESTSET2", set_dict, "teststore", True, values) self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertTrue(collection.contains("TESTSET")) self.assertTrue(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) self.assertEqual(collection.store_name("TESTSET"), "teststore") self.assertEqual(collection.store_name("TESTSET2"), "teststore") self.assertIsNone(collection.store_name("TESTSET3")) self.assertTrue(collection.is_cjk("TESTSET")) self.assertTrue(collection.is_cjk("TESTSET2")) self.assertIsNone(collection.is_cjk("TESTSET3")) aset = collection.set_list("TESTSET") self.assertIsNotNone(aset) self.assertEqual(10, collection.count_words_in_sets()) collection.remove("TESTSET2") self.assertTrue(collection.contains("TESTSET")) self.assertFalse(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) collection.empty() self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertFalse(collection.contains("TESTSET")) self.assertIsNone(collection.set_list("TESTSET")) self.assertNotEquals(collection.store_name("TESTSET"), "teststore")
def test_collection_operations(self): collection = SetCollection() collection.add_set("TESTSET", { "A": [["A", "B", "C"]], "D": [["D"]], "E": [["E", "F"]] }, "teststore") collection.add_set("TESTSET2", { "1": [["1", "2", "3"]], "4": [["4"]], "5": [["5", "6"]] }, "teststore") self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertTrue(collection.contains("TESTSET")) self.assertTrue(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) self.assertEqual("teststore", collection.storename("TESTSET")) self.assertIsNone(collection.storename("TESTMAP4")) aset = collection.set("TESTSET") self.assertIsNotNone(aset) self.assertEqual(6, collection.count_words_in_sets()) collection.remove("TESTSET2") self.assertTrue(collection.contains("TESTSET")) self.assertFalse(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) collection.empty() self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertFalse(collection.contains("TESTSET")) self.assertIsNone(collection.set("TESTSET")) self.assertNotEquals(collection.storename("TESTSET"), "teststore")
def test_collection_operations(self): collection = SetCollection() set_dict = { "A": [["A", "A B", "A C"]], "D": [["D"]], "E": [["E", "E F"]] } values = { "A": "A", "A B": "A B", "A C": "A C", "D": "D", "E": "E", "E F": "E F" } collection.add_set("TESTSET", set_dict, "teststore", False, values) set_dict = { "1": [["1", "1 2", "1 3"]], "4": [["4"]], "5": [["5", "5 6"]] } values = { "1": "1", "1 2": "1 2", "1 3": "1 3", "4": "4", "5": "5", "5 6": "5 6" } collection.add_set("TESTSET2", set_dict, "teststore", False, values) self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertTrue(collection.contains("TESTSET")) self.assertTrue(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) self.assertEqual(collection.store_name("TESTSET"), "teststore") self.assertEqual(collection.store_name("TESTSET2"), "teststore") self.assertIsNone(collection.store_name("TESTSET3")) self.assertFalse(collection.is_cjk("TESTSET")) self.assertFalse(collection.is_cjk("TESTSET2")) self.assertIsNone(collection.is_cjk("TESTSET3")) aset = collection.set_list("TESTSET") self.assertIsNotNone(aset) self.assertEqual(12, collection.count_words_in_sets()) collection.remove("TESTSET2") self.assertTrue(collection.contains("TESTSET")) self.assertFalse(collection.contains("TESTSET2")) self.assertFalse(collection.contains("TESTSET3")) collection.empty() self.assertIsNotNone(collection.sets) self.assertIsNotNone(collection.stores) self.assertFalse(collection.contains("TESTSET")) self.assertIsNone(collection.set_list("TESTSET")) self.assertNotEquals(collection.store_name("TESTSET"), "teststore")
class Brain(object): NLU_UTTERANCE = 'NLU_Matching' def __init__(self, bot, configuration: BrainConfiguration): assert (bot is not None) assert (configuration is not None) self._bot = bot self._configuration = configuration self._binaries = BinariesManager(configuration.binaries) self._braintree = BraintreeManager(configuration.braintree) self._tokenizer = Tokenizer.load_tokenizer(configuration) if configuration.debugfiles.save_errors_collection is True: errors_dict = {} else: errors_dict = None self._denormal_collection = DenormalCollection(errors_dict) self._normal_collection = NormalCollection(errors_dict) self._gender_collection = GenderCollection(errors_dict) self._person_collection = PersonCollection(errors_dict) self._person2_collection = Person2Collection(errors_dict) self._rdf_collection = RDFCollection(errors_dict) self._sets_collection = SetCollection(errors_dict) self._maps_collection = MapCollection(errors_dict) self._properties_collection = PropertiesCollection(errors_dict) self._default_variables_collection = DefaultVariablesCollection( errors_dict) self._botnames_collection = BotNamesCollection(errors_dict) self._preprocessors = PreProcessorCollection(errors_dict) self._postprocessors = PostProcessorCollection(errors_dict) self._pattern_factory = None self._template_factory = None self._security = SecurityManager(configuration.security) self._oobhandler = OOBHandler(configuration.oob) self._regex_templates = RegexTemplatesCollection(errors_dict) self._dynamics_collection = DynamicsCollection() self._aiml_parser = self.load_aiml_parser() self._nlu_collection = NluCollection(bot.client, configuration.nlu, errors_dict) self._nlu = NluRequest.load_nlu(configuration.nlu) self._nlu_utterance = None self.load(self.configuration) if configuration.debugfiles.save_errors_collection is True: storage_factory = self.bot.client.storage_factory if storage_factory.entity_storage_engine_available( StorageFactory.ERRORS_COLLECTION) is True: errors_collection_engine = storage_factory.entity_storage_engine( StorageFactory.ERRORS_COLLECTION) errors_collection_store = errors_collection_engine.errors_collection_store( ) errors_collection_store.save_errors_collection(errors_dict) def ylogger_type(self): return "brain" @property def id(self): return self._configuration.section_name @property def bot(self): return self._bot @property def configuration(self): return self._configuration @property def aiml_parser(self): return self._aiml_parser @property def denormals(self): return self._denormal_collection @property def normals(self): return self._normal_collection @property def genders(self): return self._gender_collection @property def persons(self): return self._person_collection @property def person2s(self): return self._person2_collection @property def rdf(self): return self._rdf_collection @property def sets(self): return self._sets_collection @property def maps(self): return self._maps_collection @property def properties(self): return self._properties_collection @property def default_variables(self): return self._default_variables_collection @property def botnames(self): return self._botnames_collection @property def preprocessors(self): return self._preprocessors @property def postprocessors(self): return self._postprocessors @property def pattern_factory(self): return self._pattern_factory @property def template_factory(self): return self._template_factory @property def regex_templates(self): return self._regex_templates @property def dynamics(self): return self._dynamics_collection @property def tokenizer(self): return self._tokenizer @property def nlu(self): return self._nlu @property def security(self): return self._security def load_aiml_parser(self): self._load_pattern_nodes() self._load_template_nodes() return AIMLParser(self) def load_aiml(self): YLogger.debug(self, "Loading aiml source brain") self._aiml_parser.load_aiml() def reload_aimls(self): YLogger.debug(self, "Loading aiml source brain") self._aiml_parser.empty() self._aiml_parser.load_aiml() def load(self, configuration: BrainConfiguration): self._load_properties() if self.properties.has_property("punctuation_chars") is True: self.tokenizer.set_configuration_punctuation_chars( self.properties.property("punctuation_chars")) if self.properties.has_property("before_concatenation_rule") is True: self.tokenizer.set_configuration_before_concatenation_rule( self.properties.property("before_concatenation_rule")) if self.properties.has_property("after_concatenation_rule") is True: self.tokenizer.set_configuration_after_concatenation_rule( self.properties.property("after_concatenation_rule")) YLogger.debug(self, "Loading collections") self.load_collections() YLogger.debug(self, "Loading dynamics sets, maps and vars") self.load_dynamics() YLogger.debug(self, "Loading services") self.load_services(configuration) load_aiml = True if self.configuration.binaries.load_binary is True: load_aiml = self._binaries.load_binary( self.bot.client.storage_factory) if load_aiml is False: self._aiml_parser = self._binaries.get_aiml_parser() if load_aiml is True: self.load_aiml() self._binaries.set_aiml_parser(self._aiml_parser) if configuration.binaries.save_binary is True: self._binaries.save_binary(self.bot.client.storage_factory) YLogger.debug(self, "Loading security services") self.load_security_services() YLogger.debug(self, "Loading oob processors") self._oobhandler.load_oob_processors() def dump_brain_tree(self, client_context): self._braintree.dump_brain_tree(client_context) def _load_denormals(self): self._denormal_collection.empty() self._denormal_collection.load(self.bot.client.storage_factory) def _load_normals(self): self._normal_collection.empty() self._normal_collection.load(self.bot.client.storage_factory) def _load_genders(self): self._gender_collection.empty() self._gender_collection.load(self.bot.client.storage_factory) def _load_persons(self): self._person_collection.empty() self._person_collection.load(self.bot.client.storage_factory) def _load_person2s(self): self._person2_collection.empty() self._person2_collection.load(self.bot.client.storage_factory) def _load_botnames(self): self._botnames_collection.empty() self._botnames_collection.load(self.bot.client.storage_factory) def _load_properties(self): self._properties_collection.empty() self._properties_collection.load(self.bot.client.storage_factory) def _load_default_variables(self): self._default_variables_collection.empty() self._default_variables_collection.load( self.bot.client.storage_factory) self._set_system_defined() def _set_system_defined(self): self.set_sentiment_scores(0.0, 0.5) def set_sentiment_scores(self, positivity, subjectivity): if self._default_variables_collection.has_variable( "positivity") is False: self._default_variables_collection.set_value( "positivity", str(positivity)) if self._default_variables_collection.has_variable( "subjectivity") is False: self._default_variables_collection.set_value( "subjectivity", str(subjectivity)) def _load_maps(self): self._maps_collection.empty() self._maps_collection.load(self.bot.client.storage_factory) def reload_map(self, mapname): if self._maps_collection.contains(mapname): self._maps_collection.reload(self.bot.client.storage_factory, mapname) def _load_sets(self): self._sets_collection.empty() self._sets_collection.load(self.bot.client.storage_factory) def reload_set(self, setname): if self._sets_collection.contains(setname): self._sets_collection.reload(self.bot.client.storage_factory, setname) def _load_rdfs(self): self._rdf_collection.empty() self._rdf_collection.load(self.bot.client.storage_factory) def reload_rdf(self, rdfname): if self._rdf_collection.contains(rdfname): self._rdf_collection.reload(self.bot.client.storage_factory, rdfname) def _load_regex_templates(self): self._regex_templates.load(self.bot.client.storage_factory) def _load_preprocessors(self): self._preprocessors.empty() self._preprocessors.load(self.bot.client.storage_factory) def _load_postprocessors(self): self._postprocessors.empty() self._postprocessors.load(self.bot.client.storage_factory) def _load_pattern_nodes(self): self._pattern_factory = PatternNodeFactory() self._pattern_factory.load(self.bot.client.storage_factory) def _load_template_nodes(self): self._template_factory = TemplateNodeFactory() self._template_factory.load(self.bot.client.storage_factory) def load_collections(self): self._load_denormals() self._load_normals() self._load_genders() self._load_persons() self._load_person2s() self._load_default_variables() self._load_botnames() self._load_rdfs() self._load_sets() self._load_maps() self._load_regex_templates() self._load_preprocessors() self._load_postprocessors() def load_services(self, configuration): ServiceFactory.preload_services(configuration.services) def load_security_services(self): self._security.load_security_services(self.bot.client) def load_dynamics(self): if self.configuration.dynamics is not None: self._dynamics_collection.load_from_configuration( self.configuration.dynamics) else: YLogger.debug(self, "No dynamics configuration defined...") def pre_process_question(self, client_context, question): return self.preprocessors.process(client_context, question) def post_process_response(self, client_context, response: str): return self.postprocessors.process(client_context, response) def failed_authentication(self, client_context): return self._security.failed_authentication(client_context) def authenticate_user(self, client_context): return self._security.authenticate_user(client_context) def resolve_matched_template(self, client_context, match_context): assert (client_context is not None) assert (match_context is not None) template_node = match_context.template_node() YLogger.debug(client_context, "AIML Parser evaluating template [%s]", template_node.to_string()) response = template_node.template.resolve(client_context) if self._oobhandler.oob_in_response(response) is True: if client_context.brain.template_factory.exists('oob') is True: response = self._oobhandler.handle(client_context, response) else: YLogger.debug(client_context, "OOB function is disable [%s]", response) return response def ask_question(self, client_context, sentence, srai=False, default_srai=False): assert (client_context is not None) assert (client_context.bot is not None) assert (self._aiml_parser is not None) client_context.brain = self authenticated = self.authenticate_user(client_context) if authenticated is not None: return authenticated conversation = client_context.bot.get_conversation(client_context) answer = None if conversation is not None: if srai is False and default_srai is False: self._nlu_utterance = None try: client_context.userInfo.userInfoPreProcessor( client_context, srai) except Exception: pass topic_pattern = conversation.get_topic_pattern(client_context) that_pattern = conversation.get_that_pattern(client_context, srai) original_base = None if default_srai is True and conversation.internal_base is not None: original_base = conversation.internal_base base = original_base if 'srai_histories' not in base: base['srai_histories'] = [] default_srai_root = base['srai_histories'] default_srai_root.append({}) conversation.internal_base = default_srai_root[-1] conversation.add_internal_data(conversation.internal_base, 'default_srai', True) elif srai is False: conversation.internal_data.append({}) conversation.internal_base = conversation.internal_data[-1] base = conversation.internal_base texts = self.tokenizer.words_to_texts(sentence.words) conversation.add_internal_data(base, 'question', texts) conversation.add_internal_data(base, 'topic', topic_pattern) conversation.add_internal_data(base, 'that', that_pattern) current_variables = ConversationVariables(conversation) match_context = self._aiml_parser.match_sentence( client_context, sentence, topic_pattern=topic_pattern, that_pattern=that_pattern) if match_context is not None: if len(match_context.matched_nodes ) == 3 and match_context.matched_nodes[ 0].matched_node.is_wildcard() is True: if self._aiml_parser.pattern_parser.use_nlu is False: YLogger.debug( client_context, "Matched Catgeory (*): file[%s] line[%s-%s]", match_context._template_node._filename, match_context._template_node._start_line, match_context._template_node._end_line) conversation.add_internal_matched( base, match_context._template_node) answer = self.resolve_matched_template( client_context, match_context) else: YLogger.debug(client_context, "Matched Catgeory : file[%s] line[%s-%s]", match_context._template_node._filename, match_context._template_node._start_line, match_context._template_node._end_line) try: conversation.add_internal_matched( base, match_context._template_node) answer = self.resolve_matched_template( client_context, match_context) except Exception: self.set_match_context_info(conversation, match_context) before, after = current_variables.different_variables( conversation) conversation.add_internal_data(base, 'before_variables', before) conversation.add_internal_data(base, 'after_variables', after) raise if answer is None and self._aiml_parser.pattern_parser.use_nlu is True: assert (self._nlu is not None) try: utterance = client_context.brain.tokenizer.words_to_texts( sentence.words) if default_srai is False or self.bot.configuration.default_response_srai != utterance: nluResult = conversation.current_question().property( "__SYSTEM_NLUDATA__") if nluResult is None or nluResult == "" or self._nlu_utterance != utterance: self._nlu_utterance = utterance match_context = self.multi_nlu_match( client_context, conversation, topic_pattern, that_pattern) else: client_context.match_nlu = True sentence = Sentence(client_context.brain.tokenizer, self.NLU_UTTERANCE) match_context = self._aiml_parser.match_sentence( client_context, sentence, topic_pattern=topic_pattern, that_pattern=that_pattern) client_context.match_nlu = False if match_context is not None: YLogger.debug( client_context, "Matched Catgeory (NLU): file[%s] line[%s-%s]", match_context._template_node._filename, match_context._template_node._start_line, match_context._template_node._end_line) conversation.add_internal_matched( base, match_context._template_node) answer = self.resolve_matched_template( client_context, match_context) except NotImplementedError: if match_context is not None: YLogger.debug( client_context, "Matched Catgeory (*): file[%s] line[%s-%s]", match_context._template_node._filename, match_context._template_node._start_line, match_context._template_node._end_line) conversation.add_internal_matched( base, match_context._template_node) answer = self.resolve_matched_template( client_context, match_context) except Exception: self.set_match_context_info(conversation, match_context) before, after = current_variables.different_variables( conversation) conversation.add_internal_data(base, 'before_variables', before) conversation.add_internal_data(base, 'after_variables', after) raise self.set_match_context_info(conversation, match_context) before, after = current_variables.different_variables(conversation) conversation.add_internal_data(base, 'before_variables', before) conversation.add_internal_data(base, 'after_variables', after) conversation.add_internal_data(base, 'processing_result', answer) if original_base is not None: conversation.internal_base = original_base base = conversation.internal_base conversation.add_internal_data(base, 'response', answer) try: client_context.userInfo.userInfoPostProcessor(client_context) except Exception: pass return answer def set_match_context_info(self, conversation, match_context): if match_context is not None: json_matchedNode = { 'file_name': match_context._template_node._filename, 'start_line': match_context._template_node._start_line, 'end_line': match_context._template_node._end_line } question = conversation.current_question() if question is not None: question.sentences[ question. _current_sentence_no].matched_node = json_matchedNode def multi_nlu_match(self, client_context, conversation, topic_pattern, that_pattern): nlu_list = self._nlu_collection.servers match_context = None for server_name in nlu_list: try: server = self._nlu_collection.server_info(server_name) nluResult = self._nlu.nluCall(client_context, server.url, server.apikey, self._nlu_utterance) except NotImplementedError: raise conversation.current_question().set_property( "__SYSTEM_NLUDATA__", nluResult) if nluResult is not None: client_context.match_nlu = True sentence = Sentence(client_context.brain.tokenizer, self.NLU_UTTERANCE) match_context = self._aiml_parser.match_sentence( client_context, sentence, topic_pattern=topic_pattern, that_pattern=that_pattern) client_context.match_nlu = False if match_context is not None: if len(match_context.matched_nodes) != 3 or \ match_context.matched_nodes[0].matched_node.is_wildcard() is False: break if match_context is None: conversation.current_question().set_property( "__SYSTEM_NLUDATA__", None) try: client_context.userInfo.userInfoPostProcessor(client_context) except Exception: pass return match_context
class Brain(object): def __init__(self, bot, configuration: BrainConfiguration): assert (bot is not None) assert (configuration is not None) self._questions = 0 self._bot = bot self._configuration = configuration self._binaries = BinariesManager(configuration.binaries) self._braintree = BraintreeManager(configuration.braintree) self._tokenizer = Tokenizer.load_tokenizer(configuration) self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = Person2Collection() self._rdf_collection = RDFCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._default_variables_collection = DefaultVariablesCollection() self._preprocessors = PreProcessorCollection() self._postprocessors = PostProcessorCollection() self._postquestionprocessors = PostQuestionProcessorCollection() self._pattern_factory = None self._template_factory = None self._security = SecurityManager(configuration.security) self._oobhandler = OOBHandler(configuration.oob) self._openchatbots = OpenChatBotCollection() self._regex_templates = RegexTemplatesCollection() self._dynamics_collection = DynamicsCollection() self._aiml_parser = self.load_aiml_parser() self.load(self.configuration) def ylogger_type(self): return "brain" @property def num_questions(self): return self._questions @property def id(self): return self._configuration.section_name @property def bot(self): return self._bot @property def configuration(self): return self._configuration @property def aiml_parser(self): return self._aiml_parser @property def denormals(self): return self._denormal_collection @property def normals(self): return self._normal_collection @property def genders(self): return self._gender_collection @property def persons(self): return self._person_collection @property def person2s(self): return self._person2_collection @property def rdf(self): return self._rdf_collection @property def sets(self): return self._sets_collection @property def maps(self): return self._maps_collection @property def properties(self): return self._properties_collection @property def default_variables(self): return self._default_variables_collection @property def preprocessors(self): return self._preprocessors @property def postprocessors(self): return self._postprocessors @property def postquestionprocessors(self): return self._postquestionprocessors @property def pattern_factory(self): return self._pattern_factory @property def template_factory(self): return self._template_factory @property def regex_templates(self): return self._regex_templates @property def dynamics(self): return self._dynamics_collection @property def tokenizer(self): return self._tokenizer @property def openchatbots(self): return self._openchatbots @property def security(self): return self._security def load_aiml_parser(self): self._load_pattern_nodes() self._load_template_nodes() return AIMLParser(self) def load_aiml(self): YLogger.info(self, "Loading aiml source brain") self._aiml_parser.load_aiml() def reload_aimls(self): YLogger.info(self, "Loading aiml source brain") self._aiml_parser.empty() self._aiml_parser.load_aiml() def load(self, configuration: BrainConfiguration): load_aiml = True if self.configuration.binaries.load_binary is True: load_aiml = self._binaries.load_binary( self.bot.client.storage_factory) if load_aiml is True: self.load_aiml() if configuration.binaries.save_binary is True: self._binaries.save_binary(self.bot.client.storage_factory) YLogger.info(self, "Loading collections") self.load_collections() YLogger.info(self, "Loading services") self.load_services(configuration) YLogger.info(self, "Loading openchat bots") self.load_openchatbots(configuration) YLogger.info(self, "Loading security services") self.load_security_services() YLogger.info(self, "Loading oob processors") self._oobhandler.load_oob_processors() YLogger.info(self, "Loading regex templates") self.load_regex_templates() YLogger.info(self, "Loading dynamics sets, maps and vars") self.load_dynamics() def dump_brain_tree(self, client_context): self._braintree.dump_brain_tree(client_context) def _load_denormals(self): self._denormal_collection.empty() self._denormal_collection.load(self.bot.client.storage_factory) def _load_normals(self): self._normal_collection.empty() self._normal_collection.load(self.bot.client.storage_factory) def _load_genders(self): self._gender_collection.empty() self._gender_collection.load(self.bot.client.storage_factory) def _load_persons(self): self._person_collection.empty() self._person_collection.load(self.bot.client.storage_factory) def _load_person2s(self): self._person2_collection.empty() self._person2_collection.load(self.bot.client.storage_factory) def _load_properties(self): self._properties_collection.empty() self._properties_collection.load(self.bot.client.storage_factory) def _load_default_variables(self): self._default_variables_collection.empty() self._default_variables_collection.load( self.bot.client.storage_factory) self._set_system_defined() def _set_system_defined(self): self.set_sentiment_scores(0.0, 0.5) def set_sentiment_scores(self, positivity, subjectivity): if self._default_variables_collection.has_variable( "positivity") is False: self._default_variables_collection.set_value( "positivity", str(positivity)) if self._default_variables_collection.has_variable( "subjectivity") is False: self._default_variables_collection.set_value( "subjectivity", str(subjectivity)) def _load_maps(self): self._maps_collection.empty() self._maps_collection.load(self.bot.client.storage_factory) def reload_map(self, mapname): if self._maps_collection.contains(mapname): self._maps_collection.reload(self.bot.client.storage_factory, mapname) else: YLogger.error(self, "Unknown map name [%s], unable to reload ", mapname) def _load_sets(self): self._sets_collection.empty() self._sets_collection.load(self.bot.client.storage_factory) def reload_set(self, setname): if self._sets_collection.contains(setname): self._sets_collection.reload(self.bot.client.storage_factory, setname) else: YLogger.error(self, "Unknown set name [%s], unable to reload ", setname) def _load_rdfs(self): self._rdf_collection.empty() self._rdf_collection.load(self.bot.client.storage_factory) def reload_rdf(self, rdfname): if self._rdf_collection.contains(rdfname): self._rdf_collection.reload(self.bot.client.storage_factory, rdfname) else: YLogger.error(self, "Unknown rdf name [%s], unable to reload ", rdfname) def _load_preprocessors(self): self._preprocessors.empty() self._preprocessors.load(self.bot.client.storage_factory) def _load_postprocessors(self): self._postprocessors.empty() self._postprocessors.load(self.bot.client.storage_factory) def _load_postquestionprocessors(self): self._postquestionprocessors.empty() self._postquestionprocessors.load(self.bot.client.storage_factory) def _load_pattern_nodes(self): self._pattern_factory = PatternNodeFactory() self._pattern_factory.load(self.bot.client.storage_factory) def _load_template_nodes(self): self._template_factory = TemplateNodeFactory() self._template_factory.load(self.bot.client.storage_factory) def load_collections(self): self._load_denormals() self._load_normals() self._load_genders() self._load_persons() self._load_person2s() self._load_properties() self._load_default_variables() self._load_rdfs() self._load_sets() self._load_maps() self._load_preprocessors() self._load_postprocessors() self._load_postquestionprocessors() def load_services(self, configuration): ServiceFactory.preload_services(configuration.services) def load_openchatbots(self, configuration): self._openchatbots.load_from_configuration(configuration.openchatbots) def load_security_services(self): self._security.load_security_services(self.bot.client) def load_dynamics(self): if self.configuration.dynamics is not None: self._dynamics_collection.load_from_configuration( self.configuration.dynamics) else: YLogger.debug(self, "No dynamics configuration defined...") def load_regex_templates(self): self._regex_templates.load(self.bot.client.storage_factory) def pre_process_question(self, client_context, question): return self.preprocessors.process(client_context, question) def post_process_response(self, client_context, response: str): return self.postprocessors.process(client_context, response) def post_process_question(self, client_context, question: str): return self.postquestionprocessors.process(client_context, question) def failed_authentication(self, client_context): return self._security.failed_authentication(client_context) def authenticate_user(self, client_context): return self._security.authenticate_user(client_context) def resolve_matched_template(self, client_context, match_context): assert (client_context is not None) assert (match_context is not None) template_node = match_context.template_node() YLogger.debug(client_context, "AIML Parser evaluating template [%s]", template_node.to_string()) response = template_node.template.resolve(client_context) if self._oobhandler.oob_in_response(response) is True: response = self._oobhandler.handle(client_context, response) return response def ask_question(self, client_context, sentence, srai=False): assert (client_context is not None) assert (client_context.bot is not None) assert (self._aiml_parser is not None) client_context.brain = self authenticated = self.authenticate_user(client_context) if authenticated is not None: return authenticated conversation = client_context.bot.get_conversation(client_context) if conversation is not None: self._questions += 1 topic_pattern = conversation.get_topic_pattern(client_context) that_pattern = conversation.get_that_pattern(client_context, srai) match_context = self._aiml_parser.match_sentence( client_context, sentence, topic_pattern=topic_pattern, that_pattern=that_pattern) if match_context is not None: return self.resolve_matched_template(client_context, match_context) return None
class Brain(object): ''' Object chứa logic load và xử lý các câu hội thoại @props: - _bot: Bot đang gọi brain - _tokenizer ??/ - _aiml_parser: bộ parser file aiml - _denormal_collection - _gender_collection - _person_collection - _person2_collection - _rdf_collection - _sets_collection - _maps_collection - _properties_colelction - _variables_collection - _preprocessors: - _postprocessors: - _authentication: - _authorization: - _default_oob: - _oobs - _regex_templates: - _dynamics_colllection: ''' def __init__(self, bot, configuration: BrainConfiguration): self._bot = bot self._configuration = configuration self._tokenizer = self.load_tokenizer() self._aiml_parser = self.load_aiml_parser() self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = PersonCollection() self._rdf_collection = RDFCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._variables_collection = PropertiesCollection() self._preprocessors = ProcessorLoader() self._postprocessors = ProcessorLoader() self._authentication = None self._authorisation = None self._default_oob = None self._oob = {} self._regex_templates = {} self._dynamics_collection = DynamicsCollection() self.load(self.configuration) def ylogger_type(self): return "brain" @property def id(self): return self._configuration.section_name @property def bot(self): return self._bot @property def configuration(self): return self._configuration @property def aiml_parser(self): return self._aiml_parser @property def denormals(self): return self._denormal_collection @property def normals(self): return self._normal_collection @property def genders(self): return self._gender_collection @property def persons(self): return self._person_collection @property def person2s(self): return self._person2_collection @property def rdf(self): return self._rdf_collection @property def sets(self): return self._sets_collection @property def maps(self): return self._maps_collection @property def properties(self): return self._properties_collection @property def variables(self): return self._variables_collection @property def preprocessors(self): return self._preprocessors @property def postprocessors(self): return self._postprocessors @property def authentication(self): return self._authentication @property def authorisation(self): return self._authorisation @property def default_oob(self): return self._default_oob @property def oobs(self): return self._oob @property def regex_templates(self): return self._regex_templates @property def dynamics(self): return self._dynamics_collection @property def tokenizer(self): return self._tokenizer def load_tokenizer(self): ''' Load tokenizer từ configuration ''' if self.configuration is not None and self.configuration.tokenizer.classname is not None: YLogger.info(self, "Loading tokenizer from class [%s]", self.configuration.tokenizer.classname) tokenizer_class = ClassLoader.instantiate_class( self.configuration.tokenizer.classname) return tokenizer_class(self.configuration.tokenizer.split_chars) else: return Tokenizer(self.configuration.tokenizer.split_chars) def load_aiml_parser(self): return AIMLParser(self) def load_aiml(self, configuration): ''' dùng aiml_parser để load file aiml ''' YLogger.info(self, "Loading aiml source brain") self._aiml_parser.load_aiml(configuration) def reload_aimls(self): ''' làm trống bộ parser và load lại file aiml ''' YLogger.info(self, "Loading aiml source brain") self._aiml_parser.empty() self._aiml_parser.load_aiml(self.configuration) def load_binary(self, configuration): ''' load file binary >> có thể tăng tốc độ ? ''' YLogger.info(self, "Loading binary brain from [%s]", configuration.binaries.binary_filename) try: start = datetime.datetime.now() gc.disable() bin_file = open(configuration.binaries.binary_filename, "rb") self._aiml_parser = pickle.load(bin_file) self._aiml_parser._brain = self gc.enable() bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds()) return False # Tell caller, load succeeded and skip aiml load except Exception as excep: YLogger.exception(self, "Failed to load binary file", excep) if configuration.binaries.load_aiml_on_binary_fail is True: return True # Tell caller, load failed and to load aiml directly else: raise excep def save_binary(self, configuration): YLogger.info(self, "Saving binary brain to [%s]", configuration.binaries.binary_filename) start = datetime.datetime.now() bin_file = open(configuration.binaries.binary_filename, "wb") pickle.dump(self._aiml_parser, bin_file) bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds()) def load(self, configuration: BrainConfiguration): ''' load tất cả các thể loại - aiml - collection - service - security_services - oob_processor - regex_templates - dynamics ''' load_aiml = True if self.configuration.binaries.load_binary is True: load_aiml = self.load_binary(configuration) if load_aiml is True: self.load_aiml(configuration) if configuration.binaries.save_binary is True: self.save_binary(configuration) YLogger.info(self, "Loading collections") self.load_collections(configuration) YLogger.info(self, "Loading services") self.load_services(configuration) YLogger.info(self, "Loading security services") self.load_security_services(configuration) YLogger.info(self, "Loading oob processors") self.load_oob_processors(configuration) YLogger.info(self, "Loading regex templates") self.load_regex_templates(configuration) YLogger.info(self, "Loading dynamics sets, maps and vars") self.load_dynamics(configuration) def dump_brain_tree(self): ''' ''' if self.configuration.braintree.file is not None: YLogger.debug(self, "Dumping AIML Graph as tree to [%s]", self._configuration.braintree.file) client_context = self.bot.client.create_client_context("system") self.aiml_parser.pattern_parser.save_braintree( client_context, self.configuration.braintree.file, self.configuration.braintree.content) def _load_denormals(self, configuration): if configuration.files.denormal is not None: self._denormal_collection.empty() total = self._denormal_collection.load_from_filename( configuration.files.denormal) YLogger.info(self, "Loaded a total of %d denormalisations", total) else: YLogger.warning(self, "No configuration setting for denormal") def _load_normals(self, configuration): if configuration.files.normal is not None: self._normal_collection.empty() total = self._normal_collection.load_from_filename( configuration.files.normal) YLogger.info(self, "Loaded a total of %d normalisations", total) else: YLogger.warning(self, "No configuration setting for normal") def _load_genders(self, configuration): if configuration.files.gender is not None: self._gender_collection.empty() total = self._gender_collection.load_from_filename( configuration.files.gender) YLogger.info(self, "Loaded a total of %d genderisations", total) else: YLogger.warning(self, "No configuration setting for gender") def _load_persons(self, configuration): if configuration.files.person is not None: self._person_collection.empty() total = self._person_collection.load_from_filename( configuration.files.person) YLogger.info(self, "Loaded a total of %d persons", total) else: YLogger.warning(self, "No configuration setting for person") def _load_person2s(self, configuration): if configuration.files.person2 is not None: self._person2_collection.empty() total = self._person2_collection.load_from_filename( configuration.files.person2) YLogger.info(self, "Loaded a total of %d person2s", total) else: YLogger.warning(self, "No configuration setting for person2") def _load_properties(self, configuration): if configuration.files.properties is not None: self._properties_collection.empty() total = self._properties_collection.load_from_filename( configuration.files.properties) YLogger.info(self, "Loaded a total of %d properties", total) else: YLogger.warning(self, "No configuration setting for properties") def _load_variables(self, configuration): if configuration.files.variables is not None: self._variables_collection.empty() total = self._variables_collection.load_from_filename( configuration.files.variables) YLogger.info(self, "Loaded a total of %d variables", total) else: YLogger.warning(self, "No configuration setting for variables") def _load_maps(self, configuration): self._maps_collection.empty() total = self._maps_collection.load(configuration.files.map_files) YLogger.info(self, "Loaded a total of %d maps files", total) def reload_map(self, mapname): if self._maps_collection.contains(mapname): filename = self._maps_collection.filename(mapname) self._maps_collection.reload_file(filename) def _load_sets(self, configuration): self._sets_collection.empty() total = self._sets_collection.load(configuration.files.set_files) YLogger.info(self, "Loaded a total of %d sets files", total) def reload_set(self, setname): if self._sets_collection.contains(setname): filename = self._sets_collection.filename(setname) self._sets_collection.reload_file(filename) def _load_rdfs(self, configuration): if configuration.files.rdf_files is not None and configuration.files.rdf_files.files: self._rdf_collection.empty() total = self._rdf_collection.load(configuration.files.rdf_files) YLogger.info(self, "Loaded a total of %d rdf files", total) elif configuration.files.triples is not None: self._rdf_collection.empty() total = self._rdf_collection.load_from_filename( configuration.files.triples) YLogger.info(self, "Loaded a total of %d triples", total) else: YLogger.warning(self, "No configuration setting for triples") def reload_rdf(self, rdfname): if self._rdf_collection.contains(rdfname): self._rdf_collection.reload_file(rdfname) def _load_preprocessors(self, configuration): if configuration.files.preprocessors is not None: self._preprocessors.empty() total = self._preprocessors.load(configuration.files.preprocessors) YLogger.info(self, "Loaded a total of %d pre processors", total) else: YLogger.warning(self, "No configuration setting for pre processors") def _load_postprocessors(self, configuration): if configuration.files.postprocessors is not None: self._postprocessors.empty() total = self._postprocessors.load( configuration.files.postprocessors) YLogger.info(self, "Loaded a total of %d post processors", total) else: YLogger.warning(self, "No configuration setting for post processors") def load_collections(self, configuration): self._load_denormals(configuration) self._load_normals(configuration) self._load_genders(configuration) self._load_persons(configuration) self._load_person2s(configuration) self._load_properties(configuration) self._load_variables(configuration) self._load_rdfs(configuration) self._load_sets(configuration) self._load_maps(configuration) self._load_preprocessors(configuration) self._load_postprocessors(configuration) def load_services(self, configuration): ServiceFactory.preload_services(configuration.services) def load_security_services(self, configuration): if configuration.security is not None: if configuration.security.authentication is not None: if configuration.security.authentication.classname is not None: try: classobject = ClassLoader.instantiate_class( configuration.security.authentication.classname) self._authentication = classobject( configuration.security.authentication) except Exception as excep: YLogger.exception(self, "Failed to load security services", excep) else: YLogger.debug(self, "No authentication configuration defined") if configuration.security.authorisation is not None: if configuration.security.authorisation.classname is not None: try: classobject = ClassLoader.instantiate_class( configuration.security.authorisation.classname) self._authorisation = classobject( configuration.security.authorisation) except Exception as excep: YLogger.exception( self, "Failed to instatiate authorisation class", excep) else: YLogger.debug(self, "No authorisation configuration defined") else: YLogger.debug( self, "No security configuration defined, running open...") def load_dynamics(self, configuration): if configuration.dynamics is not None: self._dynamics_collection.load_from_configuration( configuration.dynamics) else: YLogger.debug(self, "No dynamics configuration defined...") def pre_process_question(self, client_context, question): return self.preprocessors.process(client_context, question) def load_oob_processors(self, configuration): if configuration.oob is not None: if configuration.oob.default() is not None: try: YLogger.info(self, "Loading default oob") classobject = ClassLoader.instantiate_class( configuration.oob.default().classname) self._default_oob = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB Processor", excep) for oob_name in configuration.oob.oobs(): try: YLogger.info(self, "Loading oob: %s", oob_name) classobject = ClassLoader.instantiate_class( configuration.oob.oob(oob_name).classname) self._oob[oob_name] = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB", excep) def load_regex_templates(self, configuration): if configuration.files.regex_templates is not None: collection = PropertiesCollection() total = collection.load_from_filename( configuration.files.regex_templates) YLogger.info(self, "Loaded a total of %d regex templates", total) self._regex_templates.clear() for pair in collection.pairs: name = pair[0] pattern = pair[1] try: self._regex_templates[name] = re.compile( pattern, re.IGNORECASE) except Exception: YLogger.error(self, "Invalid regex template [%s]", pattern) def regex_template(self, name): if name in self._regex_templates: return self._regex_templates[name] return None def strip_oob(self, response): match = re.compile(r"(.*)(<\s*oob\s*>.*<\/\s*oob\s*>)(.*)") groupings = match.match(response) if groupings is not None: front = groupings.group(1).strip() back = groupings.group(3).strip() response = "" if front != "": response = front + " " response += back oob = groupings.group(2) return response, oob return response, None def process_oob(self, client_context, oob_command): oob_content = ET.fromstring(oob_command) if oob_content.tag == 'oob': for child in oob_content.findall('./'): if child.tag in self._oob: oob_class = self._oob[child.tag] return oob_class.process_out_of_bounds( client_context, child) return self._default_oob.process_out_of_bounds( client_context, child) return "" def post_process_response(self, client_context, response: str): return self.postprocessors.process(client_context, response) def failed_authentication(self, client_context): YLogger.error(client_context, "[%s] failed authentication!") # If we have an SRAI defined, then use that if self.authentication.configuration.denied_srai is not None: match_context = self._aiml_parser.match_sentence( client_context, Sentence(self._bot.brain.tokenizer, self.authentication.configuration.denied_srai), topic_pattern="*", that_pattern="*") # If the SRAI matched then return the result if match_context is not None: return self.resolve_matched_template(client_context, match_context) # Otherswise return the static text, which is either # User defined via config.yaml # Or use the default value BrainSecurityConfiguration.DEFAULT_ACCESS_DENIED return self.authentication.configuration.denied_text def authenticate_user(self, client_context): if self.authentication is not None: if self.authentication.authenticate(client_context) is False: return self.failed_authentication(client_context) return None def resolve_matched_template(self, client_context, match_context): template_node = match_context.template_node() YLogger.debug(client_context, "AIML Parser evaluating template [%s]", template_node.to_string()) response = template_node.template.resolve(client_context) if "<oob>" in response: response, oob = self.strip_oob(response) if oob is not None: oob_response = self.process_oob(client_context, oob) response = response + " " + oob_response return response def ask_question(self, client_context, sentence, srai=False): client_context.brain = self authenticated = self.authenticate_user(client_context) if authenticated is not None: return authenticated conversation = client_context.bot.get_conversation(client_context) topic_pattern = conversation.get_topic_pattern(client_context) that_pattern = conversation.get_that_pattern(client_context, srai) match_context = self._aiml_parser.match_sentence( client_context, sentence, topic_pattern=topic_pattern, that_pattern=that_pattern) if match_context is not None: return self.resolve_matched_template(client_context, match_context) return None
class Brain(object): def __init__(self, bot, configuration: BrainConfiguration): self._bot = bot self._configuration = configuration self._tokenizer = self.load_tokenizer() self._aiml_parser = self.load_aiml_parser() self._denormal_collection = DenormalCollection() self._normal_collection = NormalCollection() self._gender_collection = GenderCollection() self._person_collection = PersonCollection() self._person2_collection = PersonCollection() self._rdf_collection = RDFCollection() self._sets_collection = SetCollection() self._maps_collection = MapCollection() self._properties_collection = PropertiesCollection() self._variables_collection = PropertiesCollection() self._preprocessors = ProcessorLoader() self._postprocessors = ProcessorLoader() self._authentication = None self._authorisation = None self._default_oob = None self._oob = {} self._regex_templates = {} self._dynamics_collection = DynamicsCollection() self.load(self.configuration) def ylogger_type(self): return "brain" @property def id(self): return self._configuration.section_name @property def bot(self): return self._bot @property def configuration(self): return self._configuration @property def aiml_parser(self): return self._aiml_parser @property def denormals(self): return self._denormal_collection @property def normals(self): return self._normal_collection @property def genders(self): return self._gender_collection @property def persons(self): return self._person_collection @property def person2s(self): return self._person2_collection @property def rdf(self): return self._rdf_collection @property def sets(self): return self._sets_collection @property def maps(self): return self._maps_collection @property def properties(self): return self._properties_collection @property def variables(self): return self._variables_collection @property def preprocessors(self): return self._preprocessors @property def postprocessors(self): return self._postprocessors @property def authentication(self): return self._authentication @property def authorisation(self): return self._authorisation @property def default_oob(self): return self._default_oob @property def oobs(self): return self._oob @property def regex_templates(self): return self._regex_templates @property def dynamics(self): return self._dynamics_collection @property def tokenizer(self): return self._tokenizer def load_tokenizer(self): if self.configuration is not None and self.configuration.tokenizer.classname is not None: YLogger.info(self, "Loading tokenizer from class [%s]", self.configuration.tokenizer.classname) tokenizer_class = ClassLoader.instantiate_class(self.configuration.tokenizer.classname) return tokenizer_class(self.configuration.tokenizer.split_chars) else: return Tokenizer(self.configuration.tokenizer.split_chars) def load_aiml_parser(self): return AIMLParser(self) def load_aiml(self, configuration): YLogger.info(self, "Loading aiml source brain") self._aiml_parser.load_aiml(configuration) def reload_aimls(self): YLogger.info(self, "Loading aiml source brain") self._aiml_parser.empty() self._aiml_parser.load_aiml(self.configuration) def load_binary(self, configuration): YLogger.info(self, "Loading binary brain from [%s]", configuration.binaries.binary_filename) try: start = datetime.datetime.now() gc.disable() bin_file = open(configuration.binaries.binary_filename, "rb") self._aiml_parser = pickle.load(bin_file) self._aiml_parser._brain = self gc.enable() bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds()) return False # Tell caller, load succeeded and skip aiml load except Exception as excep: YLogger.exception(self, "Failed to load binary file", excep) if configuration.binaries.load_aiml_on_binary_fail is True: return True # Tell caller, load failed and to load aiml directly else: raise excep def save_binary(self, configuration): YLogger.info(self, "Saving binary brain to [%s]", configuration.binaries.binary_filename) start = datetime.datetime.now() bin_file = open(configuration.binaries.binary_filename, "wb") pickle.dump(self._aiml_parser, bin_file) bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds()) def load(self, configuration: BrainConfiguration): load_aiml = True if self.configuration.binaries.load_binary is True: load_aiml = self.load_binary(configuration) if load_aiml is True: self.load_aiml(configuration) if configuration.binaries.save_binary is True: self.save_binary(configuration) YLogger.info(self, "Loading collections") self.load_collections(configuration) YLogger.info(self, "Loading services") self.load_services(configuration) YLogger.info(self, "Loading security services") self.load_security_services(configuration) YLogger.info(self, "Loading oob processors") self.load_oob_processors(configuration) YLogger.info(self, "Loading regex templates") self.load_regex_templates(configuration) YLogger.info(self, "Loading dynamics sets, maps and vars") self.load_dynamics(configuration) def dump_brain_tree(self): if self.configuration.braintree.file is not None: YLogger.debug(self, "Dumping AIML Graph as tree to [%s]", self._configuration.braintree.file) client_context = self.bot.client.create_client_context("system") self.aiml_parser.pattern_parser.save_braintree( client_context, self.configuration.braintree.file, self.configuration.braintree.content) def _load_denormals(self, configuration): if configuration.files.denormal is not None: self._denormal_collection.empty() total = self._denormal_collection.load_from_filename(configuration.files.denormal) YLogger.info(self, "Loaded a total of %d denormalisations", total) else: YLogger.warning(self, "No configuration setting for denormal") def _load_normals(self, configuration): if configuration.files.normal is not None: self._normal_collection.empty() total = self._normal_collection.load_from_filename(configuration.files.normal) YLogger.info(self, "Loaded a total of %d normalisations", total) else: YLogger.warning(self, "No configuration setting for normal") def _load_genders(self, configuration): if configuration.files.gender is not None: self._gender_collection.empty() total = self._gender_collection.load_from_filename(configuration.files.gender) YLogger.info(self, "Loaded a total of %d genderisations", total) else: YLogger.warning(self, "No configuration setting for gender") def _load_persons(self, configuration): if configuration.files.person is not None: self._person_collection.empty() total = self._person_collection.load_from_filename(configuration.files.person) YLogger.info(self, "Loaded a total of %d persons", total) else: YLogger.warning(self, "No configuration setting for person") def _load_person2s(self, configuration): if configuration.files.person2 is not None: self._person2_collection.empty() total = self._person2_collection.load_from_filename(configuration.files.person2) YLogger.info(self, "Loaded a total of %d person2s", total) else: YLogger.warning(self, "No configuration setting for person2") def _load_properties(self, configuration): if configuration.files.properties is not None: self._properties_collection.empty() total = self._properties_collection.load_from_filename(configuration.files.properties) YLogger.info(self, "Loaded a total of %d properties", total) else: YLogger.warning(self, "No configuration setting for properties") def _load_variables(self, configuration): if configuration.files.variables is not None: self._variables_collection.empty () total = self._variables_collection.load_from_filename(configuration.files.variables) YLogger.info(self, "Loaded a total of %d variables", total) else: YLogger.warning(self, "No configuration setting for variables") def _load_maps(self, configuration): self._maps_collection.empty() total = self._maps_collection.load(configuration.files.map_files) YLogger.info(self, "Loaded a total of %d maps files", total) def reload_map(self, mapname): if self._maps_collection.contains(mapname): filename = self._maps_collection.filename(mapname) self._maps_collection.reload_file(filename) def _load_sets(self, configuration): self._sets_collection.empty() total = self._sets_collection.load(configuration.files.set_files) YLogger.info(self, "Loaded a total of %d sets files", total) def reload_set(self, setname): if self._sets_collection.contains(setname): filename = self._sets_collection.filename(setname) self._sets_collection.reload_file(filename) def _load_rdfs(self, configuration): if configuration.files.rdf_files is not None and configuration.files.rdf_files.files: self._rdf_collection.empty() total = self._rdf_collection.load(configuration.files.rdf_files) YLogger.info(self, "Loaded a total of %d rdf files", total) elif configuration.files.triples is not None: self._rdf_collection.empty() total = self._rdf_collection.load_from_filename(configuration.files.triples) YLogger.info(self, "Loaded a total of %d triples", total) else: YLogger.warning(self, "No configuration setting for triples") def reload_rdf(self, rdfname): if self._rdf_collection.contains(rdfname): self._rdf_collection.reload_file(rdfname) def _load_preprocessors(self, configuration): if configuration.files.preprocessors is not None: self._preprocessors.empty() total = self._preprocessors.load(configuration.files.preprocessors) YLogger.info(self, "Loaded a total of %d pre processors", total) else: YLogger.warning(self, "No configuration setting for pre processors") def _load_postprocessors(self, configuration): if configuration.files.postprocessors is not None: self._postprocessors.empty() total = self._postprocessors.load(configuration.files.postprocessors) YLogger.info(self, "Loaded a total of %d post processors", total) else: YLogger.warning(self, "No configuration setting for post processors") def load_collections(self, configuration): self._load_denormals(configuration) self._load_normals(configuration) self._load_genders(configuration) self._load_persons(configuration) self._load_person2s(configuration) self._load_properties(configuration) self._load_variables(configuration) self._load_rdfs(configuration) self._load_sets(configuration) self._load_maps(configuration) self._load_preprocessors(configuration) self._load_postprocessors(configuration) def load_services(self, configuration): ServiceFactory.preload_services(configuration.services) def load_security_services(self, configuration): if configuration.security is not None: if configuration.security.authentication is not None: if configuration.security.authentication.classname is not None: try: classobject = ClassLoader.instantiate_class( configuration.security.authentication.classname) self._authentication = classobject(configuration.security.authentication) except Exception as excep: YLogger.exception(self, "Failed to load security services", excep) else: YLogger.debug(self, "No authentication configuration defined") if configuration.security.authorisation is not None: if configuration.security.authorisation.classname is not None: try: classobject = ClassLoader.instantiate_class( configuration.security.authorisation.classname) self._authorisation = classobject(configuration.security.authorisation) except Exception as excep: YLogger.exception(self, "Failed to instatiate authorisation class", excep) else: YLogger.debug(self, "No authorisation configuration defined") else: YLogger.debug(self, "No security configuration defined, running open...") def load_dynamics(self, configuration): if configuration.dynamics is not None: self._dynamics_collection.load_from_configuration(configuration.dynamics) else: YLogger.debug(self, "No dynamics configuration defined...") def pre_process_question(self, client_context, question): return self.preprocessors.process(client_context, question) def load_oob_processors(self, configuration): if configuration.oob is not None: if configuration.oob.default() is not None: try: YLogger.info(self, "Loading default oob") classobject = ClassLoader.instantiate_class(configuration.oob.default().classname) self._default_oob = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB Processor", excep) for oob_name in configuration.oob.oobs(): try: YLogger.info(self, "Loading oob: %s", oob_name) classobject = ClassLoader.instantiate_class(configuration.oob.oob(oob_name).classname) self._oob[oob_name] = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB", excep) def load_regex_templates(self, configuration): if configuration.files.regex_templates is not None: collection = PropertiesCollection() total = collection.load_from_filename(configuration.files.regex_templates) YLogger.info(self, "Loaded a total of %d regex templates", total) self._regex_templates.clear() for pair in collection.pairs: name = pair[0] pattern = pair[1] try: self._regex_templates[name] = re.compile(pattern, re.IGNORECASE) except Exception: YLogger.error(self, "Invalid regex template [%s]", pattern) def regex_template(self, name): if name in self._regex_templates: return self._regex_templates[name] return None def strip_oob(self, response): match = re.compile(r"(.*)(<\s*oob\s*>.*<\/\s*oob\s*>)(.*)") groupings = match.match(response) if groupings is not None: front = groupings.group(1).strip() back = groupings.group(3).strip() response = "" if front != "": response = front + " " response += back oob = groupings.group(2) return response, oob return response, None def process_oob(self, client_context, oob_command): oob_content = ET.fromstring(oob_command) if oob_content.tag == 'oob': for child in oob_content.findall('./'): if child.tag in self._oob: oob_class = self._oob[child.tag] return oob_class.process_out_of_bounds(client_context, child) return self._default_oob.process_out_of_bounds(client_context, child) return "" def post_process_response(self, client_context, response: str): return self.postprocessors.process(client_context, response) def failed_authentication(self, client_context): YLogger.error(client_context, "[%s] failed authentication!") # If we have an SRAI defined, then use that if self.authentication.configuration.denied_srai is not None: match_context = self._aiml_parser.match_sentence(client_context, Sentence(self._bot.brain.tokenizer, self.authentication.configuration.denied_srai), topic_pattern="*", that_pattern="*") # If the SRAI matched then return the result if match_context is not None: return self.resolve_matched_template(client_context, match_context) # Otherswise return the static text, which is either # User defined via config.yaml # Or use the default value BrainSecurityConfiguration.DEFAULT_ACCESS_DENIED return self.authentication.configuration.denied_text def authenticate_user(self, client_context): if self.authentication is not None: if self.authentication.authenticate(client_context) is False: return self.failed_authentication(client_context) return None def resolve_matched_template(self, client_context, match_context): template_node = match_context.template_node() YLogger.debug(client_context, "AIML Parser evaluating template [%s]", template_node.to_string()) response = template_node.template.resolve(client_context) if "<oob>" in response: response, oob = self.strip_oob(response) if oob is not None: oob_response = self.process_oob(client_context, oob) response = response + " " + oob_response return response def ask_question(self, client_context, sentence, srai=False): client_context.brain = self authenticated = self.authenticate_user(client_context) if authenticated is not None: return authenticated conversation = client_context.bot.get_conversation(client_context) topic_pattern = conversation.get_topic_pattern(client_context) that_pattern = conversation.get_that_pattern(client_context, srai) match_context = self._aiml_parser.match_sentence(client_context, sentence, topic_pattern=topic_pattern, that_pattern=that_pattern) if match_context is not None: return self.resolve_matched_template(client_context, match_context) return None