def test_reload_from_file(self):
config = FileStorageConfiguration()
config._rdf_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "rdfs"
])
factory = StorageFactory()
storage_engine = FileStorageEngine(config)
factory._storage_engines[StorageFactory.RDF] = storage_engine
factory._store_to_engine_map[StorageFactory.RDF] = storage_engine
storage_engine.initialise()
collection = RDFCollection()
self.assertIsNotNone(collection)
collection.load(factory)
self.assertTrue(collection.has_subject("TEST1"))
self.assertTrue(collection.has_predicate("TEST1", "HASPURPOSE"))
self.assertTrue(collection.has_object("TEST1", "HASPURPOSE",
"to test"))
collection.delete_entity("TEST1", "HASPURPOSE", "to test")
self.assertFalse(
collection.has_object("TEST1", "HASPURPOSE", "to test"))
collection.reload(factory, "TESTDATA")
self.assertTrue(collection.has_subject("TEST1"))
self.assertTrue(collection.has_predicate("TEST1", "HASPURPOSE"))
self.assertTrue(collection.has_object("TEST1", "HASPURPOSE",
"to test"))
def test_apply_updates(self):
config = FileStorageConfiguration()
config._rdf_storage = FileStoreConfiguration(dirs=[
os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "rdfs"
])
tmpdir = os.path.dirname(__file__) + os.sep + "rdf_updates"
config.rdf_updates_storage._dirs = [tmpdir]
config.rdf_updates_storage._has_single_file = True
factory = StorageFactory()
storage_engine = FileStorageEngine(config)
factory._storage_engines[StorageFactory.RDF] = storage_engine
factory._store_to_engine_map[StorageFactory.RDF] = storage_engine
factory._storage_engines[StorageFactory.RDF_UPDATES] = storage_engine
factory._store_to_engine_map[
StorageFactory.RDF_UPDATES] = storage_engine
storage_engine.initialise()
updates_engine = factory.entity_storage_engine(
StorageFactory.RDF_UPDATES)
updates_store = updates_engine.rdf_updates_store()
updates_store.empty()
collection = RDFCollection()
self.assertIsNotNone(collection)
collection.load(factory)
self.assertTrue(collection.has_subject("TEST1"))
self.assertTrue(collection.has_predicate("TEST1", "HASPURPOSE"))
self.assertTrue(collection.has_object("TEST1", "HASPURPOSE",
"to test"))
collection.apply_updates()
class Brain(object):
def __init__(self, bot, configuration: BrainConfiguration):
self._bot = bot
self._configuration = configuration
self._tokenizer = self.load_tokenizer()
self._aiml_parser = self.load_aiml_parser()
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._variables_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self.configuration)
self.dump_brain_tree()
def ylogger_type(self):
return "brain"
@property
def id(self):
return self._configuration.section_name
@property
def bot(self):
return self._bot
@property
def configuration(self):
return self._configuration
@property
def aiml_parser(self):
return self._aiml_parser
@property
def denormals(self):
return self._denormal_collection
@property
def normals(self):
return self._normal_collection
@property
def genders(self):
return self._gender_collection
@property
def persons(self):
return self._person_collection
@property
def person2s(self):
return self._person2_collection
@property
def rdf(self):
return self._rdf_collection
@property
def sets(self):
return self._sets_collection
@property
def maps(self):
return self._maps_collection
@property
def properties(self):
return self._properties_collection
@property
def variables(self):
return self._variables_collection
@property
def preprocessors(self):
return self._preprocessors
@property
def postprocessors(self):
return self._postprocessors
@property
def authentication(self):
return self._authentication
@property
def authorisation(self):
return self._authorisation
@property
def default_oob(self):
return self._default_oob
@property
def oobs(self):
return self._oob
@property
def regex_templates(self):
return self._regex_templates
@property
def dynamics(self):
return self._dynamics_collection
@property
def tokenizer(self):
return self._tokenizer
def load_tokenizer(self):
if self.configuration is not None and self.configuration.tokenizer.classname is not None:
YLogger.info(self, "Loading tokenizer from class [%s]",
self.configuration.tokenizer.classname)
tokenizer_class = ClassLoader.instantiate_class(
self.configuration.tokenizer.classname)
return tokenizer_class(self.configuration.tokenizer.split_chars)
else:
return Tokenizer(self.configuration.tokenizer.split_chars)
def load_aiml_parser(self):
return AIMLParser(self)
def load_binary(self, configuration):
YLogger.info(self, "Loading binary brain from [%s]",
configuration.binaries.binary_filename)
try:
start = datetime.datetime.now()
gc.disable()
bin_file = open(configuration.binaries.binary_filename, "rb")
self._aiml_parser = pickle.load(bin_file)
gc.enable()
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain load took a total of %.2f sec",
diff.total_seconds())
return False # Tell caller, load succeeded and skip aiml load
except Exception as excep:
YLogger.exception(self, excep)
if configuration.binaries.load_aiml_on_binary_fail is True:
return True # Tell caller, load failed and to load aiml directly
else:
raise excep
def load_aiml(self, configuration):
YLogger.info(self, "Loading aiml source brain")
self._aiml_parser.load_aiml(configuration)
def save_binary(self, configuration):
YLogger.info(self, "Saving binary brain to [%s]",
configuration.binaries.binary_filename)
start = datetime.datetime.now()
bin_file = open(configuration.binaries.binary_filename, "wb")
pickle.dump(self._aiml_parser, bin_file)
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain save took a total of %.2f sec",
diff.total_seconds())
def load(self, configuration: BrainConfiguration):
load_aiml = True
if self.configuration.binaries.load_binary is True:
load_aiml = self.load_binary(configuration)
if load_aiml is True:
self.load_aiml(configuration)
if configuration.binaries.save_binary is True:
self.save_binary(configuration)
YLogger.info(self, "Loading collections")
self.load_collections(configuration)
YLogger.info(self, "Loading services")
self.load_services(configuration)
YLogger.info(self, "Loading security services")
self.load_security_services(configuration)
YLogger.info(self, "Loading oob processors")
self.load_oob_processors(configuration)
YLogger.info(self, "Loading regex templates")
self.load_regex_templates(configuration)
YLogger.info(self, "Loading dynamics sets, maps and vars")
self.load_dynamics(configuration)
def dump_brain_tree(self):
if self.configuration.braintree.file is not None:
YLogger.debug(self, "Dumping AIML Graph as tree to [%s]",
self._configuration.braintree.file)
self.aiml_parser.pattern_parser.save_braintree(
self.bot, self.clientid, self.configuration.braintree.file,
self.configuration.braintree.content)
def _load_denormals(self, configuration):
if configuration.files.denormal is not None:
total = self._denormal_collection.load_from_filename(
configuration.files.denormal)
YLogger.info(self, "Loaded a total of %d denormalisations", total)
else:
YLogger.warning(self, "No configuration setting for denormal")
def _load_normals(self, configuration):
if configuration.files.normal is not None:
total = self._normal_collection.load_from_filename(
configuration.files.normal)
YLogger.info(self, "Loaded a total of %d normalisations", total)
else:
YLogger.warning(self, "No configuration setting for normal")
def _load_genders(self, configuration):
if configuration.files.gender is not None:
total = self._gender_collection.load_from_filename(
configuration.files.gender)
YLogger.info(self, "Loaded a total of %d genderisations", total)
else:
YLogger.warning(self, "No configuration setting for gender")
def _load_persons(self, configuration):
if configuration.files.person is not None:
total = self._person_collection.load_from_filename(
configuration.files.person)
YLogger.info(self, "Loaded a total of %d persons", total)
else:
YLogger.warning(self, "No configuration setting for person")
def _load_person2s(self, configuration):
if configuration.files.person2 is not None:
total = self._person2_collection.load_from_filename(
configuration.files.person2)
YLogger.info(self, "Loaded a total of %d person2s", total)
else:
YLogger.warning(self, "No configuration setting for person2")
def _load_properties(self, configuration):
if configuration.files.properties is not None:
total = self._properties_collection.load_from_filename(
configuration.files.properties)
YLogger.info(self, "Loaded a total of %d properties", total)
else:
YLogger.warning(self, "No configuration setting for properties")
def _load_variables(self, configuration):
if configuration.files.variables is not None:
total = self._variables_collection.load_from_filename(
configuration.files.variables)
YLogger.info(self, "Loaded a total of %d variables", total)
else:
YLogger.warning(self, "No configuration setting for variables")
def _load_rdf(self, configuration):
if configuration.files.rdf_files is not None and configuration.files.rdf_files.files:
total = self._rdf_collection.load(configuration.files.rdf_files)
YLogger.info(self, "Loaded a total of %d rdf files", total)
elif configuration.files.triples is not None:
total = self._rdf_collection.load_from_filename(
configuration.files.triples)
YLogger.info(self, "Loaded a total of %d triples", total)
else:
YLogger.warning(self, "No configuration setting for triples")
def _load_sets(self, configuration):
total = self._sets_collection.load(configuration.files.set_files)
YLogger.info(self, "Loaded a total of %d sets files", total)
def _load_maps(self, configuration):
total = self._maps_collection.load(configuration.files.map_files)
YLogger.info(self, "Loaded a total of %d maps files", total)
def _load_preprocessors(self, configuration):
if configuration.files.preprocessors is not None:
total = self._preprocessors.load(configuration.files.preprocessors)
YLogger.info(self, "Loaded a total of %d pre processors", total)
else:
YLogger.warning(self,
"No configuration setting for pre processors")
def _load_postprocessors(self, configuration):
if configuration.files.postprocessors is not None:
total = self._postprocessors.load(
configuration.files.postprocessors)
YLogger.info(self, "Loaded a total of %d post processors", total)
else:
YLogger.warning(self,
"No configuration setting for post processors")
def load_collections(self, configuration):
self._load_denormals(configuration)
self._load_normals(configuration)
self._load_genders(configuration)
self._load_persons(configuration)
self._load_person2s(configuration)
self._load_properties(configuration)
self._load_variables(configuration)
self._load_rdf(configuration)
self._load_sets(configuration)
self._load_maps(configuration)
self._load_preprocessors(configuration)
self._load_postprocessors(configuration)
def load_services(self, configuration):
ServiceFactory.preload_services(configuration.services)
def load_security_services(self, configuration):
if configuration.security is not None:
if configuration.security.authentication is not None:
if configuration.security.authentication.classname is not None:
try:
classobject = ClassLoader.instantiate_class(
configuration.security.authentication.classname)
self._authentication = classobject(
configuration.security.authentication)
except Exception as excep:
YLogger.exception(self, excep)
else:
YLogger.debug(self, "No authentication configuration defined")
if configuration.security.authorisation is not None:
if configuration.security.authorisation.classname is not None:
try:
classobject = ClassLoader.instantiate_class(
configuration.security.authorisation.classname)
self._authorisation = classobject(
configuration.security.authorisation)
except Exception as excep:
YLogger.exception(self, excep)
else:
YLogger.debug(self, "No authorisation configuration defined")
else:
YLogger.debug(
self, "No security configuration defined, running open...")
def load_dynamics(self, configuration):
if configuration.dynamics is not None:
self._dynamics_collection.load_from_configuration(
configuration.dynamics)
else:
YLogger.debug(self, "No dynamics configuration defined...")
def pre_process_question(self, client_context, question):
return self.preprocessors.process(client_context, question)
def load_oob_processors(self, configuration):
if configuration.oob is not None:
if configuration.oob.default() is not None:
try:
YLogger.info(self, "Loading default oob")
classobject = ClassLoader.instantiate_class(
configuration.oob.default().classname)
self._default_oob = classobject()
except Exception as excep:
YLogger.exception(self, excep)
for oob_name in configuration.oob.oobs():
try:
YLogger.info(self, "Loading oob: %s", oob_name)
classobject = ClassLoader.instantiate_class(
configuration.oob.oob(oob_name).classname)
self._oob[oob_name] = classobject()
except Exception as excep:
YLogger.exception(self, excep)
def load_regex_templates(self, configuration):
if configuration.files.regex_templates is not None:
collection = PropertiesCollection()
total = collection.load_from_filename(
configuration.files.regex_templates)
YLogger.info(self, "Loaded a total of %d regex templates", total)
for pair in collection.pairs:
name = pair[0]
pattern = pair[1]
try:
self._regex_templates[name] = re.compile(
pattern, re.IGNORECASE)
except Exception:
YLogger.error(self, "Invalid regex template [%s]", pattern)
def regex_template(self, name):
if name in self._regex_templates:
return self._regex_templates[name]
return None
def strip_oob(self, response):
match = re.compile(r"(.*)(<\s*oob\s*>.*<\/\s*oob\s*>)(.*)")
groupings = match.match(response)
if groupings is not None:
front = groupings.group(1).strip()
back = groupings.group(3).strip()
response = ""
if front != "":
response = front + " "
response += back
oob = groupings.group(2)
return response, oob
return response, None
def process_oob(self, client_context, oob_command):
oob_content = ET.fromstring(oob_command)
if oob_content.tag == 'oob':
for child in oob_content.findall('./'):
if child.tag in self._oob:
oob_class = self._oob[child.tag]
return oob_class.process_out_of_bounds(
client_context, child)
return self._default_oob.process_out_of_bounds(
client_context, child)
return ""
def post_process_response(self, client_context, response: str):
return self.postprocessors.process(client_context, response)
def failed_authentication(self, client_context):
YLogger.error(client_context, "[%s] failed authentication!")
# If we have an SRAI defined, then use that
if self.authentication.configuration.denied_srai is not None:
match_context = self._aiml_parser.match_sentence(
client_context,
Sentence(self._bot.brain.tokenizer,
self.authentication.configuration.denied_srai),
topic_pattern="*",
that_pattern="*")
# If the SRAI matched then return the result
if match_context is not None:
return self.resolve_matched_template(client_context,
match_context)
# Otherswise return the static text, which is either
# User defined via config.yaml
# Or use the default value BrainSecurityConfiguration.DEFAULT_ACCESS_DENIED
return self.authentication.configuration.denied_text
def authenticate_user(self, client_context):
if self.authentication is not None:
if self.authentication.authenticate(client_context) is False:
return self.failed_authentication(client_context)
return None
def resolve_matched_template(self, client_context, match_context):
template_node = match_context.template_node()
YLogger.debug(client_context, "AIML Parser evaluating template [%s]",
template_node.to_string())
response = template_node.template.resolve(client_context)
if "<oob>" in response:
response, oob = self.strip_oob(response)
if oob is not None:
oob_response = self.process_oob(client_context, oob)
response = response + " " + oob_response
return response
def ask_question(self, client_context, sentence):
client_context.brain = self
authenticated = self.authenticate_user(client_context)
if authenticated is not None:
return authenticated
conversation = client_context.bot.get_conversation(client_context)
topic_pattern = conversation.get_topic_pattern()
that_pattern = conversation.get_that_pattern()
match_context = self._aiml_parser.match_sentence(
client_context,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
if match_context is not None:
return self.resolve_matched_template(client_context, match_context)
return None
class Brain(object):
def __init__(self, configuration: BrainConfiguration):
self._configuration = configuration
self._aiml_parser = AIMLParser(self)
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self._configuration)
@property
def configuration(self):
return self._configuration
@property
def aiml_parser(self):
return self._aiml_parser
@property
def denormals(self):
return self._denormal_collection
@property
def normals(self):
return self._normal_collection
@property
def genders(self):
return self._gender_collection
@property
def persons(self):
return self._person_collection
@property
def person2s(self):
return self._person2_collection
@property
def rdf(self):
return self._rdf_collection
@property
def sets(self):
return self._sets_collection
@property
def maps(self):
return self._maps_collection
@property
def properties(self):
return self._properties_collection
@property
def preprocessors(self):
return self._preprocessors
@property
def postprocessors(self):
return self._postprocessors
@property
def authentication(self):
return self._authentication
@property
def authorisation(self):
return self._authorisation
@property
def default_oob(self):
return self._default_oob
@property
def oobs(self):
return self._oob
@property
def regex_templates(self):
return self._regex_templates
@property
def dynamics(self):
return self._dynamics_collection
def load_binary(self, brain_configuration):
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading binary brain from [%s]" %
brain_configuration.binaries.binary_filename)
try:
start = datetime.datetime.now()
gc.disable()
f = open(brain_configuration.binaries.binary_filename, "rb")
self._aiml_parser = pickle.load(f)
gc.enable()
f.close()
stop = datetime.datetime.now()
diff = stop - start
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Brain load took a total of %.2f sec" %
diff.total_seconds())
load_aiml = False
except Exception as e:
logging.exception(e)
if brain_configuration.binaries.load_aiml_on_binary_fail is True:
load_aiml = True
else:
raise e
def load_aiml(self, brain_configuration):
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading aiml source brain")
self._aiml_parser.load_aiml(brain_configuration)
def save_binary(self, brain_configuration):
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Saving binary brain to [%s]" %
brain_configuration.binaries.binary_filename)
start = datetime.datetime.now()
f = open(brain_configuration.binaries.binary_filename, "wb")
pickle.dump(self._aiml_parser, f)
f.close()
stop = datetime.datetime.now()
diff = stop - start
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Brain save took a total of %.2f sec" %
diff.total_seconds())
def load(self, brain_configuration: BrainConfiguration):
if brain_configuration.binaries.load_binary is True:
self.load_binary(brain_configuration)
self.load_aiml(brain_configuration)
if brain_configuration.binaries.save_binary is True:
self.save_binary(brain_configuration)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading collections")
self.load_collections(brain_configuration)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading services")
self.load_services(brain_configuration)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading security services")
self.load_security_services(brain_configuration)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading oob processors")
self.load_oob_processors(brain_configuration)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading regex templates")
self.load_regex_templates(brain_configuration)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading dynamics sets, maps and vars")
self.load_dynamics(brain_configuration)
def _load_denormals(self, brain_configuration):
if brain_configuration.files.denormal is not None:
total = self._denormal_collection.load_from_filename(
brain_configuration.files.denormal)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d denormalisations", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for denormal")
def _load_normals(self, brain_configuration):
if brain_configuration.files.normal is not None:
total = self._normal_collection.load_from_filename(
brain_configuration.files.normal)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d normalisations", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for normal")
def _load_genders(self, brain_configuration):
if brain_configuration.files.gender is not None:
total = self._gender_collection.load_from_filename(
brain_configuration.files.gender)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d genderisations", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for gender")
def _load_persons(self, brain_configuration):
if brain_configuration.files.person is not None:
total = self._person_collection.load_from_filename(
brain_configuration.files.person)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d persons", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for person")
def _load_person2s(self, brain_configuration):
if brain_configuration.files.person2 is not None:
total = self._person2_collection.load_from_filename(
brain_configuration.files.person2)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d person2s", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for person2")
def _load_properties(self, brain_configuration):
if brain_configuration.files.properties is not None:
total = self._properties_collection.load_from_filename(
brain_configuration.files.properties)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d properties", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for properties")
def _load_rdf(self, brain_configuration):
if brain_configuration.files.rdf_files is not None and brain_configuration.files.rdf_files.files is not None:
total = self._rdf_collection.load(
brain_configuration.files.rdf_files)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d rdf files", total)
elif brain_configuration.files.triples is not None:
total = self._rdf_collection.load_from_filename(
brain_configuration.files.triples)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d triples", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for triples")
def _load_sets(self, brain_configuration):
total = self._sets_collection.load(brain_configuration.files.set_files)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d sets files", total)
def _load_maps(self, brain_configuration):
total = self._maps_collection.load(brain_configuration.files.map_files)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d maps files", total)
def _load_preprocessors(self, brain_configuration):
if brain_configuration.files.preprocessors is not None:
total = self._preprocessors.load(
brain_configuration.files.preprocessors)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d pre processors", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for pre processors")
def _load_postprocessors(self, brain_configuration):
if brain_configuration.files.postprocessors is not None:
total = self._postprocessors.load(
brain_configuration.files.postprocessors)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d post processors", total)
else:
if logging.getLogger().isEnabledFor(logging.WARNING):
logging.warning("No configuration setting for post processors")
def load_collections(self, brain_configuration):
self._load_denormals(brain_configuration)
self._load_normals(brain_configuration)
self._load_genders(brain_configuration)
self._load_persons(brain_configuration)
self._load_person2s(brain_configuration)
self._load_properties(brain_configuration)
self._load_rdf(brain_configuration)
self._load_sets(brain_configuration)
self._load_maps(brain_configuration)
self._load_preprocessors(brain_configuration)
self._load_postprocessors(brain_configuration)
def load_services(self, brain_configuration):
ServiceFactory.preload_services(brain_configuration.services)
def load_security_services(self, brain_configuration):
if brain_configuration.security is not None:
if brain_configuration.security.authentication is not None:
if brain_configuration.security.authentication.classname is not None:
try:
classobject = ClassLoader.instantiate_class(
brain_configuration.security.authentication.
classname)
self._authentication = classobject(
brain_configuration.security.authentication)
except Exception as excep:
logging.exception(excep)
else:
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("No authentication configuration defined")
if brain_configuration.security.authorisation is not None:
if brain_configuration.security.authorisation.classname is not None:
try:
classobject = ClassLoader.instantiate_class(
brain_configuration.security.authorisation.
classname)
self._authorisation = classobject(
brain_configuration.security.authorisation)
except Exception as excep:
logging.exception(excep)
else:
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("No authorisation configuration defined")
else:
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug(
"No security configuration defined, running open...")
def load_dynamics(self, brain_configuration):
if brain_configuration.dynamics is not None:
self._dynamics_collection.load_from_configuration(
brain_configuration.dynamics)
else:
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("No dynamics configuration defined...")
def pre_process_question(self, bot, clientid, question):
return self.preprocessors.process(bot, clientid, question)
def parse_last_sentences_from_response(self, response):
response = re.sub(r'<\s*br\s*/>\s*', ".", response)
response = re.sub(r'<br></br>*', ".", response)
sentences = response.split(".")
sentences = [x for x in sentences if x]
last_sentence = sentences[-1]
that_pattern = TextUtils.strip_all_punctuation(last_sentence)
that_pattern = that_pattern.strip()
return that_pattern
def load_oob_processors(self, brain_configuration):
if brain_configuration.oob is not None:
if brain_configuration.oob.default() is not None:
try:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading default oob")
classobject = ClassLoader.instantiate_class(
brain_configuration.oob.default().classname)
self._default_oob = classobject()
except Exception as excep:
logging.exception(excep)
for oob_name in brain_configuration.oob.oobs():
try:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loading oob: %s" % oob_name)
classobject = ClassLoader.instantiate_class(
brain_configuration.oob.oob(oob_name).classname)
self._oob[oob_name] = classobject()
except Exception as excep:
logging.exception(excep)
def load_regex_templates(self, brain_configuration):
if brain_configuration.files.regex_templates is not None:
collection = PropertiesCollection()
total = collection.load_from_filename(
brain_configuration.files.regex_templates)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Loaded a total of %d regex templates", total)
for pair in collection.pairs:
name = pair[0]
pattern = pair[1]
try:
self._regex_templates[name] = re.compile(pattern)
except Exception:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.error("Invalid regex template [%s]" % pattern)
def regex_template(self, name):
if name in self._regex_templates:
return self._regex_templates[name]
else:
return None
def strip_oob(self, response):
m = re.compile("(.*)(<\s*oob\s*>.*<\/\s*oob\s*>)(.*)")
g = m.match(response)
if g is not None:
front = g.group(1).strip()
back = g.group(3).strip()
response = ""
if front != "":
response = front + " "
response += back
oob = g.group(2)
return response, oob
return response, None
def process_oob(self, bot, clientid, oob_command):
oob_content = ET.fromstring(oob_command)
if oob_content.tag == 'oob':
for child in oob_content.findall('./'):
if child.tag in self._oob:
oob_class = self._oob[child.tag]
return oob_class.process_out_of_bounds(
bot, clientid, child)
else:
return self._default_oob.process_out_of_bounds(
bot, clientid, child)
return ""
def post_process_response(self, bot, clientid, response: str):
return self.postprocessors.process(bot, clientid, response)
def dump_tree(self):
self._aiml_parser.pattern_parser.root.dump(tabs="")
def ask_question(self,
bot,
clientid,
sentence,
srai=False,
brain_question_context=None):
if brain_question_context is not None:
brain_question_context.clientid = clientid
brain_question_context.srai = srai
brain_question_context.sentence = sentence
if self.authentication is not None:
if self.authentication.authenticate(clientid) is False:
if logging.getLogger().isEnabledFor(logging.ERROR):
logging.error("[%s] failed authentication!")
return self.authentication.configuration.denied_srai
conversation = bot.get_conversation(clientid)
topic_pattern = conversation.property("topic")
if topic_pattern is None:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("No Topic pattern default to [*]")
topic_pattern = "*"
else:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("Topic pattern = [%s]", topic_pattern)
if brain_question_context is not None:
brain_question_context.topic = topic_pattern
try:
that_question = conversation.previous_nth_question(1)
that_sentence = that_question.current_sentence()
# If the last response was valid, i.e not none and not empty string, then use
# that as the that_pattern, otherwise we default to '*' as pattern
if that_sentence.response is not None and that_sentence.response != '':
that_pattern = self.parse_last_sentences_from_response(
that_sentence.response)
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("That pattern = [%s]", that_pattern)
else:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("That pattern, no response, default to [*]")
that_pattern = "*"
except Exception:
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info("No That pattern default to [*]")
that_pattern = "*"
if brain_question_context is not None:
brain_question_context.that = that_pattern
match_context = self._aiml_parser.match_sentence(
bot,
clientid,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
if match_context is not None:
if brain_question_context is not None:
brain_question_context.match_context = match_context
template_node = match_context.template_node()
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("AIML Parser evaluating template [%s]",
template_node.to_string())
response = template_node.template.resolve(bot, clientid)
if brain_question_context is not None:
brain_question_context.raw_response = response
if "<oob>" in response:
response, oob = self.strip_oob(response)
if oob is not None:
oob_response = self.process_oob(bot, clientid, oob)
if brain_question_context is not None:
brain_question_context.raw_response = response
brain_question_context.oob_response = oob_response
response = response + " " + oob_response
return response
return None
class Brain(object):
NLU_UTTERANCE = 'NLU_Matching'
def __init__(self, bot, configuration: BrainConfiguration):
assert (bot is not None)
assert (configuration is not None)
self._bot = bot
self._configuration = configuration
self._binaries = BinariesManager(configuration.binaries)
self._braintree = BraintreeManager(configuration.braintree)
self._tokenizer = Tokenizer.load_tokenizer(configuration)
if configuration.debugfiles.save_errors_collection is True:
errors_dict = {}
else:
errors_dict = None
self._denormal_collection = DenormalCollection(errors_dict)
self._normal_collection = NormalCollection(errors_dict)
self._gender_collection = GenderCollection(errors_dict)
self._person_collection = PersonCollection(errors_dict)
self._person2_collection = Person2Collection(errors_dict)
self._rdf_collection = RDFCollection(errors_dict)
self._sets_collection = SetCollection(errors_dict)
self._maps_collection = MapCollection(errors_dict)
self._properties_collection = PropertiesCollection(errors_dict)
self._default_variables_collection = DefaultVariablesCollection(
errors_dict)
self._botnames_collection = BotNamesCollection(errors_dict)
self._preprocessors = PreProcessorCollection(errors_dict)
self._postprocessors = PostProcessorCollection(errors_dict)
self._pattern_factory = None
self._template_factory = None
self._security = SecurityManager(configuration.security)
self._oobhandler = OOBHandler(configuration.oob)
self._regex_templates = RegexTemplatesCollection(errors_dict)
self._dynamics_collection = DynamicsCollection()
self._aiml_parser = self.load_aiml_parser()
self._nlu_collection = NluCollection(bot.client, configuration.nlu,
errors_dict)
self._nlu = NluRequest.load_nlu(configuration.nlu)
self._nlu_utterance = None
self.load(self.configuration)
if configuration.debugfiles.save_errors_collection is True:
storage_factory = self.bot.client.storage_factory
if storage_factory.entity_storage_engine_available(
StorageFactory.ERRORS_COLLECTION) is True:
errors_collection_engine = storage_factory.entity_storage_engine(
StorageFactory.ERRORS_COLLECTION)
errors_collection_store = errors_collection_engine.errors_collection_store(
)
errors_collection_store.save_errors_collection(errors_dict)
def ylogger_type(self):
return "brain"
@property
def id(self):
return self._configuration.section_name
@property
def bot(self):
return self._bot
@property
def configuration(self):
return self._configuration
@property
def aiml_parser(self):
return self._aiml_parser
@property
def denormals(self):
return self._denormal_collection
@property
def normals(self):
return self._normal_collection
@property
def genders(self):
return self._gender_collection
@property
def persons(self):
return self._person_collection
@property
def person2s(self):
return self._person2_collection
@property
def rdf(self):
return self._rdf_collection
@property
def sets(self):
return self._sets_collection
@property
def maps(self):
return self._maps_collection
@property
def properties(self):
return self._properties_collection
@property
def default_variables(self):
return self._default_variables_collection
@property
def botnames(self):
return self._botnames_collection
@property
def preprocessors(self):
return self._preprocessors
@property
def postprocessors(self):
return self._postprocessors
@property
def pattern_factory(self):
return self._pattern_factory
@property
def template_factory(self):
return self._template_factory
@property
def regex_templates(self):
return self._regex_templates
@property
def dynamics(self):
return self._dynamics_collection
@property
def tokenizer(self):
return self._tokenizer
@property
def nlu(self):
return self._nlu
@property
def security(self):
return self._security
def load_aiml_parser(self):
self._load_pattern_nodes()
self._load_template_nodes()
return AIMLParser(self)
def load_aiml(self):
YLogger.debug(self, "Loading aiml source brain")
self._aiml_parser.load_aiml()
def reload_aimls(self):
YLogger.debug(self, "Loading aiml source brain")
self._aiml_parser.empty()
self._aiml_parser.load_aiml()
def load(self, configuration: BrainConfiguration):
self._load_properties()
if self.properties.has_property("punctuation_chars") is True:
self.tokenizer.set_configuration_punctuation_chars(
self.properties.property("punctuation_chars"))
if self.properties.has_property("before_concatenation_rule") is True:
self.tokenizer.set_configuration_before_concatenation_rule(
self.properties.property("before_concatenation_rule"))
if self.properties.has_property("after_concatenation_rule") is True:
self.tokenizer.set_configuration_after_concatenation_rule(
self.properties.property("after_concatenation_rule"))
YLogger.debug(self, "Loading collections")
self.load_collections()
YLogger.debug(self, "Loading dynamics sets, maps and vars")
self.load_dynamics()
YLogger.debug(self, "Loading services")
self.load_services(configuration)
load_aiml = True
if self.configuration.binaries.load_binary is True:
load_aiml = self._binaries.load_binary(
self.bot.client.storage_factory)
if load_aiml is False:
self._aiml_parser = self._binaries.get_aiml_parser()
if load_aiml is True:
self.load_aiml()
self._binaries.set_aiml_parser(self._aiml_parser)
if configuration.binaries.save_binary is True:
self._binaries.save_binary(self.bot.client.storage_factory)
YLogger.debug(self, "Loading security services")
self.load_security_services()
YLogger.debug(self, "Loading oob processors")
self._oobhandler.load_oob_processors()
def dump_brain_tree(self, client_context):
self._braintree.dump_brain_tree(client_context)
def _load_denormals(self):
self._denormal_collection.empty()
self._denormal_collection.load(self.bot.client.storage_factory)
def _load_normals(self):
self._normal_collection.empty()
self._normal_collection.load(self.bot.client.storage_factory)
def _load_genders(self):
self._gender_collection.empty()
self._gender_collection.load(self.bot.client.storage_factory)
def _load_persons(self):
self._person_collection.empty()
self._person_collection.load(self.bot.client.storage_factory)
def _load_person2s(self):
self._person2_collection.empty()
self._person2_collection.load(self.bot.client.storage_factory)
def _load_botnames(self):
self._botnames_collection.empty()
self._botnames_collection.load(self.bot.client.storage_factory)
def _load_properties(self):
self._properties_collection.empty()
self._properties_collection.load(self.bot.client.storage_factory)
def _load_default_variables(self):
self._default_variables_collection.empty()
self._default_variables_collection.load(
self.bot.client.storage_factory)
self._set_system_defined()
def _set_system_defined(self):
self.set_sentiment_scores(0.0, 0.5)
def set_sentiment_scores(self, positivity, subjectivity):
if self._default_variables_collection.has_variable(
"positivity") is False:
self._default_variables_collection.set_value(
"positivity", str(positivity))
if self._default_variables_collection.has_variable(
"subjectivity") is False:
self._default_variables_collection.set_value(
"subjectivity", str(subjectivity))
def _load_maps(self):
self._maps_collection.empty()
self._maps_collection.load(self.bot.client.storage_factory)
def reload_map(self, mapname):
if self._maps_collection.contains(mapname):
self._maps_collection.reload(self.bot.client.storage_factory,
mapname)
def _load_sets(self):
self._sets_collection.empty()
self._sets_collection.load(self.bot.client.storage_factory)
def reload_set(self, setname):
if self._sets_collection.contains(setname):
self._sets_collection.reload(self.bot.client.storage_factory,
setname)
def _load_rdfs(self):
self._rdf_collection.empty()
self._rdf_collection.load(self.bot.client.storage_factory)
def reload_rdf(self, rdfname):
if self._rdf_collection.contains(rdfname):
self._rdf_collection.reload(self.bot.client.storage_factory,
rdfname)
def _load_regex_templates(self):
self._regex_templates.load(self.bot.client.storage_factory)
def _load_preprocessors(self):
self._preprocessors.empty()
self._preprocessors.load(self.bot.client.storage_factory)
def _load_postprocessors(self):
self._postprocessors.empty()
self._postprocessors.load(self.bot.client.storage_factory)
def _load_pattern_nodes(self):
self._pattern_factory = PatternNodeFactory()
self._pattern_factory.load(self.bot.client.storage_factory)
def _load_template_nodes(self):
self._template_factory = TemplateNodeFactory()
self._template_factory.load(self.bot.client.storage_factory)
def load_collections(self):
self._load_denormals()
self._load_normals()
self._load_genders()
self._load_persons()
self._load_person2s()
self._load_default_variables()
self._load_botnames()
self._load_rdfs()
self._load_sets()
self._load_maps()
self._load_regex_templates()
self._load_preprocessors()
self._load_postprocessors()
def load_services(self, configuration):
ServiceFactory.preload_services(configuration.services)
def load_security_services(self):
self._security.load_security_services(self.bot.client)
def load_dynamics(self):
if self.configuration.dynamics is not None:
self._dynamics_collection.load_from_configuration(
self.configuration.dynamics)
else:
YLogger.debug(self, "No dynamics configuration defined...")
def pre_process_question(self, client_context, question):
return self.preprocessors.process(client_context, question)
def post_process_response(self, client_context, response: str):
return self.postprocessors.process(client_context, response)
def failed_authentication(self, client_context):
return self._security.failed_authentication(client_context)
def authenticate_user(self, client_context):
return self._security.authenticate_user(client_context)
def resolve_matched_template(self, client_context, match_context):
assert (client_context is not None)
assert (match_context is not None)
template_node = match_context.template_node()
YLogger.debug(client_context, "AIML Parser evaluating template [%s]",
template_node.to_string())
response = template_node.template.resolve(client_context)
if self._oobhandler.oob_in_response(response) is True:
if client_context.brain.template_factory.exists('oob') is True:
response = self._oobhandler.handle(client_context, response)
else:
YLogger.debug(client_context, "OOB function is disable [%s]",
response)
return response
def ask_question(self,
client_context,
sentence,
srai=False,
default_srai=False):
assert (client_context is not None)
assert (client_context.bot is not None)
assert (self._aiml_parser is not None)
client_context.brain = self
authenticated = self.authenticate_user(client_context)
if authenticated is not None:
return authenticated
conversation = client_context.bot.get_conversation(client_context)
answer = None
if conversation is not None:
if srai is False and default_srai is False:
self._nlu_utterance = None
try:
client_context.userInfo.userInfoPreProcessor(
client_context, srai)
except Exception:
pass
topic_pattern = conversation.get_topic_pattern(client_context)
that_pattern = conversation.get_that_pattern(client_context, srai)
original_base = None
if default_srai is True and conversation.internal_base is not None:
original_base = conversation.internal_base
base = original_base
if 'srai_histories' not in base:
base['srai_histories'] = []
default_srai_root = base['srai_histories']
default_srai_root.append({})
conversation.internal_base = default_srai_root[-1]
conversation.add_internal_data(conversation.internal_base,
'default_srai', True)
elif srai is False:
conversation.internal_data.append({})
conversation.internal_base = conversation.internal_data[-1]
base = conversation.internal_base
texts = self.tokenizer.words_to_texts(sentence.words)
conversation.add_internal_data(base, 'question', texts)
conversation.add_internal_data(base, 'topic', topic_pattern)
conversation.add_internal_data(base, 'that', that_pattern)
current_variables = ConversationVariables(conversation)
match_context = self._aiml_parser.match_sentence(
client_context,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
if match_context is not None:
if len(match_context.matched_nodes
) == 3 and match_context.matched_nodes[
0].matched_node.is_wildcard() is True:
if self._aiml_parser.pattern_parser.use_nlu is False:
YLogger.debug(
client_context,
"Matched Catgeory (*): file[%s] line[%s-%s]",
match_context._template_node._filename,
match_context._template_node._start_line,
match_context._template_node._end_line)
conversation.add_internal_matched(
base, match_context._template_node)
answer = self.resolve_matched_template(
client_context, match_context)
else:
YLogger.debug(client_context,
"Matched Catgeory : file[%s] line[%s-%s]",
match_context._template_node._filename,
match_context._template_node._start_line,
match_context._template_node._end_line)
try:
conversation.add_internal_matched(
base, match_context._template_node)
answer = self.resolve_matched_template(
client_context, match_context)
except Exception:
self.set_match_context_info(conversation,
match_context)
before, after = current_variables.different_variables(
conversation)
conversation.add_internal_data(base,
'before_variables',
before)
conversation.add_internal_data(base, 'after_variables',
after)
raise
if answer is None and self._aiml_parser.pattern_parser.use_nlu is True:
assert (self._nlu is not None)
try:
utterance = client_context.brain.tokenizer.words_to_texts(
sentence.words)
if default_srai is False or self.bot.configuration.default_response_srai != utterance:
nluResult = conversation.current_question().property(
"__SYSTEM_NLUDATA__")
if nluResult is None or nluResult == "" or self._nlu_utterance != utterance:
self._nlu_utterance = utterance
match_context = self.multi_nlu_match(
client_context, conversation, topic_pattern,
that_pattern)
else:
client_context.match_nlu = True
sentence = Sentence(client_context.brain.tokenizer,
self.NLU_UTTERANCE)
match_context = self._aiml_parser.match_sentence(
client_context,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
client_context.match_nlu = False
if match_context is not None:
YLogger.debug(
client_context,
"Matched Catgeory (NLU): file[%s] line[%s-%s]",
match_context._template_node._filename,
match_context._template_node._start_line,
match_context._template_node._end_line)
conversation.add_internal_matched(
base, match_context._template_node)
answer = self.resolve_matched_template(
client_context, match_context)
except NotImplementedError:
if match_context is not None:
YLogger.debug(
client_context,
"Matched Catgeory (*): file[%s] line[%s-%s]",
match_context._template_node._filename,
match_context._template_node._start_line,
match_context._template_node._end_line)
conversation.add_internal_matched(
base, match_context._template_node)
answer = self.resolve_matched_template(
client_context, match_context)
except Exception:
self.set_match_context_info(conversation, match_context)
before, after = current_variables.different_variables(
conversation)
conversation.add_internal_data(base, 'before_variables',
before)
conversation.add_internal_data(base, 'after_variables',
after)
raise
self.set_match_context_info(conversation, match_context)
before, after = current_variables.different_variables(conversation)
conversation.add_internal_data(base, 'before_variables', before)
conversation.add_internal_data(base, 'after_variables', after)
conversation.add_internal_data(base, 'processing_result', answer)
if original_base is not None:
conversation.internal_base = original_base
base = conversation.internal_base
conversation.add_internal_data(base, 'response', answer)
try:
client_context.userInfo.userInfoPostProcessor(client_context)
except Exception:
pass
return answer
def set_match_context_info(self, conversation, match_context):
if match_context is not None:
json_matchedNode = {
'file_name': match_context._template_node._filename,
'start_line': match_context._template_node._start_line,
'end_line': match_context._template_node._end_line
}
question = conversation.current_question()
if question is not None:
question.sentences[
question.
_current_sentence_no].matched_node = json_matchedNode
def multi_nlu_match(self, client_context, conversation, topic_pattern,
that_pattern):
nlu_list = self._nlu_collection.servers
match_context = None
for server_name in nlu_list:
try:
server = self._nlu_collection.server_info(server_name)
nluResult = self._nlu.nluCall(client_context, server.url,
server.apikey,
self._nlu_utterance)
except NotImplementedError:
raise
conversation.current_question().set_property(
"__SYSTEM_NLUDATA__", nluResult)
if nluResult is not None:
client_context.match_nlu = True
sentence = Sentence(client_context.brain.tokenizer,
self.NLU_UTTERANCE)
match_context = self._aiml_parser.match_sentence(
client_context,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
client_context.match_nlu = False
if match_context is not None:
if len(match_context.matched_nodes) != 3 or \
match_context.matched_nodes[0].matched_node.is_wildcard() is False:
break
if match_context is None:
conversation.current_question().set_property(
"__SYSTEM_NLUDATA__", None)
try:
client_context.userInfo.userInfoPostProcessor(client_context)
except Exception:
pass
return match_context
class Brain(object):
def __init__(self, bot, configuration: BrainConfiguration):
assert (bot is not None)
assert (configuration is not None)
self._questions = 0
self._bot = bot
self._configuration = configuration
self._binaries = BinariesManager(configuration.binaries)
self._braintree = BraintreeManager(configuration.braintree)
self._tokenizer = Tokenizer.load_tokenizer(configuration)
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = Person2Collection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._default_variables_collection = DefaultVariablesCollection()
self._preprocessors = PreProcessorCollection()
self._postprocessors = PostProcessorCollection()
self._postquestionprocessors = PostQuestionProcessorCollection()
self._pattern_factory = None
self._template_factory = None
self._security = SecurityManager(configuration.security)
self._oobhandler = OOBHandler(configuration.oob)
self._openchatbots = OpenChatBotCollection()
self._regex_templates = RegexTemplatesCollection()
self._dynamics_collection = DynamicsCollection()
self._aiml_parser = self.load_aiml_parser()
self.load(self.configuration)
def ylogger_type(self):
return "brain"
@property
def num_questions(self):
return self._questions
@property
def id(self):
return self._configuration.section_name
@property
def bot(self):
return self._bot
@property
def configuration(self):
return self._configuration
@property
def aiml_parser(self):
return self._aiml_parser
@property
def denormals(self):
return self._denormal_collection
@property
def normals(self):
return self._normal_collection
@property
def genders(self):
return self._gender_collection
@property
def persons(self):
return self._person_collection
@property
def person2s(self):
return self._person2_collection
@property
def rdf(self):
return self._rdf_collection
@property
def sets(self):
return self._sets_collection
@property
def maps(self):
return self._maps_collection
@property
def properties(self):
return self._properties_collection
@property
def default_variables(self):
return self._default_variables_collection
@property
def preprocessors(self):
return self._preprocessors
@property
def postprocessors(self):
return self._postprocessors
@property
def postquestionprocessors(self):
return self._postquestionprocessors
@property
def pattern_factory(self):
return self._pattern_factory
@property
def template_factory(self):
return self._template_factory
@property
def regex_templates(self):
return self._regex_templates
@property
def dynamics(self):
return self._dynamics_collection
@property
def tokenizer(self):
return self._tokenizer
@property
def openchatbots(self):
return self._openchatbots
@property
def security(self):
return self._security
def load_aiml_parser(self):
self._load_pattern_nodes()
self._load_template_nodes()
return AIMLParser(self)
def load_aiml(self):
YLogger.info(self, "Loading aiml source brain")
self._aiml_parser.load_aiml()
def reload_aimls(self):
YLogger.info(self, "Loading aiml source brain")
self._aiml_parser.empty()
self._aiml_parser.load_aiml()
def load(self, configuration: BrainConfiguration):
load_aiml = True
if self.configuration.binaries.load_binary is True:
load_aiml = self._binaries.load_binary(
self.bot.client.storage_factory)
if load_aiml is True:
self.load_aiml()
if configuration.binaries.save_binary is True:
self._binaries.save_binary(self.bot.client.storage_factory)
YLogger.info(self, "Loading collections")
self.load_collections()
YLogger.info(self, "Loading services")
self.load_services(configuration)
YLogger.info(self, "Loading openchat bots")
self.load_openchatbots(configuration)
YLogger.info(self, "Loading security services")
self.load_security_services()
YLogger.info(self, "Loading oob processors")
self._oobhandler.load_oob_processors()
YLogger.info(self, "Loading regex templates")
self.load_regex_templates()
YLogger.info(self, "Loading dynamics sets, maps and vars")
self.load_dynamics()
def dump_brain_tree(self, client_context):
self._braintree.dump_brain_tree(client_context)
def _load_denormals(self):
self._denormal_collection.empty()
self._denormal_collection.load(self.bot.client.storage_factory)
def _load_normals(self):
self._normal_collection.empty()
self._normal_collection.load(self.bot.client.storage_factory)
def _load_genders(self):
self._gender_collection.empty()
self._gender_collection.load(self.bot.client.storage_factory)
def _load_persons(self):
self._person_collection.empty()
self._person_collection.load(self.bot.client.storage_factory)
def _load_person2s(self):
self._person2_collection.empty()
self._person2_collection.load(self.bot.client.storage_factory)
def _load_properties(self):
self._properties_collection.empty()
self._properties_collection.load(self.bot.client.storage_factory)
def _load_default_variables(self):
self._default_variables_collection.empty()
self._default_variables_collection.load(
self.bot.client.storage_factory)
self._set_system_defined()
def _set_system_defined(self):
self.set_sentiment_scores(0.0, 0.5)
def set_sentiment_scores(self, positivity, subjectivity):
if self._default_variables_collection.has_variable(
"positivity") is False:
self._default_variables_collection.set_value(
"positivity", str(positivity))
if self._default_variables_collection.has_variable(
"subjectivity") is False:
self._default_variables_collection.set_value(
"subjectivity", str(subjectivity))
def _load_maps(self):
self._maps_collection.empty()
self._maps_collection.load(self.bot.client.storage_factory)
def reload_map(self, mapname):
if self._maps_collection.contains(mapname):
self._maps_collection.reload(self.bot.client.storage_factory,
mapname)
else:
YLogger.error(self, "Unknown map name [%s], unable to reload ",
mapname)
def _load_sets(self):
self._sets_collection.empty()
self._sets_collection.load(self.bot.client.storage_factory)
def reload_set(self, setname):
if self._sets_collection.contains(setname):
self._sets_collection.reload(self.bot.client.storage_factory,
setname)
else:
YLogger.error(self, "Unknown set name [%s], unable to reload ",
setname)
def _load_rdfs(self):
self._rdf_collection.empty()
self._rdf_collection.load(self.bot.client.storage_factory)
def reload_rdf(self, rdfname):
if self._rdf_collection.contains(rdfname):
self._rdf_collection.reload(self.bot.client.storage_factory,
rdfname)
else:
YLogger.error(self, "Unknown rdf name [%s], unable to reload ",
rdfname)
def _load_preprocessors(self):
self._preprocessors.empty()
self._preprocessors.load(self.bot.client.storage_factory)
def _load_postprocessors(self):
self._postprocessors.empty()
self._postprocessors.load(self.bot.client.storage_factory)
def _load_postquestionprocessors(self):
self._postquestionprocessors.empty()
self._postquestionprocessors.load(self.bot.client.storage_factory)
def _load_pattern_nodes(self):
self._pattern_factory = PatternNodeFactory()
self._pattern_factory.load(self.bot.client.storage_factory)
def _load_template_nodes(self):
self._template_factory = TemplateNodeFactory()
self._template_factory.load(self.bot.client.storage_factory)
def load_collections(self):
self._load_denormals()
self._load_normals()
self._load_genders()
self._load_persons()
self._load_person2s()
self._load_properties()
self._load_default_variables()
self._load_rdfs()
self._load_sets()
self._load_maps()
self._load_preprocessors()
self._load_postprocessors()
self._load_postquestionprocessors()
def load_services(self, configuration):
ServiceFactory.preload_services(configuration.services)
def load_openchatbots(self, configuration):
self._openchatbots.load_from_configuration(configuration.openchatbots)
def load_security_services(self):
self._security.load_security_services(self.bot.client)
def load_dynamics(self):
if self.configuration.dynamics is not None:
self._dynamics_collection.load_from_configuration(
self.configuration.dynamics)
else:
YLogger.debug(self, "No dynamics configuration defined...")
def load_regex_templates(self):
self._regex_templates.load(self.bot.client.storage_factory)
def pre_process_question(self, client_context, question):
return self.preprocessors.process(client_context, question)
def post_process_response(self, client_context, response: str):
return self.postprocessors.process(client_context, response)
def post_process_question(self, client_context, question: str):
return self.postquestionprocessors.process(client_context, question)
def failed_authentication(self, client_context):
return self._security.failed_authentication(client_context)
def authenticate_user(self, client_context):
return self._security.authenticate_user(client_context)
def resolve_matched_template(self, client_context, match_context):
assert (client_context is not None)
assert (match_context is not None)
template_node = match_context.template_node()
YLogger.debug(client_context, "AIML Parser evaluating template [%s]",
template_node.to_string())
response = template_node.template.resolve(client_context)
if self._oobhandler.oob_in_response(response) is True:
response = self._oobhandler.handle(client_context, response)
return response
def ask_question(self, client_context, sentence, srai=False):
assert (client_context is not None)
assert (client_context.bot is not None)
assert (self._aiml_parser is not None)
client_context.brain = self
authenticated = self.authenticate_user(client_context)
if authenticated is not None:
return authenticated
conversation = client_context.bot.get_conversation(client_context)
if conversation is not None:
self._questions += 1
topic_pattern = conversation.get_topic_pattern(client_context)
that_pattern = conversation.get_that_pattern(client_context, srai)
match_context = self._aiml_parser.match_sentence(
client_context,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
if match_context is not None:
return self.resolve_matched_template(client_context,
match_context)
return None
class Brain(object):
def __init__(self, bot, configuration: BrainConfiguration):
self._bot = bot
self._configuration = configuration
self._tokenizer = self.load_tokenizer()
self._aiml_parser = self.load_aiml_parser()
self._denormal_collection = DenormalCollection()
self._normal_collection = NormalCollection()
self._gender_collection = GenderCollection()
self._person_collection = PersonCollection()
self._person2_collection = PersonCollection()
self._rdf_collection = RDFCollection()
self._sets_collection = SetCollection()
self._maps_collection = MapCollection()
self._properties_collection = PropertiesCollection()
self._variables_collection = PropertiesCollection()
self._preprocessors = ProcessorLoader()
self._postprocessors = ProcessorLoader()
self._authentication = None
self._authorisation = None
self._default_oob = None
self._oob = {}
self._regex_templates = {}
self._dynamics_collection = DynamicsCollection()
self.load(self.configuration)
def ylogger_type(self):
return "brain"
@property
def id(self):
return self._configuration.section_name
@property
def bot(self):
return self._bot
@property
def configuration(self):
return self._configuration
@property
def aiml_parser(self):
return self._aiml_parser
@property
def denormals(self):
return self._denormal_collection
@property
def normals(self):
return self._normal_collection
@property
def genders(self):
return self._gender_collection
@property
def persons(self):
return self._person_collection
@property
def person2s(self):
return self._person2_collection
@property
def rdf(self):
return self._rdf_collection
@property
def sets(self):
return self._sets_collection
@property
def maps(self):
return self._maps_collection
@property
def properties(self):
return self._properties_collection
@property
def variables(self):
return self._variables_collection
@property
def preprocessors(self):
return self._preprocessors
@property
def postprocessors(self):
return self._postprocessors
@property
def authentication(self):
return self._authentication
@property
def authorisation(self):
return self._authorisation
@property
def default_oob(self):
return self._default_oob
@property
def oobs(self):
return self._oob
@property
def regex_templates(self):
return self._regex_templates
@property
def dynamics(self):
return self._dynamics_collection
@property
def tokenizer(self):
return self._tokenizer
def load_tokenizer(self):
if self.configuration is not None and self.configuration.tokenizer.classname is not None:
YLogger.info(self, "Loading tokenizer from class [%s]", self.configuration.tokenizer.classname)
tokenizer_class = ClassLoader.instantiate_class(self.configuration.tokenizer.classname)
return tokenizer_class(self.configuration.tokenizer.split_chars)
else:
return Tokenizer(self.configuration.tokenizer.split_chars)
def load_aiml_parser(self):
return AIMLParser(self)
def load_aiml(self, configuration):
YLogger.info(self, "Loading aiml source brain")
self._aiml_parser.load_aiml(configuration)
def reload_aimls(self):
YLogger.info(self, "Loading aiml source brain")
self._aiml_parser.empty()
self._aiml_parser.load_aiml(self.configuration)
def load_binary(self, configuration):
YLogger.info(self, "Loading binary brain from [%s]", configuration.binaries.binary_filename)
try:
start = datetime.datetime.now()
gc.disable()
bin_file = open(configuration.binaries.binary_filename, "rb")
self._aiml_parser = pickle.load(bin_file)
self._aiml_parser._brain = self
gc.enable()
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds())
return False # Tell caller, load succeeded and skip aiml load
except Exception as excep:
YLogger.exception(self, "Failed to load binary file", excep)
if configuration.binaries.load_aiml_on_binary_fail is True:
return True # Tell caller, load failed and to load aiml directly
else:
raise excep
def save_binary(self, configuration):
YLogger.info(self, "Saving binary brain to [%s]", configuration.binaries.binary_filename)
start = datetime.datetime.now()
bin_file = open(configuration.binaries.binary_filename, "wb")
pickle.dump(self._aiml_parser, bin_file)
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds())
def load(self, configuration: BrainConfiguration):
load_aiml = True
if self.configuration.binaries.load_binary is True:
load_aiml = self.load_binary(configuration)
if load_aiml is True:
self.load_aiml(configuration)
if configuration.binaries.save_binary is True:
self.save_binary(configuration)
YLogger.info(self, "Loading collections")
self.load_collections(configuration)
YLogger.info(self, "Loading services")
self.load_services(configuration)
YLogger.info(self, "Loading security services")
self.load_security_services(configuration)
YLogger.info(self, "Loading oob processors")
self.load_oob_processors(configuration)
YLogger.info(self, "Loading regex templates")
self.load_regex_templates(configuration)
YLogger.info(self, "Loading dynamics sets, maps and vars")
self.load_dynamics(configuration)
def dump_brain_tree(self):
if self.configuration.braintree.file is not None:
YLogger.debug(self, "Dumping AIML Graph as tree to [%s]",
self._configuration.braintree.file)
client_context = self.bot.client.create_client_context("system")
self.aiml_parser.pattern_parser.save_braintree(
client_context,
self.configuration.braintree.file,
self.configuration.braintree.content)
def _load_denormals(self, configuration):
if configuration.files.denormal is not None:
self._denormal_collection.empty()
total = self._denormal_collection.load_from_filename(configuration.files.denormal)
YLogger.info(self, "Loaded a total of %d denormalisations", total)
else:
YLogger.warning(self, "No configuration setting for denormal")
def _load_normals(self, configuration):
if configuration.files.normal is not None:
self._normal_collection.empty()
total = self._normal_collection.load_from_filename(configuration.files.normal)
YLogger.info(self, "Loaded a total of %d normalisations", total)
else:
YLogger.warning(self, "No configuration setting for normal")
def _load_genders(self, configuration):
if configuration.files.gender is not None:
self._gender_collection.empty()
total = self._gender_collection.load_from_filename(configuration.files.gender)
YLogger.info(self, "Loaded a total of %d genderisations", total)
else:
YLogger.warning(self, "No configuration setting for gender")
def _load_persons(self, configuration):
if configuration.files.person is not None:
self._person_collection.empty()
total = self._person_collection.load_from_filename(configuration.files.person)
YLogger.info(self, "Loaded a total of %d persons", total)
else:
YLogger.warning(self, "No configuration setting for person")
def _load_person2s(self, configuration):
if configuration.files.person2 is not None:
self._person2_collection.empty()
total = self._person2_collection.load_from_filename(configuration.files.person2)
YLogger.info(self, "Loaded a total of %d person2s", total)
else:
YLogger.warning(self, "No configuration setting for person2")
def _load_properties(self, configuration):
if configuration.files.properties is not None:
self._properties_collection.empty()
total = self._properties_collection.load_from_filename(configuration.files.properties)
YLogger.info(self, "Loaded a total of %d properties", total)
else:
YLogger.warning(self, "No configuration setting for properties")
def _load_variables(self, configuration):
if configuration.files.variables is not None:
self._variables_collection.empty ()
total = self._variables_collection.load_from_filename(configuration.files.variables)
YLogger.info(self, "Loaded a total of %d variables", total)
else:
YLogger.warning(self, "No configuration setting for variables")
def _load_maps(self, configuration):
self._maps_collection.empty()
total = self._maps_collection.load(configuration.files.map_files)
YLogger.info(self, "Loaded a total of %d maps files", total)
def reload_map(self, mapname):
if self._maps_collection.contains(mapname):
filename = self._maps_collection.filename(mapname)
self._maps_collection.reload_file(filename)
def _load_sets(self, configuration):
self._sets_collection.empty()
total = self._sets_collection.load(configuration.files.set_files)
YLogger.info(self, "Loaded a total of %d sets files", total)
def reload_set(self, setname):
if self._sets_collection.contains(setname):
filename = self._sets_collection.filename(setname)
self._sets_collection.reload_file(filename)
def _load_rdfs(self, configuration):
if configuration.files.rdf_files is not None and configuration.files.rdf_files.files:
self._rdf_collection.empty()
total = self._rdf_collection.load(configuration.files.rdf_files)
YLogger.info(self, "Loaded a total of %d rdf files", total)
elif configuration.files.triples is not None:
self._rdf_collection.empty()
total = self._rdf_collection.load_from_filename(configuration.files.triples)
YLogger.info(self, "Loaded a total of %d triples", total)
else:
YLogger.warning(self, "No configuration setting for triples")
def reload_rdf(self, rdfname):
if self._rdf_collection.contains(rdfname):
self._rdf_collection.reload_file(rdfname)
def _load_preprocessors(self, configuration):
if configuration.files.preprocessors is not None:
self._preprocessors.empty()
total = self._preprocessors.load(configuration.files.preprocessors)
YLogger.info(self, "Loaded a total of %d pre processors", total)
else:
YLogger.warning(self, "No configuration setting for pre processors")
def _load_postprocessors(self, configuration):
if configuration.files.postprocessors is not None:
self._postprocessors.empty()
total = self._postprocessors.load(configuration.files.postprocessors)
YLogger.info(self, "Loaded a total of %d post processors", total)
else:
YLogger.warning(self, "No configuration setting for post processors")
def load_collections(self, configuration):
self._load_denormals(configuration)
self._load_normals(configuration)
self._load_genders(configuration)
self._load_persons(configuration)
self._load_person2s(configuration)
self._load_properties(configuration)
self._load_variables(configuration)
self._load_rdfs(configuration)
self._load_sets(configuration)
self._load_maps(configuration)
self._load_preprocessors(configuration)
self._load_postprocessors(configuration)
def load_services(self, configuration):
ServiceFactory.preload_services(configuration.services)
def load_security_services(self, configuration):
if configuration.security is not None:
if configuration.security.authentication is not None:
if configuration.security.authentication.classname is not None:
try:
classobject = ClassLoader.instantiate_class(
configuration.security.authentication.classname)
self._authentication = classobject(configuration.security.authentication)
except Exception as excep:
YLogger.exception(self, "Failed to load security services", excep)
else:
YLogger.debug(self, "No authentication configuration defined")
if configuration.security.authorisation is not None:
if configuration.security.authorisation.classname is not None:
try:
classobject = ClassLoader.instantiate_class(
configuration.security.authorisation.classname)
self._authorisation = classobject(configuration.security.authorisation)
except Exception as excep:
YLogger.exception(self, "Failed to instatiate authorisation class", excep)
else:
YLogger.debug(self, "No authorisation configuration defined")
else:
YLogger.debug(self, "No security configuration defined, running open...")
def load_dynamics(self, configuration):
if configuration.dynamics is not None:
self._dynamics_collection.load_from_configuration(configuration.dynamics)
else:
YLogger.debug(self, "No dynamics configuration defined...")
def pre_process_question(self, client_context, question):
return self.preprocessors.process(client_context, question)
def load_oob_processors(self, configuration):
if configuration.oob is not None:
if configuration.oob.default() is not None:
try:
YLogger.info(self, "Loading default oob")
classobject = ClassLoader.instantiate_class(configuration.oob.default().classname)
self._default_oob = classobject()
except Exception as excep:
YLogger.exception(self, "Failed to load OOB Processor", excep)
for oob_name in configuration.oob.oobs():
try:
YLogger.info(self, "Loading oob: %s", oob_name)
classobject = ClassLoader.instantiate_class(configuration.oob.oob(oob_name).classname)
self._oob[oob_name] = classobject()
except Exception as excep:
YLogger.exception(self, "Failed to load OOB", excep)
def load_regex_templates(self, configuration):
if configuration.files.regex_templates is not None:
collection = PropertiesCollection()
total = collection.load_from_filename(configuration.files.regex_templates)
YLogger.info(self, "Loaded a total of %d regex templates", total)
self._regex_templates.clear()
for pair in collection.pairs:
name = pair[0]
pattern = pair[1]
try:
self._regex_templates[name] = re.compile(pattern, re.IGNORECASE)
except Exception:
YLogger.error(self, "Invalid regex template [%s]", pattern)
def regex_template(self, name):
if name in self._regex_templates:
return self._regex_templates[name]
return None
def strip_oob(self, response):
match = re.compile(r"(.*)(<\s*oob\s*>.*<\/\s*oob\s*>)(.*)")
groupings = match.match(response)
if groupings is not None:
front = groupings.group(1).strip()
back = groupings.group(3).strip()
response = ""
if front != "":
response = front + " "
response += back
oob = groupings.group(2)
return response, oob
return response, None
def process_oob(self, client_context, oob_command):
oob_content = ET.fromstring(oob_command)
if oob_content.tag == 'oob':
for child in oob_content.findall('./'):
if child.tag in self._oob:
oob_class = self._oob[child.tag]
return oob_class.process_out_of_bounds(client_context, child)
return self._default_oob.process_out_of_bounds(client_context, child)
return ""
def post_process_response(self, client_context, response: str):
return self.postprocessors.process(client_context, response)
def failed_authentication(self, client_context):
YLogger.error(client_context, "[%s] failed authentication!")
# If we have an SRAI defined, then use that
if self.authentication.configuration.denied_srai is not None:
match_context = self._aiml_parser.match_sentence(client_context,
Sentence(self._bot.brain.tokenizer, self.authentication.configuration.denied_srai),
topic_pattern="*",
that_pattern="*")
# If the SRAI matched then return the result
if match_context is not None:
return self.resolve_matched_template(client_context, match_context)
# Otherswise return the static text, which is either
# User defined via config.yaml
# Or use the default value BrainSecurityConfiguration.DEFAULT_ACCESS_DENIED
return self.authentication.configuration.denied_text
def authenticate_user(self, client_context):
if self.authentication is not None:
if self.authentication.authenticate(client_context) is False:
return self.failed_authentication(client_context)
return None
def resolve_matched_template(self, client_context, match_context):
template_node = match_context.template_node()
YLogger.debug(client_context, "AIML Parser evaluating template [%s]", template_node.to_string())
response = template_node.template.resolve(client_context)
if "<oob>" in response:
response, oob = self.strip_oob(response)
if oob is not None:
oob_response = self.process_oob(client_context, oob)
response = response + " " + oob_response
return response
def ask_question(self, client_context, sentence, srai=False):
client_context.brain = self
authenticated = self.authenticate_user(client_context)
if authenticated is not None:
return authenticated
conversation = client_context.bot.get_conversation(client_context)
topic_pattern = conversation.get_topic_pattern(client_context)
that_pattern = conversation.get_that_pattern(client_context, srai)
match_context = self._aiml_parser.match_sentence(client_context,
sentence,
topic_pattern=topic_pattern,
that_pattern=that_pattern)
if match_context is not None:
return self.resolve_matched_template(client_context, match_context)
return None