def _load_genders(self, configuration):
if configuration.files.gender is not None:
self._gender_collection.empty()
total = self._gender_collection.load_from_filename(configuration.files.gender)
YLogger.info(self, "Loaded a total of %d genderisations", total)
else:
YLogger.warning(self, "No configuration setting for gender")
def _load_person2s(self, configuration):
if configuration.files.person2 is not None:
self._person2_collection.empty()
total = self._person2_collection.load_from_filename(configuration.files.person2)
YLogger.info(self, "Loaded a total of %d person2s", total)
else:
YLogger.warning(self, "No configuration setting for person2")
def load_tokenizer(self):
if self.configuration is not None and self.configuration.tokenizer.classname is not None:
YLogger.info(self, "Loading tokenizer from class [%s]", self.configuration.tokenizer.classname)
tokenizer_class = ClassLoader.instantiate_class(self.configuration.tokenizer.classname)
return tokenizer_class(self.configuration.tokenizer.split_chars)
else:
return Tokenizer(self.configuration.tokenizer.split_chars)
def parse_topic(self, topic_element, namespace):
if 'name' in topic_element.attrib:
name = topic_element.attrib['name']
if name is None or not name:
raise ParserException("Topic name empty or null", xml_element=topic_element)
xml = "<topic>%s</topic>" % name
YLogger.info(self, "Topic attrib converted to %s", xml)
topic_pattern = ET.fromstring(xml)
else:
raise ParserException("Missing name attribute for topic", xml_element=topic_element)
category_found = False
num_category = 0
for child in topic_element:
tag_name, _ = self.tag_and_namespace_from_text(child.tag)
if tag_name == 'category':
self.parse_category(child, namespace, topic_pattern)
category_found = True
num_category += 1
else:
raise ParserException("Unknown child node of topic, %s" % child.tag, xml_element=topic_element)
if category_found is False:
raise ParserException("No categories in topic", xml_element=topic_element)
return num_category
def _load_variables(self, configuration):
if configuration.files.variables is not None:
self._variables_collection.empty ()
total = self._variables_collection.load_from_filename(configuration.files.variables)
YLogger.info(self, "Loaded a total of %d variables", total)
else:
YLogger.warning(self, "No configuration setting for variables")
def _load_postprocessors(self, configuration):
if configuration.files.postprocessors is not None:
self._postprocessors.empty()
total = self._postprocessors.load(configuration.files.postprocessors)
YLogger.info(self, "Loaded a total of %d post processors", total)
else:
YLogger.warning(self, "No configuration setting for post processors")
def _load_normals(self, configuration):
if configuration.files.normal is not None:
self._normal_collection.empty()
total = self._normal_collection.load_from_filename(configuration.files.normal)
YLogger.info(self, "Loaded a total of %d normalisations", total)
else:
YLogger.warning(self, "No configuration setting for normal")
def load_single_file(self, configuration):
start = datetime.datetime.now()
self._aiml_loader.load_single_file_contents(configuration.files.aiml_files.file)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Total processing time %.6f secs", diff.total_seconds())
YLogger.info(self, "Loaded a single aiml file with %d categories", self.num_categories)
def initiate_logging(self, arguments):
if arguments.logging is not None:
with open(arguments.logging, 'r+', encoding="utf-8") as yml_data_file:
logging_config = yaml.load(yml_data_file)
logging.config.dictConfig(logging_config)
YLogger.info(self, "Now logging under configuration")
else:
print("Warning. No logging configuration file defined, using defaults...")
def save_content(self):
YLogger.info(self, "Saving aiml %s to file [%s]", self._content_type, self._filename )
for entry in self._entries:
self._file.write_line(self, entry)
self._file.flush()
return len(self._entries)
def load_license_key_file(self, license_key_filename):
try:
YLogger.info(self, "Loading license key file: [%s]", license_key_filename)
with open(license_key_filename, "r", encoding="utf-8") as license_file:
for line in license_file:
self._process_license_key_line(line)
except Exception:
YLogger.error(self, "Invalid license key file [%s]", license_key_filename)
def save_binary(self, configuration):
YLogger.info(self, "Saving binary brain to [%s]", configuration.binaries.binary_filename)
start = datetime.datetime.now()
bin_file = open(configuration.binaries.binary_filename, "wb")
pickle.dump(self._aiml_parser, bin_file)
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds())
def load_learnf_files_from_directory(self, configuration):
if configuration.defaults.learnf_path is not None:
aimls_loaded = self._aiml_loader.load_dir_contents(configuration.defaults.learnf_path,
False,
configuration.files.aiml_files.extension,
filename_as_userid=True)
total_aimls_loaded = len(aimls_loaded)
YLogger.info(self, "Loaded a total of %d learnf aiml files", total_aimls_loaded)
def get_topic_pattern(self, client_context):
topic_pattern = self.property("topic")
if topic_pattern is None:
YLogger.info(client_context, "No Topic pattern default to [*]")
topic_pattern = "*"
else:
YLogger.info(client_context, "Topic pattern = [%s]", topic_pattern)
return topic_pattern
def _load_rdfs(self, configuration):
if configuration.files.rdf_files is not None and configuration.files.rdf_files.files:
self._rdf_collection.empty()
total = self._rdf_collection.load(configuration.files.rdf_files)
YLogger.info(self, "Loaded a total of %d rdf files", total)
elif configuration.files.triples is not None:
self._rdf_collection.empty()
total = self._rdf_collection.load_from_filename(configuration.files.triples)
YLogger.info(self, "Loaded a total of %d triples", total)
else:
YLogger.warning(self, "No configuration setting for triples")
def initiate_spellchecker(self):
# TODO Move this to Spelling bass class
if self.configuration is not None:
if self.configuration.spelling.classname is not None:
try:
YLogger.info(self, "Loading spelling checker from class [%s]", self.configuration.spelling.classname)
spell_class = ClassLoader.instantiate_class(self.configuration.spelling.classname)
self._spell_checker = spell_class(self.configuration.spelling)
except Exception as excep:
YLogger.exception(self, "Failed to initiate spellcheker", excep)
else:
YLogger.warning(self, "No configuration setting for spelling checker!")
def run(self):
if self.arguments.noloop is False:
YLogger.info(self, "Entering conversation loop...")
self.prior_to_run_loop()
self.run_loop()
self.post_run_loop()
else:
YLogger.debug(self, "noloop set to True, exiting...")
def load_regex_templates(self, configuration):
if configuration.files.regex_templates is not None:
collection = PropertiesCollection()
total = collection.load_from_filename(configuration.files.regex_templates)
YLogger.info(self, "Loaded a total of %d regex templates", total)
self._regex_templates.clear()
for pair in collection.pairs:
name = pair[0]
pattern = pair[1]
try:
self._regex_templates[name] = re.compile(pattern, re.IGNORECASE)
except Exception:
YLogger.error(self, "Invalid regex template [%s]", pattern)
def __init__(self, configuration):
self._filename = configuration.filename
if configuration.delete_on_start:
if os.path.exists(configuration.filename):
YLogger.info(self, "Removing %s on start up", configuration.filename)
os.remove(configuration.filename)
if configuration.file_format == 'txt':
self._file = TextFile(self._filename, encoding=configuration.encoding)
elif configuration.file_format == 'csv':
self._file = CSVFile(self._filename)
self.write_header()
else:
raise Exception ("Unknown file type [%s]", configuration.file_format)
def load_oob_processors(self, configuration):
if configuration.oob is not None:
if configuration.oob.default() is not None:
try:
YLogger.info(self, "Loading default oob")
classobject = ClassLoader.instantiate_class(configuration.oob.default().classname)
self._default_oob = classobject()
except Exception as excep:
YLogger.exception(self, "Failed to load OOB Processor", excep)
for oob_name in configuration.oob.oobs():
try:
YLogger.info(self, "Loading oob: %s", oob_name)
classobject = ClassLoader.instantiate_class(configuration.oob.oob(oob_name).classname)
self._oob[oob_name] = classobject()
except Exception as excep:
YLogger.exception(self, "Failed to load OOB", excep)
def resolve_to_string(self, client_context):
resolved = self.resolve_children_to_string(client_context)
if self._output == "logging":
YLogger.debug(client_context, "[%s] resolved to [%s]", self.to_string(), resolved)
if self._level == "debug":
YLogger.debug(client_context, resolved)
elif self._level == "warning":
YLogger.warning(client_context, resolved)
elif self._level == "error":
YLogger.error(client_context, resolved)
elif self._level == "info":
YLogger.info(client_context, resolved)
else:
YLogger.info(client_context, resolved)
else:
print(resolved)
return ""
def load_conversation(self, client_context, conversation):
YLogger.info(client_context,
"Loading conversation from Mongo [%s] [%s]",
client_context.client.id, client_context.userid)
collection = self.collection()
document = collection.find_one({
MongoConversationStore.CLIENITD:
client_context.client.id,
MongoConversationStore.USERID:
client_context.userid
})
if document:
data = document
conversation.create_from_json(
data[MongoConversationStore.CONVERSATION])
return True
return False
def run(self, app=None):
del app
if self.arguments.noloop is False:
YLogger.info(self, "Entering conversation loop...")
self.startup()
self.prior_to_run_loop()
self.run_loop()
self.post_run_loop()
self.shutdown()
else:
YLogger.debug(self, "noloop set to True, exiting...")
def initiate_spellchecker(self):
# TODO Move this to Spelling bass class
if self.configuration is not None:
if self.configuration.spelling.classname is not None:
try:
YLogger.info(self,
"Loading spelling checker from class [%s]",
self.configuration.spelling.classname)
spell_class = ClassLoader.instantiate_class(
self.configuration.spelling.classname)
self._spell_checker = spell_class(
self.configuration.spelling)
except Exception as excep:
YLogger.exception(self, "Failed to initiate spellcheker",
excep)
else:
YLogger.warning(
self, "No configuration setting for spelling checker!")
def load_binary(self, storage_factory):
YLogger.info(self, "Loading binary brain from [%s]",
StorageFactory.BINARIES)
try:
storage_engine = storage_factory.entity_storage_engine(
self._configuration.storage)
binaries_storage = storage_engine.binaries_storage()
self._aiml_parser = binaries_storage.load_binary()
return False # Tell caller, load succeeded and skip aiml load
except Exception as excep:
YLogger.exception(self, "Failed to load binary file", excep)
if self._configuration.load_aiml_on_binary_fail is True:
return True # Tell caller, load failed and to load aiml directly
else:
raise excep
def __init__(self, spelling_config=None):
SpellingChecker.__init__(self, spelling_config)
self.words = []
self.sum_of_words = 0
if spelling_config is None:
corpus_filename = os.path.dirname(__file__) + os.sep + "corpus.txt"
else:
corpus_filename = spelling_config.corpus
if os.path.exists(corpus_filename) is True:
YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename)
self.words = Counter(self._all_words(open(corpus_filename, encoding="utf-8").read()))
self.sum_of_words = sum(self.words.values())
else:
YLogger.error(self, "No spelling corpus found[%s]", corpus_filename)
def __init__(self, configuration):
self._filename = configuration.filename
if configuration.delete_on_start:
if os.path.exists(configuration.filename):
YLogger.info(self, "Removing %s on start up",
configuration.filename)
os.remove(configuration.filename)
if configuration.file_format == 'txt':
self._file = TextFile(self._filename,
encoding=configuration.encoding)
elif configuration.file_format == 'csv':
self._file = CSVFile(self._filename)
self.write_header()
else:
raise Exception("Unknown file type [%s]",
configuration.file_format)
def initiate_spellchecker(spelling_config, storage_factory):
if spelling_config.classname is not None:
try:
YLogger.info(None, "Loading spelling checker from class [%s]",
spelling_config.classname)
spell_class = ClassLoader.instantiate_class(
spelling_config.classname)
spell_checker = spell_class(spelling_config)
spell_checker.initialise(storage_factory)
return spell_checker
except Exception as excep:
YLogger.exception(None, "Failed to initiate spellcheker",
excep)
else:
YLogger.warning(None,
"No configuration setting for spelling checker!")
return None
def _send_message(self, host, port, username, password, msg):
YLogger.info(self, "Email sender starting")
server = self._smtp_server(host, port)
server.ehlo()
server.starttls()
YLogger.info(self, "Email sender logging in")
server.login(username, password)
YLogger.info(self, "Email sender sending")
server.send_message(msg)
YLogger.info(self, "Email sender quiting")
server.quit()
def upload_from_file(self, filename, format=Store.TEXT_FORMAT, commit=True, verbose=False):
YLogger.info(self, "Uploading license keys to Mongo [%s]", filename)
success = 0
count = 0
try:
YLogger.info(self, "Loading license key file: [%s]", filename)
with open(filename, "r", encoding="utf-8") as license_file:
for line in license_file:
if self._process_license_key_line(line, verbose) is True:
success += 1
count += 1
if commit is True:
self.commit()
except Exception as excep:
YLogger.exception(self, "Invalid license key file [%s]", excep, filename)
return count, success
def upload_from_file(self,
filename,
fileformat=Store.TEXT_FORMAT,
commit=True,
verbose=False):
YLogger.info(self, "Uplading spelling corpus file [%s] to Mongo",
filename)
count = success = 0
try:
count, success = self._read_corpus_from_file(filename, verbose)
except Exception as excep:
YLogger.exception(self, "Failed to load spelling corpus from [%s]",
excep, filename)
# Assume all words loaded are success, no need for additional count
return count, success
def load_spelling(self, spell_checker):
corpus_filename = self.get_storage().file
encoding = self.get_storage().encoding
if os.path.exists(corpus_filename) is True:
YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename)
try:
with open(corpus_filename, encoding=encoding) as words_file:
all_words = words_file.read()
spell_checker.add_corpus(all_words)
except Exception as e:
YLogger.exception(self, "Failed to load corpus [%s]", e,
corpus_filename)
else:
YLogger.error(self, "No spelling corpus found[%s]",
corpus_filename)
def _load(self):
bin_file_path = self._get_storage_path()
YLogger.info(self, "Loading binary brain from [%s]", bin_file_path)
start = datetime.datetime.now()
gc.disable()
with open(bin_file_path, "rb") as bin_file:
aiml_parser = pickle.load(bin_file)
bin_file.close()
gc.enable()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain load took a total of %.2f sec",
diff.total_seconds())
return aiml_parser
def get_conversation(self, client_context):
# TODO move this to Conversations base class
if client_context.userid in self._conversations:
YLogger.info(client_context, "Retrieving conversation for client %s", client_context.userid)
return self._conversations[client_context.userid]
else:
YLogger.info(client_context, "Creating new conversation for client %s", client_context.userid)
conversation = Conversation(client_context)
if client_context.brain.properties is not None:
conversation.load_initial_variables(client_context.brain.variables)
self._conversations[client_context.userid] = conversation
self.load_conversation(client_context.userid)
return conversation
def save_binary(self, aiml_parser):
try:
bin_file_path = self._get_storage_path()
YLogger.info(self, "Saving binary brain to [%s]", bin_file_path)
bin_file_dir = self._get_dir_from_path(bin_file_path)
self._ensure_dir_exists(bin_file_dir)
start = datetime.datetime.now()
with open(bin_file_path, "wb") as bin_file:
pickle.dump(aiml_parser, bin_file)
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds())
except Exception as excep:
YLogger.exception_nostack(self, "Failed to save binary file", excep)
def initiate_sentence_splitter(splitter_config):
if splitter_config.classname is not None:
try:
YLogger.info(None, "Loading sentence splitter from class [%s]",
splitter_config.classname)
splitter_class = ClassLoader.instantiate_class(
splitter_config.classname)
sentence_splitter = splitter_class(splitter_config)
return sentence_splitter
except Exception as excep:
YLogger.exception(None, "Failed to initiate sentence splitter",
excep)
else:
YLogger.warning(None,
"No configuration setting for sentence splitter!")
return None
def test_ylogger(self):
client_context = ClientContext(TestClient(), "testid")
snapshot = YLoggerSnapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(0) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.reset_snapshot()
YLogger.critical(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.fatal(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(0) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.error(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.exception(client_context, "Test Message", Exception("Test error"))
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(0) Info(0), Debug(0)")
YLogger.warning(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(0), Debug(0)")
YLogger.info(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(0)")
YLogger.debug(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(1)")
def load_oob_processors(self, configuration):
if configuration.oob is not None:
if configuration.oob.default() is not None:
try:
YLogger.info(self, "Loading default oob")
classobject = ClassLoader.instantiate_class(
configuration.oob.default().classname)
self._default_oob = classobject()
except Exception as excep:
YLogger.exception(self, excep)
for oob_name in configuration.oob.oobs():
try:
YLogger.info(self, "Loading oob: %s", oob_name)
classobject = ClassLoader.instantiate_class(
configuration.oob.oob(oob_name).classname)
self._oob[oob_name] = classobject()
except Exception as excep:
YLogger.exception(self, excep)
def resolve_to_string(self, client_context):
resolved = self.resolve_children_to_string(client_context)
if self._output == "logging":
YLogger.debug(client_context, "[%s] resolved to [%s]",
self.to_string(), resolved)
if self._level == "debug":
YLogger.debug(client_context, resolved)
elif self._level == "warning":
YLogger.warning(client_context, resolved)
elif self._level == "error":
YLogger.error(client_context, resolved)
elif self._level == "info":
YLogger.info(client_context, resolved)
else:
YLogger.info(client_context, resolved)
else:
print(resolved)
return ""
def send(self, to, subject, message, attachments=[]):
try:
if attachments:
YLogger.info(self, "Email sender adding mime attachment")
msg = MIMEMultipart()
msg.attach(MIMEText(message))
self.add_mime_attachements(msg, attachments)
else:
msg = MIMEText(message)
msg['Subject'] = subject
msg['From'] = self._from_addr
msg['To'] = to
self._send_message(self._config.host, self._config.port, self._config.username, self._config.password, msg)
except Exception as e:
YLogger.exception(self, "Email sender failed", e)
def upload_from_file(self, filename, format=Store.TEXT_FORMAT, commit=True, verbose=False):
YLogger.info(self, "Uploading %s nodes to Mongo from [%s]", self.collection_name(), filename)
count = 0
success = 0
try:
with open(filename, "r", encoding="utf-8") as file:
for line in file:
if self.process_config_line(line, verbose) is True:
success += 1
count += 1
except FileNotFoundError:
YLogger.error(self, "File not found [%s]", filename)
if commit is True:
self.commit()
return count, success
def test_ylogger(self):
client_context = ClientContext(TestClient(), "testid")
snapshot = YLoggerSnapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(0) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.reset_snapshot()
YLogger.critical(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.fatal(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(0) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.error(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(0) Warning(0) Info(0), Debug(0)")
YLogger.exception(client_context, "Test Message", Exception("Test error"))
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(0) Info(0), Debug(0)")
YLogger.warning(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(0), Debug(0)")
YLogger.info(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(0)")
YLogger.debug(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(1)")
def save_braintree(self, client_context, pattern_graph):
self._braintree = ''
# Default XML-Format
format = RedisStore.XML_FORMAT
encoding = 'UTF-8'
try:
h_key = self.create_key("binaries", format)
s_key = self._storage_engine._sessions_set_key
YLogger.info(self, "Saving brainTree to redis [%s]", h_key)
start = datetime.datetime.now()
if format == RedisStore.TEXT_FORMAT:
pattern_graph.dump(output_func=self.output_braintree, eol="\n")
elif format == RedisStore.XML_FORMAT:
self._braintree = '<?xml version="1.0" encoding="%s"?>\n' % encoding
self._braintree += '<aiml>\n'
self._braintree += pattern_graph.root.to_xml(client_context)
self._braintree += '</aiml>\n'
else:
YLogger.error(client_context,
"Unknown brainTree content type [%s]", format)
return
self.store(h_key,
s_key,
s_key,
self._braintree,
ex=self._storage_engine._expiretime)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "BrainTree save took a total of %.2f sec",
diff.total_seconds())
except Exception as e:
YLogger.exception(self, "Failed to save brainTree to Redis [%s]",
e, s_key)
def load_binary(self, configuration):
YLogger.info(self, "Loading binary brain from [%s]", configuration.binaries.binary_filename)
try:
start = datetime.datetime.now()
gc.disable()
bin_file = open(configuration.binaries.binary_filename, "rb")
self._aiml_parser = pickle.load(bin_file)
self._aiml_parser._brain = self
gc.enable()
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds())
return False # Tell caller, load succeeded and skip aiml load
except Exception as excep:
YLogger.exception(self, "Failed to load binary file", excep)
if configuration.binaries.load_aiml_on_binary_fail is True:
return True # Tell caller, load failed and to load aiml directly
else:
raise excep
def initiate_logging(self, arguments):
if arguments.logging is not None:
try:
with open(arguments.logging, 'r+',
encoding="utf-8") as yml_data_file:
logging_config = yaml.load(yml_data_file,
Loader=yaml.FullLoader)
logging.config.dictConfig(logging_config)
YLogger.info(self, "Now logging under configuration")
except Exception as excep:
YLogger.exception(self,
"Failed to open logging configuration [%s]",
excep, arguments.logging)
else:
outputLog(
self,
"Warning. No logging configuration file defined, using defaults..."
)
def add_to_lookup(self, key, value, overwrite_existing=False):
lookup = self._exists(key)
if lookup is not None:
if overwrite_existing is True:
YLogger.info(self, "Updating lookup in SQL [%s] [%s]", key,
value)
lookup.value = value
else:
YLogger.error(self, "Existing value in SQL lookup [%s] = [%s]",
key, value)
return False
else:
YLogger.debug(self, "Adding lookup to SQL [%s] = [%s]", key, value)
lookup = self._get_entity(key=key, value=value)
self._storage_engine.session.add(lookup)
self._storage_engine.session.commit()
return True
def parse_from_text(self, text):
"""
Parse an AIML text version of an aiml file and return all the cateogeries found in the file
:param text: Fully validated AIML snippet
:return list of categories parsed from file:
"""
start = datetime.datetime.now()
aiml = ET.fromstring(text)
_, namespace = AIMLParser.check_aiml_tag(aiml)
num_categories = self.parse_aiml(aiml, namespace)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Processed text with %d categories in %f.2 secs", num_categories,
diff.total_seconds())
return bool(num_categories > 0)
def add_to_lookup(self, key, value, overwrite_existing=False):
collection = self.collection()
lookup = collection.find_one({'key': key})
if lookup is not None:
if overwrite_existing is True:
YLogger.info(self, "Updating lookup in Mongo [%s] [%s]", key,
value)
lookup['value'] = value
collection.replace_one({'key': key}, lookup)
else:
YLogger.error(self,
"Existing value in Mongo lookup [%s] = [%s]",
key, value)
return False
else:
YLogger.debug(self, "Adding lookup to Mongo [%s] = [%s]", key,
value)
lookup = Lookup(key, value)
self.add_document(lookup)
return True
def upload_from_file(self,
filename,
format=Store.TEXT_FORMAT,
commit=True,
verbose=False):
YLogger.info(self, "Uploading %s to Mongo from [%s]", filename,
self.collection_name())
count = 0
success = 0
if os.path.exists(filename):
try:
with open(filename, "r") as vars_file:
for line in vars_file:
line = line.strip()
if line:
if line.startswith(
MongoPropertyStore.COMMENT) is False:
splits = line.split(
MongoPropertyStore.SPLIT_CHAR)
if len(splits) > 1:
key = splits[0].strip()
val = ":".join(splits[1:]).strip()
if verbose is True:
YLogger.debug(
self,
"Adding %s property [%s=%s] to Mongo",
self.collection_name(), key, val)
if self.add_property(key, val) is True:
success += 1
count += 1
if commit is True:
self.commit()
except Exception as excep:
YLogger.exception(self, "Failed to upload %s from %s to Mongo",
excep, self.collection_name(), filename)
return count, success
def __init__(self, spelling_config=None):
SpellingChecker.__init__(self, spelling_config)
self.words = []
self.sum_of_words = 0
if spelling_config is None:
corpus_filename = os.path.dirname(__file__) + os.sep + "corpus.txt"
else:
corpus_filename = spelling_config.corpus
if os.path.exists(corpus_filename) is True:
YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename)
self.words = Counter(
self._all_words(
open(corpus_filename, encoding="utf-8").read()))
self.sum_of_words = sum(self.words.values())
else:
YLogger.error(self, "No spelling corpus found[%s]",
corpus_filename)
def init_ping_response(ping_responder):
if ping_responder.config.host is None:
YLogger.info(None, "No REST configuration for ping responder")
print("Healthcheck now running as part of REST Service...")
return
print("Healthcheck now running as separate REST Service...")
ping_app = Flask(ping_responder.config.name)
@ping_app.route(ping_responder.config.url, methods=['GET'])
def ping():
return jsonify(ping_responder.ping())
@ping_app.route(ping_responder.config.shutdown, methods=['GET'])
def shutdown():
ping_responder.stop_ping_service()
return 'Server shutting down...'
ping_responder.start_ping_service(ping_app)
def load_aiml(self, configuration: BrainConfiguration):
if configuration.files.aiml_files is not None:
self.create_debug_storage(configuration)
if configuration.files.aiml_files.has_multiple_files():
self.load_files_from_directory(configuration)
elif configuration.files.aiml_files.has_single_file():
self.load_single_file(configuration)
else:
YLogger.info(self, "No AIML files or file defined in configuration to load")
self.save_debug_files(configuration)
self.display_debug_info(configuration)
else:
YLogger.info(self, "No AIML files or file defined in configuration to load")
def get_that_pattern(self, client_context, srai=False):
try:
that_question = None
if srai is False:
that_question = self.previous_nth_question(1)
else:
if len(self._questions) > 2:
for question in reversed(self._questions[:-2]):
if question._srai is False and question.has_response():
that_question = question
break
if that_question is not None:
that_sentence = that_question.current_sentence()
else:
that_sentence = None
# If the last response was valid, i.e not none and not empty string, then use
# that as the that_pattern, otherwise we default to '*' as pattern
if that_sentence.response is not None and that_sentence.response != '':
that_pattern = self.parse_last_sentences_from_response(that_sentence.response)
YLogger.info(client_context, "That pattern = [%s]", that_pattern)
else:
YLogger.info(client_context, "That pattern, no response, default to [*]")
that_pattern = "*"
except Exception as e:
YLogger.info(client_context, "No That pattern default to [*]")
that_pattern = "*"
return that_pattern
def get_that_pattern(self, client_context, srai=False):
try:
that_question = None
if srai is False:
that_question = self.previous_nth_question(1)
else:
if len(self._questions) > 2:
for question in reversed(self._questions[:-2]):
if question._srai is False and question.has_response():
that_question = question
break
if that_question is not None:
that_sentence = that_question.current_sentence()
else:
that_sentence = None
# If the last response was valid, i.e not none and not empty string, then use
# that as the that_pattern, otherwise we default to '*' as pattern
if that_sentence.response is not None and that_sentence.response != '':
that_pattern = self.parse_last_sentences_from_response(
that_sentence.response)
YLogger.info(client_context, "That pattern = [%s]",
that_pattern)
else:
YLogger.info(client_context,
"That pattern, no response, default to [*]")
that_pattern = "*"
except Exception as e:
YLogger.info(client_context, "No That pattern default to [*]")
that_pattern = "*"
return that_pattern
def parse_from_file(self, filename, userid="*"):
"""
Parse an AIML file and return all the cateogeries found in the file
:param filename: Name of file to parse
:return list of categories parsed from file:
"""
YLogger.info(self, "Loading aiml file: " + filename)
try:
tree = ET.parse(filename, parser=LineNumberingParser())
aiml = tree.getroot()
_, namespace = self.check_aiml_tag(aiml, filename=filename)
start = datetime.datetime.now()
num_categories = self.parse_aiml(aiml, namespace, filename, userid=userid)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Processed %s with %d categories in %f.2 secs", filename, num_categories, diff.total_seconds())
except Exception as excep:
YLogger.exception(self, "Failed to load contents of AIML file from [%s]"%filename, excep)
def load_aiml(self, configuration: BrainConfiguration):
if configuration.files.aiml_files is not None:
self.create_debug_storage(configuration)
if configuration.files.aiml_files.has_multiple_files():
self.load_files_from_directory(configuration)
self.load_learnf_files_from_directory(configuration)
elif configuration.files.aiml_files.has_single_file():
self.load_single_file(configuration)
self.load_learnf_files_from_directory(configuration)
else:
YLogger.info(self, "No AIML files or file defined in configuration to load")
self.save_debug_files(configuration)
self.display_debug_info(configuration)
else:
YLogger.info(self, "No AIML files or file defined in configuration to load")
def load_files_from_directory(self, configuration):
start = datetime.datetime.now()
total_aimls_loaded = 0
for file in configuration.files.aiml_files.files:
aimls_loaded = self._aiml_loader.load_dir_contents(file,
configuration.files.aiml_files.directories,
configuration.files.aiml_files.extension,
filename_as_userid=False)
total_aimls_loaded = len(aimls_loaded)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Total processing time %.6f secs", diff.total_seconds())
YLogger.info(self, "Loaded a total of %d aiml files with %d categories", total_aimls_loaded, self.num_categories)
if diff.total_seconds() > 0:
YLogger.info(self, "Thats approx %f aiml files per sec", total_aimls_loaded / diff.total_seconds())
def execute_oob_command(self, client_context):
YLogger.info(client_context, "URLOutOfBandProcessor: Loading=%s", self._url)
return "URL"
