def load_aiml(self, configuration: BrainConfiguration):
if configuration.files.aiml_files is not None:
self.create_debug_storage(configuration)
if configuration.files.aiml_files.has_multiple_files():
self.load_files_from_directory(configuration)
self.load_learnf_files_from_directory(configuration)
elif configuration.files.aiml_files.has_single_file():
self.load_single_file(configuration)
self.load_learnf_files_from_directory(configuration)
else:
YLogger.info(
self,
"No AIML files or file defined in configuration to load")
self.save_debug_files(configuration)
self.display_debug_info(configuration)
else:
YLogger.info(
self, "No AIML files or file defined in configuration to load")
def parse_topic(self, topic_element, namespace):
if 'name' in topic_element.attrib:
name = topic_element.attrib['name']
if name is None or not name:
raise ParserException("Topic name empty or null",
xml_element=topic_element)
xml = "<topic>%s</topic>" % name
YLogger.info(self, "Topic attrib converted to %s", xml)
topic_pattern = ET.fromstring(xml)
else:
raise ParserException("Missing name attribute for topic",
xml_element=topic_element)
category_found = False
num_category = 0
for child in topic_element:
tag_name, _ = self.tag_and_namespace_from_text(child.tag)
if tag_name == 'category':
self.parse_category(child, namespace, topic_pattern)
category_found = True
num_category += 1
else:
raise ParserException("Unknown child node of topic, %s" %
child.tag,
xml_element=topic_element)
if category_found is False:
raise ParserException("No categories in topic",
xml_element=topic_element)
return num_category
def parse_from_file(self, filename, userid="*"):
"""
Parse an AIML file and return all the cateogeries found in the file
:param filename: Name of file to parse
:return list of categories parsed from file:
"""
YLogger.info(self, "Loading aiml file: " + filename)
try:
tree = ET.parse(filename, parser=LineNumberingParser())
aiml = tree.getroot()
_, namespace = self.check_aiml_tag(aiml, filename=filename)
start = datetime.datetime.now()
num_categories = self.parse_aiml(aiml,
namespace,
filename,
userid=userid)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Processed %s with %d categories in %f.2 secs",
filename, num_categories, diff.total_seconds())
except Exception as excep:
YLogger.exception(
self,
"Failed to load contents of AIML file from [%s]" % filename,
excep)
def _load_properties(self, configuration):
if configuration.files.properties is not None:
self._properties_collection.empty()
total = self._properties_collection.load_from_filename(
configuration.files.properties)
YLogger.info(self, "Loaded a total of %d properties", total)
else:
YLogger.warning(self, "No configuration setting for properties")
def _load_person2s(self, configuration):
if configuration.files.person2 is not None:
self._person2_collection.empty()
total = self._person2_collection.load_from_filename(
configuration.files.person2)
YLogger.info(self, "Loaded a total of %d person2s", total)
else:
YLogger.warning(self, "No configuration setting for person2")
def _load_genders(self, configuration):
if configuration.files.gender is not None:
self._gender_collection.empty()
total = self._gender_collection.load_from_filename(
configuration.files.gender)
YLogger.info(self, "Loaded a total of %d genderisations", total)
else:
YLogger.warning(self, "No configuration setting for gender")
def _load_normals(self, configuration):
if configuration.files.normal is not None:
self._normal_collection.empty()
total = self._normal_collection.load_from_filename(
configuration.files.normal)
YLogger.info(self, "Loaded a total of %d normalisations", total)
else:
YLogger.warning(self, "No configuration setting for normal")
def save_content(self):
YLogger.info(self, "Saving aiml %s to file [%s]", self._content_type, self._filename )
for entry in self._entries:
self._file.write_line(self, entry)
self._file.flush()
return len(self._entries)
def _load_preprocessors(self, configuration):
if configuration.files.preprocessors is not None:
self._preprocessors.empty()
total = self._preprocessors.load(configuration.files.preprocessors)
YLogger.info(self, "Loaded a total of %d pre processors", total)
else:
YLogger.warning(self,
"No configuration setting for pre processors")
def load_single_file(self, configuration):
start = datetime.datetime.now()
self._aiml_loader.load_single_file_contents(
configuration.files.aiml_files.file)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Total processing time %.6f secs",
diff.total_seconds())
YLogger.info(self, "Loaded a single aiml file with %d categories",
self.num_categories)
def execute_oob_command(self, client_context):
if self._message is not None:
YLogger.info(client_context,
"AlarmOutOfBandProcessor: Showing alarm=%s",
self._message)
elif self._hour is not None and self._min is not None:
YLogger.info(client_context,
"AlarmOutOfBandProcessor: Setting alarm for %s:%s",
self._hour, self._min)
return "ALARM"
def get_topic_pattern(self, client_context):
topic_pattern = self.property("topic")
if topic_pattern is None:
YLogger.info(client_context, "No Topic pattern default to [*]")
topic_pattern = "*"
else:
YLogger.info(client_context, "Topic pattern = [%s]", topic_pattern)
return topic_pattern
def save_binary(self, configuration):
YLogger.info(self, "Saving binary brain to [%s]",
configuration.binaries.binary_filename)
start = datetime.datetime.now()
bin_file = open(configuration.binaries.binary_filename, "wb")
pickle.dump(self._aiml_parser, bin_file)
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain save took a total of %.2f sec",
diff.total_seconds())
def load_license_key_file(self, license_key_filename):
try:
YLogger.info(self, "Loading license key file: [%s]",
license_key_filename)
with open(license_key_filename, "r",
encoding="utf-8") as license_file:
for line in license_file:
self._process_license_key_line(line)
except Exception:
YLogger.error(self, "Invalid license key file [%s]",
license_key_filename)
def initiate_logging(self, arguments):
if arguments.logging is not None:
with open(arguments.logging, 'r+',
encoding="utf-8") as yml_data_file:
logging_config = yaml.load(yml_data_file,
Loader=yaml.FullLoader)
logging.config.dictConfig(logging_config)
YLogger.info(self, "Now logging under configuration")
else:
print(
"Warning. No logging configuration file defined, using defaults..."
)
def run(self):
if self.arguments.noloop is False:
YLogger.info(self, "Entering conversation loop...")
self.prior_to_run_loop()
self.run_loop()
self.post_run_loop()
else:
YLogger.debug(self, "noloop set to True, exiting...")
def _load_rdfs(self, configuration):
if configuration.files.rdf_files is not None and configuration.files.rdf_files.files:
self._rdf_collection.empty()
total = self._rdf_collection.load(configuration.files.rdf_files)
YLogger.info(self, "Loaded a total of %d rdf files", total)
elif configuration.files.triples is not None:
self._rdf_collection.empty()
total = self._rdf_collection.load_from_filename(
configuration.files.triples)
YLogger.info(self, "Loaded a total of %d triples", total)
else:
YLogger.warning(self, "No configuration setting for triples")
def load_learnf_files_from_directory(self, configuration):
if configuration.defaults.learnf_path is not None:
aimls_loaded = self._aiml_loader.load_dir_contents(
configuration.defaults.learnf_path,
False,
configuration.files.aiml_files.extension,
filename_as_userid=True)
total_aimls_loaded = len(aimls_loaded)
YLogger.info(self, "Loaded a total of %d learnf aiml files",
total_aimls_loaded)
def __init__(self, configuration):
self._filename = configuration.filename
if configuration.delete_on_start:
if os.path.exists(configuration.filename):
YLogger.info(self, "Removing %s on start up", configuration.filename)
os.remove(configuration.filename)
if configuration.file_format == 'txt':
self._file = TextFile(self._filename, encoding=configuration.encoding)
elif configuration.file_format == 'csv':
self._file = CSVFile(self._filename)
self.write_header()
else:
raise Exception ("Unknown file type [%s]", configuration.file_format)
def load_regex_templates(self, configuration):
if configuration.files.regex_templates is not None:
collection = PropertiesCollection()
total = collection.load_from_filename(
configuration.files.regex_templates)
YLogger.info(self, "Loaded a total of %d regex templates", total)
self._regex_templates.clear()
for pair in collection.pairs:
name = pair[0]
pattern = pair[1]
try:
self._regex_templates[name] = re.compile(
pattern, re.IGNORECASE)
except Exception:
YLogger.error(self, "Invalid regex template [%s]", pattern)
def initiate_spellchecker(self):
# TODO Move this to Spelling bass class
if self.configuration is not None:
if self.configuration.spelling.classname is not None:
try:
YLogger.info(self,
"Loading spelling checker from class [%s]",
self.configuration.spelling.classname)
spell_class = ClassLoader.instantiate_class(
self.configuration.spelling.classname)
self._spell_checker = spell_class(
self.configuration.spelling)
except Exception as excep:
YLogger.exception(self, "Failed to initiate spellcheker",
excep)
else:
YLogger.warning(
self, "No configuration setting for spelling checker!")
def __init__(self, spelling_config=None):
SpellingChecker.__init__(self, spelling_config)
self.words = []
self.sum_of_words = 0
if spelling_config is None:
corpus_filename = os.path.dirname(__file__) + os.sep + "corpus.txt"
else:
corpus_filename = spelling_config.corpus
if os.path.exists(corpus_filename) is True:
YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename)
self.words = Counter(self._all_words(open(corpus_filename, encoding="utf-8").read()))
self.sum_of_words = sum(self.words.values())
else:
YLogger.error(self, "No spelling corpus found[%s]", corpus_filename)
def __init__(self, config: BotConversationsMongodbStorageConfiguration):
super().__init__(config)
# client = MongoClient("10.0.75.2", 27017)
client = MongoClient(config.host,
config.port,
serverSelectionTimeoutMS=2000)
self._config = config
try:
if not config.name in client.list_database_names():
YLogger.info(self, "Database doesn't exist make a new one!")
print("Database doesn't exist make a new one!")
self.db = client[config.name]
except:
YLogger.warning(
self,
"No Mongo Database was found. No conversation will be saved")
print("No Mongo Database was found. No conversation will be saved")
def resolve_to_string(self, client_context):
resolved = self.resolve_children_to_string(client_context)
if self._output == "logging":
YLogger.debug(client_context, "[%s] resolved to [%s]",
self.to_string(), resolved)
if self._level == "debug":
YLogger.debug(client_context, resolved)
elif self._level == "warning":
YLogger.warning(client_context, resolved)
elif self._level == "error":
YLogger.error(client_context, resolved)
elif self._level == "info":
YLogger.info(client_context, resolved)
else:
YLogger.info(client_context, resolved)
else:
print(resolved)
return ""
def load_oob_processors(self, configuration):
if configuration.oob is not None:
if configuration.oob.default() is not None:
try:
YLogger.info(self, "Loading default oob")
classobject = ClassLoader.instantiate_class(
configuration.oob.default().classname)
self._default_oob = classobject()
except Exception as excep:
YLogger.exception(self, "Failed to load OOB Processor",
excep)
for oob_name in configuration.oob.oobs():
try:
YLogger.info(self, "Loading oob: %s", oob_name)
classobject = ClassLoader.instantiate_class(
configuration.oob.oob(oob_name).classname)
self._oob[oob_name] = classobject()
except Exception as excep:
YLogger.exception(self, "Failed to load OOB", excep)
def get_that_pattern(self, client_context, srai=False):
try:
that_question = None
if srai is False:
that_question = self.previous_nth_question(1)
else:
if len(self._questions) > 2:
for question in reversed(self._questions[:-2]):
if question._srai is False and question.has_response():
that_question = question
break
if that_question is not None:
that_sentence = that_question.current_sentence()
else:
that_sentence = None
# If the last response was valid, i.e not none and not empty string, then use
# that as the that_pattern, otherwise we default to '*' as pattern
if that_sentence.response is not None and that_sentence.response != '':
that_pattern = self.parse_last_sentences_from_response(that_sentence.response)
YLogger.info(client_context, "That pattern = [%s]", that_pattern)
else:
YLogger.info(client_context, "That pattern, no response, default to [*]")
that_pattern = "*"
except Exception as e:
YLogger.info(client_context, "No That pattern default to [*]")
that_pattern = "*"
return that_pattern
def load_binary(self, configuration):
YLogger.info(self, "Loading binary brain from [%s]",
configuration.binaries.binary_filename)
try:
start = datetime.datetime.now()
gc.disable()
bin_file = open(configuration.binaries.binary_filename, "rb")
self._aiml_parser = pickle.load(bin_file)
self._aiml_parser._brain = self
gc.enable()
bin_file.close()
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Brain load took a total of %.2f sec",
diff.total_seconds())
return False # Tell caller, load succeeded and skip aiml load
except Exception as excep:
YLogger.exception(self, "Failed to load binary file", excep)
if configuration.binaries.load_aiml_on_binary_fail is True:
return True # Tell caller, load failed and to load aiml directly
else:
raise excep
def get_conversation(self, client_context):
# TODO move this to Conversations base class
if client_context.userid in self._conversations:
YLogger.info(client_context,
"Retrieving conversation for client %s",
client_context.userid)
return self._conversations[client_context.userid]
else:
YLogger.info(client_context,
"Creating new conversation for client %s",
client_context.userid)
conversation = Conversation(client_context)
if client_context.brain.properties is not None:
conversation.load_initial_variables(
client_context.brain.variables)
self._conversations[client_context.userid] = conversation
self.load_conversation(client_context.userid)
return conversation
def load_files_from_directory(self, configuration):
start = datetime.datetime.now()
total_aimls_loaded = 0
for file in configuration.files.aiml_files.files:
aimls_loaded = self._aiml_loader.load_dir_contents(
file,
configuration.files.aiml_files.directories,
configuration.files.aiml_files.extension,
filename_as_userid=False)
total_aimls_loaded = len(aimls_loaded)
stop = datetime.datetime.now()
diff = stop - start
YLogger.info(self, "Total processing time %.6f secs",
diff.total_seconds())
YLogger.info(self,
"Loaded a total of %d aiml files with %d categories",
total_aimls_loaded, self.num_categories)
if diff.total_seconds() > 0:
YLogger.info(self, "Thats approx %f aiml files per sec",
total_aimls_loaded / diff.total_seconds())
def execute_oob_command(self, client_context):
YLogger.info(client_context,
"ScheduleOutOfBandProcessor: Scheduling=%s", self._title)
return "SCHEDULE"
