def load_aiml(self, configuration: BrainConfiguration): if configuration.files.aiml_files is not None: self.create_debug_storage(configuration) if configuration.files.aiml_files.has_multiple_files(): self.load_files_from_directory(configuration) self.load_learnf_files_from_directory(configuration) elif configuration.files.aiml_files.has_single_file(): self.load_single_file(configuration) self.load_learnf_files_from_directory(configuration) else: YLogger.info( self, "No AIML files or file defined in configuration to load") self.save_debug_files(configuration) self.display_debug_info(configuration) else: YLogger.info( self, "No AIML files or file defined in configuration to load")
def parse_topic(self, topic_element, namespace): if 'name' in topic_element.attrib: name = topic_element.attrib['name'] if name is None or not name: raise ParserException("Topic name empty or null", xml_element=topic_element) xml = "<topic>%s</topic>" % name YLogger.info(self, "Topic attrib converted to %s", xml) topic_pattern = ET.fromstring(xml) else: raise ParserException("Missing name attribute for topic", xml_element=topic_element) category_found = False num_category = 0 for child in topic_element: tag_name, _ = self.tag_and_namespace_from_text(child.tag) if tag_name == 'category': self.parse_category(child, namespace, topic_pattern) category_found = True num_category += 1 else: raise ParserException("Unknown child node of topic, %s" % child.tag, xml_element=topic_element) if category_found is False: raise ParserException("No categories in topic", xml_element=topic_element) return num_category
def parse_from_file(self, filename, userid="*"): """ Parse an AIML file and return all the cateogeries found in the file :param filename: Name of file to parse :return list of categories parsed from file: """ YLogger.info(self, "Loading aiml file: " + filename) try: tree = ET.parse(filename, parser=LineNumberingParser()) aiml = tree.getroot() _, namespace = self.check_aiml_tag(aiml, filename=filename) start = datetime.datetime.now() num_categories = self.parse_aiml(aiml, namespace, filename, userid=userid) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Processed %s with %d categories in %f.2 secs", filename, num_categories, diff.total_seconds()) except Exception as excep: YLogger.exception( self, "Failed to load contents of AIML file from [%s]" % filename, excep)
def _load_properties(self, configuration): if configuration.files.properties is not None: self._properties_collection.empty() total = self._properties_collection.load_from_filename( configuration.files.properties) YLogger.info(self, "Loaded a total of %d properties", total) else: YLogger.warning(self, "No configuration setting for properties")
def _load_person2s(self, configuration): if configuration.files.person2 is not None: self._person2_collection.empty() total = self._person2_collection.load_from_filename( configuration.files.person2) YLogger.info(self, "Loaded a total of %d person2s", total) else: YLogger.warning(self, "No configuration setting for person2")
def _load_genders(self, configuration): if configuration.files.gender is not None: self._gender_collection.empty() total = self._gender_collection.load_from_filename( configuration.files.gender) YLogger.info(self, "Loaded a total of %d genderisations", total) else: YLogger.warning(self, "No configuration setting for gender")
def _load_normals(self, configuration): if configuration.files.normal is not None: self._normal_collection.empty() total = self._normal_collection.load_from_filename( configuration.files.normal) YLogger.info(self, "Loaded a total of %d normalisations", total) else: YLogger.warning(self, "No configuration setting for normal")
def save_content(self): YLogger.info(self, "Saving aiml %s to file [%s]", self._content_type, self._filename ) for entry in self._entries: self._file.write_line(self, entry) self._file.flush() return len(self._entries)
def _load_preprocessors(self, configuration): if configuration.files.preprocessors is not None: self._preprocessors.empty() total = self._preprocessors.load(configuration.files.preprocessors) YLogger.info(self, "Loaded a total of %d pre processors", total) else: YLogger.warning(self, "No configuration setting for pre processors")
def load_single_file(self, configuration): start = datetime.datetime.now() self._aiml_loader.load_single_file_contents( configuration.files.aiml_files.file) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Total processing time %.6f secs", diff.total_seconds()) YLogger.info(self, "Loaded a single aiml file with %d categories", self.num_categories)
def execute_oob_command(self, client_context): if self._message is not None: YLogger.info(client_context, "AlarmOutOfBandProcessor: Showing alarm=%s", self._message) elif self._hour is not None and self._min is not None: YLogger.info(client_context, "AlarmOutOfBandProcessor: Setting alarm for %s:%s", self._hour, self._min) return "ALARM"
def get_topic_pattern(self, client_context): topic_pattern = self.property("topic") if topic_pattern is None: YLogger.info(client_context, "No Topic pattern default to [*]") topic_pattern = "*" else: YLogger.info(client_context, "Topic pattern = [%s]", topic_pattern) return topic_pattern
def save_binary(self, configuration): YLogger.info(self, "Saving binary brain to [%s]", configuration.binaries.binary_filename) start = datetime.datetime.now() bin_file = open(configuration.binaries.binary_filename, "wb") pickle.dump(self._aiml_parser, bin_file) bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds())
def load_license_key_file(self, license_key_filename): try: YLogger.info(self, "Loading license key file: [%s]", license_key_filename) with open(license_key_filename, "r", encoding="utf-8") as license_file: for line in license_file: self._process_license_key_line(line) except Exception: YLogger.error(self, "Invalid license key file [%s]", license_key_filename)
def initiate_logging(self, arguments): if arguments.logging is not None: with open(arguments.logging, 'r+', encoding="utf-8") as yml_data_file: logging_config = yaml.load(yml_data_file, Loader=yaml.FullLoader) logging.config.dictConfig(logging_config) YLogger.info(self, "Now logging under configuration") else: print( "Warning. No logging configuration file defined, using defaults..." )
def run(self): if self.arguments.noloop is False: YLogger.info(self, "Entering conversation loop...") self.prior_to_run_loop() self.run_loop() self.post_run_loop() else: YLogger.debug(self, "noloop set to True, exiting...")
def _load_rdfs(self, configuration): if configuration.files.rdf_files is not None and configuration.files.rdf_files.files: self._rdf_collection.empty() total = self._rdf_collection.load(configuration.files.rdf_files) YLogger.info(self, "Loaded a total of %d rdf files", total) elif configuration.files.triples is not None: self._rdf_collection.empty() total = self._rdf_collection.load_from_filename( configuration.files.triples) YLogger.info(self, "Loaded a total of %d triples", total) else: YLogger.warning(self, "No configuration setting for triples")
def load_learnf_files_from_directory(self, configuration): if configuration.defaults.learnf_path is not None: aimls_loaded = self._aiml_loader.load_dir_contents( configuration.defaults.learnf_path, False, configuration.files.aiml_files.extension, filename_as_userid=True) total_aimls_loaded = len(aimls_loaded) YLogger.info(self, "Loaded a total of %d learnf aiml files", total_aimls_loaded)
def __init__(self, configuration): self._filename = configuration.filename if configuration.delete_on_start: if os.path.exists(configuration.filename): YLogger.info(self, "Removing %s on start up", configuration.filename) os.remove(configuration.filename) if configuration.file_format == 'txt': self._file = TextFile(self._filename, encoding=configuration.encoding) elif configuration.file_format == 'csv': self._file = CSVFile(self._filename) self.write_header() else: raise Exception ("Unknown file type [%s]", configuration.file_format)
def load_regex_templates(self, configuration): if configuration.files.regex_templates is not None: collection = PropertiesCollection() total = collection.load_from_filename( configuration.files.regex_templates) YLogger.info(self, "Loaded a total of %d regex templates", total) self._regex_templates.clear() for pair in collection.pairs: name = pair[0] pattern = pair[1] try: self._regex_templates[name] = re.compile( pattern, re.IGNORECASE) except Exception: YLogger.error(self, "Invalid regex template [%s]", pattern)
def initiate_spellchecker(self): # TODO Move this to Spelling bass class if self.configuration is not None: if self.configuration.spelling.classname is not None: try: YLogger.info(self, "Loading spelling checker from class [%s]", self.configuration.spelling.classname) spell_class = ClassLoader.instantiate_class( self.configuration.spelling.classname) self._spell_checker = spell_class( self.configuration.spelling) except Exception as excep: YLogger.exception(self, "Failed to initiate spellcheker", excep) else: YLogger.warning( self, "No configuration setting for spelling checker!")
def __init__(self, spelling_config=None): SpellingChecker.__init__(self, spelling_config) self.words = [] self.sum_of_words = 0 if spelling_config is None: corpus_filename = os.path.dirname(__file__) + os.sep + "corpus.txt" else: corpus_filename = spelling_config.corpus if os.path.exists(corpus_filename) is True: YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename) self.words = Counter(self._all_words(open(corpus_filename, encoding="utf-8").read())) self.sum_of_words = sum(self.words.values()) else: YLogger.error(self, "No spelling corpus found[%s]", corpus_filename)
def __init__(self, config: BotConversationsMongodbStorageConfiguration): super().__init__(config) # client = MongoClient("10.0.75.2", 27017) client = MongoClient(config.host, config.port, serverSelectionTimeoutMS=2000) self._config = config try: if not config.name in client.list_database_names(): YLogger.info(self, "Database doesn't exist make a new one!") print("Database doesn't exist make a new one!") self.db = client[config.name] except: YLogger.warning( self, "No Mongo Database was found. No conversation will be saved") print("No Mongo Database was found. No conversation will be saved")
def resolve_to_string(self, client_context): resolved = self.resolve_children_to_string(client_context) if self._output == "logging": YLogger.debug(client_context, "[%s] resolved to [%s]", self.to_string(), resolved) if self._level == "debug": YLogger.debug(client_context, resolved) elif self._level == "warning": YLogger.warning(client_context, resolved) elif self._level == "error": YLogger.error(client_context, resolved) elif self._level == "info": YLogger.info(client_context, resolved) else: YLogger.info(client_context, resolved) else: print(resolved) return ""
def load_oob_processors(self, configuration): if configuration.oob is not None: if configuration.oob.default() is not None: try: YLogger.info(self, "Loading default oob") classobject = ClassLoader.instantiate_class( configuration.oob.default().classname) self._default_oob = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB Processor", excep) for oob_name in configuration.oob.oobs(): try: YLogger.info(self, "Loading oob: %s", oob_name) classobject = ClassLoader.instantiate_class( configuration.oob.oob(oob_name).classname) self._oob[oob_name] = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB", excep)
def get_that_pattern(self, client_context, srai=False): try: that_question = None if srai is False: that_question = self.previous_nth_question(1) else: if len(self._questions) > 2: for question in reversed(self._questions[:-2]): if question._srai is False and question.has_response(): that_question = question break if that_question is not None: that_sentence = that_question.current_sentence() else: that_sentence = None # If the last response was valid, i.e not none and not empty string, then use # that as the that_pattern, otherwise we default to '*' as pattern if that_sentence.response is not None and that_sentence.response != '': that_pattern = self.parse_last_sentences_from_response(that_sentence.response) YLogger.info(client_context, "That pattern = [%s]", that_pattern) else: YLogger.info(client_context, "That pattern, no response, default to [*]") that_pattern = "*" except Exception as e: YLogger.info(client_context, "No That pattern default to [*]") that_pattern = "*" return that_pattern
def load_binary(self, configuration): YLogger.info(self, "Loading binary brain from [%s]", configuration.binaries.binary_filename) try: start = datetime.datetime.now() gc.disable() bin_file = open(configuration.binaries.binary_filename, "rb") self._aiml_parser = pickle.load(bin_file) self._aiml_parser._brain = self gc.enable() bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds()) return False # Tell caller, load succeeded and skip aiml load except Exception as excep: YLogger.exception(self, "Failed to load binary file", excep) if configuration.binaries.load_aiml_on_binary_fail is True: return True # Tell caller, load failed and to load aiml directly else: raise excep
def get_conversation(self, client_context): # TODO move this to Conversations base class if client_context.userid in self._conversations: YLogger.info(client_context, "Retrieving conversation for client %s", client_context.userid) return self._conversations[client_context.userid] else: YLogger.info(client_context, "Creating new conversation for client %s", client_context.userid) conversation = Conversation(client_context) if client_context.brain.properties is not None: conversation.load_initial_variables( client_context.brain.variables) self._conversations[client_context.userid] = conversation self.load_conversation(client_context.userid) return conversation
def load_files_from_directory(self, configuration): start = datetime.datetime.now() total_aimls_loaded = 0 for file in configuration.files.aiml_files.files: aimls_loaded = self._aiml_loader.load_dir_contents( file, configuration.files.aiml_files.directories, configuration.files.aiml_files.extension, filename_as_userid=False) total_aimls_loaded = len(aimls_loaded) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Total processing time %.6f secs", diff.total_seconds()) YLogger.info(self, "Loaded a total of %d aiml files with %d categories", total_aimls_loaded, self.num_categories) if diff.total_seconds() > 0: YLogger.info(self, "Thats approx %f aiml files per sec", total_aimls_loaded / diff.total_seconds())
def execute_oob_command(self, client_context): YLogger.info(client_context, "ScheduleOutOfBandProcessor: Scheduling=%s", self._title) return "SCHEDULE"