def _load_genders(self, configuration): if configuration.files.gender is not None: self._gender_collection.empty() total = self._gender_collection.load_from_filename(configuration.files.gender) YLogger.info(self, "Loaded a total of %d genderisations", total) else: YLogger.warning(self, "No configuration setting for gender")
def _load_person2s(self, configuration): if configuration.files.person2 is not None: self._person2_collection.empty() total = self._person2_collection.load_from_filename(configuration.files.person2) YLogger.info(self, "Loaded a total of %d person2s", total) else: YLogger.warning(self, "No configuration setting for person2")
def load_tokenizer(self): if self.configuration is not None and self.configuration.tokenizer.classname is not None: YLogger.info(self, "Loading tokenizer from class [%s]", self.configuration.tokenizer.classname) tokenizer_class = ClassLoader.instantiate_class(self.configuration.tokenizer.classname) return tokenizer_class(self.configuration.tokenizer.split_chars) else: return Tokenizer(self.configuration.tokenizer.split_chars)
def parse_topic(self, topic_element, namespace): if 'name' in topic_element.attrib: name = topic_element.attrib['name'] if name is None or not name: raise ParserException("Topic name empty or null", xml_element=topic_element) xml = "<topic>%s</topic>" % name YLogger.info(self, "Topic attrib converted to %s", xml) topic_pattern = ET.fromstring(xml) else: raise ParserException("Missing name attribute for topic", xml_element=topic_element) category_found = False num_category = 0 for child in topic_element: tag_name, _ = self.tag_and_namespace_from_text(child.tag) if tag_name == 'category': self.parse_category(child, namespace, topic_pattern) category_found = True num_category += 1 else: raise ParserException("Unknown child node of topic, %s" % child.tag, xml_element=topic_element) if category_found is False: raise ParserException("No categories in topic", xml_element=topic_element) return num_category
def _load_variables(self, configuration): if configuration.files.variables is not None: self._variables_collection.empty () total = self._variables_collection.load_from_filename(configuration.files.variables) YLogger.info(self, "Loaded a total of %d variables", total) else: YLogger.warning(self, "No configuration setting for variables")
def _load_postprocessors(self, configuration): if configuration.files.postprocessors is not None: self._postprocessors.empty() total = self._postprocessors.load(configuration.files.postprocessors) YLogger.info(self, "Loaded a total of %d post processors", total) else: YLogger.warning(self, "No configuration setting for post processors")
def _load_normals(self, configuration): if configuration.files.normal is not None: self._normal_collection.empty() total = self._normal_collection.load_from_filename(configuration.files.normal) YLogger.info(self, "Loaded a total of %d normalisations", total) else: YLogger.warning(self, "No configuration setting for normal")
def load_single_file(self, configuration): start = datetime.datetime.now() self._aiml_loader.load_single_file_contents(configuration.files.aiml_files.file) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Total processing time %.6f secs", diff.total_seconds()) YLogger.info(self, "Loaded a single aiml file with %d categories", self.num_categories)
def initiate_logging(self, arguments): if arguments.logging is not None: with open(arguments.logging, 'r+', encoding="utf-8") as yml_data_file: logging_config = yaml.load(yml_data_file) logging.config.dictConfig(logging_config) YLogger.info(self, "Now logging under configuration") else: print("Warning. No logging configuration file defined, using defaults...")
def save_content(self): YLogger.info(self, "Saving aiml %s to file [%s]", self._content_type, self._filename ) for entry in self._entries: self._file.write_line(self, entry) self._file.flush() return len(self._entries)
def load_license_key_file(self, license_key_filename): try: YLogger.info(self, "Loading license key file: [%s]", license_key_filename) with open(license_key_filename, "r", encoding="utf-8") as license_file: for line in license_file: self._process_license_key_line(line) except Exception: YLogger.error(self, "Invalid license key file [%s]", license_key_filename)
def save_binary(self, configuration): YLogger.info(self, "Saving binary brain to [%s]", configuration.binaries.binary_filename) start = datetime.datetime.now() bin_file = open(configuration.binaries.binary_filename, "wb") pickle.dump(self._aiml_parser, bin_file) bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds())
def load_learnf_files_from_directory(self, configuration): if configuration.defaults.learnf_path is not None: aimls_loaded = self._aiml_loader.load_dir_contents(configuration.defaults.learnf_path, False, configuration.files.aiml_files.extension, filename_as_userid=True) total_aimls_loaded = len(aimls_loaded) YLogger.info(self, "Loaded a total of %d learnf aiml files", total_aimls_loaded)
def get_topic_pattern(self, client_context): topic_pattern = self.property("topic") if topic_pattern is None: YLogger.info(client_context, "No Topic pattern default to [*]") topic_pattern = "*" else: YLogger.info(client_context, "Topic pattern = [%s]", topic_pattern) return topic_pattern
def _load_rdfs(self, configuration): if configuration.files.rdf_files is not None and configuration.files.rdf_files.files: self._rdf_collection.empty() total = self._rdf_collection.load(configuration.files.rdf_files) YLogger.info(self, "Loaded a total of %d rdf files", total) elif configuration.files.triples is not None: self._rdf_collection.empty() total = self._rdf_collection.load_from_filename(configuration.files.triples) YLogger.info(self, "Loaded a total of %d triples", total) else: YLogger.warning(self, "No configuration setting for triples")
def initiate_spellchecker(self): # TODO Move this to Spelling bass class if self.configuration is not None: if self.configuration.spelling.classname is not None: try: YLogger.info(self, "Loading spelling checker from class [%s]", self.configuration.spelling.classname) spell_class = ClassLoader.instantiate_class(self.configuration.spelling.classname) self._spell_checker = spell_class(self.configuration.spelling) except Exception as excep: YLogger.exception(self, "Failed to initiate spellcheker", excep) else: YLogger.warning(self, "No configuration setting for spelling checker!")
def run(self): if self.arguments.noloop is False: YLogger.info(self, "Entering conversation loop...") self.prior_to_run_loop() self.run_loop() self.post_run_loop() else: YLogger.debug(self, "noloop set to True, exiting...")
def load_regex_templates(self, configuration): if configuration.files.regex_templates is not None: collection = PropertiesCollection() total = collection.load_from_filename(configuration.files.regex_templates) YLogger.info(self, "Loaded a total of %d regex templates", total) self._regex_templates.clear() for pair in collection.pairs: name = pair[0] pattern = pair[1] try: self._regex_templates[name] = re.compile(pattern, re.IGNORECASE) except Exception: YLogger.error(self, "Invalid regex template [%s]", pattern)
def __init__(self, configuration): self._filename = configuration.filename if configuration.delete_on_start: if os.path.exists(configuration.filename): YLogger.info(self, "Removing %s on start up", configuration.filename) os.remove(configuration.filename) if configuration.file_format == 'txt': self._file = TextFile(self._filename, encoding=configuration.encoding) elif configuration.file_format == 'csv': self._file = CSVFile(self._filename) self.write_header() else: raise Exception ("Unknown file type [%s]", configuration.file_format)
def load_oob_processors(self, configuration): if configuration.oob is not None: if configuration.oob.default() is not None: try: YLogger.info(self, "Loading default oob") classobject = ClassLoader.instantiate_class(configuration.oob.default().classname) self._default_oob = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB Processor", excep) for oob_name in configuration.oob.oobs(): try: YLogger.info(self, "Loading oob: %s", oob_name) classobject = ClassLoader.instantiate_class(configuration.oob.oob(oob_name).classname) self._oob[oob_name] = classobject() except Exception as excep: YLogger.exception(self, "Failed to load OOB", excep)
def resolve_to_string(self, client_context): resolved = self.resolve_children_to_string(client_context) if self._output == "logging": YLogger.debug(client_context, "[%s] resolved to [%s]", self.to_string(), resolved) if self._level == "debug": YLogger.debug(client_context, resolved) elif self._level == "warning": YLogger.warning(client_context, resolved) elif self._level == "error": YLogger.error(client_context, resolved) elif self._level == "info": YLogger.info(client_context, resolved) else: YLogger.info(client_context, resolved) else: print(resolved) return ""
def load_conversation(self, client_context, conversation): YLogger.info(client_context, "Loading conversation from Mongo [%s] [%s]", client_context.client.id, client_context.userid) collection = self.collection() document = collection.find_one({ MongoConversationStore.CLIENITD: client_context.client.id, MongoConversationStore.USERID: client_context.userid }) if document: data = document conversation.create_from_json( data[MongoConversationStore.CONVERSATION]) return True return False
def run(self, app=None): del app if self.arguments.noloop is False: YLogger.info(self, "Entering conversation loop...") self.startup() self.prior_to_run_loop() self.run_loop() self.post_run_loop() self.shutdown() else: YLogger.debug(self, "noloop set to True, exiting...")
def initiate_spellchecker(self): # TODO Move this to Spelling bass class if self.configuration is not None: if self.configuration.spelling.classname is not None: try: YLogger.info(self, "Loading spelling checker from class [%s]", self.configuration.spelling.classname) spell_class = ClassLoader.instantiate_class( self.configuration.spelling.classname) self._spell_checker = spell_class( self.configuration.spelling) except Exception as excep: YLogger.exception(self, "Failed to initiate spellcheker", excep) else: YLogger.warning( self, "No configuration setting for spelling checker!")
def load_binary(self, storage_factory): YLogger.info(self, "Loading binary brain from [%s]", StorageFactory.BINARIES) try: storage_engine = storage_factory.entity_storage_engine( self._configuration.storage) binaries_storage = storage_engine.binaries_storage() self._aiml_parser = binaries_storage.load_binary() return False # Tell caller, load succeeded and skip aiml load except Exception as excep: YLogger.exception(self, "Failed to load binary file", excep) if self._configuration.load_aiml_on_binary_fail is True: return True # Tell caller, load failed and to load aiml directly else: raise excep
def __init__(self, spelling_config=None): SpellingChecker.__init__(self, spelling_config) self.words = [] self.sum_of_words = 0 if spelling_config is None: corpus_filename = os.path.dirname(__file__) + os.sep + "corpus.txt" else: corpus_filename = spelling_config.corpus if os.path.exists(corpus_filename) is True: YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename) self.words = Counter(self._all_words(open(corpus_filename, encoding="utf-8").read())) self.sum_of_words = sum(self.words.values()) else: YLogger.error(self, "No spelling corpus found[%s]", corpus_filename)
def __init__(self, configuration): self._filename = configuration.filename if configuration.delete_on_start: if os.path.exists(configuration.filename): YLogger.info(self, "Removing %s on start up", configuration.filename) os.remove(configuration.filename) if configuration.file_format == 'txt': self._file = TextFile(self._filename, encoding=configuration.encoding) elif configuration.file_format == 'csv': self._file = CSVFile(self._filename) self.write_header() else: raise Exception("Unknown file type [%s]", configuration.file_format)
def initiate_spellchecker(spelling_config, storage_factory): if spelling_config.classname is not None: try: YLogger.info(None, "Loading spelling checker from class [%s]", spelling_config.classname) spell_class = ClassLoader.instantiate_class( spelling_config.classname) spell_checker = spell_class(spelling_config) spell_checker.initialise(storage_factory) return spell_checker except Exception as excep: YLogger.exception(None, "Failed to initiate spellcheker", excep) else: YLogger.warning(None, "No configuration setting for spelling checker!") return None
def _send_message(self, host, port, username, password, msg): YLogger.info(self, "Email sender starting") server = self._smtp_server(host, port) server.ehlo() server.starttls() YLogger.info(self, "Email sender logging in") server.login(username, password) YLogger.info(self, "Email sender sending") server.send_message(msg) YLogger.info(self, "Email sender quiting") server.quit()
def upload_from_file(self, filename, format=Store.TEXT_FORMAT, commit=True, verbose=False): YLogger.info(self, "Uploading license keys to Mongo [%s]", filename) success = 0 count = 0 try: YLogger.info(self, "Loading license key file: [%s]", filename) with open(filename, "r", encoding="utf-8") as license_file: for line in license_file: if self._process_license_key_line(line, verbose) is True: success += 1 count += 1 if commit is True: self.commit() except Exception as excep: YLogger.exception(self, "Invalid license key file [%s]", excep, filename) return count, success
def upload_from_file(self, filename, fileformat=Store.TEXT_FORMAT, commit=True, verbose=False): YLogger.info(self, "Uplading spelling corpus file [%s] to Mongo", filename) count = success = 0 try: count, success = self._read_corpus_from_file(filename, verbose) except Exception as excep: YLogger.exception(self, "Failed to load spelling corpus from [%s]", excep, filename) # Assume all words loaded are success, no need for additional count return count, success
def load_spelling(self, spell_checker): corpus_filename = self.get_storage().file encoding = self.get_storage().encoding if os.path.exists(corpus_filename) is True: YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename) try: with open(corpus_filename, encoding=encoding) as words_file: all_words = words_file.read() spell_checker.add_corpus(all_words) except Exception as e: YLogger.exception(self, "Failed to load corpus [%s]", e, corpus_filename) else: YLogger.error(self, "No spelling corpus found[%s]", corpus_filename)
def _load(self): bin_file_path = self._get_storage_path() YLogger.info(self, "Loading binary brain from [%s]", bin_file_path) start = datetime.datetime.now() gc.disable() with open(bin_file_path, "rb") as bin_file: aiml_parser = pickle.load(bin_file) bin_file.close() gc.enable() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds()) return aiml_parser
def get_conversation(self, client_context): # TODO move this to Conversations base class if client_context.userid in self._conversations: YLogger.info(client_context, "Retrieving conversation for client %s", client_context.userid) return self._conversations[client_context.userid] else: YLogger.info(client_context, "Creating new conversation for client %s", client_context.userid) conversation = Conversation(client_context) if client_context.brain.properties is not None: conversation.load_initial_variables(client_context.brain.variables) self._conversations[client_context.userid] = conversation self.load_conversation(client_context.userid) return conversation
def save_binary(self, aiml_parser): try: bin_file_path = self._get_storage_path() YLogger.info(self, "Saving binary brain to [%s]", bin_file_path) bin_file_dir = self._get_dir_from_path(bin_file_path) self._ensure_dir_exists(bin_file_dir) start = datetime.datetime.now() with open(bin_file_path, "wb") as bin_file: pickle.dump(aiml_parser, bin_file) bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain save took a total of %.2f sec", diff.total_seconds()) except Exception as excep: YLogger.exception_nostack(self, "Failed to save binary file", excep)
def initiate_sentence_splitter(splitter_config): if splitter_config.classname is not None: try: YLogger.info(None, "Loading sentence splitter from class [%s]", splitter_config.classname) splitter_class = ClassLoader.instantiate_class( splitter_config.classname) sentence_splitter = splitter_class(splitter_config) return sentence_splitter except Exception as excep: YLogger.exception(None, "Failed to initiate sentence splitter", excep) else: YLogger.warning(None, "No configuration setting for sentence splitter!") return None
def test_ylogger(self): client_context = ClientContext(TestClient(), "testid") snapshot = YLoggerSnapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(0) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.reset_snapshot() YLogger.critical(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.fatal(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(0) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.error(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.exception(client_context, "Test Message", Exception("Test error")) snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(0) Info(0), Debug(0)") YLogger.warning(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(0), Debug(0)") YLogger.info(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(0)") YLogger.debug(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEquals(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(1)")
def load_oob_processors(self, configuration): if configuration.oob is not None: if configuration.oob.default() is not None: try: YLogger.info(self, "Loading default oob") classobject = ClassLoader.instantiate_class( configuration.oob.default().classname) self._default_oob = classobject() except Exception as excep: YLogger.exception(self, excep) for oob_name in configuration.oob.oobs(): try: YLogger.info(self, "Loading oob: %s", oob_name) classobject = ClassLoader.instantiate_class( configuration.oob.oob(oob_name).classname) self._oob[oob_name] = classobject() except Exception as excep: YLogger.exception(self, excep)
def send(self, to, subject, message, attachments=[]): try: if attachments: YLogger.info(self, "Email sender adding mime attachment") msg = MIMEMultipart() msg.attach(MIMEText(message)) self.add_mime_attachements(msg, attachments) else: msg = MIMEText(message) msg['Subject'] = subject msg['From'] = self._from_addr msg['To'] = to self._send_message(self._config.host, self._config.port, self._config.username, self._config.password, msg) except Exception as e: YLogger.exception(self, "Email sender failed", e)
def upload_from_file(self, filename, format=Store.TEXT_FORMAT, commit=True, verbose=False): YLogger.info(self, "Uploading %s nodes to Mongo from [%s]", self.collection_name(), filename) count = 0 success = 0 try: with open(filename, "r", encoding="utf-8") as file: for line in file: if self.process_config_line(line, verbose) is True: success += 1 count += 1 except FileNotFoundError: YLogger.error(self, "File not found [%s]", filename) if commit is True: self.commit() return count, success
def test_ylogger(self): client_context = ClientContext(TestClient(), "testid") snapshot = YLoggerSnapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(0) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.reset_snapshot() YLogger.critical(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.fatal(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(0) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.error(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(0) Warning(0) Info(0), Debug(0)") YLogger.exception(client_context, "Test Message", Exception("Test error")) snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(0) Info(0), Debug(0)") YLogger.warning(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(0), Debug(0)") YLogger.info(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(0)") YLogger.debug(client_context, "Test Message") snapshot = YLogger.snapshot() self.assertIsNotNone(snapshot) self.assertEqual(str(snapshot), "Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(1)")
def save_braintree(self, client_context, pattern_graph): self._braintree = '' # Default XML-Format format = RedisStore.XML_FORMAT encoding = 'UTF-8' try: h_key = self.create_key("binaries", format) s_key = self._storage_engine._sessions_set_key YLogger.info(self, "Saving brainTree to redis [%s]", h_key) start = datetime.datetime.now() if format == RedisStore.TEXT_FORMAT: pattern_graph.dump(output_func=self.output_braintree, eol="\n") elif format == RedisStore.XML_FORMAT: self._braintree = '<?xml version="1.0" encoding="%s"?>\n' % encoding self._braintree += '<aiml>\n' self._braintree += pattern_graph.root.to_xml(client_context) self._braintree += '</aiml>\n' else: YLogger.error(client_context, "Unknown brainTree content type [%s]", format) return self.store(h_key, s_key, s_key, self._braintree, ex=self._storage_engine._expiretime) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "BrainTree save took a total of %.2f sec", diff.total_seconds()) except Exception as e: YLogger.exception(self, "Failed to save brainTree to Redis [%s]", e, s_key)
def load_binary(self, configuration): YLogger.info(self, "Loading binary brain from [%s]", configuration.binaries.binary_filename) try: start = datetime.datetime.now() gc.disable() bin_file = open(configuration.binaries.binary_filename, "rb") self._aiml_parser = pickle.load(bin_file) self._aiml_parser._brain = self gc.enable() bin_file.close() stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Brain load took a total of %.2f sec", diff.total_seconds()) return False # Tell caller, load succeeded and skip aiml load except Exception as excep: YLogger.exception(self, "Failed to load binary file", excep) if configuration.binaries.load_aiml_on_binary_fail is True: return True # Tell caller, load failed and to load aiml directly else: raise excep
def initiate_logging(self, arguments): if arguments.logging is not None: try: with open(arguments.logging, 'r+', encoding="utf-8") as yml_data_file: logging_config = yaml.load(yml_data_file, Loader=yaml.FullLoader) logging.config.dictConfig(logging_config) YLogger.info(self, "Now logging under configuration") except Exception as excep: YLogger.exception(self, "Failed to open logging configuration [%s]", excep, arguments.logging) else: outputLog( self, "Warning. No logging configuration file defined, using defaults..." )
def add_to_lookup(self, key, value, overwrite_existing=False): lookup = self._exists(key) if lookup is not None: if overwrite_existing is True: YLogger.info(self, "Updating lookup in SQL [%s] [%s]", key, value) lookup.value = value else: YLogger.error(self, "Existing value in SQL lookup [%s] = [%s]", key, value) return False else: YLogger.debug(self, "Adding lookup to SQL [%s] = [%s]", key, value) lookup = self._get_entity(key=key, value=value) self._storage_engine.session.add(lookup) self._storage_engine.session.commit() return True
def parse_from_text(self, text): """ Parse an AIML text version of an aiml file and return all the cateogeries found in the file :param text: Fully validated AIML snippet :return list of categories parsed from file: """ start = datetime.datetime.now() aiml = ET.fromstring(text) _, namespace = AIMLParser.check_aiml_tag(aiml) num_categories = self.parse_aiml(aiml, namespace) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Processed text with %d categories in %f.2 secs", num_categories, diff.total_seconds()) return bool(num_categories > 0)
def add_to_lookup(self, key, value, overwrite_existing=False): collection = self.collection() lookup = collection.find_one({'key': key}) if lookup is not None: if overwrite_existing is True: YLogger.info(self, "Updating lookup in Mongo [%s] [%s]", key, value) lookup['value'] = value collection.replace_one({'key': key}, lookup) else: YLogger.error(self, "Existing value in Mongo lookup [%s] = [%s]", key, value) return False else: YLogger.debug(self, "Adding lookup to Mongo [%s] = [%s]", key, value) lookup = Lookup(key, value) self.add_document(lookup) return True
def upload_from_file(self, filename, format=Store.TEXT_FORMAT, commit=True, verbose=False): YLogger.info(self, "Uploading %s to Mongo from [%s]", filename, self.collection_name()) count = 0 success = 0 if os.path.exists(filename): try: with open(filename, "r") as vars_file: for line in vars_file: line = line.strip() if line: if line.startswith( MongoPropertyStore.COMMENT) is False: splits = line.split( MongoPropertyStore.SPLIT_CHAR) if len(splits) > 1: key = splits[0].strip() val = ":".join(splits[1:]).strip() if verbose is True: YLogger.debug( self, "Adding %s property [%s=%s] to Mongo", self.collection_name(), key, val) if self.add_property(key, val) is True: success += 1 count += 1 if commit is True: self.commit() except Exception as excep: YLogger.exception(self, "Failed to upload %s from %s to Mongo", excep, self.collection_name(), filename) return count, success
def __init__(self, spelling_config=None): SpellingChecker.__init__(self, spelling_config) self.words = [] self.sum_of_words = 0 if spelling_config is None: corpus_filename = os.path.dirname(__file__) + os.sep + "corpus.txt" else: corpus_filename = spelling_config.corpus if os.path.exists(corpus_filename) is True: YLogger.info(self, "Loading spelling corpus [%s]", corpus_filename) self.words = Counter( self._all_words( open(corpus_filename, encoding="utf-8").read())) self.sum_of_words = sum(self.words.values()) else: YLogger.error(self, "No spelling corpus found[%s]", corpus_filename)
def init_ping_response(ping_responder): if ping_responder.config.host is None: YLogger.info(None, "No REST configuration for ping responder") print("Healthcheck now running as part of REST Service...") return print("Healthcheck now running as separate REST Service...") ping_app = Flask(ping_responder.config.name) @ping_app.route(ping_responder.config.url, methods=['GET']) def ping(): return jsonify(ping_responder.ping()) @ping_app.route(ping_responder.config.shutdown, methods=['GET']) def shutdown(): ping_responder.stop_ping_service() return 'Server shutting down...' ping_responder.start_ping_service(ping_app)
def load_aiml(self, configuration: BrainConfiguration): if configuration.files.aiml_files is not None: self.create_debug_storage(configuration) if configuration.files.aiml_files.has_multiple_files(): self.load_files_from_directory(configuration) elif configuration.files.aiml_files.has_single_file(): self.load_single_file(configuration) else: YLogger.info(self, "No AIML files or file defined in configuration to load") self.save_debug_files(configuration) self.display_debug_info(configuration) else: YLogger.info(self, "No AIML files or file defined in configuration to load")
def get_that_pattern(self, client_context, srai=False): try: that_question = None if srai is False: that_question = self.previous_nth_question(1) else: if len(self._questions) > 2: for question in reversed(self._questions[:-2]): if question._srai is False and question.has_response(): that_question = question break if that_question is not None: that_sentence = that_question.current_sentence() else: that_sentence = None # If the last response was valid, i.e not none and not empty string, then use # that as the that_pattern, otherwise we default to '*' as pattern if that_sentence.response is not None and that_sentence.response != '': that_pattern = self.parse_last_sentences_from_response(that_sentence.response) YLogger.info(client_context, "That pattern = [%s]", that_pattern) else: YLogger.info(client_context, "That pattern, no response, default to [*]") that_pattern = "*" except Exception as e: YLogger.info(client_context, "No That pattern default to [*]") that_pattern = "*" return that_pattern
def get_that_pattern(self, client_context, srai=False): try: that_question = None if srai is False: that_question = self.previous_nth_question(1) else: if len(self._questions) > 2: for question in reversed(self._questions[:-2]): if question._srai is False and question.has_response(): that_question = question break if that_question is not None: that_sentence = that_question.current_sentence() else: that_sentence = None # If the last response was valid, i.e not none and not empty string, then use # that as the that_pattern, otherwise we default to '*' as pattern if that_sentence.response is not None and that_sentence.response != '': that_pattern = self.parse_last_sentences_from_response( that_sentence.response) YLogger.info(client_context, "That pattern = [%s]", that_pattern) else: YLogger.info(client_context, "That pattern, no response, default to [*]") that_pattern = "*" except Exception as e: YLogger.info(client_context, "No That pattern default to [*]") that_pattern = "*" return that_pattern
def parse_from_file(self, filename, userid="*"): """ Parse an AIML file and return all the cateogeries found in the file :param filename: Name of file to parse :return list of categories parsed from file: """ YLogger.info(self, "Loading aiml file: " + filename) try: tree = ET.parse(filename, parser=LineNumberingParser()) aiml = tree.getroot() _, namespace = self.check_aiml_tag(aiml, filename=filename) start = datetime.datetime.now() num_categories = self.parse_aiml(aiml, namespace, filename, userid=userid) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Processed %s with %d categories in %f.2 secs", filename, num_categories, diff.total_seconds()) except Exception as excep: YLogger.exception(self, "Failed to load contents of AIML file from [%s]"%filename, excep)
def load_aiml(self, configuration: BrainConfiguration): if configuration.files.aiml_files is not None: self.create_debug_storage(configuration) if configuration.files.aiml_files.has_multiple_files(): self.load_files_from_directory(configuration) self.load_learnf_files_from_directory(configuration) elif configuration.files.aiml_files.has_single_file(): self.load_single_file(configuration) self.load_learnf_files_from_directory(configuration) else: YLogger.info(self, "No AIML files or file defined in configuration to load") self.save_debug_files(configuration) self.display_debug_info(configuration) else: YLogger.info(self, "No AIML files or file defined in configuration to load")
def load_files_from_directory(self, configuration): start = datetime.datetime.now() total_aimls_loaded = 0 for file in configuration.files.aiml_files.files: aimls_loaded = self._aiml_loader.load_dir_contents(file, configuration.files.aiml_files.directories, configuration.files.aiml_files.extension, filename_as_userid=False) total_aimls_loaded = len(aimls_loaded) stop = datetime.datetime.now() diff = stop - start YLogger.info(self, "Total processing time %.6f secs", diff.total_seconds()) YLogger.info(self, "Loaded a total of %d aiml files with %d categories", total_aimls_loaded, self.num_categories) if diff.total_seconds() > 0: YLogger.info(self, "Thats approx %f aiml files per sec", total_aimls_loaded / diff.total_seconds())
def execute_oob_command(self, client_context): YLogger.info(client_context, "URLOutOfBandProcessor: Loading=%s", self._url) return "URL"