def add_pattern_to_graph(self, pattern_element, topic_element, that_element, template_graph_root, learn=False, userid="*"):
pattern_node = self.add_pattern_to_node(pattern_element, userid=userid)
topic_node = self.add_topic_to_node(topic_element, pattern_node, userid=userid)
that_node = self.add_that_to_node(that_element, topic_node, userid=userid)
if that_node.has_template() is True:
if learn is False:
if pattern_element.text is not None:
raise DuplicateGrammarException("Dupicate grammar tree found [%s]"%(pattern_element.text.strip()))
else:
raise DuplicateGrammarException("Dupicate grammar tree found for bot/set")
else:
if pattern_element.text is not None:
YLogger.warning(self, "Dupicate grammar tree found [%s] in learn, replacing existing",
pattern_element.text.strip())
else:
YLogger.warning(self, "Dupicate grammar tree found for bot/set in learn, replacing existing")
self.add_template_to_node(template_graph_root, that_node)
else:
self.add_template_to_node(template_graph_root, that_node)
return that_node
def get_int_option(self, section, option_name, missing_value=0):
child = section.find(option_name)
if child is not None:
return self.convert_to_int(child.text)
else:
YLogger.warning(self, "Missing value for [%s] in config, return default value %d", option_name, missing_value)
return missing_value
def _load_genders(self, configuration):
if configuration.files.gender is not None:
self._gender_collection.empty()
total = self._gender_collection.load_from_filename(configuration.files.gender)
YLogger.info(self, "Loaded a total of %d genderisations", total)
else:
YLogger.warning(self, "No configuration setting for gender")
def _load_normals(self, configuration):
if configuration.files.normal is not None:
self._normal_collection.empty()
total = self._normal_collection.load_from_filename(configuration.files.normal)
YLogger.info(self, "Loaded a total of %d normalisations", total)
else:
YLogger.warning(self, "No configuration setting for normal")
def load_config_section(self, configuration_file, configuration, bot_root):
tokenizer = configuration_file.get_section(self._section_name, configuration)
if tokenizer is not None:
self._classname = configuration_file.get_option(tokenizer, "classname", missing_value="programy.parser.tokenizer.Tokenizer")
self._split_chars = configuration_file.get_option(tokenizer, "split_chars", missing_value=" ")
else:
YLogger.warning(self, "'tokenizer' section missing from bot config, using defaults")
def _load_variables(self, configuration):
if configuration.files.variables is not None:
self._variables_collection.empty ()
total = self._variables_collection.load_from_filename(configuration.files.variables)
YLogger.info(self, "Loaded a total of %d variables", total)
else:
YLogger.warning(self, "No configuration setting for variables")
def _load_postprocessors(self, configuration):
if configuration.files.postprocessors is not None:
self._postprocessors.empty()
total = self._postprocessors.load(configuration.files.postprocessors)
YLogger.info(self, "Loaded a total of %d post processors", total)
else:
YLogger.warning(self, "No configuration setting for post processors")
def load_config_section(self, configuration_file, configuration, bot_root):
scheduler = configuration_file.get_section(self._section_name, configuration)
if scheduler is not None:
self._name = configuration_file.get_option(scheduler, "name", missing_value=None)
self._debug_level = configuration_file.get_int_option(scheduler, "debug_level", missing_value=0)
self._add_listeners = configuration_file.get_bool_option(scheduler, "add_listeners", missing_value=False)
self._remove_all_jobs = configuration_file.get_bool_option(scheduler, "remove_all_jobs", missing_value=False)
if 'jobstore' in scheduler:
self._jobstore = SchedulerJobStoreConfiguration()
self._jobstore.load_config_section(configuration_file, scheduler, bot_root)
if 'threadpool' in scheduler:
self._threadpool = SchedulerThreadPoolConfiguration()
self._threadpool.load_config_section(configuration_file, scheduler, bot_root)
if 'processpool' in scheduler:
self._processpool = SchedulerProcessPoolConfiguration()
self._processpool.load_config_section(configuration_file, scheduler, bot_root)
if 'job_defaults' in scheduler:
self._job_defaults = SchedulerJobDefaultsConfiguration()
self._job_defaults.load_config_section(configuration_file, scheduler, bot_root)
else:
YLogger.warning(self, "'scheduler' section missing from client config, using to defaults")
def load_configuration(self, configuration_file, section, bot_root):
if section is not None:
bot_names = configuration_file.get_multi_option(section, "bot", missing_value="bot")
first = True
for name in bot_names:
if first is True:
config = self._bot_configs[0]
first = False
else:
config = BotConfiguration(name)
self._bot_configs.append(config)
config.load_configuration(configuration_file, bot_root)
self._license_keys = configuration_file.get_option(section, "license_keys")
if self._license_keys is not None:
self._license_keys = self.sub_bot_root(self._license_keys, bot_root)
self._bot_selector = configuration_file.get_option(section, "bot_selector")
self._scheduler.load_config_section(configuration_file, section, bot_root)
self._renderer = configuration_file.get_option(section, "renderer")
else:
YLogger.warning(self, "No bot name defined for client [%s], defaulting to 'bot'.", self.section_name)
self._bot_configs[0]._section_name = "bot"
self._bot_configs[0].load_configuration(configuration_file, bot_root)
def _load_person2s(self, configuration):
if configuration.files.person2 is not None:
self._person2_collection.empty()
total = self._person2_collection.load_from_filename(configuration.files.person2)
YLogger.info(self, "Loaded a total of %d person2s", total)
else:
YLogger.warning(self, "No configuration setting for person2")
def load_license_keys(self):
if self.configuration is not None:
if self.configuration.client_configuration.license_keys is not None:
self._license_keys.load_license_key_file(self.configuration.client_configuration.license_keys)
else:
YLogger.warning(self, "No client configuration setting for license_keys")
else:
YLogger.warning(self, "No configuration defined when loading license keys")
def load_config_section(self, configuration_file, configuration, bot_root):
service = configuration_file.get_section(self.section_name, configuration)
if service is not None:
self._classname = configuration_file.get_option(service, "classname", missing_value=None)
self._denied_srai = configuration_file.get_option(service, "denied_srai", missing_value=None)
self._denied_text = configuration_file.get_option(service, "denied_text", missing_value=BrainSecurityConfiguration.DEFAULT_ACCESS_DENIED)
else:
YLogger.warning(self, "'security' section missing from bot config, using to defaults")
def load_config_section(self, configuration_file, configuration, bot_root):
overrides = configuration_file.get_section(self._section_name, configuration)
if overrides is not None:
self._allow_system_aiml = configuration_file.get_bool_option(overrides, "allow_system_aiml", missing_value=False)
self._allow_learn_aiml = configuration_file.get_bool_option(overrides, "allow_learn_aiml", missing_value=False)
self._allow_learnf_aiml = configuration_file.get_bool_option(overrides, "allow_learnf_aiml", missing_value=False)
else:
YLogger.warning(self, "'overrides' section missing from brain config, using to defaults")
def load_config_section(self, configuration_file, configuration, bot_root):
ConversationsFileStorage = configuration_file.get_section(self._section_name, configuration)
if ConversationsFileStorage is not None:
dir = configuration_file.get_option(ConversationsFileStorage, "dir", missing_value=None)
if dir is not None:
self._dir = self.sub_bot_root(dir, bot_root)
else:
YLogger.warning(self, "'BotConversationsFileStorageConfiguration' section missing from bot config, using defaults")
def load_config_section(self, configuration_file, configuration, bot_root):
braintree = configuration_file.get_section("braintree", configuration)
if braintree is not None:
file = configuration_file.get_option(braintree, "file", missing_value=None)
if file is not None:
self._file = self.sub_bot_root(file, bot_root)
self._content = configuration_file.get_option(braintree, "content", missing_value="txt")
else:
YLogger.warning(self, "'braintree' section missing from bot config, using to defaults")
def get_storage(config):
if config.conversations.type == 'file':
return ConversationFileStorage(config.conversations.storage)
elif config.conversations.type == 'redis':
return ConversationRedisStorage(config.conversations.storage)
YLogger.warning(None, "Invalid Conversations file storage type [%s]", config.conversations.type)
return None
def parse_version(self, aiml):
if 'version' in aiml.attrib:
version = aiml.attrib['version']
if version not in ['0.9', '1.0', '1.1', '2.0']:
YLogger.warning(self, "Version number not a supported version: %s", version)
else:
YLogger.warning(self, "No version info, defaulting to 2.0")
version = "2.0"
return version
def load_users_and_groups(self):
self._users = {}
self._groups = {}
if self.configuration.usergroups is not None:
loader = UserGroupLoader()
self._users, self._groups = loader.load_users_and_groups_from_file(self.configuration.usergroups)
else:
YLogger.warning(self, "No user groups defined, authorisation tag will not work!")
def get_option(self, section, option_name, missing_value=None):
child = section.find(option_name)
if child is not None:
if not child._children:
return self._infer_type_from_string(child.text)
return child
else:
if missing_value is not None:
YLogger.warning(self, "Missing value for [%s] in config, return default value %s", option_name, missing_value)
return missing_value
def load_config_section(self, configuration_file, configuration, bot_root):
ConversationsFileStorage = configuration_file.get_section(self._section_name, configuration)
if ConversationsFileStorage is not None:
self._host = configuration_file.get_option(ConversationsFileStorage, "host", missing_value="localhost")
self._port = configuration_file.get_int_option(ConversationsFileStorage, "port", missing_value=6379)
self._password = configuration_file.get_option(ConversationsFileStorage, "password", missing_value=None)
self._prefix = configuration_file.get_option(ConversationsFileStorage, "prefix",
missing_value="program_y:bot_cache")
else:
YLogger.warning(self, "'BotConversationsRedisStorageConfiguration' section missing from bot config, using defaults")
def load_config_section(self, configuration_file, configuration, bot_root):
spelling = configuration_file.get_section(self._section_name, configuration)
if spelling is not None:
self._classname = configuration_file.get_option(spelling, "classname", missing_value=None)
self._alphabet = configuration_file.get_option(spelling, "alphabet", missing_value=None)
corpus = configuration_file.get_option(spelling, "corpus", missing_value=None)
self._corpus = self.sub_bot_root(corpus, bot_root)
self._check_before = configuration_file.get_bool_option(spelling, "check_before", missing_value=False)
self._check_and_retry = configuration_file.get_option(spelling, "check_and_retry", missing_value=False)
else:
YLogger.warning(self, "'spelling' section missing from bot config, using defaults")
def _load_rdfs(self, configuration):
if configuration.files.rdf_files is not None and configuration.files.rdf_files.files:
self._rdf_collection.empty()
total = self._rdf_collection.load(configuration.files.rdf_files)
YLogger.info(self, "Loaded a total of %d rdf files", total)
elif configuration.files.triples is not None:
self._rdf_collection.empty()
total = self._rdf_collection.load_from_filename(configuration.files.triples)
YLogger.info(self, "Loaded a total of %d triples", total)
else:
YLogger.warning(self, "No configuration setting for triples")
def load_config_section(self, configuration_file, configuration, bot_root):
service = configuration_file.get_section(self.section_name, configuration)
if service is not None:
self._classname = configuration_file.get_option(service, "classname", missing_value=None)
self._method = configuration_file.get_option(service, "method", missing_value=None)
self._host = configuration_file.get_option(service, "host", missing_value=None)
self._port = configuration_file.get_option(service, "port", missing_value=None)
self._url = configuration_file.get_option(service, "url", missing_value=None)
self.load_additional_key_values(configuration_file, service)
else:
YLogger.warning(self, "'services' section missing from brain config, using to defaults")
def load_config_section(self, configuration_file, configuration, bot_root):
binaries = configuration_file.get_section("defaults", configuration)
if binaries is not None:
self._default_get = configuration_file.get_option(binaries, "default-get", missing_value=None)
self._default_property = configuration_file.get_option(binaries, "default-property", missing_value=None)
self._default_map = configuration_file.get_option(binaries, "default-map", missing_value=None)
learnf_path = configuration_file.get_option(binaries, "learnf-path", missing_value=None)
if learnf_path is not None:
self._learnf_path = self.sub_bot_root(learnf_path, bot_root)
else:
YLogger.warning(self, "'defaults' section missing from bot config, using default defaults")
def load_config_section(self, configuration_file, configuration, bot_root):
debugfile_config = configuration_file.get_option(configuration, self._section_name)
if debugfile_config is not None:
file = configuration_file.get_option(debugfile_config, "file", missing_value=None)
if file is not None:
self._file = self.sub_bot_root(file, bot_root)
self._file_format = configuration_file.get_option(debugfile_config, "format", missing_value="txt")
self._encoding = configuration_file.get_option(debugfile_config, "encoding", missing_value="utf-8")
self._delete_on_start = configuration_file.get_bool_option(debugfile_config, "delete_on_start", missing_value=False)
else:
YLogger.warning(self, "'%s' section missing from aiml files config, using to defaults", self.section_name)
def load_config_section(self, configuration_file, configuration, bot_root):
services = configuration_file.get_section("services", configuration)
if services is not None:
service_keys = configuration_file.get_keys(services)
for name in service_keys:
service = BrainServiceConfiguration(name)
service.load_config_section(configuration_file, services, bot_root)
self._services[name] = service
else:
YLogger.warning(self, "Config section [services] missing from Brain, no services loaded")
def get_storage_config(type, config_name, configuration_file, configuration, bot_root):
if type == 'file':
storage = BotConversationsFileStorageConfiguration(config_name=config_name)
storage.load_config_section(configuration_file, configuration, bot_root)
return storage
elif type == 'redis':
storage = BotConversationsRedisStorageConfiguration(config_name=config_name)
storage.load_config_section(configuration_file, configuration, bot_root)
return storage
YLogger.warning(None, "Invalid Conversations file storage type [%s]", type)
return None
def _process_license_key_line(self, line):
line = line.strip()
if line:
if line.startswith('#') is False:
splits = line.split("=")
if len(splits) > 1:
key_name = splits[0].strip()
# If key has = signs in it, then combine all elements past the first
key = "".join(splits[1:]).strip()
self._keys[key_name] = key
else:
YLogger.warning(self, "Invalid license key [%s]", line)
def initiate_spellchecker(self):
# TODO Move this to Spelling bass class
if self.configuration is not None:
if self.configuration.spelling.classname is not None:
try:
YLogger.info(self, "Loading spelling checker from class [%s]", self.configuration.spelling.classname)
spell_class = ClassLoader.instantiate_class(self.configuration.spelling.classname)
self._spell_checker = spell_class(self.configuration.spelling)
except Exception as excep:
YLogger.exception(self, "Failed to initiate spellcheker", excep)
else:
YLogger.warning(self, "No configuration setting for spelling checker!")
def resolve_to_string(self, client_context):
if client_context.brain.configuration.overrides.allow_system_aiml is True:
command = self.resolve_children_to_string(client_context)
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = []
for line in process.stdout.readlines():
byte_string = line.decode("utf-8")
result.append(byte_string.strip())
process.wait()
resolved = " ".join(result)
else:
YLogger.warning(client_context, "System command node disabled in config")
resolved = ""
YLogger.debug(client_context, "[%s] resolved to [%s]", self.to_string(), resolved)
return resolved
def load_configuration(self, configuration_file, section, bot_root):
if section is not None:
bot_names = configuration_file.get_multi_option(section, "bot", missing_value="bot")
first = True
for name in bot_names:
if first is True:
config = self._bot_configs[0]
first = False
else:
config = BotConfiguration(name)
self._bot_configs.append(config)
config.load_configuration(configuration_file, bot_root)
self._license_keys = configuration_file.get_option(section, "license_keys")
if self._license_keys is not None:
self._license_keys = self.sub_bot_root(self._license_keys, bot_root)
self._bot_selector = configuration_file.get_option(section, "bot_selector")
self._scheduler.load_config_section(configuration_file, section, bot_root)
else:
YLogger.warning(self, "No bot name defined for client [%s], defaulting to 'bot'.", self.section_name)
self._bot_configs[0]._section_name = "bot"
self._bot_configs[0].load_configuration(configuration_file, bot_root)
def get_multi_file_option(self,
section,
option_name,
bot_root,
missing_value=None,
subs: Substitutions = None):
if missing_value is None:
missing_value = []
if option_name in section:
values = section[option_name]
splits = values.split('\n')
multis = []
for value in splits:
if value is not None and value != '':
value = self._replace_subs(subs, value)
multis.append(value.replace('$BOT_ROOT', bot_root))
return multis
YLogger.warning(
self, "Missing value for [%s] in config, return default value",
option_name)
return missing_value
def load_config_section(self,
configuration_file,
configuration,
bot_root,
subs: Substitutions = None):
oobs = configuration_file.get_section("oob", configuration)
if oobs is not None:
oob_keys = configuration_file.get_keys(oobs)
for name in oob_keys:
oob = BrainOOBConfiguration(name)
oob.load_config_section(configuration_file,
oobs,
bot_root,
subs=subs)
if name == 'default':
self._default = oob
else:
self._oobs[name] = oob
else:
YLogger.warning(
self,
"Config section [oobs] missing from Brain, no oobs loaded")
def extract_attribute(self, json_data, name, data_type, time_period=None):
if name in json_data:
return json_data[name]
else:
if data_type == MetOfficeWeatherReport.OBSERVATION:
YLogger.warning(
self,
'%s attribute missing from ObservationDataPoint data point',
name)
elif data_type == MetOfficeWeatherReport.FORECAST:
if time_period == metoffer.THREE_HOURLY:
YLogger.warning(
self,
'%s attribute missing from three hourly forecast data point',
name)
if time_period == metoffer.DAILY:
YLogger.warning(
self,
'%s attribute missing from daily forecast data point',
name)
return None
def value(self, key):
if key in self._additionals:
return self._additionals[key]
else:
YLogger.warning(self, "Configuration key [%s] does not exist", key)
return None
def combine_answers(self, answers, srai):
# The simplest heuristic is to pull answers through set data structure and deduplicate pure
# duplicates
len_before = len(answers)
new_answers = []
for i in range(len(answers)):
if not (answers[i] in new_answers):
new_answers.append(answers[i])
answers = new_answers
len_after = len(answers)
if len_after != len_before:
YLogger.warning(
self,
"Sentence DeDuplicator stripped duplicated answers: %d %d",
len_before, len_after)
final_sentences = []
# TODO it would be better to reference to config's bot.default_response setting for
# the IDK response, but
# 1) it is not clear what is the best practice to reference variables from YAML.
# 2) config consumed by SentenceJoinerDeDuplicator has no acces to BotConfiguration where
# IDK is specified
IDK_SENTENCE = "Sorry, I don't have an answer for that!"
# reversed `answers` to join answers in correct order
for sentence in answers:
if sentence and sentence.lower() != IDK_SENTENCE.lower():
# Sometimes sentence can be already merged list of answers which may contain
# duplicated IDKs. So we make cleaning here.
if IDK_SENTENCE.lower() in sentence.lower():
sentence, _ = re.subn(IDK_SENTENCE,
"",
sentence,
flags=re.IGNORECASE)
sentence = sentence.strip()
if not sentence:
# if sentence is empty after IDK removal
continue
# Capitalise the start of each sentence
if sentence[0].isalpha():
sentence = sentence[0].upper() + sentence[1:]
# If it ends with a terminator, keep the terminator, otherwise add a full stop
if self.ends_with_terminator(sentence):
final_sentences.append(sentence)
else:
if srai is False:
final_sentences.append(sentence +
self._configuration.terminator)
else:
final_sentences.append(sentence)
# return just the first answer that satisfies the criteria of allowed answer (not IDK).
# break
if len(final_sentences) == 0:
# if we here means all answers are IDKs, choose the only one:
final_sentences = [IDK_SENTENCE]
for i, sent in enumerate(final_sentences):
if "socialbot running inside" in sent:
final_sentences = [sent]
if len(final_sentences) > 1:
final_sentences = [final_sentences[-1]]
return " ".join([sentence for sentence in final_sentences])
def parse_expression(self, graph, expression):
self._parse_node_with_attrib(graph, expression, "index", "1")
if self.children:
YLogger.warning(self, "<response> node should not contain child text, use <response /> or <response></response> only")
def parse_expression(self, graph, expression):
mode_count = 0
if 'method' in expression.attrib:
YLogger.warning(self, "'method' attrib not supported in sraix, moved to config, see documentation")
if 'query' in expression.attrib:
YLogger.warning(self, "'query' attrib not supported in sraix, moved to config, see documentation")
if 'header' in expression.attrib:
YLogger.warning(self, "'header' attrib not supported in sraix, moved to config, see documentation")
if 'body' in expression.attrib:
YLogger.warning(self, "'body' attrib not supported in sraix, moved to config, see documentation")
if 'botName' in expression.attrib:
bot_name = expression.attrib['botName']
if graph.aiml_parser.brain.botnames.botInfo(bot_name) is None:
raise ParserException("BotName[%s] not found" % bot_name, xml_element=expression, nodename='sraix')
mode_count += 1
self._botName = bot_name
if 'locale' in expression.attrib:
YLogger.warning(self, "'locale' attrib not supported in sraix, moved to config, see documentation")
if 'time' in expression.attrib:
YLogger.warning(self, "'time' attrib not supported in sraix, moved to config, see documentation")
if 'userId' in expression.attrib:
YLogger.warning(self, "'userId' attrib not supported in sraix, moved to config, see documentation")
if 'topic' in expression.attrib:
YLogger.warning(self, "'topic' attrib not supported in sraix, moved to config, see documentation")
if 'deleteVariable' in expression.attrib:
YLogger.warning(self, "'deleteVariable' attrib not supported in sraix, moved to config, see documentation")
if 'metadata' in expression.attrib:
YLogger.warning(self, "'metadata' attrib not supported in sraix, moved to config, see documentation")
if 'config' in expression.attrib:
YLogger.warning(self, "'config' attrib not supported in sraix, moved to config, see documentation")
if 'service' in expression.attrib:
service_name = expression.attrib['service']
if ServiceFactory.service_exists(service_name) is False:
raise ParserException("Service[%s] not found" % service_name, xml_element=expression, nodename='sraix')
mode_count += 1
self._service = service_name
if 'default' in expression.attrib:
self._default = expression.attrib['default']
head_text = self.get_text_from_element(expression)
self.parse_text(graph, head_text)
for child in expression:
tag_name = TextUtils.tag_from_text(child.tag)
if tag_name == 'host':
mode_count += 1
self._host = self._parse_template_node(graph, child, 'host', False)
elif tag_name == 'method':
self._method = self._parse_template_node(graph, child, 'method', False)
elif tag_name == 'query':
self._query = self._parse_template_node(graph, child, 'query', True)
elif tag_name == 'header':
self._header = self._parse_template_node(graph, child, 'header', True)
elif tag_name == 'body':
self._body = self._parse_template_node(graph, child, 'body', False)
elif tag_name == 'botName':
YLogger.warning(self, "'botName' element not supported in sraix, moved to config, see documentation")
elif tag_name == 'locale':
self._locale = self._parse_template_node(graph, child, 'locale', False)
elif tag_name == 'time':
self._time = self._parse_template_node(graph, child, 'time', False)
elif tag_name == 'userId':
self._userId = self._parse_template_node(graph, child, 'userId', False)
elif tag_name == 'topic':
self._topic = self._parse_template_node(graph, child, 'topic', False)
elif tag_name == 'deleteVariable':
self._deleteVariable = self._parse_template_node(graph, child, 'deleteVariable', False)
elif tag_name == 'metadata':
self._metadata = self._parse_template_node(graph, child, 'metadata', False)
elif tag_name == 'config':
self._config = self._parse_template_node(graph, child, 'config', True)
elif tag_name == 'service':
YLogger.warning(self, "'service' element not supported in sraix, moved to config, see documentation")
elif tag_name == 'default':
YLogger.warning(self, "'default' element not supported in sraix, moved to config, see documentation")
else:
graph.parse_tag_expression(child, self)
tail_text = self.get_tail_from_element(child)
self.parse_text(graph, tail_text)
if mode_count == 0:
raise ParserException("Missing type attribute or host element", xml_element=expression, nodename='sraix')
elif mode_count > 1:
raise ParserException("Node has Multiple type attribute or host element", xml_element=expression, nodename='sraix')
def test_ylogger(self):
client_context = ClientContext(TestClient(), "testid")
snapshot = YLoggerSnapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(0) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)"
)
YLogger.reset_snapshot()
YLogger.critical(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(0) Error(0) Exception(0) Warning(0) Info(0), Debug(0)"
)
YLogger.fatal(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(1) Error(0) Exception(0) Warning(0) Info(0), Debug(0)"
)
YLogger.error(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(1) Error(1) Exception(0) Warning(0) Info(0), Debug(0)"
)
YLogger.exception(client_context, "Test Message",
Exception("Test error"))
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(1) Error(1) Exception(1) Warning(0) Info(0), Debug(0)"
)
YLogger.warning(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(0), Debug(0)"
)
YLogger.info(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(0)"
)
YLogger.debug(client_context, "Test Message")
snapshot = YLogger.snapshot()
self.assertIsNotNone(snapshot)
self.assertEqual(
str(snapshot),
"Critical(1) Fatal(1) Error(1) Exception(1) Warning(1) Info(1), Debug(1)"
)
def parse_expression(self, graph, expression):
if 'format' in expression.attrib:
self.interval_format = self.parse_attrib_value_as_word_node(graph, expression, 'format')
head_text = self.get_text_from_element(expression)
self.parse_text(graph, head_text)
for child in expression:
tag_name = TextUtils.tag_from_text(child.tag)
if tag_name == 'format':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self.interval_format = node
elif tag_name == 'style':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self.style = node
elif tag_name == 'from':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self.interval_from = node
elif tag_name == 'to':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self.interval_to = node
else:
graph.parse_tag_expression(child, self)
tail_text = self.get_tail_from_element(child)
self.parse_text(graph, tail_text)
if self.interval_format is None or len(self.interval_format.children) == 0:
YLogger.warning(self, "Interval node, format missing, defaulting to '%c'")
self.interval_format = graph.get_word_node('%c')
if self.style is None or len(self.style.children) == 0:
raise ParserException("interval_style node missing", xml_element=expression, nodename='interval')
elif len(self.style.children) == 1 and type(self.style.children[0]) is TemplateWordNode:
style = self.style.children[0].word
if style not in ["years", "months", "weeks", "days", "hours", "minutes", "seconds", "microseconds", "ymd", "hms", "ymdhms"]:
raise ParserException(("invalid interval stile [%s]" % style), xml_element=expression, nodename='interval')
if self.interval_from is None or len(self.interval_from.children) == 0:
raise ParserException("interval_from node missing", xml_element=expression, nodename='interval')
if self.interval_to is None or len(self.interval_to.children) == 0:
raise ParserException("interval_to node missing", xml_element=expression, nodename='interval')
def _get_storage_path(self):
if len(self.storage_engine.configuration.rdf_updates_storage.dirs) > 1:
YLogger.warning(self, "RDF-Modified Storage has multiple folders specified, using first only")
return self.storage_engine.configuration.rdf_updates_storage.dirs[0]
def load_config_section(self, configuration_file, configuration, bot_root, subs: Substitutions = None):
dirs_config = configuration_file.get_option(configuration, self.section_name)
if dirs_config is not None:
self.extract_configuration(configuration_file, dirs_config, bot_root, subs=subs)
else:
YLogger.warning(self, "'%s' section missing from bot config, using to defaults", self.section_name)
def parse_aiml(self, aiml_xml, namespace, filename=None, userid="*"):
self.parse_version(aiml_xml)
categories_found = False
num_category = 0
for expression in aiml_xml:
if hasattr(expression, "_start_line_number"):
start_line = str(expression._start_line_number)
else:
start_line = 0
if hasattr(expression, "_end_line_number"):
end_line = str(expression._end_line_number)
else:
end_line = 0
tag_name, namespace = self.tag_and_namespace_from_text(
expression.tag)
if tag_name == 'topic':
try:
num_topic_categories = self.parse_topic(
expression, namespace, filename)
num_category += num_topic_categories
categories_found = True
except DuplicateGrammarException as dupe_excep:
self.handle_aiml_duplicate(dupe_excep, filename,
expression)
except ParserException as parser_excep:
self.handle_aiml_error(parser_excep, filename, expression)
elif tag_name == 'category':
try:
if (self._num_categories + self._user_categories
) >= self.brain.bot.configuration.max_categories:
self._category_over = True
raise ParserException(
("Max categories [%d] exceeded" %
self.brain.bot.configuration.max_categories),
xml_element=expression)
parsed = self.parse_category(expression,
namespace,
userid=userid)
if parsed[4] is not None:
parsed[4].set_fileInfo(filename, start_line, end_line)
categories_found = True
num_category += 1
except DuplicateGrammarException as dupe_excep:
self.handle_aiml_duplicate(dupe_excep, filename,
expression)
except ParserException as parser_excep:
self.handle_aiml_error(parser_excep, filename, expression)
else:
parser_excep = ParserException("Unknown top level tag [%s]" %
tag_name,
xml_element=expression)
self.handle_aiml_error(parser_excep, filename, expression)
if self._category_over is True:
break
if categories_found is False:
YLogger.warning(self, "no categories in aiml file [%s]", filename)
return num_category
def load_config_section(self,
configuration_file,
configuration,
bot_root,
subs: Substitutions = None):
storage = configuration_file.get_section(self._section_name,
configuration)
if storage is not None:
entities = configuration_file.get_section("entities", storage)
entity_types = configuration_file.get_child_section_keys(
"entities", storage)
for entity in entity_types:
entity_config = configuration_file.get_section(
entity, entities)
self._entity_store[entity] = entity_config
stores = configuration_file.get_section("stores", storage)
store_names = configuration_file.get_child_section_keys(
"stores", storage)
for store in store_names:
store_config = configuration_file.get_section(store, stores)
keys = configuration_file.get_keys(store_config)
if 'type' not in keys:
YLogger.error(
None,
"'type' section missing from client config stores element [%s], ignoring config",
store)
continue
if 'config' not in keys:
YLogger.error(
None,
"'config' section missing from client config stores element [%s], ignoring config",
store)
continue
type = configuration_file.get_option(store_config,
'type',
subs=subs)
if type == 'sql':
config = SQLStorageConfiguration()
config.load_config_section(configuration_file,
store_config,
bot_root,
subs=subs)
elif type == 'mongo':
config = MongoStorageConfiguration()
config.load_config_section(configuration_file,
store_config,
bot_root,
subs=subs)
elif type == 'redis':
config = RedisStorageConfiguration()
config.load_config_section(configuration_file,
store_config,
bot_root,
subs=subs)
elif type == 'file':
config = FileStorageConfiguration()
config.load_config_section(configuration_file,
store_config,
bot_root,
subs=subs)
elif type == 'logger':
config = LoggerStorageConfiguration()
config.load_config_section(configuration_file,
store_config,
bot_root,
subs=subs)
self._store_configs[store] = config
else:
YLogger.warning(
self,
"'storage' section missing from client config, using to defaults"
)
self._entity_store = {}
self.add_default_entities(self._entity_store)
self._store_configs = {}
self.add_default_stores(self._store_configs)
def load_config_section(self, configuration_file, configuration, bot_root, subs: Substitutions = None):
oob = configuration_file.get_section(self.section_name, configuration)
if oob is not None:
self._classname = configuration_file.get_option(oob, "classname", missing_value=None, subs=subs)
else:
YLogger.warning(self, "'oob' section missing from brain config, using to defaults")
def load_configuration(self,
configuration_file,
bot_root,
subs: Substitutions = None):
bot = configuration_file.get_section(self.section_name)
if bot is not None:
self._default_response = configuration_file.get_option(
bot,
"default_response",
BotConfiguration.DEFAULT_RESPONSE,
subs=subs)
self._default_response_srai = configuration_file.get_option(
bot,
"default_response_srai",
BotConfiguration.DEFAULT_RESPONSE_SRAI,
subs=subs)
self._empty_string = configuration_file.get_option(
bot,
"empty_string",
BotConfiguration.DEFAULT_EMPTY_STRING,
subs=subs)
self._exit_response = configuration_file.get_option(
bot,
"exit_response",
BotConfiguration.DEFAULT_EXIT_RESPONSE,
subs=subs)
self._exit_response_srai = configuration_file.get_option(
bot,
"exit_response_srai",
BotConfiguration.DEFAULT_EXIT_RESPONSE_SRAI,
subs=subs)
self._initial_question = configuration_file.get_option(
bot,
"initial_question",
BotConfiguration.DEFAULT_INITIAL_QUESTION,
subs=subs)
self._initial_question_srai = configuration_file.get_option(
bot,
"initial_question_srai",
BotConfiguration.DEFAULT_INITIAL_QUESTION_SRAI,
subs=subs)
self._override_properties = configuration_file.get_option(
bot,
"override_properties",
BotConfiguration.DEFAULT_OVERRIDE_PREDICATES,
subs=subs)
self._max_question_recursion = configuration_file.get_int_option(
bot,
"max_question_recursion",
BotConfiguration.DEFAULT_MAX_QUESTION_RECURSION,
subs=subs)
self._max_question_timeout = configuration_file.get_int_option(
bot,
"max_question_timeout",
BotConfiguration.DEFAULT_MAX_QUESTION_TIMEOUT,
subs=subs)
self._max_search_depth = configuration_file.get_int_option(
bot,
"max_search_depth",
BotConfiguration.DEFAULT_MAX_SEARCH_DEPTH,
subs=subs)
self._max_search_timeout = configuration_file.get_int_option(
bot,
"max_search_timeout",
BotConfiguration.DEFAULT_MAX_SEARCH_TIMEOUT,
subs=subs)
self._tab_parse_output = configuration_file.get_bool_option(
bot,
"tab_parse_output",
BotConfiguration.DEFAULT_TAB_PARSE_OUTPUT,
subs=subs)
self._spelling.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
self._conversations.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
self._splitter.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
self._joiner.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
self._from_translator.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
self._to_translator.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
self._sentiment.load_config_section(configuration_file,
bot,
bot_root,
subs=subs)
else:
YLogger.warning(
self, "Config section [%s] missing, using default values",
self.section_name)
self.load_configurations(configuration_file, bot, bot_root, subs)
def parse_expression(self, graph, expression):
if 'format' in expression.attrib:
self._interval_format = graph.get_word_node(
expression.attrib['format'])
if 'style' in expression.attrib:
self._style = graph.get_word_node(expression.attrib['style'])
if 'from' in expression.attrib:
self._interval_from = graph.get_word_node(
expression.attrib['from'])
if 'to' in expression.attrib:
self._interval_to = graph.get_word_node(expression.attrib['to'])
head_text = self.get_text_from_element(expression)
self.parse_text(graph, head_text)
for child in expression:
tag_name = TextUtils.tag_from_text(child.tag)
if tag_name == 'format':
self._interval_format = graph.get_word_node(
self.get_text_from_element(child))
elif tag_name == 'style':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self._style = node
elif tag_name == 'from':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self._interval_from = node
elif tag_name == 'to':
node = graph.get_base_node()
node.parse_text(graph, self.get_text_from_element(child))
for sub_child in child:
graph.parse_tag_expression(sub_child, node)
node.parse_text(graph, self.get_text_from_element(child))
self._interval_to = node
else:
raise ParserException("No children allows in interval")
tail_text = self.get_tail_from_element(child)
self.parse_text(graph, tail_text)
if self.interval_format is None:
YLogger.warning(
self, "Interval node, format missing, defaulting to '%c'!")
self._interval_format = TemplateWordNode("%c")
if self.style is None:
YLogger.warning(
self, "style node, format missing, defaulting to 'days'!")
self._style = TemplateWordNode("days")
if self.interval_from is None:
raise ParserException("interval_from node, format missing !")
if self.interval_to is None:
raise ParserException(self, "interval_to node, format missing !")
def set_attrib(self, attrib_name, attrib_value):
if attrib_name != 'timeout':
raise ParserException("Invalid attribute name %s for this node", attrib_name)
YLogger.warning(self, "System node timeout attrib currently ignored")
self._timeout = attrib_value
def load_config_section(self, configuration_file, configuration, bot_root):
service = configuration_file.get_section(self.section_name, configuration)
if service is not None:
self._classname = configuration_file.get_option(service, "classname", missing_value=None)
else:
YLogger.warning(self, "'oob' section missing from brain config, using to defaults")
def add_key(self, name, value):
if name in self._keys:
YLogger.warning(self, "License key [%s], already exists", name)
self._keys[name] = value
def add_substitute(self, name, value):
if name in self._substitutes:
YLogger.warning(self, "Substitute [%s], already exists", name)
self._substitutes[name] = value
def parse_expression(self, graph, expression):
if 'host' in expression.attrib:
YLogger.warning(self, "'host' attrib not supported in sraix, moved to config, see documentation")
if 'botid' in expression.attrib:
YLogger.warning(self, "'botid' attrib not supported in sraix, moved to config, see documentation")
if 'hint' in expression.attrib:
YLogger.warning(self, "'hint' attrib not supported in sraix, moved to config, see documentation")
if 'apikey' in expression.attrib:
YLogger.warning(self, "'apikey' attrib not supported in sraix, moved to config, see documentation")
if 'service' in expression.attrib:
self.service = expression.attrib['service']
head_text = self.get_text_from_element(expression)
self.parse_text(graph, head_text)
for child in expression:
tag_name = TextUtils.tag_from_text(child.tag)
if tag_name == 'host':
YLogger.warning(self, "'host' element not supported in sraix, moved to config, see documentation")
elif tag_name == 'botid':
YLogger.warning(self, "'botid' element not supported in sraix, moved to config, see documentation")
elif tag_name == 'hint':
YLogger.warning(self, "'hint' element not supported in sraix, moved to config, see documentation")
elif tag_name == 'apikey':
YLogger.warning(self, "'apikey' element not supported in sraix, moved to config, see documentation")
elif tag_name == 'service':
self.service = self.get_text_from_element(child)
else:
graph.parse_tag_expression(child, self)
tail_text = self.get_tail_from_element(child)
self.parse_text(graph, tail_text)
if self.service is None:
raise ParserException("SRAIX node, service attribute missing !")
def load_configuration_section(self,
configuration_file,
section,
bot_root,
subs: Substitutions = None):
assert (configuration_file is not None)
if section is not None:
self._description = configuration_file.get_option(
section,
"description",
missing_value='ProgramY AIML2.0 Client',
subs=subs)
bot_names = configuration_file.get_multi_option(
section, "bot", missing_value="bot", subs=subs)
first = True
for name in bot_names:
if first is True:
config = self._bot_configs[0]
first = False
else:
config = BotConfiguration(name)
self._bot_configs.append(config)
config.load_configuration(configuration_file,
bot_root,
subs=subs)
self._bot_selector = configuration_file.get_option(
section,
"bot_selector",
missing_value="programy.clients.client.DefaultBotSelector",
subs=subs)
self._scheduler.load_config_section(configuration_file,
section,
bot_root,
subs=subs)
self._storage.load_config_section(configuration_file,
section,
bot_root,
subs=subs)
self._renderer = configuration_file.get_option(section,
"renderer",
subs=subs)
self._email.load_config_section(configuration_file,
section,
bot_root,
subs=subs)
self._triggers.load_config_section(configuration_file,
section,
bot_root,
subs=subs)
else:
YLogger.warning(
self,
"No bot name defined for client [%s], defaulting to 'bot'.",
self.section_name)
self._bot_configs[0]._section_name = "bot"
self._bot_configs[0].load_configuration(configuration_file,
bot_root,
subs=subs)
def get_int_option(self, section, option_name, missing_value=0):
if option_name in section:
return section[option_name]
else:
YLogger.warning(self, "Missing value for [%s] in config, return default value %d", option_name, missing_value)
return missing_value
