def assert_upload_from_file(self, store):
store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "nodes" + os.sep + "pattern_nodes.conf")
collection = PatternNodeFactory()
store.load(collection)
self.assertEqual(12, len(collection.nodes))
self.assertTrue(collection.exists("zeroormore"))
def test_load_variables(self):
config = FileStorageConfiguration()
config._pattern_nodes_storage = FileStoreConfiguration(file=os.path.dirname(__file__) + os.sep + "data" + os.sep + "nodes" + os.sep + "pattern_nodes.conf", format="text", encoding="utf-8", delete_on_start=False)
engine = FileStorageEngine(config)
engine.initialise()
store = FilePatternNodeStore(engine)
collection = PatternNodeFactory()
store.load(collection)
self.assertEqual(12, len(collection.nodes))
self.assertTrue(collection.exists("zeroormore"))
class PatternGraph(object):
def __init__(self, aiml_parser, root_node=None):
self._aiml_parser = aiml_parser
self.load_pattern_node_factory()
if root_node is None:
YLogger.debug(self, "Defaulting root to PatternRootNode")
self._root_node = self._pattern_factory.get_root_node()
else:
if root_node.is_root() is False:
raise ParserException(
"Root node needs to be of base type PatternRootNode")
self._root_node = root_node
def load_pattern_node_factory(self):
pattern_nodes = self._aiml_parser.brain.configuration.nodes.pattern_nodes
self._pattern_factory = PatternNodeFactory()
self._pattern_factory.load_nodes_config_from_file(pattern_nodes)
@property
def root(self):
return self._root_node
@property
def aiml_parser(self):
return self._aiml_parser
@property
def pattern_factory(self):
return self._pattern_factory
def empty(self):
YLogger.debug(self, "Defaulting root to PatternRootNode")
self._empty_children(self._root_node)
self._root_node = self._pattern_factory.get_root_node()
def _empty_children(self, node):
for child in node.children:
self._empty_children(child)
child.children.clear()
def node_from_text(self, word, userid="*"):
if word.startswith("$"):
node_class = self._pattern_factory.new_node_class('priority')
return node_class(word[1:], userid)
elif PatternZeroOrMoreWildCardNode.is_wild_card(word):
node_class = self._pattern_factory.new_node_class('zeroormore')
return node_class(word, userid)
elif PatternOneOrMoreWildCardNode.is_wild_card(word):
node_class = self._pattern_factory.new_node_class('oneormore')
return node_class(word, userid)
node_class = self._pattern_factory.new_node_class('word')
return node_class(word, userid)
def node_from_element(self, element, userid="*"):
node_name = TextUtils.tag_from_text(element.tag)
if self._pattern_factory.exists(node_name) is False:
raise ParserException("Unknown node name [%s]" % node_name)
text = None
if element.text is not None:
text = TextUtils.strip_whitespace(element.text)
node_class_instance = self._pattern_factory.new_node_class(node_name)
node_instance = node_class_instance(element.attrib, text, userid)
return node_instance
def _parse_text(self, pattern_text, current_node, userid="*"):
stripped = pattern_text.strip()
words = self._aiml_parser.brain.tokenizer.texts_to_words(stripped)
for word in words:
if word != '': # Blank nodes add no value, ignore them
word = TextUtils.strip_whitespace(word)
new_node = self.node_from_text(word, userid=userid)
current_node = current_node.add_child(new_node)
return current_node
def get_text_from_element(self, element):
text = element.text
if text is not None:
text = TextUtils.strip_whitespace(text)
if text == "":
return None
return text
return None
def get_tail_from_element(self, element):
text = element.tail
if text is not None:
text = TextUtils.strip_whitespace(text)
if text == "":
return None
return text
return None
def add_pattern_to_node(self, pattern_element, userid="*"):
try:
head_text = self.get_text_from_element(pattern_element)
if head_text is not None:
current_node = self._parse_text(head_text,
self._root_node,
userid=userid)
else:
current_node = self._root_node
for sub_element in pattern_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
return current_node
except ParserException as parser_excep:
parser_excep.xml_element = pattern_element
raise parser_excep
def add_topic_to_node(self, topic_element, base_node, userid="*"):
try:
current_node = self._pattern_factory.new_node_class('topic')(
userid)
current_node = base_node.add_topic(current_node)
head_text = self.get_text_from_element(topic_element)
if head_text is not None:
current_node = self._parse_text(head_text, current_node)
added_child = False
for sub_element in topic_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
added_child = True
if head_text is None:
if added_child is False:
raise ParserException("Topic node text is empty",
xml_element=topic_element)
return current_node
except ParserException as parser_excep:
parser_excep.xml_element = topic_element
raise parser_excep
def add_that_to_node(self, that_element, base_node, userid="*"):
try:
current_node = self._pattern_factory.new_node_class('that')(userid)
current_node = base_node.add_that(current_node)
head_text = self.get_text_from_element(that_element)
if head_text is not None:
current_node = self._parse_text(
TextUtils.strip_all_punctuation(head_text), current_node)
added_child = False
for sub_element in that_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
added_child = True
if head_text is None:
if added_child is False:
raise ParserException("That node text is empty",
xml_element=that_element)
return current_node
except ParserException as parser_excep:
parser_excep.xml_element = that_element
raise parser_excep
def add_template_to_node(self,
template_graph_root,
current_node,
userid="*"):
template_node = self._pattern_factory.new_node_class('template')(
template_graph_root, userid)
current_node = current_node.add_child(template_node,
replace_existing=True)
return current_node
def add_pattern_to_graph(self,
pattern_element,
topic_element,
that_element,
template_graph_root,
learn=False,
userid="*"):
pattern_node = self.add_pattern_to_node(pattern_element, userid=userid)
topic_node = self.add_topic_to_node(topic_element,
pattern_node,
userid=userid)
that_node = self.add_that_to_node(that_element,
topic_node,
userid=userid)
if that_node.has_template() is True:
if learn is False:
if pattern_element.text is not None:
raise DuplicateGrammarException(
"Dupicate grammar tree found [%s]" %
(pattern_element.text.strip()))
else:
raise DuplicateGrammarException(
"Dupicate grammar tree found for bot/set")
else:
if pattern_element.text is not None:
YLogger.warning(
self,
"Dupicate grammar tree found [%s] in learn, replacing existing",
pattern_element.text.strip())
else:
YLogger.warning(
self,
"Dupicate grammar tree found for bot/set in learn, replacing existing"
)
self.add_template_to_node(template_graph_root, that_node)
else:
self.add_template_to_node(template_graph_root, that_node)
return that_node
def count_words_in_patterns(self):
counter = [0]
self._count_words_in_children(self._root_node, counter)
return counter[0]
def _count_words_in_children(self, node, counter):
for child in node.children:
counter[0] += 1
self._count_words_in_children(child, counter)
def dump(self, output_func=YLogger.debug, eol="", verbose=True):
self.root.dump("", output_func, eol, verbose)
output_func(self, "")
def save_braintree(self, client_context, filename, content):
if content == 'txt':
with open(filename, "w+", encoding="utf-8") as dump_file:
self.dump(output_func=dump_file.write, eol="\n")
elif content == 'xml':
braintree = '<?xml version="1.0" encoding="UTF-8"?>\n'
braintree += '<aiml>\n'
braintree += self.root.to_xml(client_context)
braintree += '</aiml>\n'
with open(filename, "w+", encoding="utf-8") as dump_file:
dump_file.write(braintree)
else:
YLogger.error(client_context,
"Unknown braintree content type [%s]", content)
class PatternGraph(object):
def __init__(self, root_node=None, pattern_factory=None):
if root_node is None:
logging.debug("Defaulting root to PatternRootNode")
self._root_node = PatternRootNode()
else:
if root_node.is_root() is False:
raise ParserException(
"Root node needs to be of base type PatternRootNode")
self._root_node = root_node
if pattern_factory is None:
logging.debug("Defaulting node factory to PatternNodeFactory")
self._pattern_factory = PatternNodeFactory()
self._pattern_factory.load_nodes_config_from_file(
"dummy_config.conf")
else:
if isinstance(pattern_factory, PatternNodeFactory) is False:
raise ParserException(
"Pattern factory needs to be base class of PatternNodeFactory"
)
self._pattern_factory = pattern_factory
@property
def root(self):
return self._root_node
def node_from_text(self, word):
if word.startswith("$"):
node_class = self._pattern_factory.new_node_class('priority')
return node_class(word[1:])
elif PatternZeroOrMoreWildCardNode.is_wild_card(word):
node_class = self._pattern_factory.new_node_class('zeroormore')
return node_class(word)
elif PatternOneOrMoreWildCardNode.is_wild_card(word):
node_class = self._pattern_factory.new_node_class('oneormore')
return node_class(word)
else:
node_class = self._pattern_factory.new_node_class('word')
return node_class(word)
def node_from_element(self, element):
node_name = element.tag
if self._pattern_factory.exists(node_name) is False:
raise ParserException("Unknown node name [%s]" % node_name)
node_instance = self._pattern_factory.new_node_class(node_name)
if 'name' in element.attrib:
return node_instance(element.attrib['name'])
else:
return node_instance(TextUtils.strip_whitespace(element.text))
def _parse_text(self, pattern_text, current_node):
stripped = pattern_text.strip()
words = stripped.split(" ")
for word in words:
if word != '': # Blank nodes add no value, ignore them
word = TextUtils.strip_whitespace(word)
new_node = self.node_from_text(word)
current_node = current_node.add_child(new_node)
return current_node
def get_text_from_element(self, element):
text = element.text
if text is not None:
text = TextUtils.strip_whitespace(text)
if text == "":
return None
return text
return None
def get_tail_from_element(self, element):
text = element.tail
if text is not None:
text = TextUtils.strip_whitespace(text)
if text == "":
return None
return text
return None
def add_pattern_to_node(self, pattern_element):
try:
head_text = self.get_text_from_element(pattern_element)
if head_text is not None:
current_node = self._parse_text(head_text, self._root_node)
else:
current_node = self._root_node
for sub_element in pattern_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
return current_node
except ParserException as parser_excep:
parser_excep._xml_element = pattern_element
raise parser_excep
def add_topic_to_node(self, topic_element, base_node):
try:
current_node = self._pattern_factory.new_node_class('topic')()
current_node = base_node.add_topic(current_node)
head_text = self.get_text_from_element(topic_element)
if head_text is not None:
current_node = self._parse_text(head_text, current_node)
added_child = False
for sub_element in topic_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
added_child = True
if head_text is None:
if added_child is False:
raise ParserException("Topic node text is empty",
xml_element=topic_element)
return current_node
except ParserException as parser_excep:
parser_excep._xml_element = topic_element
raise parser_excep
def add_that_to_node(self, that_element, base_node):
try:
current_node = self._pattern_factory.new_node_class('that')()
current_node = base_node.add_that(current_node)
head_text = self.get_text_from_element(that_element)
if head_text is not None:
current_node = self._parse_text(
TextUtils.strip_whitespace(head_text), current_node)
added_child = False
for sub_element in that_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
added_child = True
if head_text is None:
if added_child is False:
raise ParserException("That node text is empty",
xml_element=that_element)
return current_node
except ParserException as parser_excep:
parser_excep._xml_element = that_element
raise parser_excep
def add_template_to_node(self, template_graph_root, current_node):
template_node = self._pattern_factory.new_node_class('template')(
template_graph_root)
current_node = current_node.add_child(template_node)
return current_node
def add_pattern_to_graph(self, pattern_element, topic_element,
that_element, template_graph_root):
current_node = self.add_pattern_to_node(pattern_element)
current_node = self.add_topic_to_node(topic_element, current_node)
current_node = self.add_that_to_node(that_element, current_node)
if current_node.has_template():
raise DuplicateGrammarException(
"Duplicate grammar tree found [%s] in [%s] with template [%s]"
% (pattern_element.text, tostring(pattern_element),
current_node.template.template.to_xml(None, None)))
else:
self.add_template_to_node(template_graph_root, current_node)
def count_words_in_patterns(self):
counter = [0]
self._count_words_in_children(self._root_node, counter)
return counter[0]
def _count_words_in_children(self, node, counter):
for child in node.children:
counter[0] += 1
self._count_words_in_children(child, counter)
def dump(self, output_func=logging.debug, eol="", verbose=True):
self.root.dump("", output_func, eol, verbose)
output_func("")
def dump_to_file(self, filename):
with open(filename, "w+") as dump_file:
self.dump(output_func=dump_file.write, eol="\n")
class PatternGraph(object):
def __init__(self, aiml_parser, root_node=None):
self._aiml_parser = aiml_parser
self.load_pattern_node_factory()
if root_node is None:
YLogger.debug(self, "Defaulting root to PatternRootNode")
self._root_node = self._pattern_factory.get_root_node()
else:
if root_node.is_root() is False:
raise ParserException("Root node needs to be of base type PatternRootNode")
self._root_node = root_node
def load_pattern_node_factory(self):
pattern_nodes = self._aiml_parser.brain.configuration.nodes.pattern_nodes
self._pattern_factory = PatternNodeFactory()
self._pattern_factory.load_nodes_config_from_file(pattern_nodes)
@property
def root(self):
return self._root_node
@property
def aiml_parser(self):
return self._aiml_parser
@property
def pattern_factory(self):
return self._pattern_factory
def empty(self):
YLogger.debug(self, "Defaulting root to PatternRootNode")
self._empty_children(self._root_node)
self._root_node = self._pattern_factory.get_root_node()
def _empty_children(self, node):
for child in node.children:
self._empty_children(child)
child.children.clear()
def node_from_text(self, word, userid="*"):
if word.startswith("$"):
node_class = self._pattern_factory.new_node_class('priority')
return node_class(word[1:], userid)
elif PatternZeroOrMoreWildCardNode.is_wild_card(word):
node_class = self._pattern_factory.new_node_class('zeroormore')
return node_class(word, userid)
elif PatternOneOrMoreWildCardNode.is_wild_card(word):
node_class = self._pattern_factory.new_node_class('oneormore')
return node_class(word, userid)
node_class = self._pattern_factory.new_node_class('word')
return node_class(word, userid)
def node_from_element(self, element, userid="*"):
node_name = TextUtils.tag_from_text(element.tag)
if self._pattern_factory.exists(node_name) is False:
raise ParserException("Unknown node name [%s]"%node_name)
text = None
if element.text is not None:
text = TextUtils.strip_whitespace(element.text)
node_class_instance = self._pattern_factory.new_node_class(node_name)
node_instance = node_class_instance(element.attrib, text, userid)
return node_instance
def _parse_text(self, pattern_text, current_node, userid="*"):
stripped = pattern_text.strip()
words = self._aiml_parser.brain.tokenizer.texts_to_words(stripped)
for word in words:
if word != '': # Blank nodes add no value, ignore them
word = TextUtils.strip_whitespace(word)
new_node = self.node_from_text(word, userid=userid)
current_node = current_node.add_child(new_node)
return current_node
def get_text_from_element(self, element):
text = element.text
if text is not None:
text = TextUtils.strip_whitespace(text)
if text == "":
return None
return text
return None
def get_tail_from_element(self, element):
text = element.tail
if text is not None:
text = TextUtils.strip_whitespace(text)
if text == "":
return None
return text
return None
def add_pattern_to_node(self, pattern_element, userid="*"):
try:
head_text = self.get_text_from_element(pattern_element)
if head_text is not None:
current_node = self._parse_text(head_text, self._root_node, userid=userid)
else:
current_node = self._root_node
for sub_element in pattern_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
return current_node
except ParserException as parser_excep:
parser_excep.xml_element = pattern_element
raise parser_excep
def add_topic_to_node(self, topic_element, base_node, userid="*"):
try:
current_node = self._pattern_factory.new_node_class('topic')(userid)
current_node = base_node.add_topic(current_node)
head_text = self.get_text_from_element(topic_element)
if head_text is not None:
current_node = self._parse_text(head_text, current_node)
added_child = False
for sub_element in topic_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
added_child = True
if head_text is None:
if added_child is False:
raise ParserException("Topic node text is empty", xml_element=topic_element)
return current_node
except ParserException as parser_excep:
parser_excep.xml_element = topic_element
raise parser_excep
def add_that_to_node(self, that_element, base_node, userid="*"):
try:
current_node = self._pattern_factory.new_node_class('that')(userid)
current_node = base_node.add_that(current_node)
head_text = self.get_text_from_element(that_element)
if head_text is not None:
current_node = self._parse_text(TextUtils.strip_whitespace(head_text), current_node)
added_child = False
for sub_element in that_element:
new_node = self.node_from_element(sub_element)
current_node = current_node.add_child(new_node)
tail_text = self.get_tail_from_element(sub_element)
if tail_text is not None:
current_node = self._parse_text(tail_text, current_node)
added_child = True
if head_text is None:
if added_child is False:
raise ParserException("That node text is empty", xml_element=that_element)
return current_node
except ParserException as parser_excep:
parser_excep.xml_element = that_element
raise parser_excep
def add_template_to_node(self, template_graph_root, current_node, userid="*"):
template_node = self._pattern_factory.new_node_class('template')(template_graph_root, userid)
current_node = current_node.add_child(template_node, replace_existing=True)
return current_node
def add_pattern_to_graph(self, pattern_element, topic_element, that_element, template_graph_root, learn=False, userid="*"):
pattern_node = self.add_pattern_to_node(pattern_element, userid=userid)
topic_node = self.add_topic_to_node(topic_element, pattern_node, userid=userid)
that_node = self.add_that_to_node(that_element, topic_node, userid=userid)
if that_node.has_template() is True:
if learn is False:
if pattern_element.text is not None:
raise DuplicateGrammarException("Dupicate grammar tree found [%s]"%(pattern_element.text.strip()))
else:
raise DuplicateGrammarException("Dupicate grammar tree found for bot/set")
else:
if pattern_element.text is not None:
YLogger.warning(self, "Dupicate grammar tree found [%s] in learn, replacing existing",
pattern_element.text.strip())
else:
YLogger.warning(self, "Dupicate grammar tree found for bot/set in learn, replacing existing")
self.add_template_to_node(template_graph_root, that_node)
else:
self.add_template_to_node(template_graph_root, that_node)
return that_node
def count_words_in_patterns(self):
counter = [0]
self._count_words_in_children(self._root_node, counter)
return counter[0]
def _count_words_in_children(self, node, counter):
for child in node.children:
counter[0] += 1
self._count_words_in_children(child, counter)
def dump(self, output_func=YLogger.debug, eol="", verbose=True):
self.root.dump("", output_func, eol, verbose)
output_func(self, "")
def save_braintree(self, client_context, filename, content):
if content == 'txt':
with open(filename, "w+", encoding="utf-8") as dump_file:
self.dump(output_func=dump_file.write, eol="\n")
elif content == 'xml':
braintree = '<?xml version="1.0" encoding="UTF-8"?>\n'
braintree += '<aiml>\n'
braintree += self.root.to_xml(client_context)
braintree += '</aiml>\n'
with open(filename, "w+", encoding="utf-8") as dump_file:
dump_file.write(braintree)
else:
YLogger.error(client_context, "Unknown braintree content type [%s]", content)