def build_tree_for_template(self, template: Template) -> TemplateTree:
""" Converts the learnt state into a template tree for the given template """
# Check cache
if template in self._built_trees:
return self._built_trees[template]
# If no children, just childless template tree
if template not in self._template_children:
resulting_tree = TemplateTree(template)
else:
# If children, first convert children
children: Set[TemplateTree] = {
self.build_tree_for_template(c)
for c in self._template_children[template]
}
descendants = {
desc
for descs in [c.get_strict_descendants() for c in children]
for desc in descs
}
children_without_already_included_descendants = children.difference(
descendants)
resulting_tree = TemplateTree(
template, children_without_already_included_descendants)
# Cache built tree
self._built_trees[template] = resulting_tree
return resulting_tree
def test_3_line_learner(self):
learner = TemplateLatticeLearner(minimal_variables=True)
dataset = ["hello world", "hi world", "hello universe"]
template_tree = learner.learn(dataset)
expected = TemplateTree(
Template.from_string("[SLOT]"),
[
TemplateTree(
Template.from_string("[SLOT] world"),
[
TemplateTree(Template.from_string(s))
for s in ["hello world", "hi world"]
],
),
TemplateTree(
Template.from_string("hello [SLOT]"),
[
TemplateTree(Template.from_string(s))
for s in ["hello world", "hello universe"]
],
),
],
)
print(template_tree_visualiser.render_tree_string(template_tree))
self.assertEqual(expected, template_tree)
def create_tree_node_representation(tree: TemplateTree):
return Node(
tree.get_template().to_flat_string(),
children=[
create_tree_node_representation(child)
for child in tree.get_children()
],
)
def _merge_template_trees(child1: TemplateTree, child2: TemplateTree,
minimal_variables) -> TemplateTree:
new_template_tree = TemplateTree(
template=_merge_templates(child1.get_template(), child2.get_template(),
minimal_variables),
children=[child1, child2],
)
return new_template_tree
def test_2_line_learner(self):
learner = TemplateLatticeLearner(minimal_variables=True)
dataset = ["hello world", "hi world"]
template_tree = learner.learn(dataset)
expected_top_template = Template.from_string("[SLOT] world")
expected = TemplateTree(
expected_top_template,
[TemplateTree(Template.from_string(s)) for s in dataset],
)
print(template_tree_visualiser.render_tree_string(template_tree))
self.assertEqual(expected_top_template, template_tree.get_template())
self.assertEqual(expected, template_tree)
def test_equals_new_leaves(self):
""" Test if Template Trees are equal if different leaves are used by constructing new trees from scratch"""
s1 = TemplateTree(Template.from_string("a b c d"))
s2 = TemplateTree(Template.from_string("a b e d"))
s3 = TemplateTree(Template.from_string("a b f d"))
s4 = TemplateTree(Template.from_string("g b h d"))
u1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"), [s1, s2]
)
u2 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"), [s3, u1]
)
u2_selfs3 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"), [self.s3, u1]
)
u3 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"), [s4, u2]
)
self.assertEqual(self.s1, s1)
self.assertEqual(self.s2, s2)
self.assertEqual(self.s3, s3)
self.assertEqual(self.s4, s4)
self.assertEqual(self.u1, u1)
self.assertEqual(self.u2, u2_selfs3)
self.assertEqual(self.u2, u2)
self.assertEqual(self.u3, u3)
def _to_template_trees(lines: Collection[str]) -> List[TemplateTree]:
template_trees = [
TemplateTree(
template=template,
children=[],
) for template in _to_templates(lines)
]
return template_trees
def test_disallow_empty_string_hard(self):
dataset = [
"I saw her on the quiet hill",
"I saw her on the tall hill",
"I saw her on the hill",
"He likes cute cats",
"He likes nice cats",
"He likes cats",
]
learner = TemplateLatticeLearner(minimal_variables=True,
allow_empty_string=False)
template_tree = learner.learn(dataset)
expected = TemplateTree(
Template.from_string("[SLOT]"),
[
TemplateTree(
Template.from_string("He likes [SLOT]"),
[
TemplateTree(
Template.from_string("He likes [SLOT] cats"),
[
TemplateTree(Template.from_string(s)) for s in
["He likes cute cats", "He likes nice cats"]
],
),
TemplateTree(Template.from_string("He likes cats")),
],
),
TemplateTree(
Template.from_string("I saw her on the [SLOT]"),
[
TemplateTree(
Template.from_string(
"I saw her on the [SLOT] hill"),
[
TemplateTree(Template.from_string(s))
for s in [
"I saw her on the tall hill",
"I saw her on the quiet hill",
]
],
),
TemplateTree(
Template.from_string("I saw her on the hill")),
],
),
],
)
print(template_tree_visualiser.render_tree_string(template_tree))
self.assertEqual(expected, template_tree)
def test_equals(self):
""" Tests the TemplateTree __eq__ """
e1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s1, self.s2],
)
self.assertEqual(e1, self.u1)
self.assertEqual(e1, e1)
self.assertEqual(self.t3, self.t3)
self.assertNotEqual(e1, self.u2)
self.assertNotEqual(e1, self.t1)
def test_disallow_empty_string_simple_2(self):
dataset = [
"He likes cute cats",
"He likes nice cats",
"He likes cats",
"This is another sentence",
]
learner = TemplateLatticeLearner(minimal_variables=True,
allow_empty_string=False)
template_tree = learner.learn(dataset)
expected = TemplateTree(
Template.from_string("[SLOT]"),
[
TemplateTree(
Template.from_string("He likes [SLOT]"),
[
TemplateTree(
Template.from_string("He likes [SLOT] cats"),
[
TemplateTree(Template.from_string(s)) for s in
["He likes cute cats", "He likes nice cats"]
],
),
TemplateTree(Template.from_string("He likes cats")),
],
),
TemplateTree(Template.from_string("This is another sentence")),
],
)
print(template_tree_visualiser.render_tree_string(template_tree))
self.assertEqual(expected, template_tree)
def test_disallow_empty_string_simple(self):
""" Checks whether disallowing empty string in learning works"""
learner = TemplateLatticeLearner(minimal_variables=True,
allow_empty_string=False)
dataset = ["I am a human", "I am a nice human", "I am a bad human"]
template_tree = learner.learn(dataset)
expected = TemplateTree(
Template.from_string("I am a [SLOT]"),
[
TemplateTree(
Template.from_string("I am a [SLOT] human"),
[
TemplateTree(Template.from_string(s))
for s in ["I am a nice human", "I am a bad human"]
],
),
TemplateTree(Template.from_string("I am a human"), ),
],
)
print(template_tree_visualiser.render_tree_string(template_tree))
self.assertEqual(expected, template_tree)
def test_4_line_learner_longer_second(self):
learner = TemplateLatticeLearner(minimal_variables=True,
words_per_leaf_slot=2)
dataset = [
"hello world", "hi world", "hello solar system", "hi solar system"
]
template_tree = learner.learn(dataset)
expected = TemplateTree(
Template.from_string("[SLOT]"),
[
TemplateTree(
Template.from_string("[SLOT] world"),
[
TemplateTree(Template.from_string(s))
for s in ["hello world", "hi world"]
],
),
TemplateTree(
Template.from_string("[SLOT] solar system"),
[
TemplateTree(Template.from_string(s))
for s in ["hello solar system", "hi solar system"]
],
),
TemplateTree(
Template.from_string("hello [SLOT]"),
[
TemplateTree(Template.from_string(s))
for s in ["hello world", "hello solar system"]
],
),
TemplateTree(
Template.from_string("hi [SLOT]"),
[
TemplateTree(Template.from_string(s))
for s in ["hi world", "hi solar system"]
],
),
],
)
print(template_tree_visualiser.render_tree_string(template_tree))
self.assertEqual(expected, template_tree)
def test_get_slot_content_mappings(self):
self.assertEqual(set(), self.s1.get_slot_content_mappings())
slot1 = NamedTemplateSlot("x")
slot2 = NamedTemplateSlot("y")
a = TemplateString("a")
b = TemplateString("b")
c = TemplateString("c")
# Simple tree
simple_tree = TemplateTree(
Template([a, slot1]), [TemplateTree(Template([a, b]), [])]
)
simple_slot_contents = simple_tree.get_slot_content_mappings()
self.assertEqual(1, len(simple_slot_contents))
simple_slot_content = list(simple_slot_contents)[0]
self.assertTrue(slot1 in simple_slot_content)
self.assertTrue(slot1 in simple_slot_content.keys())
self.assertEqual(Template([b]), simple_slot_content[slot1])
self.assertEqual({SlotAssignment({slot1: Template([b])})}, simple_slot_contents)
# Two slot tree
two_slot_tree = TemplateTree(
Template([slot1, b, slot2]), [TemplateTree(Template([a, b, c]), [])]
)
two_slot_tree_contents = two_slot_tree.get_slot_content_mappings()
self.assertEqual(
{SlotAssignment({slot1: Template([a]), slot2: Template([c])})},
two_slot_tree_contents,
)
# Test tree
u1_slot = self.u1.get_template().get_slots()[0]
self.assertEqual(
{
SlotAssignment({u1_slot: Template([TemplateString("c")])}),
SlotAssignment({u1_slot: Template([TemplateString("e")])}),
},
self.u1.get_slot_content_mappings(),
)
def test_reduce_depth(self):
# Depth 1
u4_reduced_1 = self.u4.reduce_depth(1)
self.assertEqual(1, u4_reduced_1.get_depth())
self.assertEqual(
TemplateTree(
self.u4.get_template(), [self.s1, self.s2, self.s3, self.s4, self.s5]
),
u4_reduced_1,
)
# Depth 2
u4_reduced_2 = self.u4.reduce_depth(2)
self.assertEqual(2, u4_reduced_2.get_depth())
self.assertEqual(
TemplateTree(self.u4.get_template(), [self.u1, self.s3, self.s4, self.s5]),
u4_reduced_2,
)
# Depth 3
u4_reduced_3 = self.u4.reduce_depth(3)
self.assertEqual(3, u4_reduced_3.get_depth())
self.assertEqual(
TemplateTree(self.u4.get_template(), [self.u2, self.s4, self.s5]),
u4_reduced_3,
)
# Depth 4
u4_reduced_4 = self.u4.reduce_depth(4)
self.assertEqual(4, u4_reduced_4.get_depth())
self.assertEqual(
TemplateTree(self.u4.get_template(), [self.u3, self.s5]), u4_reduced_4,
)
# T Tree
t3_reduced_1 = self.t3.reduce_depth(1)
self.assertEqual(1, t3_reduced_1.get_depth())
self.assertEqual(
TemplateTree(
self.t3.get_template(), [self.s1, self.s2, self.s3, self.s4, self.s5]
),
t3_reduced_1,
)
# T Tree Depth 2
t3_reduced_2 = self.t3.reduce_depth(2)
self.assertEqual(2, t3_reduced_2.get_depth())
self.assertEqual(
TemplateTree(self.t3.get_template(), [self.t1, self.s4, self.s5]),
t3_reduced_2,
)
def _name_and_simplify_tree(
learned_tree: TemplateTree, relative_similarity_threshold: float
) -> Tuple[SlotValues, TemplateTree]:
"""
Gives a name to all unnamed slots, and simplifies under the independence between slots assumption
"""
# Give all slots a unique name
slot_name_generator = alphabetic_slot_name_iterator()
named_lattice = learned_tree.name_slots_automatically(slot_name_generator)
# Find what every slot maps to
possible_slot_values = named_lattice.get_slot_values()
# Merge similar slots together, to reduce the number of unique slots
merged_slot_values = possible_slot_values.merge_slots(
relative_similarity_threshold=relative_similarity_threshold, )
# Use merged slots to reduce variables in the template tree
replacements = merged_slot_values.get_replacements()
merged_slots_tree = named_lattice.name_template_slots(replacements)
return merged_slot_values, merged_slots_tree
class TemplateTreeLearner(unittest.TestCase):
def setUp(self) -> None:
random.seed(123)
self.s1 = TemplateTree(Template.from_string("a b c d"))
self.s2 = TemplateTree(Template.from_string("a b e d"))
self.s3 = TemplateTree(Template.from_string("a b f d"))
self.s4 = TemplateTree(Template.from_string("g b h d"))
self.s5 = TemplateTree(Template.from_string("h i j d"))
self.all_s = [self.s1, self.s2, self.s3, self.s4, self.s5]
# Uncompressed tree (one of multiple possible)
self.u1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s1, self.s2],
)
self.u2 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s3, self.u1],
)
self.u3 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"),
[self.s4, self.u2],
)
self.u4 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [self.s5, self.u3]
)
# Collapsed tree
self.t1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s1, self.s2, self.s3],
)
self.t2 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"),
[self.s4, self.t1],
)
self.t3 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [self.s5, self.t2]
)
def test_reduce_depth(self):
# Depth 1
u4_reduced_1 = self.u4.reduce_depth(1)
self.assertEqual(1, u4_reduced_1.get_depth())
self.assertEqual(
TemplateTree(
self.u4.get_template(), [self.s1, self.s2, self.s3, self.s4, self.s5]
),
u4_reduced_1,
)
# Depth 2
u4_reduced_2 = self.u4.reduce_depth(2)
self.assertEqual(2, u4_reduced_2.get_depth())
self.assertEqual(
TemplateTree(self.u4.get_template(), [self.u1, self.s3, self.s4, self.s5]),
u4_reduced_2,
)
# Depth 3
u4_reduced_3 = self.u4.reduce_depth(3)
self.assertEqual(3, u4_reduced_3.get_depth())
self.assertEqual(
TemplateTree(self.u4.get_template(), [self.u2, self.s4, self.s5]),
u4_reduced_3,
)
# Depth 4
u4_reduced_4 = self.u4.reduce_depth(4)
self.assertEqual(4, u4_reduced_4.get_depth())
self.assertEqual(
TemplateTree(self.u4.get_template(), [self.u3, self.s5]), u4_reduced_4,
)
# T Tree
t3_reduced_1 = self.t3.reduce_depth(1)
self.assertEqual(1, t3_reduced_1.get_depth())
self.assertEqual(
TemplateTree(
self.t3.get_template(), [self.s1, self.s2, self.s3, self.s4, self.s5]
),
t3_reduced_1,
)
# T Tree Depth 2
t3_reduced_2 = self.t3.reduce_depth(2)
self.assertEqual(2, t3_reduced_2.get_depth())
self.assertEqual(
TemplateTree(self.t3.get_template(), [self.t1, self.s4, self.s5]),
t3_reduced_2,
)
def test_collapse(self):
""" Test if collapsing a tree gives the desired result """
# For top level of a node
self.assertEqual(self.t1, self.u2.collapse())
# Bit lower
self.assertEqual(self.t2, self.u3.collapse())
# For full tree
collapsed_u4 = self.u4.collapse()
t3 = self.t3
self.assertEqual(self.t3, collapsed_u4)
def test_collapse_same_children(self):
""" Tests if collapsing a tree with children with similar templates will merge correctly """
ss1 = TemplateTree(Template.from_string("a b c c d"))
ss2 = TemplateTree(Template.from_string("c b e e d"))
ss3 = TemplateTree(Template.from_string("h h h b f d"))
ss4 = TemplateTree(Template.from_string("i i i b g d"))
ss5 = TemplateTree(Template.from_string("j k l l d"))
us1 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"), [ss1, ss2]
)
us2 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"), [ss3, ss4]
)
us3 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [us1, us2, ss5]
)
ts1 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"),
[ss1, ss2, ss3, ss4],
)
ts2 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [ts1, ss5]
)
collapsed_u = us3.collapse()
self.assertEqual(ts2, collapsed_u)
def test_equals(self):
""" Tests the TemplateTree __eq__ """
e1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s1, self.s2],
)
self.assertEqual(e1, self.u1)
self.assertEqual(e1, e1)
self.assertEqual(self.t3, self.t3)
self.assertNotEqual(e1, self.u2)
self.assertNotEqual(e1, self.t1)
def test_equals_new_leaves(self):
""" Test if Template Trees are equal if different leaves are used by constructing new trees from scratch"""
s1 = TemplateTree(Template.from_string("a b c d"))
s2 = TemplateTree(Template.from_string("a b e d"))
s3 = TemplateTree(Template.from_string("a b f d"))
s4 = TemplateTree(Template.from_string("g b h d"))
u1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"), [s1, s2]
)
u2 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"), [s3, u1]
)
u2_selfs3 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"), [self.s3, u1]
)
u3 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"), [s4, u2]
)
self.assertEqual(self.s1, s1)
self.assertEqual(self.s2, s2)
self.assertEqual(self.s3, s3)
self.assertEqual(self.s4, s4)
self.assertEqual(self.u1, u1)
self.assertEqual(self.u2, u2_selfs3)
self.assertEqual(self.u2, u2)
self.assertEqual(self.u3, u3)
def test_get_descendent_leaves(self):
""" Test if all leaves are properly found"""
self.assertEqual({self.s1, self.s2}, set(self.u1.get_descendant_leaves()))
self.assertEqual(
{self.s1, self.s2, self.s3}, set(self.u2.get_descendant_leaves())
)
self.assertEqual(
{self.s1, self.s2, self.s3, self.s4}, set(self.u3.get_descendant_leaves())
)
self.assertEqual(set(self.all_s), set(self.u4.get_descendant_leaves()))
self.assertEqual(
{self.s1, self.s2, self.s3}, set(self.t1.get_descendant_leaves())
)
self.assertEqual(
{self.s1, self.s2, self.s3, self.s4}, set(self.t2.get_descendant_leaves())
)
self.assertEqual(set(self.all_s), set(self.t3.get_descendant_leaves()))
def test_get_slot_content(self):
self.assertEqual(
{(Template.from_string("c"),), (Template.from_string("e"),)},
self.u1.get_slot_contents_tuples(),
)
self.assertEqual(
{(Template.from_string("[SLOT]"),), (Template.from_string("f"),)},
self.u2.get_slot_contents_tuples(),
)
self.assertEqual(
{
(Template.from_string("c"),),
(Template.from_string("e"),),
(Template.from_string("f"),),
},
self.t1.get_slot_contents_tuples(),
)
def test_get_descendent_leaves_slot_content(self):
# Same tests as before without recursion
self.assertEqual(
{(Template.from_string("c"),), (Template.from_string("e"),)},
self.u1.get_descendent_leaves_slot_content_tuples(),
)
self.assertEqual(
{
(Template.from_string("c"),),
(Template.from_string("e"),),
(Template.from_string("f"),),
},
self.t1.get_descendent_leaves_slot_content_tuples(),
)
# New tests
self.assertEqual(
{
(Template.from_string("c"),),
(Template.from_string("e"),),
(Template.from_string("f"),),
},
self.u2.get_descendent_leaves_slot_content_tuples(),
)
def test_get_slot_content_mappings(self):
self.assertEqual(set(), self.s1.get_slot_content_mappings())
slot1 = NamedTemplateSlot("x")
slot2 = NamedTemplateSlot("y")
a = TemplateString("a")
b = TemplateString("b")
c = TemplateString("c")
# Simple tree
simple_tree = TemplateTree(
Template([a, slot1]), [TemplateTree(Template([a, b]), [])]
)
simple_slot_contents = simple_tree.get_slot_content_mappings()
self.assertEqual(1, len(simple_slot_contents))
simple_slot_content = list(simple_slot_contents)[0]
self.assertTrue(slot1 in simple_slot_content)
self.assertTrue(slot1 in simple_slot_content.keys())
self.assertEqual(Template([b]), simple_slot_content[slot1])
self.assertEqual({SlotAssignment({slot1: Template([b])})}, simple_slot_contents)
# Two slot tree
two_slot_tree = TemplateTree(
Template([slot1, b, slot2]), [TemplateTree(Template([a, b, c]), [])]
)
two_slot_tree_contents = two_slot_tree.get_slot_content_mappings()
self.assertEqual(
{SlotAssignment({slot1: Template([a]), slot2: Template([c])})},
two_slot_tree_contents,
)
# Test tree
u1_slot = self.u1.get_template().get_slots()[0]
self.assertEqual(
{
SlotAssignment({u1_slot: Template([TemplateString("c")])}),
SlotAssignment({u1_slot: Template([TemplateString("e")])}),
},
self.u1.get_slot_content_mappings(),
)
def test_get_all_descendent_slots_breadth_first(self):
self.assertEqual(1, len(self.u1.get_all_descendent_slots_breadth_first()))
self.assertEqual(2, len(self.u2.get_all_descendent_slots_breadth_first()))
self.assertEqual(4, len(self.u3.get_all_descendent_slots_breadth_first()))
self.assertEqual(5, len(self.u4.get_all_descendent_slots_breadth_first()))
self.assertEqual(4, len(self.t3.get_all_descendent_slots_breadth_first()))
def test_collapse_using_slot_values(self):
hello = TemplateString("hello")
hey = TemplateString("hey")
world = TemplateString("world")
universe = TemplateString("universe")
h1 = TemplateTree(Template([hello, world]))
h2 = TemplateTree(Template([hey, world]))
h3 = TemplateTree(Template([hello, universe]))
h4 = TemplateTree(Template([hey, universe]))
slot_a = NamedTemplateSlot("A")
slot_b = NamedTemplateSlot("B")
slot_c = NamedTemplateSlot("C")
expected = TemplateTree(Template([slot_a, slot_b]), [h1, h2, h3, h4])
expected_values = SlotValues(
{
slot_a: {Template([hello]), Template([hey])},
slot_b: {Template([world]), Template([universe])},
}
)
# Test first argument
hello_t = Template([hello, slot_b])
hello_tt = TemplateTree(hello_t, [h1, h3])
hey_t = Template([hey, slot_b])
hey_tt = TemplateTree(hey_t, [h2, h4])
greeting_t = Template([slot_a, slot_b])
greeting_tt = TemplateTree(greeting_t, [hello_tt, hey_tt])
self.assertTrue(greeting_t.encompasses(hey_t, expected_values))
self.assertTrue(greeting_t.encompasses(hello_t, expected_values))
self.assertFalse(hello_t.encompasses(greeting_t, expected_values))
self.assertEqual(
expected_values, greeting_tt.calculated_merged_independent_slot_values()
)
self.assertEqual(
expected, greeting_tt.collapse_using_slot_values(expected_values)
)
# Do same, but for second argument
world_t = Template([slot_a, world])
world_tt = TemplateTree(world_t, [h1, h2])
universe_t = Template([slot_a, universe])
universe_tt = TemplateTree(universe_t, [h3, h4])
place_t = Template([slot_a, slot_b])
place_tt = TemplateTree(place_t, [world_tt, universe_tt])
self.assertEqual(
expected_values, place_tt.calculated_merged_independent_slot_values()
)
self.assertEqual(expected, place_tt.collapse_using_slot_values(expected_values))
# Test mix
mix_tt = TemplateTree(place_t, [world_tt, hey_tt, h3])
self.assertEqual(
expected_values, mix_tt.calculated_merged_independent_slot_values()
)
self.assertEqual(expected, mix_tt.collapse_using_slot_values(expected_values))
# Now with some noise
noise = Template([TemplateString("noise")])
noise_tt = TemplateTree(noise)
noise_t = Template([slot_c])
full_noise_tt = TemplateTree(noise_t, [greeting_tt, noise_tt])
noise_values = SlotValues(
{
slot_a: {Template([hello]), Template([hey])},
slot_b: {Template([world]), Template([universe])},
slot_c: {Template([slot_a, slot_b]), noise},
}
)
collapsed_full_noise = full_noise_tt.collapse_using_slot_values(noise_values)
self.assertEqual(
noise_values, full_noise_tt.calculated_merged_independent_slot_values(),
)
self.assertEqual(
TemplateTree(Template([slot_c]), [expected, noise_tt]),
collapsed_full_noise,
)
def test_collapse_using_slot_values(self):
hello = TemplateString("hello")
hey = TemplateString("hey")
hi = TemplateString("hi")
h1 = TemplateTree(Template([hello, hello]))
h2 = TemplateTree(Template([hey, hello]))
h3 = TemplateTree(Template([hello, hi]))
h4 = TemplateTree(Template([hi, hello]))
h5 = TemplateTree(Template([hi, hi]))
hello_t = Template([hello])
hey_t = Template([hey])
hi_t = Template([hi])
slot_a = NamedTemplateSlot("A")
slot_b = NamedTemplateSlot("B")
slot_c = NamedTemplateSlot("C")
slot_d = NamedTemplateSlot("D")
slot_e = NamedTemplateSlot("E")
slot_f = NamedTemplateSlot("F")
t1 = TemplateTree(Template([hello, slot_a]), [h1, h3])
t2 = TemplateTree(Template([slot_b, hello]), [h1, h2, h4])
t3 = TemplateTree(Template([slot_c, hi]), [h3, h5])
t4 = TemplateTree(Template([hi, slot_d]), [h4, h5])
t5 = TemplateTree(Template([slot_e, slot_f]), [t1, t2, t3, t4])
slot_values = SlotValues(
{
slot_a: {Template([slot_e])},
slot_b: {Template([slot_e])},
slot_c: {Template([slot_e])},
slot_d: {Template([slot_e])},
slot_e: {hello_t, hi_t, hey_t},
slot_f: {Template([slot_e])},
}
)
self.assertEqual(
slot_values,
t5.get_slot_values().merge_slots(relative_similarity_threshold=0.01),
)
renamed_tree = t5.name_template_slots(
{
slot_a: slot_e,
slot_b: slot_e,
slot_c: slot_e,
slot_d: slot_e,
slot_f: slot_e,
}
)
collapsed_tree = renamed_tree.collapse_using_slot_values(slot_values)
self.assertEqual(Template([slot_e, slot_e]), collapsed_tree.get_template())
self.assertEqual(
{tt.get_template() for tt in [h1, h2, h3, h4, h5]},
{tt.get_template() for tt in collapsed_tree.get_children()},
)
def setUp(self) -> None:
random.seed(123)
self.s1 = TemplateTree(Template.from_string("a b c d"))
self.s2 = TemplateTree(Template.from_string("a b e d"))
self.s3 = TemplateTree(Template.from_string("a b f d"))
self.s4 = TemplateTree(Template.from_string("g b h d"))
self.s5 = TemplateTree(Template.from_string("h i j d"))
self.all_s = [self.s1, self.s2, self.s3, self.s4, self.s5]
# Uncompressed tree (one of multiple possible)
self.u1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s1, self.s2],
)
self.u2 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s3, self.u1],
)
self.u3 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"),
[self.s4, self.u2],
)
self.u4 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [self.s5, self.u3]
)
# Collapsed tree
self.t1 = TemplateTree(
Template.from_string("a b [SLOT] d", slot_token="[SLOT]"),
[self.s1, self.s2, self.s3],
)
self.t2 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"),
[self.s4, self.t1],
)
self.t3 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [self.s5, self.t2]
)
def test_collapse_using_slot_values(self):
hello = TemplateString("hello")
hey = TemplateString("hey")
hi = TemplateString("hi")
h1 = TemplateTree(Template([hello, hello]))
h2 = TemplateTree(Template([hey, hello]))
h3 = TemplateTree(Template([hello, hi]))
h4 = TemplateTree(Template([hi, hello]))
h5 = TemplateTree(Template([hi, hi]))
hello_t = Template([hello])
hey_t = Template([hey])
hi_t = Template([hi])
slot_a = NamedTemplateSlot("A")
slot_b = NamedTemplateSlot("B")
slot_c = NamedTemplateSlot("C")
slot_d = NamedTemplateSlot("D")
slot_e = NamedTemplateSlot("E")
slot_f = NamedTemplateSlot("F")
t1 = TemplateTree(Template([hello, slot_a]), [h1, h3])
t2 = TemplateTree(Template([slot_b, hello]), [h1, h2, h4])
t3 = TemplateTree(Template([slot_c, hi]), [h3, h5])
t4 = TemplateTree(Template([hi, slot_d]), [h4, h5])
t5 = TemplateTree(Template([slot_e, slot_f]), [t1, t2, t3, t4])
slot_values = SlotValues(
{
slot_a: {Template([slot_e])},
slot_b: {Template([slot_e])},
slot_c: {Template([slot_e])},
slot_d: {Template([slot_e])},
slot_e: {hello_t, hi_t, hey_t},
slot_f: {Template([slot_e])},
}
)
self.assertEqual(
slot_values,
t5.get_slot_values().merge_slots(relative_similarity_threshold=0.01),
)
renamed_tree = t5.name_template_slots(
{
slot_a: slot_e,
slot_b: slot_e,
slot_c: slot_e,
slot_d: slot_e,
slot_f: slot_e,
}
)
collapsed_tree = renamed_tree.collapse_using_slot_values(slot_values)
self.assertEqual(Template([slot_e, slot_e]), collapsed_tree.get_template())
self.assertEqual(
{tt.get_template() for tt in [h1, h2, h3, h4, h5]},
{tt.get_template() for tt in collapsed_tree.get_children()},
)
def test_collapse_using_slot_values(self):
hello = TemplateString("hello")
hey = TemplateString("hey")
world = TemplateString("world")
universe = TemplateString("universe")
h1 = TemplateTree(Template([hello, world]))
h2 = TemplateTree(Template([hey, world]))
h3 = TemplateTree(Template([hello, universe]))
h4 = TemplateTree(Template([hey, universe]))
slot_a = NamedTemplateSlot("A")
slot_b = NamedTemplateSlot("B")
slot_c = NamedTemplateSlot("C")
expected = TemplateTree(Template([slot_a, slot_b]), [h1, h2, h3, h4])
expected_values = SlotValues(
{
slot_a: {Template([hello]), Template([hey])},
slot_b: {Template([world]), Template([universe])},
}
)
# Test first argument
hello_t = Template([hello, slot_b])
hello_tt = TemplateTree(hello_t, [h1, h3])
hey_t = Template([hey, slot_b])
hey_tt = TemplateTree(hey_t, [h2, h4])
greeting_t = Template([slot_a, slot_b])
greeting_tt = TemplateTree(greeting_t, [hello_tt, hey_tt])
self.assertTrue(greeting_t.encompasses(hey_t, expected_values))
self.assertTrue(greeting_t.encompasses(hello_t, expected_values))
self.assertFalse(hello_t.encompasses(greeting_t, expected_values))
self.assertEqual(
expected_values, greeting_tt.calculated_merged_independent_slot_values()
)
self.assertEqual(
expected, greeting_tt.collapse_using_slot_values(expected_values)
)
# Do same, but for second argument
world_t = Template([slot_a, world])
world_tt = TemplateTree(world_t, [h1, h2])
universe_t = Template([slot_a, universe])
universe_tt = TemplateTree(universe_t, [h3, h4])
place_t = Template([slot_a, slot_b])
place_tt = TemplateTree(place_t, [world_tt, universe_tt])
self.assertEqual(
expected_values, place_tt.calculated_merged_independent_slot_values()
)
self.assertEqual(expected, place_tt.collapse_using_slot_values(expected_values))
# Test mix
mix_tt = TemplateTree(place_t, [world_tt, hey_tt, h3])
self.assertEqual(
expected_values, mix_tt.calculated_merged_independent_slot_values()
)
self.assertEqual(expected, mix_tt.collapse_using_slot_values(expected_values))
# Now with some noise
noise = Template([TemplateString("noise")])
noise_tt = TemplateTree(noise)
noise_t = Template([slot_c])
full_noise_tt = TemplateTree(noise_t, [greeting_tt, noise_tt])
noise_values = SlotValues(
{
slot_a: {Template([hello]), Template([hey])},
slot_b: {Template([world]), Template([universe])},
slot_c: {Template([slot_a, slot_b]), noise},
}
)
collapsed_full_noise = full_noise_tt.collapse_using_slot_values(noise_values)
self.assertEqual(
noise_values, full_noise_tt.calculated_merged_independent_slot_values(),
)
self.assertEqual(
TemplateTree(Template([slot_c]), [expected, noise_tt]),
collapsed_full_noise,
)
def test_collapse_same_children(self):
""" Tests if collapsing a tree with children with similar templates will merge correctly """
ss1 = TemplateTree(Template.from_string("a b c c d"))
ss2 = TemplateTree(Template.from_string("c b e e d"))
ss3 = TemplateTree(Template.from_string("h h h b f d"))
ss4 = TemplateTree(Template.from_string("i i i b g d"))
ss5 = TemplateTree(Template.from_string("j k l l d"))
us1 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"), [ss1, ss2]
)
us2 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"), [ss3, ss4]
)
us3 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [us1, us2, ss5]
)
ts1 = TemplateTree(
Template.from_string("[SLOT] b [SLOT] d", slot_token="[SLOT]"),
[ss1, ss2, ss3, ss4],
)
ts2 = TemplateTree(
Template.from_string("[SLOT] d", slot_token="[SLOT]"), [ts1, ss5]
)
collapsed_u = us3.collapse()
self.assertEqual(ts2, collapsed_u)