def test_get_rule_unifiers_10(self):
rule_a = qcfg_rule.rule_from_string("jump and NT_1 ### jump NT_1")
rule_b = qcfg_rule.rule_from_string("NT_1 and NT_2 ### NT_1 NT_2")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string("jump ### jump")
self.assertEqual({expected}, unifiers)
def test_get_rule_unifiers_repeated(self):
rule_a = qcfg_rule.rule_from_string(
"jump and jump twice ### JUMP JUMP JUMP")
rule_b = qcfg_rule.rule_from_string("jump twice ### JUMP JUMP")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
self.assertEmpty(unifiers)
def test_get_rule_unifiers_1(self):
rule_b = qcfg_rule.rule_from_string("bar ### bar")
rule_a = qcfg_rule.rule_from_string("foo bar ### foo bar")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string("foo NT_1 ### foo NT_1")
self.assertEqual({expected}, unifiers)
def test_get_rule_unifiers_11(self):
rule_a = qcfg_rule.rule_from_string(
"NT_1 right twice ### I_TURN_RIGHT NT_1 I_TURN_RIGHT NT_1")
rule_b = qcfg_rule.rule_from_string("NT_1 right ### I_TURN_RIGHT NT_1")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string("NT_1 twice ### NT_1 NT_1")
self.assertEqual({expected}, unifiers)
def test_parse(self):
tokens = ["dax", "twice"]
rules = [
qcfg_rule.rule_from_string("dax ### DAX"),
qcfg_rule.rule_from_string("NT_1 twice ### NT_1 NT_1"),
]
parses = qcfg_parser.parse(tokens, rules, _node_fn,
_postprocess_cell_fn)
self.assertEqual(parses, ["DAX DAX"])
def test_generate_derivation_1(self):
goal_rule = qcfg_rule.rule_from_string("foo bar ### foo bar")
rules = {
qcfg_rule.rule_from_string("foo NT_1 ### foo NT_1"),
qcfg_rule.rule_from_string("bar ### bar"),
}
derivations = derivation_utils.generate_derivation(
_get_config(), goal_rule, rules)
self.assertEqual(derivations, rules)
def test_get_rule_unifiers_single_nt(self):
rule_a = qcfg_rule.rule_from_string(
"xyz NT_1 ### bar ( foo ( NT_1 ) )")
rule_b = qcfg_rule.rule_from_string("NT_1 ### foo ( NT_1 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string("xyz NT_1 ### bar ( NT_1 )")
self.assertEqual({expected}, unifiers)
def test_get_rule_unifiers_7(self):
rule_a = qcfg_rule.rule_from_string(
"NT_1 ' s NT_2 director ### NT_2 & film.director.film ( NT_1 )")
rule_b = qcfg_rule.rule_from_string("NT_1 ' s NT_2 ### NT_2 ( NT_1 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"NT_1 director ### NT_1 & film.director.film")
self.assertEqual({expected}, unifiers)
def test_parse_single_nt(self):
tokens = ["list", "the", "states"]
rules = [
qcfg_rule.rule_from_string("list the states ### state"),
qcfg_rule.rule_from_string("NT_1 ### answer ( NT_1 )"),
]
parses = qcfg_parser.parse(tokens, rules, _node_fn,
_postprocess_cell_fn)
self.assertCountEqual(parses, ["state", "answer ( state )"])
def test_get_rule_unifiers_12(self):
rule_a = qcfg_rule.rule_from_string("foo NT_1 NT_2 ### NT_2 foo NT_1")
rule_b = qcfg_rule.rule_from_string("foo ### foo")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"NT_1 NT_2 NT_3 ### NT_3 NT_1 NT_2")
self.assertEqual({expected}, unifiers)
def test_can_parse(self):
rules = [
qcfg_rule.rule_from_string("dax ### DAX"),
qcfg_rule.rule_from_string("NT_1 twice ### NT_1 NT_1"),
]
can_parse = qcfg_parser.can_parse(source="dax twice",
target="DAX DAX",
rules=rules)
self.assertTrue(can_parse)
def test_get_rule_unifiers_no_split(self):
rule_a = qcfg_rule.rule_from_string(
"NT_1 have a major city named NT_2 ### intersection ( NT_1 , loc_1 ( intersection ( major , intersection ( city , NT_2 ) ) ) )"
)
rule_b = qcfg_rule.rule_from_string(
"NT_1 named NT_2 ### intersection ( NT_1 , NT_2 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
self.assertEmpty(unifiers)
def test_rhs_can_maybe_derive_1(self):
rule_a = qcfg_rule.rule_from_string(
"who is NT_1 ' s boss ? ### ( Yield :output ( FindManager :recipient ( NT_1 ) ) )"
)
rule_b = qcfg_rule.rule_from_string(
"who is NT_1 ? ### ( Yield :output ( NT_1 ) )")
self.assertTrue(
rule_utils.rhs_can_maybe_derive(rule_b.source, rule_a.source))
self.assertTrue(
rule_utils.rhs_can_maybe_derive(rule_b.target, rule_a.target))
def test_repeated_target_nts_2(self):
rule_a = qcfg_rule.rule_from_string(
"x NT_1 y z NT_2 ### X NT_2 Y NT_2 Z NT_2 W NT_1")
rule_b = qcfg_rule.rule_from_string("z NT_1 ### NT_1 Z NT_1")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
# Should not include rule like `x NT_1 y NT_2 ### X NT_2 Y NT_2 W NT_1`.
self.assertEmpty(unifiers)
def test_get_rule_unifiers_no_split_2(self):
rule_a = qcfg_rule.rule_from_string(
"NT_1 NT_2 NT_3 ### answer ( intersection ( NT_3 , loc_1 ( intersection ( NT_2 , NT_1 ) ) ) )"
)
rule_b = qcfg_rule.rule_from_string(
"NT_1 NT_2 ### intersection ( NT_1 , NT_2 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
self.assertEmpty(unifiers)
def test_get_rule_unifiers_6(self):
rule_a = qcfg_rule.rule_from_string(
"written and edited ### film.film.edited_by , film.film.written_by"
)
rule_b = qcfg_rule.rule_from_string(
"NT_1 and edited ### film.film.edited_by , NT_1")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"written ### film.film.written_by")
self.assertEqual({expected}, unifiers)
def test_get_rule_unifiers_5(self):
rule_a = qcfg_rule.rule_from_string(
"Who influenced M2 ' s Dutch child ### answer ( a(person) & influenced ( nat ( m_059j2 ) & parent ( M2 ) ) )"
)
rule_b = qcfg_rule.rule_from_string("Dutch ### nat ( m_059j2 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"Who influenced M2 ' s NT_1 child ### answer ( a(person) & influenced ( NT_1 & parent ( M2 ) ) )"
)
self.assertEqual({expected}, unifiers)
def test_example_converter(self):
config = test_utils.get_test_config()
rules = [
qcfg_rule.rule_from_string("foo NT_1 ### foo NT_1"),
qcfg_rule.rule_from_string("bar ### bar"),
qcfg_rule.rule_from_string("foo bar ### foo bar"),
qcfg_rule.rule_from_string("foo bar ### baz"),
]
converter = example_converter.ExampleConverter(rules, config)
example = ("foo bar", "foo bar")
tf_example = converter.convert(example)
print(tf_example)
feature_dict = tf_example.feature_lists.feature_list
expected_keys = {
"node_type_list",
"node_idx_list",
"rhs_emb_idx_list",
"lhs_emb_idx_list",
"num_nodes",
}
self.assertEqual(feature_dict.keys(), expected_keys)
for key in expected_keys:
for feature in feature_dict[key].feature:
print("%s: %s" % (key, feature.int64_list.value))
self.assertEqual(feature_dict["num_nodes"].feature[0].int64_list.value,
[6])
self.assertEqual(
feature_dict["node_type_list"].feature[0].int64_list.value,
[1, 1, 1, 1, 1, 2, 0, 0])
self.assertEqual(
feature_dict["rhs_emb_idx_list"].feature[0].int64_list.value,
[-1, -1, 1, 0, 0, -1, 0, 0])
self.assertEqual(
feature_dict["rhs_emb_idx_list"].feature[1].int64_list.value,
[-1, -1, -1, -1, -1, -1, 0, 0])
self.assertEqual(
feature_dict["lhs_emb_idx_list"].feature[0].int64_list.value,
[-1, -1, 0, 3, 1, -1, 0, 0])
self.assertEqual(
feature_dict["lhs_emb_idx_list"].feature[1].int64_list.value,
[-1, -1, -1, -1, -1, -1, 0, 0])
self.assertEqual(
feature_dict["node_idx_list"].feature[0].int64_list.value,
[-1, -1, 0, 1, 2, 3, 0, 0])
self.assertEqual(
feature_dict["node_idx_list"].feature[1].int64_list.value,
[-1, -1, -1, -1, -1, 4, 0, 0])
def test_get_rule_unifiers_13(self):
rule_a = qcfg_rule.rule_from_string(
"what NT_1 is the NT_2 in the m0 ### answer ( NT_2 ( intersection ( NT_1 , loc_2 ( m0 ) ) ) )"
)
rule_b = qcfg_rule.rule_from_string("in the m0 ### loc_2 ( m0 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"what NT_1 is the NT_2 NT_3 ### answer ( NT_2 ( intersection ( NT_1 , NT_3 ) ) )"
)
self.assertEqual({expected}, unifiers)
def test_generate_derivation_2(self):
goal_rule = qcfg_rule.rule_from_string(
"foo bar and NT_1 ### foo bar NT_1")
rules = [
qcfg_rule.rule_from_string("foo NT_1 ### foo NT_1"),
qcfg_rule.rule_from_string("bar ### bar"),
qcfg_rule.rule_from_string("NT_1 and NT_2 ### NT_1 NT_2"),
qcfg_rule.rule_from_string("foo bar ### foo bar"),
]
derivations = derivation_utils.generate_derivation(
_get_config(), goal_rule, set(rules))
self.assertEqual(derivations, {rules[0], rules[1], rules[2]})
def test_can_parse_3_nts(self):
rules = [
qcfg_rule.rule_from_string("jump ### JUMP"),
qcfg_rule.rule_from_string("walk ### WALK"),
qcfg_rule.rule_from_string("dax ### DAX"),
qcfg_rule.rule_from_string(
"NT_1 and NT_2 after NT_3 ### NT_1 NT_3 NT_2"),
]
can_parse = qcfg_parser.can_parse(source="jump and walk after dax",
target="JUMP DAX WALK",
rules=rules)
self.assertTrue(can_parse)
def test_repeated_target_nts_3(self):
rule_a = qcfg_rule.rule_from_string(
"NT_1 and jump twice ### NT_1 NT_1 JUMP")
rule_b = qcfg_rule.rule_from_string("NT_1 and NT_2 ### NT_1 NT_2")
rule_c = qcfg_rule.rule_from_string("jump twice ### NT_1 JUMP")
# Should not include `rule_c`.
self.assertEmpty(
unification_utils.get_rule_unifiers(rule_a, rule_b, _get_config()))
# Should not include `rule_b`.
self.assertEmpty(
unification_utils.get_rule_unifiers(rule_a, rule_c, _get_config()))
def test_get_rule_unifiers_14(self):
rule_a = qcfg_rule.rule_from_string(
"state has NT_1 major rivers NT_2 ### NT_1 ( state , NT_2 , intersection ( major , river ) )"
)
rule_b = qcfg_rule.rule_from_string(
"state has NT_1 NT_2 NT_3 ### NT_1 ( state , NT_3 , NT_2 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"major rivers ### intersection ( major , river )")
self.assertEqual({expected}, unifiers)
def test_get_rule_unifiers_4(self):
rule_a = qcfg_rule.rule_from_string(
"Which NT_1 was NT_2 ' s star ### answer ( film.actor.film/ns:film.performance.film ( NT_2 ) & NT_1 )"
)
rule_b = qcfg_rule.rule_from_string(
"NT_1 ' s star ### film.actor.film/ns:film.performance.film ( NT_1 )"
)
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"Which NT_1 was NT_2 ### answer ( NT_2 & NT_1 )")
self.assertEqual({expected}, unifiers)
def test_parse_3_nts(self):
tokens = ["jump", "and", "walk", "after", "dax"]
rules = [
qcfg_rule.rule_from_string("jump ### JUMP"),
qcfg_rule.rule_from_string("walk ### WALK"),
qcfg_rule.rule_from_string("dax ### DAX"),
qcfg_rule.rule_from_string(
"NT_1 and NT_2 after NT_3 ### NT_1 NT_3 NT_2"),
]
parses = qcfg_parser.parse(tokens, rules, _node_fn,
_postprocess_cell_fn)
self.assertEqual(parses, ["JUMP DAX WALK"])
def test_get_rule_unifiers_2(self):
rule_a = qcfg_rule.rule_from_string(
"what is NT_1 traversed by NT_2 ### answer ( intersection ( NT_1 , traverse_1 ( NT_2 ) ) )"
)
rule_b = qcfg_rule.rule_from_string("what is NT_1 ### answer ( NT_1 )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"NT_1 traversed by NT_2 ### intersection ( NT_1 , traverse_1 ( NT_2 ) )"
)
self.assertEqual({expected}, unifiers)
def test_get_rule_unifiers_3(self):
rule_a = qcfg_rule.rule_from_string(
"NT_1 traversed by the NT_2 river ### intersection ( NT_1 , traverse_1 ( intersection ( NT_2 , river ) ) )"
)
rule_b = qcfg_rule.rule_from_string(
"NT_1 river ### intersection ( NT_1 , river )")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
expected = qcfg_rule.rule_from_string(
"NT_1 traversed by the NT_2 ### intersection ( NT_1 , traverse_1 ( NT_2 ) )"
)
self.assertEqual({expected}, unifiers)
def test_can_parse_single_nt(self):
rules = [
qcfg_rule.rule_from_string("list the states ### state"),
qcfg_rule.rule_from_string("NT_1 ### answer ( NT_1 )"),
]
can_parse = qcfg_parser.can_parse(source="list the states",
target="state",
rules=rules)
self.assertTrue(can_parse)
can_parse = qcfg_parser.can_parse(source="list the states",
target="answer ( state )",
rules=rules)
self.assertTrue(can_parse)
def test_repeated_target_nts_1(self):
rule_a = qcfg_rule.rule_from_string(
"x NT_1 y z NT_2 ### X NT_2 Y NT_2 Z NT_2 W NT_1")
rule_b = qcfg_rule.rule_from_string(
"x NT_1 y NT_2 ### X NT_2 Y NT_2 W NT_1")
unifiers = unification_utils.get_rule_unifiers(rule_a, rule_b,
_get_config())
# Should not include rule like `z NT_1 ### NT_1 Z NT_1` that has been formed
# by replacing a sub-span of the target in rule_a which does not include
# all of the NT_2s.
self.assertEmpty(unifiers)
def setUp(self):
super(SamplerUtilsTest, self).setUp()
self.rules = [
qcfg_rule.rule_from_string("foo NT_1 ### foo NT_1"),
qcfg_rule.rule_from_string("bar ### bar"),
qcfg_rule.rule_from_string("foo bar ### foo bar"),
]
self.target_grammar_rules = [
target_grammar.TargetCfgRule.from_string("ROOT => foo ##NT"),
target_grammar.TargetCfgRule.from_string("NT => bar")
]
self.augment_config = augment_test_utils.get_test_config()
self.model_config = model_test_utils.get_test_config()
self.model_dir = self.get_temp_dir()
