class LevinTest(unittest.TestCase):
def setUp(self):
self.lexicon = vol.lexicon.clone()
self.learner = WordLearner(self.lexicon,
Compressor(self.lexicon.ontology,
**EC_kwargs),
bootstrap=True)
def test_functional(self):
# We should get five derived categories.
self.learner.compress_lexicon()
self.assertEquals(len(self.learner.lexicon._derived_categories), 2)
# Learn examples
for sentence, scene, answer in vol.examples:
sentence = sentence.split()
model = Model(scene, self.ontology)
weighted_results = self.learner.update_with_distant(
sentence, model, answer)
final_sem = weighted_results[0][0].label()[0].semantics()
print(" ".join(sentence), len(weighted_results), final_sem)
print("\t", model.evaluate(final_sem))
def main(args, lexicon, ontology, examples):
compressor = Compressor(ontology, **
EC_kwargs) if not args.no_compress else None
learner = WordLearner(lexicon, compressor, bootstrap=not args.no_bootstrap)
# No-op if `--no-compress`
learner.compress_lexicon()
for sentence, scene, answer in examples:
print("\n\n")
sentence = sentence.split()
model = Model(scene, ontology)
weighted_results = learner.update_with_example(sentence, model, answer)
final_sem = weighted_results[0][0].label()[0].semantics()
print(" ".join(sentence), len(weighted_results), final_sem)
print("\t", model.evaluate(final_sem))
class LevinTest(unittest.TestCase):
def setUp(self):
lexicon = levin.lexicon.clone()
self.learner = WordLearner(lexicon,
Compressor(lexicon.ontology, **EC_kwargs),
bootstrap=True)
def test_functional(self):
expected_derived = {
# Levin verb classes
"9.1": {"set", "put"},
"9.2": {"lay", "hang"},
"9.4": {"drop", "hoist"},
"9.5": {"pour", "spill"},
"9.7": {"spray", "load"},
"9.8": {"fill", "stuff"},
# PPs
"contact": {"on", "onto"},
}
#######
self.learner.compress_lexicon()
#######
# self.assertEquals(set(frozenset(token._token for token in tokens)
# for _, tokens in self.learner.lexicon._derived_categories.values()),
# set(frozenset(xs) for xs in expected_derived.values()))
# OK, now try to bootstrap with an example.
for sentence, scene, answer in levin.examples:
sentence = sentence.split()
model = Model(scene, self.learner.ontology)
weighted_results = self.learner.update_with_distant(
sentence, model, answer)
final_sem = weighted_results[0][0].label()[0].semantics()
print(" ".join(sentence), len(weighted_results), final_sem)
print("\t", model.evaluate(final_sem))
def eval_model(compress=True, bootstrap=True, **learner_kwargs):
L.info("Building model.")
lexicon = initial_lexicon.clone()
compressor = Compressor(lexicon.ontology, **
EC_kwargs) if compress else None
learner = WordLearner(lexicon,
compressor,
bootstrap=bootstrap,
**learner_kwargs)
if compress:
learner.compress_lexicon()
# transitive verb -> causative semantics
eval_bootstrap_example(learner, examples[0], "gorps", None)
eval_oneshot_example(learner, examples[0], "gorps", None)
# intransitive verb -> non-causative semantics
eval_bootstrap_example(learner, examples[1], "florps", None)
eval_oneshot_example(learner, examples[1], "florps", None)
def eval_model(bootstrap=True, **learner_kwargs):
L.info("Building model.")
learner_kwargs["limit_induction"] = True
pprint.pprint(learner_kwargs)
lexical_limit = learner_kwargs.pop("lexical_limit", 1000)
learner_kwargs["prune_entries"] = lexical_limit
learner_kwargs["zero_shot_limit"] = lexical_limit
default_weight = learner_kwargs.pop("weight_init")
lexicon = initial_lexicon.clone()
learner = WordLearner(lexicon,
compressor=None,
bootstrap=bootstrap,
**learner_kwargs)
###########
# Prepare online evaluation.
#
# Track 1) correct LF computation on arbitrary sentences, and 2) 2AFC
# syntactic bootstrapping performance.
eval_results_generic = []
eval_results_bootstrap = []
def do_online_eval():
parser = learner.make_parser()
n_generic_correct = 0
for example in dev_examples:
sentence, model, _ = prep_example(learner, example)
ground_truth = example[1]
results = parser.parse(sentence)
try:
result = model.evaluate(results[0].label()[0].semantics(),
no_lookup=True)
# print("\t", result == ground_truth, ground_truth, result)
if result == ground_truth:
n_generic_correct += 1
except:
pass
# except:
# print("\t\t=-=-=-=---=-=-=here", ground_truth, results[0].label()[0].semantics())
# print_exc()
# continue
n_bootstrap_correct = 0
for example, expected_idx in test_2afc_examples:
sentence, model1, model2 = prep_example_2afc(learner, example)
posterior = learner.predict_zero_shot_2afc(sentence, model1,
model2)
if posterior.argmax().scene == [model1,
model2][expected_idx].scene:
n_bootstrap_correct += 1
print(n_generic_correct / len(dev_examples))
print(n_bootstrap_correct / len(test_2afc_examples))
eval_results_generic.append(n_generic_correct / len(dev_examples))
eval_results_bootstrap.append(n_bootstrap_correct /
len(test_2afc_examples))
###########
for example in training_examples:
sentence, model, _ = prep_example(learner, example)
print("------ ", " ".join(sentence))
try:
learner.update_with_cross_situational(sentence, model)
except:
print_exc()
L.info("Abandoning cross-situational update for sentence.")
# Parse failed. That's okay.
continue
learner.lexicon.debug_print()
do_online_eval()
print("Online general accuracies:", eval_results_generic)
print("Online bootstrap accuracies:", eval_results_bootstrap)
for example, expected_idx in test_2afc_examples:
sentence = example[0]
print("------ ", sentence)
eval_2afc_zeroshot(learner, example, expected_idx)
with (args.out_dir / "results.jsonl").open("a") as out_f:
json.dump(
{
"params": learner_kwargs,
"accuracies": {
"general": eval_results_generic,
"bootstrap": eval_results_bootstrap
}
}, out_f)
def setUp(self):
self.lexicon = voo.lexicon.clone()
self.learner = WordLearner(self.lexicon,
Compressor(self.lexicon.ontology,
**EC_kwargs),
bootstrap=True)
def eval_model(compress=True, bootstrap=True, **learner_kwargs):
L.info("Building model.")
pprint(learner_kwargs)
default_weight = learner_kwargs.pop("weight_init")
lexicon = levin.make_lexicon(default_weight=default_weight)
compressor = Compressor(lexicon.ontology, **EC_kwargs) if compress else None
learner = WordLearner(lexicon, compressor, bootstrap=bootstrap,
**learner_kwargs)
# Run compression.
learner.compress_lexicon()
try:
PP_CONTACT_CATEGORY, _ = learner.lexicon._derived_categories["D0"]
PUT_CATEGORY, _ = learner.lexicon._derived_categories["D1"]
DROP_CATEGORY, _ = learner.lexicon._derived_categories["D2"]
POUR_CATEGORY, _ = learner.lexicon._derived_categories["D3"]
FILL_CATEGORY, _ = learner.lexicon._derived_categories["D4"]
# sanity check
_assert(str(PP_CONTACT_CATEGORY.base) == "PP", "PP contact derived cat has correct base")
_assert(str(PUT_CATEGORY.base) == "S", "Put verb derived cat has correct base")
_assert(str(DROP_CATEGORY.base) == "S", "Drop verb derived cat has correct base")
_assert(str(FILL_CATEGORY.base) == "S", "Fill verb derived cat has correct base")
_assert(str(POUR_CATEGORY.base) == "S", "Pour verb derived cat has correct base")
except KeyError:
_assert(False, "Derived categories not available", False)
PP_CONTACT_CATEGORY = None
PUT_CATEGORY = None
DROP_CATEGORY = None
FILL_CATEGORY = None
POUR_CATEGORY = None
# Constructions in which different derived verb cats appear
locative_construction = learner.lexicon.parse_category("S/N/PP")
locative_construction._arg = PP_CONTACT_CATEGORY
constructions = [
locative_construction,
learner.lexicon.parse_category("S/N/PP"),
learner.lexicon.parse_category("S/N"),
]
###########
# Run initial weight updates.
for example in examples[:3]:
sentence, model, answer = prep_example(learner, example)
learner.update_with_distant(sentence, model, answer)
# Ensure that derived categories are present in the highest-scoring entries'
# yields.
expected = [("put", PUT_CATEGORY), ("fill", FILL_CATEGORY)]
for token, expected_top_yield in expected:
entries = learner.lexicon._entries[token]
top_entry = max(entries, key=lambda e: e.weight())
top_yield = get_yield(top_entry.categ())
_assert(top_yield == expected_top_yield,
"Top-scoring category for '%s' has yield %s: %s"
% (token, expected_top_yield, top_yield))
# Zero-shot predictions with bootstrapping in known frames
def make_extra(target):
def extra_check(token, cand_cats, cand_joint):
top_cat, top_expr = cand_joint.argmax()
_assert(top_cat.arg() == target,
"Top cat for %s should have first arg of type %s: %s" %
(token, target, top_cat))
return extra_check
eval_bootstrap_example(learner, examples[3], "place", PUT_CATEGORY,
bootstrap=bootstrap, extra=make_extra(PP_CONTACT_CATEGORY))
eval_oneshot_example(learner, examples[3], "place", PUT_CATEGORY,
extra=make_extra(PP_CONTACT_CATEGORY))
print(compute_alternations(learner, constructions))
eval_bootstrap_example(learner, examples[4], "cover", FILL_CATEGORY,
bootstrap=bootstrap, extra=make_extra(learner.lexicon.parse_category("PP")))
eval_oneshot_example(learner, examples[4], "cover", FILL_CATEGORY,
extra=make_extra(learner.lexicon.parse_category("PP")))
print(compute_alternations(learner, constructions))
# Learn a novel frame for the fill class.
# Skip 0-shot asserts -- don't expect to have correct guess for an entirely
# new frame.
eval_bootstrap_example(learner, examples[5], "fill", FILL_CATEGORY, bootstrap=bootstrap,
asserts=False)
eval_oneshot_example(learner, examples[5], "fill", FILL_CATEGORY)
eval_oneshot_example(learner, examples[6], "fill", FILL_CATEGORY)
print(compute_alternations(learner, constructions))
# Zero-shot predictions for the newly learned frame.
eval_bootstrap_example(learner, examples[7], "stuff", FILL_CATEGORY, bootstrap=bootstrap)
eval_oneshot_example(learner, examples[7], "stuff", FILL_CATEGORY)
print(compute_alternations(learner, constructions))
eval_oneshot_example(learner, examples[8], "lower", DROP_CATEGORY)
eval_oneshot_example(learner, examples[9], "raise", DROP_CATEGORY,
extra=make_extra(PP_CONTACT_CATEGORY))
print(compute_alternations(learner, constructions))
eval_oneshot_example(learner, examples[10], "drip", POUR_CATEGORY,
extra=make_extra(PP_CONTACT_CATEGORY))
###########
# Produce alternation table.
table = compute_alternations(learner, constructions)
print(table)
table.to_csv(args.out_dir / "alternations.csv")
plt.clf()
sns.heatmap(table)
plt.savefig(args.out_dir / "alternations.png")