def test_that_grammar_equality_works_as_expected():
rulebooks1 = rulebooks.copy()
rulebooks2 = rulebooks.copy()
rulebooks2[3] = {"c": 0.3, "d": 0.7}
rulebooks3 = rulebooks.copy()
rulebooks3[0] = {(1, 1): 1.0}
grammar0 = Grammar(rulebooks)
grammar1 = Grammar(rulebooks1)
grammar2 = Grammar(rulebooks2)
grammar3 = Grammar(rulebooks3)
assert grammar0 == grammar0
assert grammar0 == grammar1
assert grammar1 == grammar1
assert grammar2 == grammar2
assert grammar0 != grammar2
assert grammar2 != grammar0
assert grammar0 != grammar3
assert grammar1 != grammar3
assert grammar2 != grammar3
def test_that_sample_returns_trees_with_different_terminals():
grammar = Grammar(rulebooks)
trees = [grammar.sample_tree(root=0) for _ in range(100)]
words = ["".join(tree.terminals) for tree in trees]
assert len(set(words)) > 1
def test_that_occupancy_probabilities_are_in_the_unit_interval():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
fillprob = grammar.compute_occupancy_probabilities(tree.terminals)
assert np.all(fillprob >= 0.0 - 1e-12)
assert np.all(fillprob <= 1.0 + 1e-12)
def test_that_occupancy_probabilities_are_not_zero_at_filled_slots():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
isfilled = tree.get_occupancy_matrix(dtype=float)
fillprob = grammar.compute_occupancy_probabilities(tree.terminals, root=0)
assert np.all(isfilled[fillprob == 0.0] == 0) # p = 0 ==> not filled
assert np.all(fillprob[isfilled == 1.0] > 0) # same: filled ==> p > 0
def test_that_total_likelihood_exceeds_single_largest_likelihood():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree()
sentence = tree.terminals
maximal = grammar.compute_maximal_probabilities(sentence)
summed = grammar.compute_inside_probabilities(sentence)
assert np.all(maximal <= summed + 1e-12), (sentence, maximal, summed)
def test_that_grammar_computes_probabilities_in_the_right_range():
grammar = Grammar(rulebooks)
for tree in grammar.sample_tree():
logprob = grammar.logprob_tree(tree)
prob = grammar.prob_tree(tree)
assert -np.inf < logprob <= 0.0
assert 0.0 < prob <= 1.0
assert np.isclose(np.exp(logprob), prob)
def test_that_grammar_from_rulebooks_compiles_alphabet_correctly():
rulebooks = {0: {"a": 0.5, "d": 0.5}, 1: {"c": 0.3, "b": 0.7}}
grammar = Grammar(rulebooks)
assert grammar.alphabet == tuple("abcd")
grammar = Grammar(rulebooks, alphabet=None) # same as above
assert grammar.alphabet == tuple("abcd")
grammar = Grammar(rulebooks, alphabet="abcdX")
assert grammar.alphabet == tuple("abcdX")
grammar = Grammar(rulebooks, alphabet=list("abcdX"))
assert grammar.alphabet == tuple("abcdX")
def test_that_conditional_sampling_returns_different_trees():
grammar = Grammar(rulebooks)
word = "abba" # structually ambiguous under the grammar
inner = grammar.compute_inside_probabilities(word)
trees = [
grammar.conditionally_sample_tree(word, inner, root=0)
for _ in range(30)
]
# we sample different trees, but they all have the same leaves:
assert set(["".join(tree.terminals) for tree in trees]) == set([word])
comparisons = pairwise_equalities(trees)
assert any(comparisons) # it virtually impossible they're all identical
assert not all(comparisons) # they cannot all be different either
def test_that_occupancy_probabilities_are_zero_and_one_where_expected():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
fillprob = grammar.compute_occupancy_probabilities(tree.terminals)
assert np.allclose(fillprob.diagonal(), 1.0) # terminals have parents
should_be_upper = fillprob - np.diag(fillprob.diagonal())
assert np.allclose(np.tril(should_be_upper), 0) # no p>0 below diagonal
# each branching gives rise to one extra nonterminals and one
# extra terminal; exactly `size` nonterminals are not going to
# branch out. Together, this implies the following constraint:
expected_probsum = 2 * tree.size - 1
computed_probsum = np.sum(fillprob)
assert np.isclose(expected_probsum, computed_probsum)
def test_that_emission_probabilities_are_in_the_right_range():
# first define a grammar with some overlap between the
# terminals that can be produced by different nonterminals:
rulebooks = {
0: {
(1, 1): 0.3,
(1, 2): 0.4,
'a': 0.3
},
1: {
(1, 2): 0.2,
(2, 1): 0.3,
'b': 0.4,
'a': 0.1
},
2: {
'a': 0.3,
'b': 0.7
}
}
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
sentence = tree.terminals
inside = grammar.compute_inside_probabilities(sentence)
outside = grammar.compute_outside_probabilities(inside, initial=0)
probsums = grammar.sum_emission_probabilities(sentence, outside)
assert np.all(probsums >= 0.0) # but may be larger than 1.0
for (nt, letter) in tree.iter_emissions():
idx = grammar.alphabet.index(letter)
assert probsums[nt, idx] > 0.0 # must be positive if it happened
def sample_random_grammar_like(grammar):
transitions = np.random.gamma(1.0, size=grammar.transitions.shape)
emissions = np.random.gamma(1.0, size=grammar.emissions.shape)
transitions /= np.sum(transitions, axis=(1, 2), keepdims=True)
emissions /= np.sum(emissions, axis=1, keepdims=True)
transitions *= 0.3
emissions *= 0.7
return Grammar(transitions=transitions,
emissions=emissions,
alphabet=grammar.alphabet)
def create_random_grammar(num_nonterminals=4):
N = num_nonterminals
transitions = np.random.gamma(1.0, size=(N, N, N))
transitions /= np.sum(transitions, axis=(1, 2), keepdims=True)
emissions = np.random.gamma(1.0, size=(N, 2))
emissions /= np.sum(emissions, axis=1, keepdims=True)
# make sure that the typical sentence is short:
transitions *= 0.25
emissions *= 0.75
return Grammar(transitions=transitions,
emissions=emissions,
alphabet=[+1, -1])
def test_that_most_probable_tree_is_most_probable():
grammar = Grammar(rulebooks)
actual_tree = grammar.sample_tree(root=0)
actual_logprob = grammar.logprob_tree(actual_tree)
sentence = actual_tree.terminals
inner = grammar.compute_inside_probabilities(sentence)
max_logprob = np.sum(inner[0, -1, :])
best_tree = grammar.compute_most_likely_tree(sentence, root=0)
best_logprob = grammar.logprob_tree(best_tree)
assert actual_logprob <= best_logprob + 1e-14, (actual_logprob,
best_logprob)
assert best_logprob <= max_logprob # total likelihood across trees
for _ in range(10):
random_tree = grammar.conditionally_sample_tree(sentence,
inner,
root=0)
random_logprob = grammar.logprob_tree(random_tree)
assert random_logprob <= best_logprob + 1e-14, (random_logprob,
best_logprob)
def test_that_most_probable_always_returns_a_tree_of_the_same_logprob():
"""
Because of slight numerical instabilities, the most-likely-tree
method does _not_ in fact always return the same tree when there
are multiple trees with the same or very nearly the same likelihood.
The best we can ask for is that the method returns _a_ best tree,
defined as one with the biggest likelihood, or nearly the biggest.
"""
grammar = Grammar(rulebooks)
actual_tree = grammar.sample_tree()
sentence = actual_tree.terminals
inner = grammar.compute_inside_probabilities(sentence)
best_tree = grammar.compute_most_likely_tree(sentence, inner, root=0)
best_logprob = grammar.logprob_tree(best_tree)
for _ in range(20):
tree = grammar.compute_most_likely_tree(sentence, inner, root=0)
assert np.isclose(grammar.logprob_tree(tree), best_logprob)
def test_that_transition_probabilities_are_in_the_right_range():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
sentence = tree.terminals
inside = grammar.compute_inside_probabilities(sentence)
outside = grammar.compute_outside_probabilities(inside, initial=0)
probsums = grammar.sum_transition_probabilities(inside, outside)
assert np.all(probsums >= 0.0) # but may be larger than 1.0
for triple in tree.iter_transitions():
assert probsums[triple] > 0.0
def create_random_grammar(num_nonterminals=4):
alphabet = tuple("abcdefghijklmnopqrstuvwxyz")
N = num_nonterminals
T = len(alphabet)
transitions = np.random.gamma(1.0, size=(N, N, N))
transitions /= np.sum(transitions, axis=(1, 2), keepdims=True)
emissions = np.random.gamma(1.0, size=(N, T))
emissions /= np.sum(emissions, axis=1, keepdims=True)
# make sure that the typical sentence is short:
transitions *= 0.25
emissions *= 0.75
return Grammar(transitions=transitions,
emissions=emissions,
alphabet=alphabet)
def test_that_transition_probabilities_are_well_calibrated():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
sentence = tree.terminals
inside = grammar.compute_inside_probabilities(sentence)
outside = grammar.compute_outside_probabilities(inside, initial=0)
probsums = grammar.sum_transition_probabilities(inside, outside)
num_samples = 400
occursums = np.zeros_like(probsums)
for _ in range(num_samples):
randtree = grammar.conditionally_sample_tree(sentence, inside, root=0)
for triple in randtree.iter_transitions():
occursums[triple] += 1
occursums /= num_samples
assert np.allclose(probsums, occursums, atol=0.1)
def test_that_emission_probabilities_are_well_calibrated():
# first define a grammar with some overlap between the
# terminals that can be produced by different nonterminals:
rulebooks = {
0: {
(1, 1): 0.3,
(1, 2): 0.4,
'a': 0.3
},
1: {
(1, 2): 0.2,
(2, 1): 0.3,
'b': 0.4,
'a': 0.1
},
2: {
'a': 0.3,
'b': 0.7
}
}
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
sentence = tree.terminals
inside = grammar.compute_inside_probabilities(sentence)
outside = grammar.compute_outside_probabilities(inside, initial=0)
probsums = grammar.sum_emission_probabilities(sentence, outside)
num_samples = 400
occursums = np.zeros_like(probsums)
for _ in range(num_samples):
randtree = grammar.conditionally_sample_tree(sentence, inside, root=0)
for (nt, letter) in randtree.iter_emissions():
idx = grammar.alphabet.index(letter)
occursums[nt, idx] += 1
occursums /= num_samples
assert np.allclose(probsums, occursums, atol=0.1)
def test_that_conditional_probability_agrees_with_alternative_computation():
grammar = Grammar(rulebooks)
tree = grammar.sample_tree(root=0)
word = tree.terminals
# compute the total likelihood of the letters, summed over all trees:
inner = grammar.compute_inside_probabilities(word)
word_likelihood = inner[0, -1, 0] # assume root head is 0
assert word_likelihood > 0
# compute the conditional probability of the tree given the letters:
conditional_logprob = grammar.logprob_tree(tree) - np.log(word_likelihood)
conditional_prob = np.exp(conditional_logprob)
# perform the same computation using the method:
by_method = grammar.conditional_prob_tree(tree, root=0)
assert np.isclose(conditional_prob, by_method)
def test_that_parameter_estimation_is_consistent():
grammar = Grammar(rulebooks)
sumemits = np.zeros_like(grammar.emissions)
sumtrans = np.zeros_like(grammar.transitions)
for _ in range(400):
tree = grammar.sample_tree(root=0)
word = tree.terminals
if len(word) > 25:
continue # introduces slight bias, but saves a ton of time
inside = grammar.compute_inside_probabilities(word)
outside = grammar.compute_outside_probabilities(inside, initial=0)
sumemits += grammar.sum_emission_probabilities(word, outside)
sumtrans += grammar.sum_transition_probabilities(inside, outside)
norms = np.sum(sumemits, axis=1) + np.sum(sumtrans, axis=(1, 2))
sumemits /= norms[:, None]
sumtrans /= norms[:, None, None]
assert np.allclose(grammar.emissions, sumemits, atol=0.1)
assert np.allclose(grammar.transitions, sumtrans, atol=0.1)
def test_that_tree_probs_agree_with_explicit_computations():
# define the grammar here to make sure it stays put:
frozen = {
0: {
(1, 1): 0.4,
(1, 2): 0.6
},
1: {
(1, 2): 0.2,
(2, 1): 0.3,
'a': 0.5
},
2: {
(1, 3): 0.1,
(1, 2): 0.2,
'b': 0.7
},
3: {
'c': 0.6,
'd': 0.4
}
}
grammar = Grammar(frozen)
tree1 = Tree(1, ["a"])
prob1 = grammar.prob_tree(tree1)
assert np.isclose(prob1, 0.5)
tree2 = Tree(2, ["b"])
prob2 = grammar.prob_tree(tree2)
assert np.isclose(prob2, 0.7)
tree3 = Tree(1, [tree1, tree2]) # 1 --> (1, 2)
prob3 = grammar.prob_tree(tree3)
assert np.isclose(prob3, 0.2 * prob1 * prob2)
tree4 = Tree(0, [tree1, tree3]) # 0 --> (1, 1)
prob4 = grammar.prob_tree(tree4)
assert np.isclose(prob4, 0.4 * prob1 * prob3)
def test_that_tree_prob_is_well_calibrated():
grammar = Grammar(rulebooks)
while True:
tree = grammar.sample_tree(root=0)
word = tree.terminals
if len(word) <= 10: # it takes forever with long words
break
inner = grammar.compute_inside_probabilities(word)
conditional_prob = grammar.conditional_prob_tree(tree, inner, root=0)
# empiricall estimate the conditional frequency of the tree:
sample = lambda: grammar.conditionally_sample_tree(word, inner, root=0)
num_samples = 400
empirical = np.mean([tree == sample() for _ in range(num_samples)])
# compute the standard deviation of the average, and cast a wide net:
std = 0.25 / num_samples**0.5
atol = 5.0 * std # 5 stds ~ about one in a million chance
# check that the empirical and theoretical frequencies roughly agree:
assert np.isclose(empirical, conditional_prob, atol=atol)
emissions = np.random.gamma(1.0, size=grammar.emissions.shape)
transitions /= np.sum(transitions, axis=(1, 2), keepdims=True)
emissions /= np.sum(emissions, axis=1, keepdims=True)
transitions *= 0.3
emissions *= 0.7
return Grammar(transitions=transitions,
emissions=emissions,
alphabet=grammar.alphabet)
if __name__ == "__main__":
reality = Grammar(rulebooks)
model = sample_random_grammar_like(reality)
for updatestep in range(100):
print("--- Update %s ---\n" % (updatestep + 1,))
# initialize counts with virtual observations:
sumemits = 10.0 * model.emissions.copy()
sumtrans = 10.0 * model.transitions.copy()
print("Training . . .")
true_likelihoods = []
model_likelihoods = []
for _ in tqdm(range(1000), leave=False, unit="words"):
actual_tree = reality.sample_tree(root=0)
emissions = np.random.gamma(1.0, size=(N, 2))
emissions /= np.sum(emissions, axis=1, keepdims=True)
# make sure that the typical sentence is short:
transitions *= 0.25
emissions *= 0.75
return Grammar(transitions=transitions,
emissions=emissions,
alphabet=[+1, -1])
if __name__ == "__main__":
true_grammar = Grammar(true_rulebooks)
estimated_grammar = create_random_grammar()
for updateidx in range(30):
print("--- Step %s ---" % (updateidx + 1))
# initialize accumulators with a few virtual observations:
sumtrans = 0.5 * estimated_grammar.transitions.copy()
sumemits = 0.5 * estimated_grammar.emissions.copy()
losses = []
for _ in tqdm(range(400), leave=False):
tree = true_grammar.sample_tree()
sentence = tree.terminals
if len(sentence) > 25: