def latent_viterbi_derivation(self, debug=False):
manager = PyDerivationManager(self.grammar, self.nontMap)
manager.convert_chart_to_hypergraph(self.chart, self.disco_grammar, debug=False)
if debug:
manager.serialize(b'/tmp/my_debug_hypergraph.hg')
if isinstance(self.la, list):
la = self.la[0]
else:
la = self.la
vit_der = manager.latent_viterbi_derivation(0, la, self.grammar, debug=debug)
# if len(self.input) < 15 and not debug:
# for weight, der in self.k_best_derivation_trees():
# if der != vit_der:
# print(weight, der, vit_der)
# vit_der2 = self.latent_viterbi_derivation(debug=True)
# print("vit2", vit_der2)
# if vit_der2 != vit_der:
# print("first and second viterbi derivation differ")
# if vit_der2 == der:
# print("second viterbi derivation = 1-best-disco-dop derivation")
# print("##############################", flush=True)
# break
# # raise Exception("too much to read")
if vit_der is not None:
vit_der = LCFRSDerivationWrapper(vit_der)
return vit_der
def __projection_based_derivation_tree(self, la, variational=False, op=prod):
if self.nontMap is None:
print("A nonterminal map is required for weight projection based parsing!")
return None
manager = PyDerivationManager(self.grammar, self.nontMap)
manager.convert_chart_to_hypergraph(self.chart, self.disco_grammar, debug=False)
if self.grammarInfo is not None:
assert manager.is_consistent_with_grammar(self.grammarInfo)
manager.set_io_cycle_limit(200)
manager.set_io_precision(0.000001)
if not isinstance(la, list):
la = [la]
edge_weights = None
for l in la:
edge_weights_l = py_edge_weight_projection(l, manager, variational=variational, debug=self.debug,
log_mode=self.log_mode)
if edge_weights is None:
edge_weights = edge_weights_l
else:
if self.log_mode:
edge_weights = [w1 + w2 for w1, w2 in zip(edge_weights, edge_weights_l)]
else:
edge_weights = [op(w1, w2) for w1, w2 in zip(edge_weights, edge_weights_l)]
if self.debug:
nans = 0
infs = 0
zeros = 0
for weight in edge_weights:
if weight == float("nan"):
nans += 1
if weight == float("inf") or weight == float("-inf"):
infs += 1
if weight == 0.0:
zeros += 1
print("[", len(edge_weights), nans, infs, zeros, "]")
if len(edge_weights) < 100:
print(edge_weights)
der = manager.viterbi_derivation(0, edge_weights, self.grammar, op=op, log_mode=self.log_mode)
if der is None:
print("p", end="")
der = self.latent_viterbi_derivation(debug=self.debug)
if der is not None:
der = LCFRSDerivationWrapper(der)
if der is None:
_, der = next(self.k_best_derivation_trees())
return der
def test_individual_parsing_stages(self):
grammar = self.build_grammar()
for r in transform_grammar(grammar):
pprint(r)
rule_list = list(transform_grammar(grammar))
pprint(rule_list)
disco_grammar = Grammar(rule_list, start=grammar.start())
print(disco_grammar)
inp = ["a"] * 3
estimates = 'SXlrgaps', getestimates(disco_grammar, 40, grammar.start())
print(type(estimates))
chart, msg = parse(inp, disco_grammar, estimates=estimates)
print(chart)
print(msg)
chart.filter()
print("filtered chart")
print(disco_grammar.nonterminals)
print(type(disco_grammar.nonterminals))
print(chart)
# print(help(chart))
root = chart.root()
print("root", root, type(root))
print(chart.indices(root))
print(chart.itemstr(root))
print(chart.stats())
print("root label", chart.label(root))
print(root, chart.itemid1(chart.label(root), chart.indices(root)))
for i in range(1, chart.numitems() + 1):
print(i, chart.label(i), chart.indices(i), chart.numedges(i))
if True or len(chart.indices(i)) > 1:
for edge_num in range(chart.numedges(i)):
edge = chart.getEdgeForItem(i, edge_num)
if isinstance(edge, tuple):
print("\t", disco_grammar.nonterminalstr(chart.label(i)) + "[" + str(i) + "]", "->", ' '.join([disco_grammar.nonterminalstr(chart.label(j)) + "[" + str(j) + "]" for j in [edge[1], edge[2]] if j != 0]))
else:
print("\t", disco_grammar.nonterminalstr(chart.label(i)) + "[" + str(i) + "]", "->", inp[edge])
print(chart.getEdgeForItem(root, 0))
# print(lazykbest(chart, 5))
manager = PyDerivationManager(grammar)
manager.convert_chart_to_hypergraph(chart, disco_grammar, debug=True)
file = tempfile.mktemp()
print(file)
manager.serialize(bytes(file, encoding="utf-8"))
gi = PyGrammarInfo(grammar, manager.get_nonterminal_map())
sm = PyStorageManager()
la = build_PyLatentAnnotation_initial(grammar, gi, sm)
vec = py_edge_weight_projection(la, manager, variational=True, debug=True, log_mode=False)
print(vec)
self.assertEqual([1.0, 1.0, 1.0, 0.5, 0.5, 0.5, 0.5, 0.25, 0.25, 0.25, 0.25, 1.0], vec)
vec = py_edge_weight_projection(la, manager, variational=False, debug=True, log_mode=False)
print(vec)
self.assertEqual([1.0, 1.0, 1.0, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 1.0], vec)
der = manager.viterbi_derivation(0, vec, grammar)
print(der)
# print(disco_grammar.rulenos)
# print(disco_grammar.numrules)
# print(disco_grammar.lexicalbylhs)
# print(disco_grammar.lexicalbyword)
# print(disco_grammar.lexicalbynum)
# print(disco_grammar.origrules, type(disco_grammar.origrules))
# print(disco_grammar.numbinary)
# print(disco_grammar.numunary)
# print(disco_grammar.toid)
# print(disco_grammar.tolabel)
# print(disco_grammar.bitpar)
# striplabelre = re.compile(r'-\d+$')
# msg = disco_grammar.getmapping(None, None)
# disco_grammar.getrulemapping(disco_grammar, striplabelre)
# mapping = disco_grammar.rulemapping
# print(mapping)
# for idx, group in enumerate(mapping):
# print("Index", idx)
# for elem in group:
# print(grammar.rule_index(elem))
# for _, item in zip(range(20), chart.parseforest):
# edge = chart.parseforest[item]
# print(item, item.binrepr(), item.__repr__(), item.lexidx())
# print(type(edge))
for _ in range(5):
vec2 = py_edge_weight_projection(la, manager, debug=True, log_mode=True)
print(vec2)