def check_semirings(graph):
weights = [10.0] * len(graph.edges)
weights2 = [0.5] * len(graph.edges)
potentials = np.array(weights)
edge_marg = pydecode.marginals(graph, potentials,
weight_type=pydecode.Viterbi)
log_potentials = np.array(weights)
potentials = np.array(weights)
chart = pydecode.inside(graph, log_potentials)
chart2 = pydecode.inside(graph, potentials)
# Array-form.
for node in graph.nodes:
nt.assert_equal(chart[node.id], chart2[node.id])
# Matrix-form.
numpy.testing.assert_array_almost_equal(
chart, chart2, decimal=4)
marg = pydecode.marginals(graph, log_potentials)
marg2 = pydecode.marginals(graph, potentials)
for edge in graph.edges:
nt.assert_almost_equal(marg[edge.id], marg2[edge.id])
potentials = np.array(weights2)
def structure_path(self, graph, parse):
"""
Helper method for debugging. Checks that a graph contains parse.
Parameters
-----------
graph : hypergraph
parse : nltk.Tree
"""
parts = self.transform_structure(parse)
#labels = [self.encoder[part]
#print parts
label_weights = np.zeros(len(self.encoder)+20, dtype=np.int8)
for part in parts:
#assert
if not (tuple(part) in self.encoder): print part
# print part[-1], [self.grammar.nonterms[nt] for nt in self.grammar.rule_nonterms(part[-1])]
label_weights[self.encoder[tuple(part)]] = 1
weights = pydecode.transform(graph, label_weights)
part_set = set([self.encoder[tuple(part)] for part in parts])
for edge in graph.edges:
if edge.label == -1:
weights[edge.id] = 1
else:
if edge.label in part_set:
part_set.remove(edge.label)
bad_parts = self.transform_labels([part for part in part_set])
# print part_set, bad_parts, [self.grammar.rule_nonterms(p[-1]) for p in bad_parts]
#assert not part_set, [self.transform_labels([part for part in part_set])]
chart = pydecode.inside(graph, weights, weight_type=pydecode.Boolean)
for edge in graph.edges:
if edge.label != -1 and weights[edge.id] == 1 or edge.head.id == graph.root.id:
# print len(edge.tail), edge.label
for node in edge.tail:
# print node.id
if chart[node.id] != 1:
pass
# print self.transform_labels([edge.label])
# assert(False)
# assert chart[edge.head.id] == 1
# print chart
if not chart[graph.root.id]:
print "fail"
else:
print "good"
return
def structure_path(self, graph, parse):
"""
Helper method for debugging. Checks that a graph contains parse.
Parameters
-----------
graph : hypergraph
parse : nltk.Tree
"""
parts = self.transform_structure(parse)
#labels = [self.encoder[part]
#print parts
label_weights = np.zeros(len(self.encoder) + 20, dtype=np.int8)
for part in parts:
#assert
if not (tuple(part) in self.encoder): print part
# print part[-1], [self.grammar.nonterms[nt] for nt in self.grammar.rule_nonterms(part[-1])]
label_weights[self.encoder[tuple(part)]] = 1
weights = pydecode.transform(graph, label_weights)
part_set = set([self.encoder[tuple(part)] for part in parts])
for edge in graph.edges:
if edge.label == -1:
weights[edge.id] = 1
else:
if edge.label in part_set:
part_set.remove(edge.label)
bad_parts = self.transform_labels([part for part in part_set])
# print part_set, bad_parts, [self.grammar.rule_nonterms(p[-1]) for p in bad_parts]
#assert not part_set, [self.transform_labels([part for part in part_set])]
chart = pydecode.inside(graph, weights, weight_type=pydecode.Boolean)
for edge in graph.edges:
if edge.label != -1 and weights[
edge.id] == 1 or edge.head.id == graph.root.id:
# print len(edge.tail), edge.label
for node in edge.tail:
# print node.id
if chart[node.id] != 1:
pass
# print self.transform_labels([edge.label])
# assert(False)
# assert chart[edge.head.id] == 1
# print chart
if not chart[graph.root.id]:
print "fail"
else:
print "good"
return
def check_outside(graph, pot):
"""
Test outside chart properties.
"""
print graph
path = pydecode.best_path(graph, pot)
chart = pydecode.inside(graph, pot)
print pot.shape, path.v.shape
best = pot.T * path.v
print path.v
print best
nt.assert_almost_equal(best, chart[graph.root.id])
nt.assert_not_equal(best, 0.0)
out_chart = pydecode.outside(graph, pot, chart)
# Array-form
for vertex in graph.vertices:
other = chart[vertex.id] + out_chart[vertex.id]
nt.assert_less_equal(other, best + 1e-4,
"%f %f %d %f %f"%(other, best, vertex.id,
chart[vertex.id], out_chart[vertex.id]))
# Matrix-form
m = chart + out_chart
assert (m < best + 1e4).all()
# for node in graph.nodes:
# other = chart[node] * out_chart[node]
# nt.assert_less_equal(other, best + 1e-4)
print chart
print out_chart
for edge in path.edges:
for node in edge.tail:
if node.is_terminal:
other = out_chart[node.id]
nt.assert_almost_equal(other, best)
def special_decode(self, method, problem, hypergraph, scores, constraints,
scorer):
if method == "CUBE":
groups = [node.label.i for node in hypergraph.nodes]
ins = ph.inside(hypergraph, scores)
out = ph.outside(hypergraph, scores, ins)
beam_chart = ph.beam_search_BinaryVector(
hypergraph, scores, constraints.to_binary_potentials(),
out, -10000, groups, [1000] * len(groups), cube_pruning=True)
return beam_chart.path(0)
elif method == "BEAM":
groups = [node.label.i for node in hypergraph.nodes]
ins = ph.inside(hypergraph, scores)
out = ph.outside(hypergraph, scores, ins)
beam_chart = ph.beam_search_BinaryVector(
hypergraph, scores, constraints.to_binary_potentials(),
out, -10000, groups, [1000] * len(groups))
return beam_chart.path(0)
elif method == "MULTIDFA":
old = hypergraph
old_hmap = None
for j in range(problem.size):
states = 2
symbols = 2
dfa = ph.DFA(states, symbols, [{0:0, 1:1} , {0:1}], [1])
vec = [(1 if (edge.head.label.j == j) else 0)
for edge in old.edges]
counts = ph.CountingPotentials(old).from_vector(vec)
hmap = ph.extend_hypergraph_by_dfa(old, counts, dfa)
old = hmap.domain_hypergraph
old.labeling = ph.Labeling(old, [hmap[node].label
for node in old.nodes],
None)
#new_scores = old_scores.up_project(old, hmap)
if old_hmap is not None:
old_hmap = old_hmap.compose(hmap)
else:
old_hmap = hmap
# old_scores = new_scores
new_scores = scores.up_project(old, old_hmap)
#new_scores = self.potentials(old, scorer)
return ph.best_path(old, new_scores)
elif method == "BIGDFA":
old = hypergraph
states = 2**problem.size
symbols = problem.size + 1
final_state = 0
for i in range(problem.size):
final_state |= 2**i
transitions = \
[{j : i | 2**j for j in range(symbols) if i & 2**j == 0}
for i in range(states)]
dfa = ph.DFA(states, symbols,
transitions,
[final_state])
vec = [edge.head.label.j for edge in old.edges]
counts = ph.CountingPotentials(old).from_vector(vec)
hmap = ph.extend_hypergraph_by_dfa(old, counts, dfa)
old = hmap.domain_hypergraph
old.labeling = ph.Labeling(old, [hmap[node].label
for node in old.nodes],
None)
new_scores = scores.up_project(old, hmap)
return ph.best_path(old, new_scores)
def special_decode(self, method, problem, hypergraph, scores, constraints,
scorer):
if method == "CUBE":
groups = [node.label.i for node in hypergraph.nodes]
ins = ph.inside(hypergraph, scores)
out = ph.outside(hypergraph, scores, ins)
beam_chart = ph.beam_search_BinaryVector(
hypergraph,
scores,
constraints.to_binary_potentials(),
out,
-10000,
groups, [1000] * len(groups),
cube_pruning=True)
return beam_chart.path(0)
elif method == "BEAM":
groups = [node.label.i for node in hypergraph.nodes]
ins = ph.inside(hypergraph, scores)
out = ph.outside(hypergraph, scores, ins)
beam_chart = ph.beam_search_BinaryVector(
hypergraph, scores, constraints.to_binary_potentials(), out,
-10000, groups, [1000] * len(groups))
return beam_chart.path(0)
elif method == "MULTIDFA":
old = hypergraph
old_hmap = None
for j in range(problem.size):
states = 2
symbols = 2
dfa = ph.DFA(states, symbols, [{0: 0, 1: 1}, {0: 1}], [1])
vec = [(1 if (edge.head.label.j == j) else 0)
for edge in old.edges]
counts = ph.CountingPotentials(old).from_vector(vec)
hmap = ph.extend_hypergraph_by_dfa(old, counts, dfa)
old = hmap.domain_hypergraph
old.labeling = ph.Labeling(
old, [hmap[node].label for node in old.nodes], None)
#new_scores = old_scores.up_project(old, hmap)
if old_hmap is not None:
old_hmap = old_hmap.compose(hmap)
else:
old_hmap = hmap
# old_scores = new_scores
new_scores = scores.up_project(old, old_hmap)
#new_scores = self.potentials(old, scorer)
return ph.best_path(old, new_scores)
elif method == "BIGDFA":
old = hypergraph
states = 2**problem.size
symbols = problem.size + 1
final_state = 0
for i in range(problem.size):
final_state |= 2**i
transitions = \
[{j : i | 2**j for j in range(symbols) if i & 2**j == 0}
for i in range(states)]
dfa = ph.DFA(states, symbols, transitions, [final_state])
vec = [edge.head.label.j for edge in old.edges]
counts = ph.CountingPotentials(old).from_vector(vec)
hmap = ph.extend_hypergraph_by_dfa(old, counts, dfa)
old = hmap.domain_hypergraph
old.labeling = ph.Labeling(
old, [hmap[node].label for node in old.nodes], None)
new_scores = scores.up_project(old, hmap)
return ph.best_path(old, new_scores)
def check_inside(graph, pot):
"""
Test inside chart gen.
"""
inside = pydecode.inside(graph, pot)
__author__ = 'Superuser'
import pydecode
import numpy as np
n = 10
chart = np.zeros(10)
chart[0] = 0
chart[1] = chart[0] + 1
for item in range(2, n):
chart[item] = chart[item-1] + chart[item-2]
chart
chart = pydecode.ChartBuilder(np.arange(10))
chart.init(0)
chart.set(1, [[0]], labels=[0])
for item in range(2, n):
chart.set(item, [[item-1, item-2]])
graph = chart.finish()
weights = pydecode.transform(graph, np.array([1.0]))
inside = pydecode.inside(graph, weights)
inside
pydecode.draw(graph, np.array(["+"]*10), graph.node_labeling)
__author__ = 'Superuser'
import pydecode
import numpy as np
n = 10
chart = np.zeros(10)
chart[0] = 0
chart[1] = chart[0] + 1
for item in range(2, n):
chart[item] = chart[item - 1] + chart[item - 2]
chart
chart = pydecode.ChartBuilder(np.arange(10))
chart.init(0)
chart.set(1, [[0]], labels=[0])
for item in range(2, n):
chart.set(item, [[item - 1, item - 2]])
graph = chart.finish()
weights = pydecode.transform(graph, np.array([1.0]))
inside = pydecode.inside(graph, weights)
inside
pydecode.draw(graph, np.array(["+"] * 10), graph.node_labeling)