def fn(W):
"Evaluate numerator and denominator of risk."
g = Hypergraph()
g.root = root
for e, [_, r, f] in list(E.items()):
p = LogVal(np.exp(f.dot(W).to_real()))
g.edge(Semiring1(p, p * r), *e)
B = g.inside(Semiring1.Zero)
Q = B[g.root]
return Q.p.to_real(), Q.r.to_real(), (Q.r / Q.p).to_real()
def brute_force(derivations, E):
"Brute-force enumeration method for computing (Z, rbar, sbar, tbar)."
Z = LogVal(0)
rbar = LogVal(0)
sbar = LogValVector()
tbar = LogValVector()
for d in derivations:
#print
#print 'Derivation:', d
rd = LogVal(0)
pd = LogVal(1)
sd = LogValVector()
for [x, [y, z]] in tree_edges(d):
(p, r, s) = E[x, y, z]
#print (x,y,z), tuple(x.to_real() for x in (p, r, s))
pd *= p
rd += r
sd += s
#print 'p=%s,r=%s,s=%s' % tuple(x.to_real() for x in (pd, rd, sd))
Z += pd
rbar += pd * rd
sbar += pd * sd
tbar += pd * rd * sd
return Semiring2(Z, rbar, sbar, tbar)
def sample(forest, B, v=None):
""" Sample from parse forest. """
if v is None:
v = forest.root
edges = forest.incoming[v]
if not edges:
# base case (leaf), nothing to sample
return v
# sample incoming edge, p(e|head) \propto edge.weight * (\prod_{b in e.body} beta[b])
Z = LogVal.Zero()
cs = []
for e in edges:
p = e.weight
for y in e.body:
p *= B[y]
Z += p
cs.append(Z.to_real())
# sample one of the incoming edges
i = np.array(cs).searchsorted(uniform(0, cs[-1]))
e = edges[i]
return Tree(v, [sample(forest, B, y) for y in e.body])
def _test_sample_tree(example, grammar, N):
# gold = {(X,I,K) for (X,I,K) in example.gold_items if (I,K) in example.nodes}
print()
_forest = parse_forest(example, grammar)
# apply temperature to grammar rules
forest = Hypergraph()
forest.root = _forest.root
for e in _forest.edges:
c = LogVal.Zero()
c.logeq(e.weight)
forest.edge(c, e.head, *e.body)
# run inside-outside
B, A = sum_product(forest)
Z = B[forest.root]
# compute marginals and recall from samples
# sample_recall = 0.0
m = defaultdict(float)
for _ in iterview(range(N)):
t = sample(forest, B)
for s in t.subtrees():
x = s.label()
m[x] += 1.0 / N
# xx = rename(grammar, x)
# sample_recall += (xx in gold) * 1.0 / N
# convert node names and marginalize-out time index
IO = defaultdict(float)
for x in forest.incoming:
IO[x] += (B[x] * A[x] / Z).to_real()
# check marginals
threshold = 1e-4
for x in IO:
(I, K, X, T) = x
if K - I > 1:
a = IO[x]
b = m[x]
if a > threshold or b > threshold:
print('[%s %s %8s, %s] %7.3f %7.3f' \
% (I, K, X, T, a, b))
assert abs(a - b) < 0.05
def One(cls):
return cls(LogVal.One(), LogVal.Zero())
def Zero(cls):
return cls(LogVal.Zero(), LogVal.Zero())
def One():
return Semiring1(LogVal.One(), LogVal.Zero())
def One():
return Semiring2(LogVal.One(), LogVal.Zero(), LogValVector(),
LogValVector())
def fdcheck(E, root, eps=1e-4):
"""Finite-difference approximation of gradient of numerator and denominator wrt
edge probability.
"""
def fn(W):
"Evaluate numerator and denominator of risk."
g = Hypergraph()
g.root = root
for e, [_, r, f] in list(E.items()):
p = LogVal(np.exp(f.dot(W).to_real()))
g.edge(Semiring1(p, p * r), *e)
B = g.inside(Semiring1.Zero)
Q = B[g.root]
return Q.p.to_real(), Q.r.to_real(), (Q.r / Q.p).to_real()
features = {k for [_, _, f] in E.values() for k in f}
W = LogValVector({k: LogVal(np.random.uniform(-1, 1)) for k in features})
# For gradient of risk we use <p, p*r, D[p], r*D[p]>, but my code computes
# <p, p*r, p*s, p*r*s>, so we pass in s=D[p]/p.
#
# D[p] = D[exp(f.dot(W))] = exp(s.dot(W))*D[f.dot(W)] = exp(s.dot(W))*f
#
# therefore D[p]/p = f
if 0:
E1 = {}
for e, [_, r, f] in list(E.items()):
p = LogVal(np.exp(f.dot(W).to_real()))
E1[e] = (p, r, f * p)
#S = secondorder_expectation_semiring(E, root)
from hypergraphs.insideout3 import inside_outside_speedup
khat, xhat = inside_outside_speedup(E1, root)
else:
E1 = {}
for e, [_, r, f] in list(E.items()):
p = LogVal(np.exp(f.dot(W).to_real()))
E1[e] = (p, r, f)
#S = secondorder_expectation_semiring(E, root)
from hypergraphs.insideout import inside_outside_speedup
khat, xhat = inside_outside_speedup(E1, root)
ad_Z = xhat.s
ad_rbar = xhat.t
Z = khat.p
rbar = khat.r
ad_risk = ad_rbar / Z - rbar * ad_Z / Z / Z
dd = []
for k in features:
was = W[k]
W.x[k] = was + LogVal(eps)
b_Z, b_rbar, b_risk = fn(W)
W.x[k] = was - LogVal(eps)
a_Z, a_rbar, a_risk = fn(W)
W.x[k] = was
fd_rbar = (b_rbar - a_rbar) / (2 * eps)
fd_Z = (b_Z - a_Z) / (2 * eps)
fd_risk = (b_risk - a_risk) / (2 * eps)
dd.append({
'key': k,
'ad_risk': ad_risk[k].to_real(),
'fd_risk': fd_risk,
'ad_Z': ad_Z[k].to_real(),
'fd_Z': fd_Z,
'ad_rbar': ad_rbar[k].to_real(),
'fd_rbar': fd_rbar
})
from arsenal.maths import compare
from pandas import DataFrame
df = DataFrame(dd)
compare(df.fd_Z, df.ad_Z, alphabet=df.key).show()
compare(df.fd_rbar, df.ad_rbar, alphabet=df.key).show()
compare(df.fd_risk, df.ad_risk, alphabet=df.key).show()
def small():
# Define the set of valid derivations.
D = [
Tree('(0,3)', [Tree('(0,2)', ['(0,1)', '(1,2)']), '(2,3)']),
Tree('(0,3)', ['(0,1)', Tree('(1,3)', ['(1,2)', '(2,3)'])]),
]
# Define the set of edges and the associated (p,r,s) values that are local
# to each edge.
E = {
('(0,3)', '(0,2)', '(2,3)'): (
LogVal(10),
LogVal(1),
LogValVector({
'023': LogVal(1),
'23': LogVal(1)
}),
),
('(0,2)', '(0,1)', '(1,2)'): (
LogVal(10),
LogVal(1),
LogValVector({'012': LogVal(1)}),
),
('(0,3)', '(0,1)', '(1,3)'): (
LogVal(20),
LogVal(1),
LogValVector({'013': LogVal(1)}),
),
('(1,3)', '(1,2)', '(2,3)'): (
LogVal(10),
LogVal(3),
LogValVector({
'123': LogVal(1),
'23': LogVal(1)
}),
),
('(0,1)', ): (LogVal(1), LogVal(0), LogValVector()),
('(1,2)', ): (LogVal(1), LogVal(0), LogValVector()),
('(2,3)', ): (LogVal(1), LogVal(0), LogValVector()),
}
# Define the root of all derivations hypergraph.
root = '(0,3)'
assert all(d.label() == root for d in D), \
'All derivations must have a common root node.'
assert all(isinstance(k, tuple) for k in E)
return root, D, E