def choose_m(n, c1, c2, sampler):
p = 0.0
weights = [0] * (n + 1)
for i in xrange(n):
for x in c1:
p += logprob.elog(sampler.posterior(x))
sampler.count(x)
weights[n] = p
for i in xrange(n):
for x in c1:
sampler.discount(x)
p -= logprob.elog(sampler.posterior(x))
for x in c2:
p += logprob.elog(sampler.posterior(x))
sampler.count(x)
weights[n - i - 1] = p
# discount all c2's
for i in xrange(n):
for x in c2:
sampler.discount(x)
for i in xrange(n + 1):
weights[i] += lncr(n, i)
weights = [logprob.eexp(w) for w in weights]
#print(weights)
return discrete(weights)
def rule_size_likelihood(self):
result = 0.0
for rule, count in self.counts.iteritems():
l = rule_size(rule)
f_n_l = float(self.rule_size_counts[l])
l_prob = rule_size_prob(rule)
result += count * logprob.elog(float(count) * l_prob / f_n_l)
return result
def write_viterbi_tree(self, hg, treefilename):
treefile = open(treefilename, 'w')
hg.set_semiring(hypergraph.SHORTEST_PATH)
hg.set_functions(
lambda edge: -elog(
self.counter.get_prob(edge.rule, self.length_factor)), None,
None)
treefile.write(hg.root.best_paths()[0].tree_str())
treefile.close()
def compute_expected_counts(self, hg):
hg.set_semiring(hypergraph.LOGPROB)
hg.set_functions(
lambda edge: elog(
self.counter.get_prob(edge.rule, self.length_factor)), None,
None)
hg.inside()
hg.outside()
for edge in hg.edges():
self.counter.add_count(edge.rule,
eexp(edge.posterior() - hg.root.inside))
def dp_likelihood(self):
n = 0.0
counts = {}
result = 0.0
alpha = FLAGS.alpha
for r, count in self.counts.iteritems():
for _ in range(count):
n_r = float(counts.get(r, 0.0))
result += logprob.elog((n_r + alpha * base(r)) / (n + alpha))
counts[r] = counts.get(r, 0.0) + 1.0
n += 1.0
return result
def compute_cobleu(self):
if logger.level >= 1:
logger.writeln()
logger.writeln('----CoBLEU----')
cobleu = 0
for n in range(self.max_n):
if logger.level >= 1:
logger.writeln('%s-gram: %s/%s' % \
(n+1,
self.expected_clipped_counts[n],
self.expected_counts[n]))
precision = self.expected_clipped_counts[n] / self.expected_counts[
n]
cobleu += logprob.elog(precision)
cobleu /= self.max_n
# brevity
cobleu += min(0, 1 - self.ref_length / self.expected_counts[0])
if logger.level >= 1:
logger.writeln('---- %s ----' % cobleu)
return cobleu
def weight(self, rule):
if is_glue(rule):
return -elog(rule.feats[0])
else:
return 0
def weight(self, rule):
if is_ghkm(rule):
return -elog(rule.feats[5])
else:
return 0
def weight(self, rule):
if is_phrase(rule):
return -elog(rule.feats[3])
else:
return 0
def weight(self, rule):
return -elog(rule.feats[0])
def log(self, f):
"""return a function which is log f"""
return lambda x: logprob.elog(f(x))
