def choose_m(n, c1, c2, sampler): p = 0.0 weights = [0] * (n + 1) for i in xrange(n): for x in c1: p += logprob.elog(sampler.posterior(x)) sampler.count(x) weights[n] = p for i in xrange(n): for x in c1: sampler.discount(x) p -= logprob.elog(sampler.posterior(x)) for x in c2: p += logprob.elog(sampler.posterior(x)) sampler.count(x) weights[n - i - 1] = p # discount all c2's for i in xrange(n): for x in c2: sampler.discount(x) for i in xrange(n + 1): weights[i] += lncr(n, i) weights = [logprob.eexp(w) for w in weights] #print(weights) return discrete(weights)
def rule_size_likelihood(self): result = 0.0 for rule, count in self.counts.iteritems(): l = rule_size(rule) f_n_l = float(self.rule_size_counts[l]) l_prob = rule_size_prob(rule) result += count * logprob.elog(float(count) * l_prob / f_n_l) return result
def write_viterbi_tree(self, hg, treefilename): treefile = open(treefilename, 'w') hg.set_semiring(hypergraph.SHORTEST_PATH) hg.set_functions( lambda edge: -elog( self.counter.get_prob(edge.rule, self.length_factor)), None, None) treefile.write(hg.root.best_paths()[0].tree_str()) treefile.close()
def compute_expected_counts(self, hg): hg.set_semiring(hypergraph.LOGPROB) hg.set_functions( lambda edge: elog( self.counter.get_prob(edge.rule, self.length_factor)), None, None) hg.inside() hg.outside() for edge in hg.edges(): self.counter.add_count(edge.rule, eexp(edge.posterior() - hg.root.inside))
def dp_likelihood(self): n = 0.0 counts = {} result = 0.0 alpha = FLAGS.alpha for r, count in self.counts.iteritems(): for _ in range(count): n_r = float(counts.get(r, 0.0)) result += logprob.elog((n_r + alpha * base(r)) / (n + alpha)) counts[r] = counts.get(r, 0.0) + 1.0 n += 1.0 return result
def compute_cobleu(self): if logger.level >= 1: logger.writeln() logger.writeln('----CoBLEU----') cobleu = 0 for n in range(self.max_n): if logger.level >= 1: logger.writeln('%s-gram: %s/%s' % \ (n+1, self.expected_clipped_counts[n], self.expected_counts[n])) precision = self.expected_clipped_counts[n] / self.expected_counts[ n] cobleu += logprob.elog(precision) cobleu /= self.max_n # brevity cobleu += min(0, 1 - self.ref_length / self.expected_counts[0]) if logger.level >= 1: logger.writeln('---- %s ----' % cobleu) return cobleu
def weight(self, rule): if is_glue(rule): return -elog(rule.feats[0]) else: return 0
def weight(self, rule): if is_ghkm(rule): return -elog(rule.feats[5]) else: return 0
def weight(self, rule): if is_phrase(rule): return -elog(rule.feats[3]) else: return 0
def weight(self, rule): return -elog(rule.feats[0])
def log(self, f): """return a function which is log f""" return lambda x: logprob.elog(f(x))