def consume(self, word): """Updates the current nodes by searching for all nodes which are reachable from the current nodes by a path consisting of any number of epsilons and exactly one ``word`` label. If there is no such arc, we set the predictor in an invalid state. In this case, all subsequent ``predict_next`` calls will return the empty set. Args: word (int): Word on an outgoing arc from the current node """ # Add all epsilon reachable states d = {} # Collect distances to nodes reachable by word for weight, node in self.cur_nodes: for arc in self.cur_fst.arcs(node): if arc.olabel == word: d[arc.nextstate] = d.get(arc.nextstate, []) + [ weight + self.weight_factor * w2f(arc.weight) ] # Add epsilon reachable states prev_d = {} while len(prev_d) != len(d): prev_d = copy.copy(d) for node, weights in prev_d.iteritems(): for arc in self.cur_fst.arcs(node): weight = utils.log_sum(weights) if arc.olabel == EPS_ID: d[arc.nextstate] = d.get(arc.nextstate, []) + [ weight + self.weight_factor * w2f(arc.weight) ] self.cur_nodes = [(utils.log_sum(weights), n) for n, weights in d.iteritems()]
def breakdown2score_bayesian_loglin(working_score, score_breakdown, full=False, prev_score=None): """Like bayesian combination scheme, but uses loglinear model combination rather than linear interpolation weights TODO: Implement incremental version of it, write weights into breakdowns. """ if not score_breakdown: return working_score acc = [] prev_alphas = [] # list of all alpha_i,k # Write priors to alphas for (p, w) in score_breakdown[0]: prev_alphas.append(np.log(w)) for pos in score_breakdown: # for each position in the hypothesis alphas = [] sub_acc = [] # for each predictor (p: p_k(w_i|h_i), w: prior p(k)) for k, (p, w) in enumerate(pos): alpha = prev_alphas[k] + p alphas.append(alpha) sub_acc.append(p + alpha) acc.append(utils.log_sum(sub_acc) - utils.log_sum(alphas)) prev_alphas = alphas return sum(acc)
def write_hypos(self, all_hypos, sen_indices): """Writes ngram files for each sentence in ``all_hypos``. Args: all_hypos (list): list of nbest lists of hypotheses sen_indices (list): List of sentence indices (0-indexed) Raises: OSError. If the directory could not be created IOError. If something goes wrong while writing to the disk """ _mkdir(self.path, "ngram") for sen_idx, hypos in zip(sen_indices, all_hypos): sen_idx += 1 total = utils.log_sum([hypo.total_score for hypo in hypos]) normed_scores = [hypo.total_score - total for hypo in hypos] ngrams = defaultdict(dict) # Collect ngrams for hypo_idx, hypo in enumerate(hypos): sen_eos = [utils.GO_ID] + hypo.trgt_sentence + [utils.EOS_ID] for pos in xrange(1, len(sen_eos) + 1): hist = sen_eos[:pos] for order in xrange(self.min_order, self.max_order + 1): ngram = ' '.join(map(str, hist[-order:])) ngrams[ngram][hypo_idx] = True with open(self.file_pattern % sen_idx, "w") as f: for ngram, hypo_indices in ngrams.iteritems(): ngram_score = np.exp( utils.log_sum([ normed_scores[hypo_idx] for hypo_idx in hypo_indices ])) f.write("%s : %f\n" % (ngram, min(1.0, ngram_score)))
def write_hypos(self, all_hypos, sen_indices): """Writes ngram files for each sentence in ``all_hypos``. Args: all_hypos (list): list of nbest lists of hypotheses sen_indices (list): List of sentence indices (0-indexed) Raises: OSError. If the directory could not be created IOError. If something goes wrong while writing to the disk """ _mkdir(self.path, "ngram") for sen_idx, hypos in zip(sen_indices, all_hypos): sen_idx += 1 total = utils.log_sum([hypo.total_score for hypo in hypos]) normed_scores = [hypo.total_score - total for hypo in hypos] ngrams = defaultdict(dict) # Collect ngrams for hypo_idx, hypo in enumerate(hypos): sen_eos = [utils.GO_ID] + hypo.trgt_sentence + [utils.EOS_ID] for pos in xrange(1, len(sen_eos) + 1): hist = sen_eos[:pos] for order in xrange(self.min_order, self.max_order + 1): ngram = ' '.join(map(str, hist[-order:])) ngrams[ngram][hypo_idx] = True with open(self.file_pattern % sen_idx, "w") as f: for ngram, hypo_indices in ngrams.iteritems(): ngram_score = np.exp(utils.log_sum( [normed_scores[hypo_idx] for hypo_idx in hypo_indices])) f.write("%s : %f\n" % (ngram, min(1.0, ngram_score)))
def breakdown2score_bayesian(working_score, score_breakdown, full=False, prev_score=None): """This realizes score combination following the Bayesian LM interpolation scheme from (Allauzen and Riley, 2011) Bayesian Language Model Interpolation for Mobile Speech Input By setting K=T we define the predictor weights according the score the predictors give to the current partial hypothesis. The initial predictor weights are used as priors. TODO could make more efficient use of ``working_score`` Args: working_score (float): Working combined score, which is the weighted sum of the scores in ``score_breakdown``. Not used. score_breakdown (list): Breakdown of the combined score into predictor scores full (bool): If True, reevaluate all time steps. If False, assume that this function has been called in the previous time step. Returns: float. Bayesian interpolated predictor scores """ if not score_breakdown or working_score == utils.NEG_INF: return working_score alphas = [np.log(w) for (_, w) in score_breakdown[0]] if full: acc = [] for pos in score_breakdown: # for each position in the hypothesis for k, (p, _) in enumerate(pos): alphas[k] += p alpha_part = utils.log_sum(alphas) scores = [ alphas[k] - alpha_part + p for k, (p, _) in enumerate(pos) ] acc.append(utils.log_sum(scores)) return sum(acc) else: if len(score_breakdown) == 1: scores = [np.log(w) + p for p, w in score_breakdown[0]] return utils.log_sum(scores) working_score = prev_score for k, (p, w) in enumerate(score_breakdown[-2]): alphas[k] = np.log(w) + p alpha_norm = alphas - utils.log_sum(alphas) scores = [ alpha_norm[k] + p for k, (p, w) in enumerate(score_breakdown[-1]) ] updated_breakdown = [(p, np.exp(alpha_norm[k])) for k, (p, w) in enumerate(score_breakdown[-1])] score_breakdown[-1] = updated_breakdown working_score += utils.log_sum(scores) return working_score
def _combine_posteriors_norm_reduced(self, non_zero_words, posteriors, unk_probs, pred_weights, top_n=0): """Combine predictor posteriors according the normalization scheme ``CLOSED_VOCAB_SCORE_NORM_REDUCED``. For more information on closed vocabulary predictor score normalization see the documentation on the ``CLOSED_VOCAB_SCORE_NORM_*`` vars. Args: non_zero_words (set): All words with positive probability posteriors: Predictor posterior distributions calculated with ``predict_next()`` unk_probs: UNK probabilities of the predictors, calculated with ``get_unk_probability`` pred_weights (list): Predictor weights top_n (int): Not implemented! Returns: combined,score_breakdown: like in ``apply_predictors()`` """ n_predictors = len(self.predictors) score_breakdown_raw = {} for trgt_word in non_zero_words: score_breakdown_raw[trgt_word] = [(utils.common_get( posteriors[idx], trgt_word, unk_probs[idx]), w) for idx, w in enumerate(pred_weights)] sums = [] for idx in range(n_predictors): sums.append(utils.log_sum([preds[idx][0] for preds in score_breakdown_raw.values()])) return self._combine_posteriors_with_renorm(score_breakdown_raw, sums)
def predict_next(self): """Uses the outgoing arcs from all current node to build up the scores for the next word. This method does not follow epsilon arcs: ``consume`` updates ``cur_nodes`` such that all reachable arcs with word ids are connected directly with a node in ``cur_nodes``. If there are multiple arcs with the same word, we use the log sum of the arc weights as score. Returns: dict. Set of words on outgoing arcs from the current node together with their scores, or an empty set if we currently have no active nodes or fst. """ scorelst = {} for weight, node in self.cur_nodes: for arc in self.cur_fst.arcs(node): if arc.olabel != EPS_ID: scorelst[arc.olabel] = scorelst.get(arc.olabel, []) + [ weight + self.weight_factor * w2f(arc.weight) ] scores = { word: utils.log_sum(weights) for word, weights in scorelst.iteritems() } return self.finalize_posterior(scores, self.use_weights, self.normalize_scores)
def predict_next(self): """Looks up ngram scores via self.scores. """ cur_hist_length = len(self.history) this_scores = [[] for _ in xrange(cur_hist_length + 1)] this_unk_scores = [[] for _ in xrange(cur_hist_length + 1)] for pos in xrange(len(self.scores)): this_scores[0].append(self.scores[pos]) this_unk_scores[0].append(self.unk_scores[pos]) acc = 0.0 for order, word in enumerate(self.history): if pos + order + 1 >= len(self.scores): break acc += utils.common_get(self.scores[pos + order], word, self.unk_scores[pos + order]) this_scores[order + 1].append(acc + self.scores[pos + order + 1]) this_unk_scores[order + 1].append(acc + self.unk_scores[pos + order + 1]) combined_scores = [] combined_unk_scores = [] for order, (scores, unk_scores) in enumerate(zip(this_scores, this_unk_scores)): if scores and order + 1 >= self.min_order: score_matrix = np.vstack(scores) combined_scores.append(logsumexp(score_matrix, axis=0)) combined_unk_scores.append(utils.log_sum(unk_scores)) if not combined_scores: self.cur_unk_score = 0.0 return {} self.cur_unk_score = sum(combined_unk_scores) return sum(combined_scores)
def _combine_posteriors_norm_reduced(self, non_zero_words, posteriors, unk_probs, pred_weights, top_n=0): """Combine predictor posteriors according the normalization scheme ``CLOSED_VOCAB_SCORE_NORM_REDUCED``. For more information on closed vocabulary predictor score normalization see the documentation on the ``CLOSED_VOCAB_SCORE_NORM_*`` vars. Args: non_zero_words (set): All words with positive probability posteriors: Predictor posterior distributions calculated with ``predict_next()`` unk_probs: UNK probabilities of the predictors, calculated with ``get_unk_probability`` pred_weights (list): Predictor weights top_n (int): Not implemented! Returns: combined,score_breakdown: like in ``apply_predictors()`` """ n_predictors = len(self.predictors) score_breakdown_raw = {} for trgt_word in non_zero_words: score_breakdown_raw[trgt_word] = [(utils.common_get( posteriors[idx], trgt_word, unk_probs[idx]), w) for idx, w in enumerate(pred_weights)] sums = [] for idx in xrange(n_predictors): sums.append(utils.log_sum([preds[idx][0] for preds in score_breakdown_raw.itervalues()])) return self._combine_posteriors_with_renorm(score_breakdown_raw, sums)
def predict_next(self): """Looks up ngram scores via self.scores. """ cur_hist_length = len(self.history) this_scores = [[] for _ in xrange(cur_hist_length+1)] this_unk_scores = [[] for _ in xrange(cur_hist_length+1)] for pos in xrange(len(self.scores)): this_scores[0].append(self.scores[pos]) this_unk_scores[0].append(self.unk_scores[pos]) acc = 0.0 for order, word in enumerate(self.history): if pos + order + 1 >= len(self.scores): break acc += utils.common_get( self.scores[pos + order], word, self.unk_scores[pos + order]) this_scores[order+1].append(acc + self.scores[pos + order + 1]) this_unk_scores[order+1].append( acc + self.unk_scores[pos + order + 1]) combined_scores = [] combined_unk_scores = [] for order, (scores, unk_scores) in enumerate(zip(this_scores, this_unk_scores)): if scores and order + 1 >= self.min_order: score_matrix = np.vstack(scores) combined_scores.append(logsumexp(score_matrix, axis=0)) combined_unk_scores.append(utils.log_sum(unk_scores)) if not combined_scores: self.cur_unk_score = 0.0 return {} self.cur_unk_score = sum(combined_unk_scores) return sum(combined_scores)
def _get_next_hypos_renorm(self, hypos, scores): """Get hypotheses of the next time step. Args: hypos (list): List of hypotheses scores (list): hypo scores with heuristic estimates Return: list. List with hypotheses. """ probs = (1.0 - self.smooth_factor) * np.exp( scores - utils.log_sum(scores)) \ + self.smooth_factor / float(len(scores)) lengths = [len(hypo.trgt_sentence) for hypo in hypos] logging.debug("%d candidates min_length=%d max_length=%d" % (len(lengths), min(lengths), max(lengths))) ngrams = [] for hypo in hypos: ngram_list = [] for order in xrange(self.min_order, self.max_order + 1): ngram_list.append( set([ " ".join( map(str, hypo.trgt_sentence[start:start + order])) for start in xrange(len(hypo.trgt_sentence)) ])) ngrams.append(ngram_list) exp_bleus = [] for hyp_ngrams, hyp_length in zip(ngrams, lengths): precisions = np.array([ self._compute_bleu(hyp_ngrams, ref_ngrams, hyp_length, ref_length) for ref_ngrams, ref_length in zip(ngrams, lengths) ]) exp_bleus.append(precisions * probs) next_hypos = [] if self.selection_strategy == 'oracle_bleu': for _ in xrange(min(self.beam_size, len(hypos))): idx = np.argmax(np.sum(exp_bleus, axis=1)) bleu = np.sum(exp_bleus[idx]) logging.debug("Selected (score=%f expected_bleu=%f): %s" % (scores[idx], bleu, hypos[idx].trgt_sentence)) hypos[idx].bleu = -bleu next_hypos.append(hypos[idx]) gained_bleus = exp_bleus[idx] for update_idx in xrange(len(exp_bleus)): exp_bleus[update_idx] = np.maximum(exp_bleus[update_idx], gained_bleus) else: # selection strategy 'bleu' total_exp_bleus = np.sum(exp_bleus, axis=1) for idx in utils.argmax_n(total_exp_bleus, self.beam_size): hypos[idx].bleu = total_exp_bleus[idx] next_hypos.append(hypos[idx]) logging.debug( "Selected (score=%f expected_bleu=%f): %s" % (scores[idx], hypos[idx].bleu, hypos[idx].trgt_sentence)) return next_hypos
def breakdown2score_bayesian(working_score, score_breakdown): """This realizes score combination following the Bayesian LM interpolation scheme from (Allauzen and Riley, 2011) Bayesian Language Model Interpolation for Mobile Speech Input By setting K=T we define the predictor weights according the score the predictors give to the current partial hypothesis. The initial predictor weights are used as priors. This function is designed to be assigned to the globals ``breakdown2score_partial`` or ``breakdown2score_full``. TODO could make more efficient use of ``working_score`` Args: working_score (float): Working combined score, which is the weighted sum of the scores in ``score_breakdown``. Not used. score_breakdown (list): Breakdown of the combined score into predictor scores Returns: float. Bayesian interpolated predictor scores """ if not score_breakdown: return working_score acc = [] prev_alphas = [] # list of all alpha_i,k # Write priors to alphas for (p, w) in score_breakdown[0]: prev_alphas.append(np.log(w)) for pos in score_breakdown: # for each position in the hypothesis alphas = [] sub_acc = [] # for each predictor (p: p_k(w_i|h_i), w: prior p(k)) for k, (p, w) in enumerate(pos): alpha = prev_alphas[k] + p alphas.append(alpha) sub_acc.append(p + alpha) acc.append(utils.log_sum(sub_acc) - utils.log_sum(alphas)) prev_alphas = alphas return sum(acc)
def breakdown2score_bayesian(working_score, score_breakdown): """This realizes score combination following the Bayesian LM interpolation scheme from (Allauzen and Riley, 2011) Bayesian Language Model Interpolation for Mobile Speech Input By setting K=T we define the predictor weights according the score the predictors give to the current partial hypothesis. The initial predictor weights are used as priors. This function is designed to be assigned to the globals ``breakdown2score_partial`` or ``breakdown2score_full``. TODO could make more efficient use of ``working_score`` Args: working_score (float): Working combined score, which is the weighted sum of the scores in ``score_breakdown``. Not used. score_breakdown (list): Breakdown of the combined score into predictor scores Returns: float. Bayesian interpolated predictor scores """ if not score_breakdown: return working_score acc = [] prev_alphas = [] # list of all alpha_i,k # Write priors to alphas for (p,w) in score_breakdown[0]: prev_alphas.append(np.log(w)) for pos in score_breakdown: # for each position in the hypothesis alphas = [] sub_acc = [] # for each predictor (p: p_k(w_i|h_i), w: prior p(k)) for k,(p,w) in enumerate(pos): alpha = prev_alphas[k] + p alphas.append(alpha) sub_acc.append(p + alpha) acc.append(utils.log_sum(sub_acc) - utils.log_sum(alphas)) prev_alphas = alphas return sum(acc)
def _get_next_hypos_renorm(self, hypos, scores): """Get hypotheses of the next time step. Args: hypos (list): List of hypotheses scores (list): hypo scores with heuristic estimates Return: list. List with hypotheses. """ probs = (1.0 - self.smooth_factor) * np.exp( scores - utils.log_sum(scores)) \ + self.smooth_factor / float(len(scores)) lengths = [len(hypo.trgt_sentence) for hypo in hypos] logging.debug("%d candidates min_length=%d max_length=%d" % (len(lengths), min(lengths), max(lengths))) ngrams = [] for hypo in hypos: ngram_list = [] for order in xrange(self.min_order, self.max_order+1): ngram_list.append(set([ " ".join(map(str, hypo.trgt_sentence[start:start+order])) for start in xrange(len(hypo.trgt_sentence))])) ngrams.append(ngram_list) exp_bleus = [] for hyp_ngrams, hyp_length in zip(ngrams, lengths): precisions = np.array([self._compute_bleu( hyp_ngrams, ref_ngrams, hyp_length, ref_length) for ref_ngrams, ref_length in zip(ngrams, lengths)]) exp_bleus.append(precisions * probs) next_hypos = [] if self.selection_strategy == 'oracle_bleu': for _ in xrange(min(self.beam_size, len(hypos))): idx = np.argmax(np.sum(exp_bleus, axis=1)) bleu = np.sum(exp_bleus[idx]) logging.debug("Selected (score=%f expected_bleu=%f): %s" % (scores[idx], bleu, hypos[idx].trgt_sentence)) hypos[idx].bleu = -bleu next_hypos.append(hypos[idx]) gained_bleus = exp_bleus[idx] for update_idx in xrange(len(exp_bleus)): exp_bleus[update_idx] = np.maximum(exp_bleus[update_idx], gained_bleus) else: # selection strategy 'bleu' total_exp_bleus = np.sum(exp_bleus, axis=1) for idx in utils.argmax_n(total_exp_bleus, self.beam_size): hypos[idx].bleu = total_exp_bleus[idx] next_hypos.append(hypos[idx]) logging.debug("Selected (score=%f expected_bleu=%f): %s" % (scores[idx], hypos[idx].bleu, hypos[idx].trgt_sentence)) return next_hypos
def finalize_posterior(self, scores, use_weights, normalize_scores): """This method can be used to enforce the parameters use_weights normalize_scores in predictors with dict posteriors. Args: scores (dict): unnormalized log valued scores use_weights (bool): Set to false to replace all values in ``scores`` with 0 (= log 1) normalize_scores: Set to true to make the exp of elements in ``scores`` sum up to 1""" if not scores: # empty scores -> pass through return scores if not use_weights: scores = dict.fromkeys(scores, 0.0) if normalize_scores: log_sum = utils.log_sum(scores.itervalues()) ret = {k: v - log_sum for k, v in scores.iteritems()} return ret return scores
def breakdown2score_bayesian(working_score, score_breakdown, full=False): """This realizes score combination following the Bayesian LM interpolation scheme from (Allauzen and Riley, 2011) Bayesian Language Model Interpolation for Mobile Speech Input By setting K=T we define the predictor weights according the score the predictors give to the current partial hypothesis. The initial predictor weights are used as priors. This function is designed to be assigned to the globals ``breakdown2score_partial`` or ``breakdown2score_full``. TODO could make more efficient use of ``working_score`` Args: working_score (float): Working combined score, which is the weighted sum of the scores in ``score_breakdown``. Not used. score_breakdown (list): Breakdown of the combined score into predictor scores full (bool): If True, reevaluate all time steps. If False, assume that this function has been called in the previous time step. Returns: float. Bayesian interpolated predictor scores """ if not score_breakdown or working_score == NEG_INF: return working_score if full: acc = [] alphas = [] # list of all alpha_i,k # Write priors to alphas for (p, w) in score_breakdown[0]: alphas.append(np.log(w)) for pos in score_breakdown: # for each position in the hypothesis for k, (p, w) in enumerate(pos): alphas[k] += p alpha_part = utils.log_sum(alphas) scores = [alphas[k] - alpha_part + p for k, (p, w) in enumerate(pos)] acc.append(utils.log_sum(scores)) return sum(acc) else: # Incremental: Alphas are in predictor weights if len(score_breakdown) == 1: scores = [np.log(w) + p for p, w in score_breakdown[0]] return utils.log_sum(scores) priors = [s[1] for s in score_breakdown[0]] last_score = sum([w * s[0] for w, s in zip(priors, score_breakdown[-1])]) working_score -= last_score # Now, working score does not include the last time step anymore # Compute updated alphas alphas = [np.log(p) for p in priors] for pos in score_breakdown[:-1]: for k, (p, _) in enumerate(pos): alphas[k] += p alpha_part = utils.log_sum(alphas) scores = [alphas[k] - alpha_part + p for k, (p, w) in enumerate(score_breakdown[-1])] updated_breakdown = [(p, np.exp(alphas[k] - alpha_part)) for k, (p, w) in enumerate(score_breakdown[-1])] score_breakdown[-1] = updated_breakdown working_score += utils.log_sum(scores) return working_score
def breakdown2score_bayesian_state_dependent(working_score, score_breakdown, full=False, prev_score=None, lambdas=None): """This realizes score combination following the Bayesian LM interpolation scheme from (Allauzen and Riley, 2011) Bayesian Language Model Interpolation for Mobile Speech Input By setting K=T we define the predictor weights according the score the predictors give to the current partial hypothesis. The initial predictor weights are used as priors . Unlike breakdown2score_bayesian, define state-independent weights which affect how much state-dependent mixture weights (alphas) are affected by scores from the other model. Makes more efficient use of working_score and calculated priors when used incrementally. Args: working_score (float): Working combined score, which is the weighted sum of the scores in ``score_breakdown``. Not used. score_breakdown (list): Breakdown of the combined score into predictor scores full (bool): If True, reevaluate all time steps. If False, assume that this function has been called in the previous time step. prev_score: score of hypothesis without final step lambdas: np array of domain-task weights Returns: float. Bayesian interpolated predictor scores """ if not score_breakdown or working_score == utils.NEG_INF: return working_score if full: acc = [] alphas = [np.log(w) for (_, w) in score_breakdown[0]] for pos in score_breakdown: # for each position in the hypothesis for k, (p_k, _) in enumerate(pos): alphas[k] += p_k alpha_prob = np.exp(alphas - utils.log_sum(alphas)) alpha_prob_lambdas = np.zeros_like(alpha_prob) for k in range(len(alpha_prob)): for t in range(len(alpha_prob)): alpha_prob_lambdas[k] += alpha_prob[t] * lambdas[k, t] scores = [np.log(alpha_prob_lambdas[k]) + p for k, (p, _) in enumerate(pos)] acc.append(utils.log_sum(scores)) return sum(acc) else: if len(score_breakdown) == 1: scores = [np.log(w) + p for p, w in score_breakdown[0]] return utils.log_sum(scores) working_score = prev_score alphas = [np.log(w) for (_, w) in score_breakdown[-2]] for k, (p_k, _) in enumerate(score_breakdown[-2]): alphas[k] += p_k alpha_prob = np.exp(alphas - utils.log_sum(alphas)) alpha_prob_lambdas = np.zeros_like(alpha_prob) for k in range(len(alpha_prob)): for t in range(len(alpha_prob)): alpha_prob_lambdas[k] += alpha_prob[t] * lambdas[k, t] scores = [np.log(alpha_prob_lambdas[k]) + p for k, (p, _) in enumerate(score_breakdown[-1])] updated_breakdown = [(p, alpha_prob[k]) for k, (p, _) in enumerate(score_breakdown[-1])] score_breakdown[-1] = updated_breakdown working_score += utils.log_sum(scores) return working_score