def next_for_guessed_oov_token(self, prev_tags_set: set, lword: str, guesser: HashSuffixGuesser): rrr = dict() tag_probs = dict() guessed_tags = guesser.tag_log_probabilities(lword) pruned_guessed_tags = self.prune_guessed_tags(guessed_tags) for prev_tags in prev_tags_set: for guess in pruned_guessed_tags: emission_prob = guess[1] tag = guess[0] tag_trans_prob = self.model.compiled_data.tag_transition_model.log_prob( prev_tags.token_list, tag) apriori_prob = math.log( self.model.compiled_data.apriori_tag_probs[tag]) tag_probs[tag] = (tag_trans_prob, emission_prob - apriori_prob) rrr[prev_tags] = tag_probs return rrr
def next_for_guessed_oov_token(self, prev_tags_set: set, lword: str, guesser: HashSuffixGuesser): rrr = dict() tag_probs = dict() guessed_tags = guesser.tag_log_probabilities(lword) pruned_guessed_tags = self.prune_guessed_tags(guessed_tags) for prev_tags in prev_tags_set: for guess in pruned_guessed_tags: emission_prob = guess[1] tag = guess[0] tag_trans_prob = self.model.compiled_data.tag_transition_model.log_prob( prev_tags.token_list, tag) apriori_prob = math.log(self.model.compiled_data.apriori_tag_probs[tag]) tag_probs[tag] = (tag_trans_prob, emission_prob - apriori_prob) rrr[prev_tags] = tag_probs return rrr