def next_for_guessed_oov_token(self, prev_tags_set: set, lword: str,
guesser: HashSuffixGuesser):
rrr = dict()
tag_probs = dict()
guessed_tags = guesser.tag_log_probabilities(lword)
pruned_guessed_tags = self.prune_guessed_tags(guessed_tags)
for prev_tags in prev_tags_set:
for guess in pruned_guessed_tags:
emission_prob = guess[1]
tag = guess[0]
tag_trans_prob = self.model.compiled_data.tag_transition_model.log_prob(
prev_tags.token_list, tag)
apriori_prob = math.log(
self.model.compiled_data.apriori_tag_probs[tag])
tag_probs[tag] = (tag_trans_prob, emission_prob - apriori_prob)
rrr[prev_tags] = tag_probs
return rrr
def next_for_guessed_oov_token(self, prev_tags_set: set,
lword: str,
guesser: HashSuffixGuesser):
rrr = dict()
tag_probs = dict()
guessed_tags = guesser.tag_log_probabilities(lword)
pruned_guessed_tags = self.prune_guessed_tags(guessed_tags)
for prev_tags in prev_tags_set:
for guess in pruned_guessed_tags:
emission_prob = guess[1]
tag = guess[0]
tag_trans_prob = self.model.compiled_data.tag_transition_model.log_prob(
prev_tags.token_list, tag)
apriori_prob = math.log(self.model.compiled_data.apriori_tag_probs[tag])
tag_probs[tag] = (tag_trans_prob, emission_prob - apriori_prob)
rrr[prev_tags] = tag_probs
return rrr
