def best_match(i, text):
candidates = enumerate(reversed(cost[max(0, i - max_word):i]))
_pick_me = []
for k, cost_pr in candidates:
_pick_me.append((cost_pr + word_cost.get(correction(text[i - k - 1:i]), 9e999), k + 1))
cost_pr, k = min(tuple(_pick_me))
return cost_pr, k
def correct_match(match):
# print(match)
word = match.group()
word = delete_rule(word)
num_loc = [f.span() for f in re.finditer('[0-9]', word)]
chr_loc = [f.span() for f in re.finditer('[a-zA-Z]', word)]
if bool(num_loc) and len(word) < 5 and bool(chr_loc):
letters = ''.join(re.findall('[a-zA-Z]', word))
numbers = ''.join(re.findall('[0-9]', word))
num_end_loc = num_loc[-1][1]
if num_end_loc == len(word):
new_word = letters + numbers
return case_of(new_word)(correction(letters.lower()) + numbers)
else:
new_word = numbers + letters
return case_of(new_word)(numbers + correction(letters.lower()))
else:
return case_of(word)(correction(word.lower()))
def connect_split_word_main(word_need_connect):
def _max_total(_candidates_list):
total_score = 0
uni_candidate, bi_candidate = _candidates_list
uni_candidate = list(uni_candidate)
bi_candidate = list(bi_candidate)
if bi_candidate is not []:
for x in range(len(bi_candidate)):
if uni_score_dict[uni_candidate[x]]:
p_x = uni_score_dict[uni_candidate[x]]
p_xy = bi_score_dict[bi_candidate[x]]
p_y_x = p_xy / p_x
total_score += p_y_x
else:
total_score += 0
else:
total_score += uni_score_dict[uni_candidate[0]]
return total_score
words = word_need_connect.split(' ')
candidates_list = get_candidates_list(words)
candidate_group = []
candidate_dict = {}
for candi in tqdm(candidates_list):
candi_condition = []
new_candi = [correction(c.lower()) for c in candi]
candi_bi = list(ngrams(new_candi, 2))
candi_uni = list(ngrams(new_candi, 1))
candi_condition.append(tuple(candi_uni))
candi_condition.append(tuple(candi_bi))
candidate_group.append(candi_condition)
candidate_dict[tuple(candi_condition)] = new_candi
tokens = sentence_corpus_to_tokens()
uni_score_dict, bi_score_dict = get_score_dicts_from_tokens(tokens)
connected_words = ' '.join(candidate_dict[tuple(
max(candidate_group, key=_max_total))])
return connected_words
def segment_correct_text(text):
global word_cost
global max_word
global cost
cleaned_text = clean_words_in_dict()
word_cost = dict((k, np.log((i + 1) * np.log(len(cleaned_text)))) for i, k in enumerate(cleaned_text))
max_word = max(len(x) for x in cleaned_text)
cost = [0]
for i in range(1, len(text) + 1):
c, k = best_match(i, text)
cost.append(c)
out = []
i = len(text)
while i > 0:
c, k = best_match(i, text)
assert c == cost[i]
out.append(correction(text[i - k:i]))
i -= k
final = " ".join(reversed(out))
return final