def do_we_need_to_search_in_dictionary(dictionary, word): for each_word in dictionary: decomposed_dict = WordLetterProcessingHelperMethod.decompose_word_into_letters( each_word) decomposed_act = WordLetterProcessingHelperMethod.decompose_word_into_letters( word) norm_dict = WordLetterProcessingHelperMethod.normalize(decomposed_dict) norm_act = WordLetterProcessingHelperMethod.normalize(decomposed_act) if len(norm_dict) != len(norm_act): raise ValueError( "Bug Found In 'do_we_need_to_search_in_dictionary'") if sorted(norm_dict) == sorted(norm_act): return False for each_word in dictionary: decomposed_dict = WordLetterProcessingHelperMethod.decompose_word_into_letters( each_word) decomposed_act = WordLetterProcessingHelperMethod.decompose_word_into_letters( word) norm_dict = WordLetterProcessingHelperMethod.normalize(decomposed_dict) norm_act = WordLetterProcessingHelperMethod.normalize(decomposed_act) for x in range(0, len(norm_act)): # compare letters before last letter if x < (len(norm_act) - 1): if norm_dict[x] != norm_act[x]: # so diff is in first or middle letters break else: # so diff is in last letter so ignore it return False return True
def prepare_master_object(selected_sentence, rnn_op, exp_op, location, undiac_words): list_of_master_object = [] list_of_word_len = [] total_length = 0 letter_counter = 0 if len(rnn_op) != len(exp_op) != len(location): raise Exception("bug found in data") for each_word in undiac_words: if each_word != 'space': decomposed_word = WordLetterProcessingHelperMethod.decompose_word_into_letters( each_word) total_length += len(decomposed_word) list_of_word_len.append(total_length) master = MasterObject() for (each_rnn_char, each_exp_char, each_location) in (zip(rnn_op, exp_op, location)): if each_rnn_char == 'space': continue master.undiac_char = WordLetterProcessingHelperMethod.remove_diacritics_from_this( each_rnn_char) master.rnn_diac_char = each_rnn_char decomposed_result_1 = WordLetterProcessingHelperMethod.decompose_diac_char_into_char_and_diacritics\ (each_rnn_char) if len(decomposed_result_1) == 1: master.rnn_diac = '' elif len(decomposed_result_1) == 2 and decomposed_result_1[1] != u'ْ': master.rnn_diac = decomposed_result_1[1] elif len(decomposed_result_1) == 2 and decomposed_result_1[1] == u'ْ': master.rnn_diac = '' master.rnn_diac_char = master.undiac_char elif len(decomposed_result_1) == 3: master.rnn_diac = decomposed_result_1[1] + decomposed_result_1[2] master.exp_diac_char = each_exp_char decomposed_result_2 = WordLetterProcessingHelperMethod.decompose_diac_char_into_char_and_diacritics\ (each_exp_char) if len(decomposed_result_2) == 1: master.exp_diac = '' elif len(decomposed_result_2) == 2 and decomposed_result_2[1] != u'ْ': master.exp_diac = decomposed_result_2[1] elif len(decomposed_result_2) == 2 and decomposed_result_2[1] == u'ْ': master.exp_diac = '' master.exp_diac_char = master.undiac_char elif len(decomposed_result_2) == 3: master.exp_diac = decomposed_result_2[1] + decomposed_result_2[2] master.location_in_word = each_location master.location_in_sent = letter_counter master.sentence = selected_sentence index = 0 for each_length_index in range(0, len(list_of_word_len)): index = each_length_index if list_of_word_len[each_length_index] >= (letter_counter + 1): break master.undiac_word = undiac_words[index] if each_location == 'first' and list_of_word_len[index] != 1: master.has_next_char = True master.has_prev_char = False elif each_location == 'first' and len(list(master.undiac_word)) == 1: master.has_next_char = False master.has_prev_char = False elif each_location == 'middle': master.has_next_char = True master.has_prev_char = True elif each_location == 'last': master.has_next_char = False master.has_prev_char = True list_of_master_object.append(deepcopy(master)) letter_counter += 1 list_of_rnn_words = WordLetterProcessingHelperMethod.reform_word_from_version_2( list_of_master_object) st_range = 0 for each_number in list_of_word_len: en_range = each_number for index in range(st_range, en_range): try: list_of_master_object[index].rnn_diac_word = list_of_rnn_words[ 0] list_of_master_object[index].exp_diac_word = selected_sentence[ 0] except: x = 1 del list_of_rnn_words[0] del selected_sentence[0] st_range = en_range return list_of_master_object
def get_diac_version_with_smallest_dist(list_of_objects): list_of_actual_words_after_dictionary_correction = [] list_of_undiac_objects = WordLetterProcessingHelperMethod.remove_diacritics_from( list_of_objects) list_of_undiac_words = WordLetterProcessingHelperMethod.reform_word_from( list_of_undiac_objects) diacritized_rnn_op_words = WordLetterProcessingHelperMethod.reform_word_from( list_of_objects) if len(diacritized_rnn_op_words) != len(list_of_undiac_words): raise Exception( "error appeared in get_diac_version_with_smallest_dist") for each_corrected_word, each_un_diacritized_word in zip( diacritized_rnn_op_words, list_of_undiac_words): minimum_error = 100000000 dictionary_diacritized_words = DBHelperMethod.\ get_dictionary_all_diacritized_version_of(each_un_diacritized_word) if len(dictionary_diacritized_words) == 0: dictionary_diacritized_words.append(each_corrected_word) dictionary_diacritized_words_after_sukun_correction = SukunCorrection.\ sukun_correction_for_list_of_words(dictionary_diacritized_words) if do_we_need_to_search_in_dictionary( dictionary_diacritized_words_after_sukun_correction, each_corrected_word): for each_word in dictionary_diacritized_words_after_sukun_correction: error_count = 0 decomposed_dic_word = WordLetterProcessingHelperMethod.decompose_word_into_letters( each_word) decomposed_act_word = WordLetterProcessingHelperMethod.decompose_word_into_letters( each_corrected_word) norm_dic_word = WordLetterProcessingHelperMethod.normalize( decomposed_dic_word) norm_act_word = WordLetterProcessingHelperMethod.normalize( decomposed_act_word) for each_diacritized_version_letter, each_current_word_letter in zip( norm_dic_word, norm_act_word): if (len(each_diacritized_version_letter) - len(each_current_word_letter) == 1) or ((len(each_diacritized_version_letter) - len(each_current_word_letter) == -1)): error_count += 1 elif (len(each_diacritized_version_letter) - len(each_current_word_letter) == 2) or ((len(each_diacritized_version_letter) - len(each_current_word_letter) == -2)): error_count += 2 else: for each_item_in_diacritized_version, each_item_in_current_word in \ zip(each_diacritized_version_letter, each_current_word_letter): if each_item_in_diacritized_version != each_item_in_current_word: error_count += 1 if error_count < minimum_error: minimum_error = error_count selected_dictionary_word = each_word list_of_actual_words_after_dictionary_correction.append( selected_dictionary_word) else: list_of_actual_words_after_dictionary_correction.append( each_corrected_word) chars_after_dic_correction = WordLetterProcessingHelperMethod.convert_list_of_words_to_list_of_chars( list_of_actual_words_after_dictionary_correction) if len(list_of_objects) != len(chars_after_dic_correction): raise Exception("Error Happened Here") for x in range(0, len(list_of_objects)): list_of_objects[x].letter = chars_after_dic_correction[x] return list_of_objects
def get_diac_version_with_smallest_dist_no_db_access_version_2( master_object, dic_words): # rnn_diac_words = [each_object.rnn_diac_word for each_object in master_object] # rnn_diac_words = [x[0] for x in groupby(rnn_diac_words1)] # rnn_diac_words = master_object[0].sentence undiac_words = [] rnn_diac_words = [] for each_word in master_object[0].sentence: undiac_words.append( WordLetterProcessingHelperMethod.remove_diacritics_from_this_word( each_word)) for each_word in undiac_words: for each_object in master_object: if each_object.undiac_word == each_word: rnn_diac_words.append(each_object.rnn_diac_word) break #undiac_words = [each_object.undiac_word for each_object in master_object] #undiac_words = [x[0] for x in groupby(undiac_words)] selected_dictionary_word = '' selected_norm_dictionary_word = '' output = [] for each_word, undiac_word in zip(rnn_diac_words, undiac_words): rows, cols = np.where(dic_words == undiac_word) dictionary_diacritized_words = [] dictionary_diacritized_words = (dic_words[rows, 0]).tolist() # no dictionary data found if len(dictionary_diacritized_words) == 0: output.append(extract_data_in_req_format(WordLetterProcessingHelperMethod.normalize \ (WordLetterProcessingHelperMethod.decompose_word_into_letters( each_word)), each_word)) else: dict_words_after_sukun_correction = SukunCorrection. \ sukun_correction_for_list_of_words(dictionary_diacritized_words) if not (do_we_need_to_search_in_dictionary( dict_words_after_sukun_correction, each_word)): output.append(extract_data_in_req_format(WordLetterProcessingHelperMethod.normalize \ (WordLetterProcessingHelperMethod.decompose_word_into_letters(each_word)), each_word)) else: minimum_error = 100000000 for each_dic_word in dict_words_after_sukun_correction: error_count = 0 norm_dic_word = WordLetterProcessingHelperMethod.normalize \ (WordLetterProcessingHelperMethod.decompose_word_into_letters(each_dic_word)) norm_act_word = WordLetterProcessingHelperMethod.normalize \ (WordLetterProcessingHelperMethod.decompose_word_into_letters(each_word)) # unify last char because it depend on context norm_dic_word[-1] = norm_act_word[-1] for each_dic_letter, each_act_letter in zip( norm_dic_word, norm_act_word): if each_dic_letter[0] != each_act_letter[ 0] or each_dic_letter[1] != each_act_letter[1]: error_count += 1 if error_count < minimum_error: minimum_error = error_count selected_norm_dictionary_word = norm_dic_word selected_dictionary_word = each_dic_word output.append( extract_data_in_req_format(selected_norm_dictionary_word, selected_dictionary_word)) merged = list(itertools.chain(*output)) if len(merged) != len(master_object): Exception("bug found") for index, (each_merged_object) in enumerate(merged): master_object[index].rnn_diac_char = each_merged_object.letter master_object[index].rnn_diac = each_merged_object.diac master_object[index].rnn_diac_word = each_merged_object.word return master_object