def copied_code_from_translate_Akkadian(): train_texts, dev_texts, sign_to_id, tran_to_id, id_to_sign, id_to_tran = preprocess( ) # Run the HMM. hmm_train(train_texts, dev_texts) # lambda1, lambda2 = hmm_train(train_texts, dev_texts) (lambda1, lambda2, _, _, _, _) = load_object_from_file(Path("../output/hmm_model.pkl")) memm_path = Path("../output/memm_model.pkl") memm_from_file = load_object_from_file(memm_path) (logreg, vec, idx_to_tag_dict) = memm_from_file extra_decoding_arguments = build_extra_decoding_arguments(train_texts) #dump_object_to_file(predictor, "predictor") pred_path = Path("../output/predictor_lr_03_test_96_8.pkl") predictor_from_file = load_object_from_file(pred_path) #dump_object_to_file(model, "model") model_path = Path("../output/model_lr_03_test_96_8.pkl") model_from_file = load_object_from_file(model_path) return lambda1, lambda2, logreg, vec, idx_to_tag_dict, extra_decoding_arguments, sign_to_id, id_to_tran, \ predictor_from_file, model_from_file
def main(): """ Loads all models' learned data and open an interpreter for transliterating sentences of signs from input :return: nothing, never stops """ most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \ load_object_from_file(hmm_path) logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path) model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path) gamma1 = 0.4 gamma2 = 0.2 """ Sennacherib = "𒁹𒀭𒌍𒋀𒈨𒌍𒌷𒁀" """ while True: sentence = input("write here:") if sentence == "": continue overall_classifier(sentence, gamma1, gamma2, total_tokens, q_bi_counts, q_uni_counts, q, e, S, most_common_tag, possible_tags, lambda1, lambda2, logreg, vec, idx_to_tag_dict, predictor, model, id_to_tran, sign_to_id, True)
def check_results(train_texts, dev_texts, test_texts, sign_to_id, id_to_tran): """ Prints the accuracy of the trained biLSTM models :param train_texts: texts used for train :param dev_texts: texts used for dev :param test_texts: texts used for test :param sign_to_id: dictionary mapping signs to ids :param id_to_tran: dictionary mapping ids to transliterations :return: nothing """ predictor_from_file = load_object_from_file(predictor_path) model_from_file = load_object_from_file(model_path) print(compute_accuracy(train_texts, BiLSTM_predict, model_from_file, predictor_from_file, sign_to_id, id_to_tran)) print(compute_accuracy(dev_texts, BiLSTM_predict, model_from_file, predictor_from_file, sign_to_id, id_to_tran)) print(compute_accuracy(test_texts, BiLSTM_predict, model_from_file, predictor_from_file, sign_to_id, id_to_tran))
def biLSTM_train_and_test(corpora): """ Trains biLSTM model, stores all data and print the accuracy :return: nothing, stores everything in bilstm_model.pkl """ biLSTM_train_and_store(corpora) model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path) print(compute_accuracy(test_texts, BiLSTM_predict, model, predictor, sign_to_id, id_to_tran))
def memm_train_and_test(corpora): """ Trains MEMM model, stores all data and print the accuracy :return: nothing, stores everything in memm_model.pkl """ memm_train_and_store(corpora) logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path) print(compute_accuracy(test_texts, memm_greedy, logreg, vec, idx_to_tag_dict))
def hmm_train_and_test(corpora): """ Trains HMM model, stores all data and print the accuracy :return: nothing, stores everything in hmm_model.pkl """ hmm_train_and_store(corpora) most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \ load_object_from_file(hmm_path) print(compute_accuracy(test_texts, hmm_viterbi, total_tokens, q_bi_counts, q_uni_counts, q, e, S, most_common_tag, possible_tags, lambda1, lambda2))
def transliterate_bilstm_top3(sentence): """ Transliterate signs using biLSTM :param sentence: signs to be transliterated :return: 3 top transliterations of the sentence with their scores """ sentence = sanitize(sentence) model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path) tag_logits = predictor.predict(sentence_to_allen_format(sentence, sign_to_id, True))['tag_logits'] prediction1, prediction2, prediction3, score1, score2, score3 = logits_to_trans(tag_logits, model, id_to_tran) return list_to_tran(prediction1), list_to_tran(prediction2), list_to_tran(prediction3)
def transliterate_bilstm(sentence): """ Transliterate signs using biLSTM :param sentence: signs to be transliterated :return: transliteration of the sentence """ sentence = sanitize(sentence) model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path) tag_logits = predictor.predict(sentence_to_allen_format(sentence, sign_to_id, True))['tag_logits'] biLSTM_predicted_tags, _, _, _, _, _ = logits_to_trans(tag_logits, model, id_to_tran) return list_to_tran(biLSTM_predicted_tags)
def transliterate_memm(sentence): """ Transliterate signs using MEMM :param sentence: signs to be transliterated :return: transliteration of the sentence """ sentence = sanitize(sentence) logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path) MEMM_predicted_tags = memm_greedy(sentence_to_HMM_format(sentence), logreg, vec, idx_to_tag_dict) return list_to_tran(MEMM_predicted_tags)
def transliterate_memm(sentence): """ Transliterate signs using MEMM :param sentence: signs to be transliterated :return: transliteration of the sentence """ sentences = [sanitize(line) for line in sentence.splitlines() if len(sanitize(line)) > 0] logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path) MEMM_predicted_tags_list = [memm_greedy(sentence_to_HMM_format(s), logreg, vec, idx_to_tag_dict) for s in sentences] tran_list = [list_to_tran(MEMM_predicted_tags) for MEMM_predicted_tags in MEMM_predicted_tags_list] return ''.join(tran_list)
def transliterate_hmm(sentence): """ Transliterate signs using HMM :param sentence: signs to be transliterated :return: transliteration of the sentence """ sentence = sanitize(sentence) most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \ load_object_from_file(hmm_path) HMM_predicted_tags = hmm_viterbi(sentence_to_HMM_format(sentence), total_tokens, q_bi_counts, q_uni_counts, q, e, S, most_common_tag, possible_tags, lambda1, lambda2) return list_to_tran(HMM_predicted_tags)
def transliterate_hmm(sentence): """ Transliterate signs using HMM :param sentence: signs to be transliterated :return: transliteration of the sentence """ sentences = [sanitize(line) for line in sentence.splitlines() if len(sanitize(line)) > 0] most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \ load_object_from_file(hmm_path) HMM_predicted_tags_list = [hmm_viterbi(sentence_to_HMM_format(s), total_tokens, q_bi_counts, q_uni_counts, q, e, S, most_common_tag, possible_tags, lambda1, lambda2) for s in sentences] tran_list = [list_to_tran(HMM_predicted_tags) for HMM_predicted_tags in HMM_predicted_tags_list] return ''.join(tran_list)