def copied_code_from_translate_Akkadian():
train_texts, dev_texts, sign_to_id, tran_to_id, id_to_sign, id_to_tran = preprocess(
)
# Run the HMM.
hmm_train(train_texts, dev_texts)
# lambda1, lambda2 = hmm_train(train_texts, dev_texts)
(lambda1, lambda2, _, _, _,
_) = load_object_from_file(Path("../output/hmm_model.pkl"))
memm_path = Path("../output/memm_model.pkl")
memm_from_file = load_object_from_file(memm_path)
(logreg, vec, idx_to_tag_dict) = memm_from_file
extra_decoding_arguments = build_extra_decoding_arguments(train_texts)
#dump_object_to_file(predictor, "predictor")
pred_path = Path("../output/predictor_lr_03_test_96_8.pkl")
predictor_from_file = load_object_from_file(pred_path)
#dump_object_to_file(model, "model")
model_path = Path("../output/model_lr_03_test_96_8.pkl")
model_from_file = load_object_from_file(model_path)
return lambda1, lambda2, logreg, vec, idx_to_tag_dict, extra_decoding_arguments, sign_to_id, id_to_tran, \
predictor_from_file, model_from_file
def main():
"""
Loads all models' learned data and open an interpreter for transliterating sentences of signs from input
:return: nothing, never stops
"""
most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \
load_object_from_file(hmm_path)
logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path)
model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path)
gamma1 = 0.4
gamma2 = 0.2
"""
Sennacherib = "𒁹𒀭𒌍𒋀𒈨𒌍𒌷𒁀"
"""
while True:
sentence = input("write here:")
if sentence == "":
continue
overall_classifier(sentence, gamma1, gamma2, total_tokens, q_bi_counts, q_uni_counts,
q, e, S, most_common_tag, possible_tags, lambda1, lambda2, logreg, vec, idx_to_tag_dict, predictor, model,
id_to_tran, sign_to_id, True)
def check_results(train_texts, dev_texts, test_texts, sign_to_id, id_to_tran):
"""
Prints the accuracy of the trained biLSTM models
:param train_texts: texts used for train
:param dev_texts: texts used for dev
:param test_texts: texts used for test
:param sign_to_id: dictionary mapping signs to ids
:param id_to_tran: dictionary mapping ids to transliterations
:return: nothing
"""
predictor_from_file = load_object_from_file(predictor_path)
model_from_file = load_object_from_file(model_path)
print(compute_accuracy(train_texts, BiLSTM_predict, model_from_file, predictor_from_file, sign_to_id, id_to_tran))
print(compute_accuracy(dev_texts, BiLSTM_predict, model_from_file, predictor_from_file, sign_to_id, id_to_tran))
print(compute_accuracy(test_texts, BiLSTM_predict, model_from_file, predictor_from_file, sign_to_id, id_to_tran))
def biLSTM_train_and_test(corpora):
"""
Trains biLSTM model, stores all data and print the accuracy
:return: nothing, stores everything in bilstm_model.pkl
"""
biLSTM_train_and_store(corpora)
model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path)
print(compute_accuracy(test_texts, BiLSTM_predict, model, predictor, sign_to_id, id_to_tran))
def memm_train_and_test(corpora):
"""
Trains MEMM model, stores all data and print the accuracy
:return: nothing, stores everything in memm_model.pkl
"""
memm_train_and_store(corpora)
logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path)
print(compute_accuracy(test_texts, memm_greedy, logreg, vec, idx_to_tag_dict))
def hmm_train_and_test(corpora):
"""
Trains HMM model, stores all data and print the accuracy
:return: nothing, stores everything in hmm_model.pkl
"""
hmm_train_and_store(corpora)
most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \
load_object_from_file(hmm_path)
print(compute_accuracy(test_texts, hmm_viterbi, total_tokens, q_bi_counts, q_uni_counts, q, e, S, most_common_tag,
possible_tags, lambda1, lambda2))
def transliterate_bilstm_top3(sentence):
"""
Transliterate signs using biLSTM
:param sentence: signs to be transliterated
:return: 3 top transliterations of the sentence with their scores
"""
sentence = sanitize(sentence)
model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path)
tag_logits = predictor.predict(sentence_to_allen_format(sentence, sign_to_id, True))['tag_logits']
prediction1, prediction2, prediction3, score1, score2, score3 = logits_to_trans(tag_logits, model, id_to_tran)
return list_to_tran(prediction1), list_to_tran(prediction2), list_to_tran(prediction3)
def transliterate_bilstm(sentence):
"""
Transliterate signs using biLSTM
:param sentence: signs to be transliterated
:return: transliteration of the sentence
"""
sentence = sanitize(sentence)
model, predictor, sign_to_id, id_to_tran, test_texts = load_object_from_file(bilstm_path)
tag_logits = predictor.predict(sentence_to_allen_format(sentence, sign_to_id, True))['tag_logits']
biLSTM_predicted_tags, _, _, _, _, _ = logits_to_trans(tag_logits, model, id_to_tran)
return list_to_tran(biLSTM_predicted_tags)
def transliterate_memm(sentence):
"""
Transliterate signs using MEMM
:param sentence: signs to be transliterated
:return: transliteration of the sentence
"""
sentence = sanitize(sentence)
logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path)
MEMM_predicted_tags = memm_greedy(sentence_to_HMM_format(sentence), logreg, vec, idx_to_tag_dict)
return list_to_tran(MEMM_predicted_tags)
def transliterate_memm(sentence):
"""
Transliterate signs using MEMM
:param sentence: signs to be transliterated
:return: transliteration of the sentence
"""
sentences = [sanitize(line) for line in sentence.splitlines() if len(sanitize(line)) > 0]
logreg, vec, idx_to_tag_dict, test_texts = load_object_from_file(memm_path)
MEMM_predicted_tags_list = [memm_greedy(sentence_to_HMM_format(s), logreg, vec, idx_to_tag_dict) for s in sentences]
tran_list = [list_to_tran(MEMM_predicted_tags) for MEMM_predicted_tags in MEMM_predicted_tags_list]
return ''.join(tran_list)
def transliterate_hmm(sentence):
"""
Transliterate signs using HMM
:param sentence: signs to be transliterated
:return: transliteration of the sentence
"""
sentence = sanitize(sentence)
most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \
load_object_from_file(hmm_path)
HMM_predicted_tags = hmm_viterbi(sentence_to_HMM_format(sentence), total_tokens, q_bi_counts, q_uni_counts, q, e,
S, most_common_tag, possible_tags, lambda1, lambda2)
return list_to_tran(HMM_predicted_tags)
def transliterate_hmm(sentence):
"""
Transliterate signs using HMM
:param sentence: signs to be transliterated
:return: transliteration of the sentence
"""
sentences = [sanitize(line) for line in sentence.splitlines() if len(sanitize(line)) > 0]
most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts = \
load_object_from_file(hmm_path)
HMM_predicted_tags_list = [hmm_viterbi(sentence_to_HMM_format(s), total_tokens, q_bi_counts, q_uni_counts, q, e,
S, most_common_tag, possible_tags, lambda1, lambda2) for s in sentences]
tran_list = [list_to_tran(HMM_predicted_tags) for HMM_predicted_tags in HMM_predicted_tags_list]
return ''.join(tran_list)
