def evaluate_testset(input_file, model_file):
sentences = merge_input_files(input_file)
eric = eric_nlp.Eric_nlp()
eric.load_model(model_file)
sp = depparse.init_stanza("en")
results = analyze_sentences(sentences, eric, sp)
list_to_file(
results, "D:\\Bachelor\\eric\\output\\testrestults_survey_8_to_11.txt")
def get_function_match(self, sentence):
if not self.ft:
print("ERROR: No model loaded for eric_nlp object")
return
if not self.stanza_pipeline:
self.stanza_pipeline = depparse.init_stanza(self.language)
#tuple (fct_id, tuple(similarity in percent, matched key_sentence_original, matched_keysentence_with_replaced_placeholders))
similarity_result = self.get_similarity_result(sentence)
similarity_result_similarity = similarity_result[1][0]
similarity_result_matched_sentence = similarity_result[1][1]
similarity_result_matched_sentence_preprocessed = similarity_result[1][
2]
print(
f" THRESHOLDS: {self.depparse_threshold} :: {self.deny_threshold}"
)
print(f" PASSED SENTENCE: '{sentence}'")
print(f"SIMILARITY CHOSE: {similarity_result}")
result = similarity_result[0]
if similarity_result_similarity < self.depparse_threshold:
tree = self.stanza_pipeline(sentence).sentences[0]
depparse_ranking = depparse.get_matching_dictionary_trees(
tree, self)
try:
if len(depparse_ranking) == 0:
depparse_result = ("none", [])
else:
#tuple (fct_id, matching tree template)
depparse_result = (depparse_ranking[0][0],
depparse_ranking[0][1])
result = depparse_result[0]
print(f" DEPPARSE CHOSE: {depparse_result}")
except:
print("could not work with depparse ranking:")
print(depparse_ranking)
quit()
else:
print(
f"only SIMILARITY: {similarity_result[0]} :: {similarity_result_similarity}"
)
depparse_result = ("none", [
f"No depparsing necessary; similarity >= depparse_threshold ({similarity_result_similarity} >= {self.depparse_threshold})"
])
if depparse_result[
0] == "none" and similarity_result_similarity < self.deny_threshold:
result = "none"
return result, similarity_result, depparse_result
def tester(kind,
eric,
models,
methods,
output_path,
output_name,
in_files,
merge_in_files,
stanza_pipeline="",
lang="en"):
if merge_in_files:
test_inputs = {f"{output_name}_merged": merge_input_files(in_files)}
else:
test_inputs = {
f"{output_name}_{x+1}": read_input_from_file(f)
for x, f in enumerate(in_files)
}
print("done reading files")
# test_inputs = {"sample": [
# ("whatif", "what would happen if they were in first class instead"),
# ("when", "when is a person died survived")
# ]}
if kind == "similarity":
for test_name, test_sentences in test_inputs.items():
print(f"similarity test: {test_name}")
for model in models:
if eric.model != model:
eric.load_model(model)
for method in methods:
similarity_tester(method, 10, eric,
f"{output_path}{test_name}.txt",
test_sentences)
elif kind == "depparse":
sp = depparse.init_stanza(
lang) if not stanza_pipeline else stanza_pipeline
for test_name, test_sentences in test_inputs.items():
print(f"depparse test: {test_name}")
depparse_tester(eric, sp, f"{output_path}{test_name}.txt",
test_sentences)
else:
print(f"unknown kind: {kind}")
def init_depparsing(self, language):
self.stanza_pipeline = depparse.init_stanza(language)
for row in data:
one_line = "".join(word.ljust(col_width) for word in row)
ret_val.append(one_line)
else:
for row in data:
one_line = "".join(word.center(col_width) for word in row)
ret_val.append(one_line)
return ret_val
def test_sentences_from_console(eric):
print("Please provide input:")
while True:
usr_in = input()
if usr_in.lower() in [
"exit", "exit()", "quit", "quit()", "end", "end()"
]:
break
else:
result = eric.map_to_eric_function(usr_in)
print(f"result: {result}")
if __name__ == "__main__":
model, method = "wiki.en.bin", "minkowski"
eric = Eric_nlp()
eric.method = method
eric.load_model(f"data\\{model}")
eric.stanza_pipeline = depparse.init_stanza("en")
test_sentences_from_console(eric)
def evaluate_fasttext_models(model_files, model_path, output_path, sentences):
eric = eric_nlp.Eric_nlp()
sp = depparse.init_stanza("en")
comparison_dict = {"model": "accuracy"}
for model in model_files:
print("=" * 20)
print(f"loading model: {model_path}{model}")
time_start = time.time()
eric.load_model(f"{model_path}{model}")
time_end = time.time()
model_load_string = f" Model Load Time: {time_end - time_start}s"
disable_print_function()
output, sent_count, correct_count, correct_ratio, confusion = analyze_sentences(
sentences, eric, sp, sendback_accuracy=True)
enable_print_function()
output.insert(8, model_load_string)
confusion_output = []
tab = "\t"
#v contains the counts how often k got predicted when v.key should have been predicted. so e.g. if result = predict, then v["predict"] has the true positives
column_names = "\t\t"
for k, v in confusion.items():
out_str = f"{k}{tab}"
column_names += f"{k}{tab}"
for k2, v2 in v.items():
out_str += f"{v2}{tab}"
confusion_output.append(out_str)
confusion_output.insert(0, column_names)
confusion_output.append("")
confusion_output.append("PRECISION AND RECALL PER FUNCTION")
for k, v in confusion.items():
confusion_output.append(f"[{k}]")
tp = confusion[k][k] #true positive
#tp_fp is sum of tp and the times k was predicted, when a different function was gold
tp_fp = 0
for k_predicted in v.values():
tp_fp += k_predicted
#tp_fn is the number of all the times k should have been predicte, i.e. number of correct predictions + all the times something else was predicted when it should have been k
tp_fn = 0
for row in confusion.values():
tp_fn += row[k]
#CALCULATE
precision = tp / tp_fp if tp_fp > 0 else "n/A"
recall = tp / tp_fn if tp_fn > 0 else "n/A"
f_measure = 2 * (precision * recall) / (
precision +
recall) if precision != "n/A" and recall != "n/A" else "n/A"
confusion_output.append(f"Precision: {precision}")
confusion_output.append(f" Recall: {recall}")
confusion_output.append(f"F-Measure: {f_measure}")
confusion_output.append("")
comparison_dict[model] = correct_ratio
list_to_file(output, f"{output_path}{model}.txt")
list_to_file(confusion_output, f"{output_path}{model}_confusion.txt")
print(f"{correct_count}/{sent_count} correct. {correct_ratio}%")
print()
print("COMPARISON RESULTS:")
for key, val in comparison_dict.items():
print(f"{key}\t\t{val}")
def similarity_depparse_combination(eric,
model,
method,
output_path,
output_name,
in_files,
merge_in_files,
stanza_pipeline="",
lang="en"):
sp = depparse.init_stanza(lang) if not stanza_pipeline else stanza_pipeline
eric = eric_nlp.Eric_nlp()
eric.method = method
print(f"eric model '{model}' loading")
eric.load_model(model)
print(f"eric model '{model}' loaded")
if merge_in_files:
test_inputs = {f"{output_name}_merged": merge_input_files(in_files)}
else:
test_inputs = {
f"{output_name}_{x+1}": read_input_from_file(f)
for x, f in enumerate(in_files)
}
print("done reading files")
print("\n")
all_out = []
correct_count = 0
correct_count_no_depparse = 0
wrong_but_denied_count = 0
for test_name, test_tuples in test_inputs.items():
print(f"{test_name}:")
for gold, sentence in test_tuples:
out = []
depparse_necessary = False
preprocessed_sentence = eric.preprocessing(sentence, "usr_input")
similarity_ranking = eric.get_similarity_result(
preprocessed_sentence, limit=5)
#tuple (fct_id, matching similarity)
similarity_result = (similarity_ranking[0][0],
similarity_ranking[0][1])
result = similarity_result[0]
if similarity_result[1] < eric.depparse_threshold:
depparse_necessary = True
preprocessed_depparse_sentence = eric.preprocessing(
sentence, "usr_input")
tree = sp(preprocessed_depparse_sentence).sentences[0]
depparse_ranking = depparse.get_matching_dictionary_trees(
tree, eric)
try:
if len(depparse_ranking) == 0:
depparse_result = ("none", [])
else:
#tuple (fct_id, matching tree template)
depparse_result = (depparse_ranking[0][0],
depparse_ranking[0][1])
result = depparse_result[0]
except:
print("could not work with depparse ranking:")
print(depparse_ranking)
quit()
else:
depparse_result = ("none", [
f"No depparsing necessary; similarity >= depparse_threshold ({similarity_result[1]} >= {eric.depparse_threshold})"
])
if depparse_result[0] == "none" and similarity_result[
1] < eric.deny_threshold:
result = "none"
if result == gold:
correctness = True
correctness_phrase = "CORRECT"
correct_count += 1
else:
correctness_phrase = "WRONG"
correctness = False
if result == "none":
wrong_but_denied_count += 1
out.extend([
"", f"{'/'*40}", f" sentence: {sentence}",
f" result: {result}", f" gold: {gold}",
f" correctness: {correctness_phrase}",
f" did depparsing: {depparse_necessary}",
f" simil-result: {similarity_result}",
f"depparse-result: {depparse_result}"
])
if depparse_necessary:
results_were_equal = "True" if depparse_result[
0] == similarity_result[0] else "False"
out.extend([f" results equal: {results_were_equal}"])
else:
if correctness:
correct_count_no_depparse += 1
all_out.extend(out)
for o in out:
print(o)
sentence_count = len(test_tuples)
correct_percent = correct_count * 100.0 / sentence_count
wrong_count = sentence_count - correct_count
wrong_percent = wrong_count * 100.0 / sentence_count
preface = [
f" CORRECT: {correct_count} / {sentence_count} ({correct_percent}%)",
f" WRONG: {wrong_count} / {sentence_count} ({wrong_percent}%)",
f"Correct without depparsing: {correct_count_no_depparse}",
f" Wrong but denied: {wrong_but_denied_count}", "\n"
]
preface.extend(all_out)
list_to_file(preface, f"{output_path}{test_name}.txt")