def validation_loss_calculation(filename, opt_thr, el_mode):
if args.predictions_folder is not None:
printPredictions.process_file(el_mode, filename, opt_thr)
ensemble_fixed = []
ensemble_acc = [] # final_scores and similarity_scores. all the rest are fixed
for model_num, model_folder in enumerate(args.output_folder): # for all ensemble models
model, handles = create_input_pipeline(el_mode, model_folder,
[filename])
retrieve_l = [model.final_scores, model.similarity_scores,
model.cand_entities_len, model.cand_entities,
model.begin_span, model.end_span, model.spans_len,
model.begin_gm, model.end_gm,
model.ground_truth, model.ground_truth_len,
model.words_len, model.chunk_id,
# model.similarity_scores,
model.words, model.chars, model.chars_len,
model.log_cand_entities_scores]
elem_idx = 0
while True:
try:
result_l = model.sess.run(
retrieve_l, feed_dict={model.input_handle_ph: handles[0], model.dropout: 1})
if model_num == 0:
ensemble_fixed.append(result_l[2:])
ensemble_acc.append(result_l[:2])
else:
ensemble_acc[elem_idx][0] += result_l[0]
ensemble_acc[elem_idx][1] += result_l[1]
elem_idx += 1
except tf.errors.OutOfRangeError:
break
model.close_session()
evaluator = Evaluator(opt_thr, name=filename)
number_of_models = len(args.output_folder)
for (final_scores, similarity_scores), fixed in zip(ensemble_acc, ensemble_fixed):
final_scores /= number_of_models
similarity_scores /= number_of_models
metrics_calculation_and_prediction_printing(evaluator,
final_scores, similarity_scores, *fixed, el_mode,
printPredictions=printPredictions)
if printPredictions:
printPredictions.file_ended()
print(filename)
micro_f1, macro_f1 = evaluator.print_log_results(None, -1, el_mode)
return macro_f1
def validation_loss_calculation(model, iterator, dataset_handle, opt_thr, el_mode, name=""):
if args.print_predictions:
printPredictions.process_file(el_mode, name, opt_thr)
model.sess.run(iterator.initializer)
evaluator = Evaluator(opt_thr, name=name)
while True:
try:
retrieve_l = [model.final_scores,
model.cand_entities_len, model.cand_entities,
model.begin_span, model.end_span, model.spans_len,
model.begin_gm, model.end_gm,
model.ground_truth, model.ground_truth_len,
model.words_len, model.chunk_id,
model.words, model.chars, model.chars_len]
scores_retrieve_l, scores_names_l = [], []
if model.args.nn_components.find("lstm") != -1:
scores_retrieve_l.append(model.similarity_scores)
scores_names_l.append("lstm")
if model.args.nn_components.find("pem") != -1:
scores_retrieve_l.append(model.log_cand_entities_scores)
scores_names_l.append("logpem")
if model.args.nn_components.find("attention") != -1:
scores_retrieve_l.append(model.attention_scores)
scores_names_l.append("attention")
if model.args.nn_components.find("global") != -1:
scores_retrieve_l.append(model.final_scores_before_global)
scores_names_l.append("before_global")
scores_retrieve_l.append(model.global_voting_scores)
scores_names_l.append("global_voting")
global_pairwise_scores = []
if args.print_global_voters:
global_pairwise_scores.append(model.gmask)
global_pairwise_scores.append(model.pure_entity_embeddings)
retrieve_l.append(scores_retrieve_l)
retrieve_l.append(global_pairwise_scores)
result_l = model.sess.run(
retrieve_l, feed_dict={model.input_handle_ph: dataset_handle, model.dropout: 1})
metrics_calculation_and_prediction_printing(evaluator, *result_l, scores_names_l, el_mode,
printPredictions=printPredictions)
except tf.errors.OutOfRangeError:
if args.print_predictions:
printPredictions.file_ended()
print(name)
micro_f1, macro_f1 = evaluator.print_log_results(None, -1, el_mode)
break
return macro_f1