def multipule_eval_for_liblinear(test_corpus_dir, feature_map_character, feature_map_numeric, feature_show, args):
num_docs_having_motif = {}
tfidf_flag = args.tfidf
exno = args.experiment_no
# 分類器が正しい判断をした回数を保存する．つまりCAP(gold motif tag, candidate by classifier)
num_of_correct_decision = {}
precision_sum = 0
recall_sum = 0
F_sum = 0
h_loss_sum = 0
subset_acc_sum = 0
ex_p_sum = 0
ex_r_sum = 0
ex_f_sum = 0
acc_sum = 0
classifier_return_1_sum = 0
for test_file in load_files(test_corpus_dir):
# ============================================================
try:
motif_stack, tokens_stack = file_loader_json_doc(test_file)
out_libsvm_format(tokens_stack, feature_map_character, feature_map_numeric, feature_show, tfidf_flag)
result_map = call_liblinear.eval_with_liblinear(exno)
gold_map = calc_p_r_f_old(
num_docs_having_motif,
result_map,
motif_stack,
num_of_correct_decision,
precision_sum,
recall_sum,
F_sum,
)
h_loss_sum = calc_h_loss(result_map, gold_map, h_loss_sum)
subset_acc_sum = calc_subset_acc(result_map, gold_map, subset_acc_sum)
ex_p_sum, ex_r_sum, ex_f_sum, acc_sum = calc_p_r_f(
result_map, gold_map, ex_p_sum, ex_r_sum, ex_f_sum, acc_sum
)
print "Predict Result", result_map
print "Gold", gold_map
print "Union", [label for label, value in result_map.items() if label in gold_map and value == 1]
print u"-" * 40
classifier_return_1_sum += get_the_num_of_1_classifier(result_map)
# ------------------------------------------------------------
except ValueError:
print "File decode error in some reason"
# ============================================================
num_of_files = len(load_files(test_corpus_dir))
h_loss = h_loss_sum / num_of_files
subset_acc = float(subset_acc_sum) / num_of_files
ex_p = ex_p_sum / num_of_files
ex_r = ex_r_sum / num_of_files
ex_f = ex_f_sum / num_of_files
acc = acc_sum / num_of_files
classifier_return_1 = float(classifier_return_1_sum) / num_of_files
print "-" * 30
print "RESULT for {} files classification".format(len(load_files(test_corpus_dir)))
print "Hamming Loss:{}".format(h_loss)
print "Subset Accuracy(classification accuracy):{}".format(subset_acc)
print "example-based precision:{} example-based recall:{} example-based F:{} accuracy:{}".format(
ex_p, ex_r, ex_f, acc
)
print "Average number of classifier which returns 1:{}".format(classifier_return_1)
def eval_on_single_file(
test_corpus_dir, feature_map_numeric, feature_map_character, tfidf_score_map, classifier_path_list, args
):
"""
RETURN void
"""
stop = args.stop
feature_show = args.feature_show
tfidf_flag = args.tfidf
exno = args.experiment_no
# ------------------------------------------------------------
test_filepath = test_corpus_dir
# ------------------------------------------------------------
if args.persian_test == True:
tokens_stack, motif_stack = file_loader(test_filepath, stop)
elif args.dutch_test == True:
tokens_stack, motif_stack = file_loader_dutch(test_filepath, stop)
test_data_list = unify_stack(tokens_stack, motif_stack)
# ------------------------------------------------------------
if args.arow == True:
# この関数はまだ未改修
out_libsvm_format(tokens_stack, feature_map_character, feature_map_numeric)
result_map = eval_with_arow()
elif args.liblinear == True:
out_libsvm_format(tokens_stack, feature_map_character, feature_map_numeric, feature_show, tfidf_flag)
result_map = call_liblinear.eval_with_liblinear(exno)
elif args.mulan == True:
call_mulan.out_mulan_file(test_data_list, feature_map_character, feature_map_numeric, tfidf_score_map, args)
model_type = "RAkEL"
arff_train = "../get_thompson_motif/classifier/mulan/exno{}.arff".format(exno)
modelsavepath = "../get_thompson_motif/classifier/mulan/exno{}.model".format(exno)
arff_test = "./arff_and_xml/test_{}.arff".format(args.experiment_no)
xml_file = "./arff_and_xml/test_{}.xml".format(args.experiment_no)
call_mulan.mulan_command(
model_type, arff_train, xml_file, arff_test, modelsavepath, args.experiment_no, args.reduce_method
)
sys.exit("Stil not implemented")
else:
# この関数はまだ未改修
test_matrix = construct_input_matrix(tokens_stack, feature_map_character, feature_map_numeric)
result_map = predict_labels(test_matrix, classifier_path_list)
# ------------------------------------------------------------
gold_map = {}
list_gold_cap_result = []
for gold_motif in motif_stack:
alphabet_label = gold_motif[0]
gold_map[alphabet_label] = 1
gold_cap_result = 0
for result_label in result_map:
if result_label in gold_map and result_map[result_label] == 1:
gold_cap_result += 1
list_gold_cap_result.append(result_label)
print "-" * 30
print "RESULT\nresult of classifiers:{}\ngold:{}\ncorrect estimation:{}\n".format(
result_map, gold_map, list_gold_cap_result
)
if 1 in result_map.values():
precision = float(gold_cap_result) / len([label for label in result_map.values() if label == 1])
else:
precision = 0
recall = float(gold_cap_result) / len(gold_map)
if not precision == 0 and not recall == 0:
F = float(2 * precision * recall) / (precision + recall)
else:
F = 0
print "Precision:{}\nRecall:{}\nF:{}".format(precision, recall, F)
