def multipule_eval_for_liblinear(test_corpus_dir, feature_map_character, feature_map_numeric, feature_show, args): num_docs_having_motif = {} tfidf_flag = args.tfidf exno = args.experiment_no # 分類器が正しい判断をした回数を保存する.つまりCAP(gold motif tag, candidate by classifier) num_of_correct_decision = {} precision_sum = 0 recall_sum = 0 F_sum = 0 h_loss_sum = 0 subset_acc_sum = 0 ex_p_sum = 0 ex_r_sum = 0 ex_f_sum = 0 acc_sum = 0 classifier_return_1_sum = 0 for test_file in load_files(test_corpus_dir): # ============================================================ try: motif_stack, tokens_stack = file_loader_json_doc(test_file) out_libsvm_format(tokens_stack, feature_map_character, feature_map_numeric, feature_show, tfidf_flag) result_map = call_liblinear.eval_with_liblinear(exno) gold_map = calc_p_r_f_old( num_docs_having_motif, result_map, motif_stack, num_of_correct_decision, precision_sum, recall_sum, F_sum, ) h_loss_sum = calc_h_loss(result_map, gold_map, h_loss_sum) subset_acc_sum = calc_subset_acc(result_map, gold_map, subset_acc_sum) ex_p_sum, ex_r_sum, ex_f_sum, acc_sum = calc_p_r_f( result_map, gold_map, ex_p_sum, ex_r_sum, ex_f_sum, acc_sum ) print "Predict Result", result_map print "Gold", gold_map print "Union", [label for label, value in result_map.items() if label in gold_map and value == 1] print u"-" * 40 classifier_return_1_sum += get_the_num_of_1_classifier(result_map) # ------------------------------------------------------------ except ValueError: print "File decode error in some reason" # ============================================================ num_of_files = len(load_files(test_corpus_dir)) h_loss = h_loss_sum / num_of_files subset_acc = float(subset_acc_sum) / num_of_files ex_p = ex_p_sum / num_of_files ex_r = ex_r_sum / num_of_files ex_f = ex_f_sum / num_of_files acc = acc_sum / num_of_files classifier_return_1 = float(classifier_return_1_sum) / num_of_files print "-" * 30 print "RESULT for {} files classification".format(len(load_files(test_corpus_dir))) print "Hamming Loss:{}".format(h_loss) print "Subset Accuracy(classification accuracy):{}".format(subset_acc) print "example-based precision:{} example-based recall:{} example-based F:{} accuracy:{}".format( ex_p, ex_r, ex_f, acc ) print "Average number of classifier which returns 1:{}".format(classifier_return_1)
def eval_on_single_file( test_corpus_dir, feature_map_numeric, feature_map_character, tfidf_score_map, classifier_path_list, args ): """ RETURN void """ stop = args.stop feature_show = args.feature_show tfidf_flag = args.tfidf exno = args.experiment_no # ------------------------------------------------------------ test_filepath = test_corpus_dir # ------------------------------------------------------------ if args.persian_test == True: tokens_stack, motif_stack = file_loader(test_filepath, stop) elif args.dutch_test == True: tokens_stack, motif_stack = file_loader_dutch(test_filepath, stop) test_data_list = unify_stack(tokens_stack, motif_stack) # ------------------------------------------------------------ if args.arow == True: # この関数はまだ未改修 out_libsvm_format(tokens_stack, feature_map_character, feature_map_numeric) result_map = eval_with_arow() elif args.liblinear == True: out_libsvm_format(tokens_stack, feature_map_character, feature_map_numeric, feature_show, tfidf_flag) result_map = call_liblinear.eval_with_liblinear(exno) elif args.mulan == True: call_mulan.out_mulan_file(test_data_list, feature_map_character, feature_map_numeric, tfidf_score_map, args) model_type = "RAkEL" arff_train = "../get_thompson_motif/classifier/mulan/exno{}.arff".format(exno) modelsavepath = "../get_thompson_motif/classifier/mulan/exno{}.model".format(exno) arff_test = "./arff_and_xml/test_{}.arff".format(args.experiment_no) xml_file = "./arff_and_xml/test_{}.xml".format(args.experiment_no) call_mulan.mulan_command( model_type, arff_train, xml_file, arff_test, modelsavepath, args.experiment_no, args.reduce_method ) sys.exit("Stil not implemented") else: # この関数はまだ未改修 test_matrix = construct_input_matrix(tokens_stack, feature_map_character, feature_map_numeric) result_map = predict_labels(test_matrix, classifier_path_list) # ------------------------------------------------------------ gold_map = {} list_gold_cap_result = [] for gold_motif in motif_stack: alphabet_label = gold_motif[0] gold_map[alphabet_label] = 1 gold_cap_result = 0 for result_label in result_map: if result_label in gold_map and result_map[result_label] == 1: gold_cap_result += 1 list_gold_cap_result.append(result_label) print "-" * 30 print "RESULT\nresult of classifiers:{}\ngold:{}\ncorrect estimation:{}\n".format( result_map, gold_map, list_gold_cap_result ) if 1 in result_map.values(): precision = float(gold_cap_result) / len([label for label in result_map.values() if label == 1]) else: precision = 0 recall = float(gold_cap_result) / len(gold_map) if not precision == 0 and not recall == 0: F = float(2 * precision * recall) / (precision + recall) else: F = 0 print "Precision:{}\nRecall:{}\nF:{}".format(precision, recall, F)