def circle_results(file): t = PrettyTable(['Description', 'Total', 'True Pos.', 'False Pos.', 'True Neg.', 'False Neg.', 'Precision', 'Sensitivity', 'Dice Coeff.', 'Jaccard Ind.', 'Jaccard Dist.']) f = open(file, 'r') for line in f: totals = line.split() description = totals[0] tp = totals[1] fp = totals[2] tn = totals[3] fn = totals[4] total = int(tp) + int(fp) + int(tn) + int(fn) precision = evaluation.precision(tp, fp) sensitivity = evaluation.sensitivity(tp, fn) # fmeasure = evaluation.fmeasure(tp, fp, fn) dicecoeff = evaluation.dicecoeff(tp, fp, fn) jaccardindex = evaluation.jaccardindex(tp, fp, fn) jaccarddifference = 1 - jaccardindex t.add_row([description, str(total), str(tp), str(fp), str(tn), str(fn), str(precision), str(sensitivity), str(dicecoeff), str(jaccardindex), str(jaccarddifference)]) print "Circle Detection\n" print t now = "\n" + str(datetime.now()) + "\n" data = t.get_string() r = open('circle_test_results.txt', 'ab') r.write(now) r.write(data) r.write("\n") r.close()
def evaluation(self, Tr_neg, Te, positions=[5, 10, 15]): from evaluation import precision from evaluation import recall from evaluation import nDCG prec = np.zeros(len(positions)) rec = np.zeros(len(positions)) ndcg = np.zeros(len(positions)) map_value, auc_value = 0.0, 0.0 for u in Te: val = np.dot(self.U[u, :], self.V.transpose()) inx = Tr_neg[u]["items"] A = set(Te[u]) B = set(inx) - A # compute precision and recall ii = np.argsort(val[inx])[::-1][:max(positions)] prec += precision(Te[u], inx[ii], positions) rec += recall(Te[u], inx[ii], positions) # ndcg += nDCGAtN(Te[u], inx[ii], 10) ndcg += np.array([nDCG(Te[u], inx[ii], p) for p in positions]) # compute map and AUC pos_inx = np.array(list(A)) neg_inx = np.array(list(B)) map_value += BPR.mean_average_precision(pos_inx, neg_inx, val) auc_value += BPR.auc_computation(pos_inx, neg_inx, val) return map_value, auc_value, ndcg, prec, rec
def predict_with_fixed_value(forward,comment,like,submission=True): # type check if isinstance(forward,int) and isinstance(forward,int) and isinstance(forward,int): pass else: raise TypeError("forward,comment,like should be type 'int' ") traindata,testdata = loadData() # score on the training set train_real_pred = traindata[['forward','comment','like']] train_real_pred['fp'],train_real_pred['cp'],train_real_pred['lp'] = forward,comment,like print ("Score on the training set:{0:.2f}%".format(precision(train_real_pred.values)*100)) # predict on the test data with fixed value, generate submission file if submission: test_pred = testdata[['uid','mid']] test_pred['fp'],test_pred['cp'],test_pred['lp'] = forward,comment,like result = [] filename = "weibo_predict_{}_{}_{}.txt".format(forward,comment,like) for _,row in test_pred.iterrows(): result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4])) f = open(filename,'w') f.writelines(result) f.close() print ('generate submission file "{}"'.format(filename))
def test_octavo( num_points, classes, xbound, ybound, zbound, max_depth, min_node_size, min_loss, expected, ): xy_parent = data_for_tests.make_octavo( num_points, classes, xbound, ybound, zbound ).values X = xy_parent[:, :-1] y = xy_parent[:, -1] forest = random_forest.grow_random_forest( X, y, num_trees=20, max_features=2, min_node_size=1 ) predictions = random_forest.forest_predict(forest, X) targets = y tfpns = evaluation.tfpn(predictions, targets) cm = evaluation.make_confusion_matrix(*tfpns, percentage=True) result = np.array( [evaluation.precision(cm), evaluation.sensitivity(cm), evaluation.fpr(cm)] ) expected = np.array(expected) assert np.any(np.abs(expected - result) < 0.01)
def evaluation(self, Tr_neg, Te, positions=[5, 10, 15]): prec = np.zeros(len(positions)) rec = np.zeros(len(positions)) map_value, auc_value, ndcg = 0.0, 0.0, 0.0 for u in Te: val = self.M[u, :] inx = Tr_neg[u]['items'] A = set(Te[u]) B = set(inx) - A # compute precision and recall ii = np.argsort(val[inx])[::-1][:max(positions)] prec += precision(Te[u], inx[ii], positions) rec += recall(Te[u], inx[ii], positions) ndcg_user = nDCG(Te[u], inx[ii], 10) # compute map and AUC pos_inx = np.array(list(A)) neg_inx = np.array(list(B)) map_user = mean_average_precision(pos_inx, neg_inx, val) auc_user = auc_computation(pos_inx, neg_inx, val) ndcg += ndcg_user map_value += map_user auc_value += auc_user # outf.write(" ".join([str(map_user), str(auc_user), str(ndcg_user)])+"\n") # outf.close() return map_value/len(Te.keys()), auc_value/len(Te.keys()), ndcg/len(Te.keys()), prec/len(Te.keys()), rec/len(Te.keys())
def evaluation(self, Tr_neg, Te, positions=[5, 10, 15]): prec = np.zeros(len(positions)) rec = np.zeros(len(positions)) map_value, auc_value, ndcg = 0.0, 0.0, 0.0 for u in Te: val = self.M[u, :] inx = Tr_neg[u]['items'] A = set(Te[u]) B = set(inx) - A # compute precision and recall ii = np.argsort(val[inx])[::-1][:max(positions)] prec += precision(Te[u], inx[ii], positions) rec += recall(Te[u], inx[ii], positions) ndcg_user = nDCG(Te[u], inx[ii], 10) # compute map and AUC pos_inx = np.array(list(A)) neg_inx = np.array(list(B)) map_user = mean_average_precision(pos_inx, neg_inx, val) auc_user = auc_computation(pos_inx, neg_inx, val) ndcg += ndcg_user map_value += map_user auc_value += auc_user # outf.write(" ".join([str(map_user), str(auc_user), str(ndcg_user)])+"\n") # outf.close() return map_value / len(Te.keys()), auc_value / len( Te.keys()), ndcg / len(Te.keys()), prec / len( Te.keys()), rec / len(Te.keys())
def test(test_model, test_data, test_labels, show_mistake=False): test_predictions = test_model.predict(test_data, verbose=0) # PRINT WRONG PREDICTIONS if show_mistake: for i in range(len(test_predictions)): stress_probability = test_predictions[i][1] score = abs(test_labels[i][1] - stress_probability) if score > 0: seq = "" for j in range(len(test_data[i])): seq += idx_to_word[test_data[i][j]].strip() + " " print(seq, ",", score, ",", test_labels[i][1], stress_probability) # TEST PERFORMANCE res_accu = eval.accuracy(test_predictions, test_labels) res_f1 = eval.fscore(test_predictions, test_labels) res_recall = eval.recall(test_predictions, test_labels) res_precision = eval.precision(test_predictions, test_labels) print('Test Accuracy: %.3f' % res_accu) print('Test F1-score: %.3f' % res_f1) print('Test Recall: %.3f' % res_recall) print('Test Precision: %.3f' % res_precision) return res_accu, res_f1, res_recall, res_precision
def predict_with_fixed_value(forward,comment,like,submission=True): # type check if isinstance(forward,int) and isinstance(forward,int) and isinstance(forward,int): pass else: raise TypeError("forward,comment,like should be type 'int' ") traindata,testdata = loadData() #score on the training set train_real_pred = traindata[['forward','comment','like']] train_real_pred['fp'],train_real_pred['cp'],train_real_pred['lp'] = forward,comment,like print "Score on the training set:{0:.2f}%".format(precision(train_real_pred.values)*100) #predict on the test data with fixed value, generate submission file if submission: test_pred = testdata[['uid','mid']] test_pred['fp'],test_pred['cp'],test_pred['lp'] = forward,comment,like result = [] filename = "weibo_predict_{}_{}_{}.txt".format(forward,comment,like) for _,row in test_pred.iterrows(): result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4])) f = open(filename,'w') f.writelines(result) f.close() print 'generate submission file "{}"'.format(filename)
def online_evaluate(gtmat, pred): pred_labels = torch.argmax(pred.cpu(), dim=1).long() gt_labels = gtmat.view(-1).cpu().numpy() pred_labels = pred_labels.numpy() acc = accuracy(gt_labels, pred_labels) pre = precision(gt_labels, pred_labels) rec = recall(gt_labels, pred_labels) return acc, pre, rec
def predict_with_stat(stat="median",submission=True): """ stat: string min,max,mean,median """ stat_dic = genUidStat() # 載入資料 并設定欄位 traindata,testdata = loadData() # get stat for each uid forward,comment,like = [],[],[] for uid in traindata['uid']: if uid in stat_dic: forward.append(int(stat_dic[uid]["forward_"+stat])) comment.append(int(stat_dic[uid]["comment_"+stat])) like.append(int(stat_dic[uid]["like_"+stat])) else: forward.append(0) comment.append(0) like.append(0) # score on the training set train_real_pred = traindata[['forward','comment','like']] train_real_pred['fp'],train_real_pred['cp'],train_real_pred['lp'] = forward,comment,like print ("Score on the training set:{0:.2f}%".format(precision(train_real_pred.values)*100)) #predict on the test data with fixed value, generate submission file if submission: test_pred = testdata[['uid','mid']] forward,comment,like = [],[],[] for uid in testdata['uid']: if uid in stat_dic: forward.append(int(stat_dic[uid]["forward_"+stat])) comment.append(int(stat_dic[uid]["comment_"+stat])) like.append(int(stat_dic[uid]["like_"+stat])) else: forward.append(0) comment.append(0) like.append(0) test_pred['fp'],test_pred['cp'],test_pred['lp'] = forward,comment,like result = [] filename = "weibo_predict_{}.txt".format(stat) for _,row in test_pred.iterrows(): result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4])) f = open(filename,'w') f.writelines(result) f.close() print ('generate submission file "{}"'.format(filename))
def score(uid_data,pred): """ uid_data: pd.DataFrame pred: list, [fp,cp,lp] """ uid_real_pred = uid_data[['forward','comment','like']] uid_real_pred['fp'] = pred[0] uid_real_pred['cp'] = pred[1] uid_real_pred['lp'] = pred[2] return precision(uid_real_pred.values)
def _get_best_thres_ary(self, pd_prob_mat, gt_mat): thres_ary = [] for k in xrange(len(self.lbs)): f_val_max = -np.inf best_thres = None for thres in np.arange(0., 1. + 1e-6, 0.01): prec = eva.precision(pd_prob_mat[:, k], gt_mat[:, k], thres) rec = eva.recall(pd_prob_mat[:, k], gt_mat[:, k], thres) f_val = eva.f_value(prec, rec) if f_val > f_val_max: f_val_max = f_val best_thres = thres thres_ary.append(best_thres) return thres_ary
def predict_with_stat(stat="median",submission=True): """ stat: string min,max,mean,median """ stat_dic = genUidStat() traindata,testdata = loadData() #get stat for each uid forward,comment,like = [],[],[] for uid in traindata['uid']: if stat_dic.has_key(uid): forward.append(int(stat_dic[uid]["forward_"+stat])) comment.append(int(stat_dic[uid]["comment_"+stat])) like.append(int(stat_dic[uid]["like_"+stat])) else: forward.append(0) comment.append(0) like.append(0) #score on the training set train_real_pred = traindata[['forward','comment','like']] train_real_pred['fp'],train_real_pred['cp'],train_real_pred['lp'] = forward,comment,like print "Score on the training set:{0:.2f}%".format(precision(train_real_pred.values)*100) #predict on the test data with fixed value, generate submission file if submission: test_pred = testdata[['uid','mid']] forward,comment,like = [],[],[] for uid in testdata['uid']: if stat_dic.has_key(uid): forward.append(int(stat_dic[uid]["forward_"+stat])) comment.append(int(stat_dic[uid]["comment_"+stat])) like.append(int(stat_dic[uid]["like_"+stat])) else: forward.append(0) comment.append(0) like.append(0) test_pred['fp'],test_pred['cp'],test_pred['lp'] = forward,comment,like result = [] filename = "weibo_predict_{}.txt".format(stat) for _,row in test_pred.iterrows(): result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4])) f = open(filename,'w') f.writelines(result) f.close() print 'generate submission file "{}"'.format(filename)
def predict_by_search(submission=True): traindata,testdata = loadData() uid_best_pred = search_all_uid() print "search done,now predict on traindata and testdata..." #predict traindata with uid's best fp,cp,lp forward,comment,like = [],[],[] for uid in traindata['uid']: if uid_best_pred.has_key(uid): forward.append(int(uid_best_pred[uid][0])) comment.append(int(uid_best_pred[uid][1])) like.append(int(uid_best_pred[uid][2])) else: forward.append(0) comment.append(0) like.append(0) #score on the traindata train_real_pred = traindata[['forward','comment','like']] train_real_pred['fp'],train_real_pred['cp'],train_real_pred['lp'] = forward,comment,like print "Score on the training set:{0:.2f}%".format(precision(train_real_pred.values)*100) if submission: test_pred = testdata[['uid','mid']] forward,comment,like = [],[],[] for uid in testdata['uid']: if uid_best_pred.has_key(uid): forward.append(int(uid_best_pred[uid][0])) comment.append(int(uid_best_pred[uid][1])) like.append(int(uid_best_pred[uid][2])) else: forward.append(0) comment.append(0) like.append(0) test_pred['fp'],test_pred['cp'],test_pred['lp'] = forward,comment,like #generate submission file result = [] filename = "weibo_predict_search.txt" for _,row in test_pred.iterrows(): result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4])) f = open(filename,'w') f.writelines(result) f.close() print 'generate submission file "{}"'.format(filename)
def test_diagonal_ndim(num_points, dim, max_features, expected, precision_bound): xy_parent = data_for_tests.make_diagonal_ndim(num_points, dim).values X = xy_parent[:, :-1] y = xy_parent[:, -1] forest = random_forest.grow_random_forest( X, y, num_trees=30, max_depth=20, max_features=max_features, min_node_size=1 ) predictions = random_forest.forest_predict(forest, X) targets = y tfpns = evaluation.tfpn(predictions, targets) cm = evaluation.make_confusion_matrix(*tfpns, percentage=True) result = np.array( [evaluation.precision(cm), evaluation.sensitivity(cm), evaluation.fpr(cm)] ) expected = np.array(expected) print(precision_bound) assert np.any(np.abs(expected - result) < precision_bound)
def evaluate_link(class_match_set, class_nonmatch_set, true_match_set, all_comparisons): # Linkage evaluation linkage_result = evaluation.confusion_matrix(class_match_set, class_nonmatch_set, true_match_set, all_comparisons) accuracy = evaluation.accuracy(linkage_result) precision = evaluation.precision(linkage_result) recall = evaluation.recall(linkage_result) fmeasure = evaluation.fmeasure(linkage_result) print('Linkage evaluation:') print(' Accuracy: %.6f' % (accuracy)) print(' Precision: %.6f' % (precision)) print(' Recall: %.6f' % (recall)) print(' F-measure: %.6f' % (fmeasure)) print('')
def evaluate(self): if self.model_used != 'vectoriel': self.popup(EVALUATION_ERR_MESSAGE, EVALUATION_ERR_INFO, EVALUATION_ERR_TITLE) self.rappelLabel.setText("Rappel :" + "-") self.precisionLabel.setText("Precision :" + "-") return retrived_documents = self.retrieved_doc pertinent_documents = [ item.text() for item in self.collectionItems if item.checkState() ] nb_pertinent_retrived_docu = evaluation.nb_pertinent_retrived_doc( retrived_documents, pertinent_documents) rappel = precision = None try: rappel = evaluation.rappel(nb_pertinent_retrived_docu, len(pertinent_documents)) except ZeroDivisionError: self.popup(NO_SELECTED_DOC_MESSAGE, NO_SELECTED_DOC_INFO, NO_SELECTED_DOC_TITLE) self.rappelLabel.setText("Rappel :" + "-") self.precisionLabel.setText("Precision :" + "-") try: precision = evaluation.precision(nb_pertinent_retrived_docu, len(retrived_documents)) except: self.popup(NO_DOCUMENTS_MESSAGE, NO_DOCUMENTS_INFO, NO_DOCUMENTS_TITLE) self.rappelLabel.setText("Rappel :" + "-") self.precisionLabel.setText("Precision :" + "-") if rappel != None and precision != None: self.rappelLabel.setText("Rappel :" + "%.2f" % rappel) self.precisionLabel.setText("Precision :" + "%.2f" % precision)
l_regr.fit(X_train, l_y_train) f_y_predict = f_regr.predict(X_test) c_y_predict = c_regr.predict(X_test) l_y_predict = l_regr.predict(X_test) # The coefficients print ("Coefficients: \n") print f_regr.coef_ print c_regr.coef_ print l_regr.coef_ # The mean square error print ("Residual sum of squares: %.2f" % np.mean((f_y_predict - f_y_test) ** 2)) print ("Residual sum of squares: %.2f" % np.mean((c_y_predict - c_y_test) ** 2)) print ("Residual sum of squares: %.2f" % np.mean((l_y_predict - l_y_test) ** 2)) # Explained variance score: 1 is perfect prediction print ("Variance score: %.2f" % f_regr.score(X_test, f_y_test)) print ("Variance score: %.2f" % c_regr.score(X_test, c_y_test)) print ("Variance score: %.2f" % l_regr.score(X_test, l_y_test)) predict_and_real = [] for i in range(0, test_num): predict_and_real.append([f_y_predict[i], c_y_predict[i], l_y_predict[i], f_y_test[i], c_y_test[i], l_y_test[i]]) predict_and_real0 = [] for i in range(0, test_num): predict_and_real0.append([0, 0, 0, f_y_test[i], c_y_test[i], l_y_test[i]]) print "Predict: {:.2f}%".format(precision(predict_and_real)) print "Zero baseline: {:.2f}%".format(precision(predict_and_real0))
l_sum2 += lc_list[i] ** 2 f_std = math.sqrt(f_sum2 / N - avg_map[uid][0] ** 2) c_std = math.sqrt(c_sum2 / N - avg_map[uid][1] ** 2) l_std = math.sqrt(l_sum2 / N - avg_map[uid][2] ** 2) std_map[uid] = [f_std, c_std, l_std] predict_and_real_med.append([med_map[uid][0], med_map[uid][1], med_map[uid][2], line[1], line[2], line[3]]) predict_and_real_avg.append([avg_map[uid][0], avg_map[uid][1], avg_map[uid][2], line[1], line[2], line[3]]) predict_and_real_std.append([med_map[uid][0], med_map[uid][1], med_map[uid][2], line[1], line[2], line[3]]) std_thres = 5 if std_map[uid][0] > std_thres and 0 < predict_and_real_std[-1][0]: predict_and_real_std[-1][0] += 5 if std_map[uid][1] > std_thres and 0 < predict_and_real_std[-1][1]: predict_and_real_std[-1][1] += 5 if std_map[uid][2] > std_thres and 0 < predict_and_real_std[-1][2]: predict_and_real_std[-1][2] += 5 predict_and_real_0_baseline.append([0, 0, 0, line[1], line[2], line[3]]) # print(predict_and_real_med[-1]) print "Median predict: {:.2f}%".format(precision(predict_and_real_med)) print "Average predict: {:.2f}%".format(precision(predict_and_real_avg)) print "Median and STD predict: {:.2f}%".format(precision(predict_and_real_std)) print "Zero baseline: {:.2f}%".format(precision(predict_and_real_0_baseline)) cursor.close() cnx.close() except mysql.connector.Error as err: print(err.msg)
def main(): # reading in import argparse parser = argparse.ArgumentParser() parser.add_argument("--data_dir", default='data/sampling', help='determine the base dir of the dataset document') parser.add_argument("--sample_n", default=1000, type=int, help='starting image index of preprocessing') parser.add_argument("--evidence_n", default=20, type=int, help='how many top/bottom tiles to pick from') parser.add_argument("--repl_n", default=3, type=int, help='how many resampled replications') parser.add_argument("--image_split", action='store_true', help='if use image_split') parser.add_argument("--batch_size", default=50, type=int, help="batch size") parser.add_argument("--stage_two", action='store_true', help='if only use stage two patients') parser.add_argument("--changhai", action='store_true', help='if use additional data') args = parser.parse_args() feature_size = 32 #gpu = "cuda:0" gpu = None # 5-folds cross validation dataloader = CVDataLoader(args, gpu, feature_size) n_epoch = 800 lr = 0.0005 if args.stage_two: weight_decay = 0.008 else: weight_decay = 0.005 manytimes_n = 8 if not os.path.isdir('figure'): os.mkdir('figure') if not os.path.isdir(os.path.join(args.data_dir, 'model')): os.mkdir(os.path.join(args.data_dir, 'model')) acc_folds = [] auc_folds = [] c_index_folds = [] f1_folds = [] f1_folds_pos = [] total_round = 0 model_count = 0 loss_function = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(0.8)) for _ in range(manytimes_n): # averaging for i in range(5): train_history = [] test_history = [] minimum_loss = None auc_fold = None acc_fold = None early_stop_count = 0 model = Predictor(evidence_size=args.evidence_n, layers=(100, 50, 1), feature_size=feature_size) # model.apply(weight_init) if gpu: model = model.to(gpu) optimizer = torch.optim.RMSprop(model.parameters(), lr=lr, weight_decay=weight_decay) # optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) dataloader.set_fold(i) X_test, Y_test, df_test = dataloader.get_test() # X_train, Y_train, df_train = dataloader.get_train() print('starting fold %d' % i) for epoch in range(n_epoch): #result = model(X_train) #loss = nn.functional.binary_cross_entropy(result, Y_train) + nn.functional.mse_loss(result, Y_train) # loss = nn.functional.mse_loss(result, Y_train) #loss.backward() #optimizer.step() #optimizer.zero_grad() # batch input for X_train_batch, Y_train_batch, df_train_batch in dataloader: # print(X_train_batch.shape) result = model(X_train_batch) loss = loss_function(result, Y_train_batch) loss.backward() optimizer.step() optimizer.zero_grad() X_train, Y_train, df_train = X_train_batch, Y_train_batch, df_train_batch if epoch % 20 == 0: result_test = model(X_test) loss_test = loss_function(result_test, Y_test) #loss_test = nn.functional.mse_loss(result_test, Y_test) acc_train, acc_test = accuracy(result, Y_train), accuracy( result_test, Y_test) auc_train, auc_test = auc(result, Y_train), auc( result_test, Y_test) if args.changhai: c_index_train, c_index_test = 0, 0 else: c_index_train, c_index_test = c_index( result, df_train), c_index(result_test, df_test) recall_train, recall_test = recall(result, Y_train), recall( result_test, Y_test) precision_train, precision_test = precision( result, Y_train), precision(result_test, Y_test) f1_train_pos, f1_test_pos = f1(result, Y_train), f1( result_test, Y_test) f1_train, f1_test = f1(result, Y_train, negative=True), f1(result_test, Y_test, negative=True) train_history.append( (epoch, loss, acc_train, auc_train, c_index_train)) test_history.append( (epoch, loss_test, acc_test, auc_test, c_index_test)) if epoch % 40 == 0: print( "%s epoch:%d loss:%.3f/%.3f acc:%.3f/%.3f auc:%.3f/%.3f c_index:%.3f/%.3f recall:%.3f/%.3f prec:%.3f/%.3f f1:%.3f/%.3f f1(neg):%.3f/%.3f" % (time.strftime( '%m.%d %H:%M:%S', time.localtime( time.time())), epoch, loss, loss_test, acc_train, acc_test, auc_train, auc_test, c_index_train, c_index_test, recall_train, recall_test, precision_train, precision_test, f1_train_pos, f1_test_pos, f1_train, f1_test)) # early stop if minimum_loss is None or minimum_loss * 0.995 > loss_test: # if minimum_loss is None or minimum_loss > loss_test: if f1_train == 0: continue minimum_loss = loss_test auc_fold = auc_test acc_fold = acc_test c_index_fold = c_index_test f1_fold_pos = f1_test_pos f1_fold = f1_test early_stop_count = 0 elif auc_test > auc_fold and auc_test > 0.5 and acc_test >= acc_fold: minimum_loss = loss_test auc_fold = auc_test acc_fold = acc_test c_index_fold = c_index_test f1_fold_pos = f1_test_pos f1_fold = f1_test early_stop_count = 0 else: early_stop_count += 1 if early_stop_count > 2 and epoch > 100: if args.stage_two: if auc_fold > 0.55: print('early stop at epoch %d' % epoch) break elif early_stop_count > 3: print('early stop at epoch %d' % epoch) break if epoch > 500: optimizer = torch.optim.RMSprop( model.parameters(), lr * 0.6, weight_decay=weight_decay * 1.2) train_history = np.array(train_history) test_history = np.array(test_history) acc_folds.append(acc_fold) auc_folds.append(auc_fold) f1_folds.append(f1_fold) f1_folds_pos.append(f1_fold_pos) c_index_folds.append(c_index_fold) plt.plot(train_history[:, 0], train_history[:, 1], label='train') plt.plot(test_history[:, 0], test_history[:, 1], label='test') plt.legend() plt.savefig('figure/sample_%d_fold%d.png' % (args.sample_n, i)) plt.cla() if acc_fold > 0.7 and auc_fold > 0.6 and model_count < 10: model.save(args.data_dir + "/model/model_%d" % model_count) model_count += 1 print("acc:%.3f\tauc:%.3f\tc_index:%.3f\tf1:%.3f" % (acc_fold, auc_fold, c_index_fold, f1_fold)) total_round += 1 if gpu: del dataloader.X_train, dataloader.Y_train, dataloader.X_test, dataloader.Y_test del X_test, Y_test, X_train, Y_train, model, optimizer torch.cuda.empty_cache() print('CV-acc:%.3f CV-auc:%.3f CV-c-index:%.3f f1:%.3f f1(neg):%.3f' % (sum(acc_folds) / 5 / manytimes_n, sum(auc_folds) / 5 / manytimes_n, sum(c_index_folds) / 5 / manytimes_n, sum(f1_folds_pos) / 5 / manytimes_n, sum(f1_folds) / 5 / manytimes_n))
pq = evaluation.pairs_quality(cand_rec_id_pair_list, true_match_set) print('Blocking evaluation:') print(' Reduction ratio: %.3f' % (rr)) print(' Pairs completeness: %.3f' % (pc)) print(' Pairs quality: %.3f' % (pq)) print('') # Linkage evaluation # linkage_result = evaluation.confusion_matrix(class_match_set, class_nonmatch_set, true_match_set, all_comparisons) accuracy = evaluation.accuracy(linkage_result) precision = evaluation.precision(linkage_result) recall = evaluation.recall(linkage_result) fmeasure = evaluation.fmeasure(linkage_result) print('Linkage evaluation:') print(' Accuracy: %.3f' % (accuracy)) print(' Precision: %.3f' % (precision)) print(' Recall: %.3f' % (recall)) print(' F-measure: %.3f' % (fmeasure)) print('') linkage_time = loading_time + blocking_time + comparison_time + \ classification_time print('Total runtime required for linkage: %.3f sec' % (linkage_time)) # -----------------------------------------------------------------------------
# Get cross validation accuracy for 5-fold cv print("Ionosphere validation accuracy (default parameters):") evaluation.cross_validation(5, ionosphere_train_features, ionosphere_train_labels, model=LogisticRegression) # Grid search for optimal hyperparameters print("Ionosphere grid search hyperparameters:") ionosphere_max_val_acc, ionosphere_arg_max = evaluation.grid_search(learning_rates=lrs, epsilons=eps, lambdas=lamdas, x=ionosphere_train_features, y=ionosphere_train_labels, model=LogisticRegression) # Accuracy on test split - train with best hyperparameters print("Ionosphere test accuracy:") logistic_ionosphere = LogisticRegression(ionosphere_train_features, ionosphere_train_labels) logistic_ionosphere.fit(lr=ionosphere_arg_max[0], eps=ionosphere_arg_max[1], regularization=ionosphere_arg_max[2]) ionosphere_prediction = logistic_ionosphere.predict(ionosphere_test_features) cm_ionosphere = evaluation.confusion_matrix(ionosphere_test_labels, ionosphere_prediction) print("Accuracy:", evaluation.accuracy(cm_ionosphere), "Precision:", evaluation.precision(cm_ionosphere), "Recall:", evaluation.true_positive(cm_ionosphere), "F1:", evaluation.f_score(cm_ionosphere)) # 5-fold CV for naive bayes print("Ionosphere validation accuracy (naive bayes):") evaluation.cross_validation_naive(5, ionosphere_dataset.train_data, NaiveBayes, ionosphere_dataset.label_column, ionosphere_dataset.feature_columns) naive_ionosphere = NaiveBayes(ionosphere_dataset.train_data, ionosphere_dataset.label_column, continuous=ionosphere_dataset.feature_columns) print("Ionosphere test accuracy (naive bayes):") ionosphere_pred_naive = ionosphere_dataset.test_data.apply(naive_ionosphere.predict, axis=1) cm_ionosphere_naive = evaluation.confusion_matrix(ionosphere_test_labels, ionosphere_pred_naive.to_numpy()) print("Accuracy:", evaluation.accuracy(cm_ionosphere_naive), "Precision:", evaluation.precision(cm_ionosphere_naive), "Recall:", evaluation.true_positive(cm_ionosphere_naive), "F1:", evaluation.f_score(cm_ionosphere_naive)) # Abalone -----
def calculate(method): global dataset a=0.0 b=0.0 c=0.0 ''' #setting 4 user_Mu = np.loadtxt("./genre7-9/data/d11/user.csv",delimiter=",").astype(np.int64) user_Mv = np.loadtxt("./genre7-9/data/d22/user.csv",delimiter=",").astype(np.int64) test_user = np.loadtxt("./genre7-9/data/d12/user.csv",delimiter=",").astype(np.int64) item_Mu = np.loadtxt("./genre7-9/data/d11/item.csv",delimiter=",").astype(np.int64) item_Mv = np.loadtxt("./genre7-9/data/d22/item.csv",delimiter=",").astype(np.int64) test_item = np.loadtxt("./genre7-9/data/d12/item.csv",delimiter=",").astype(np.int64) data_Mu = np.loadtxt("./genre7-9/data/d11/data.csv",delimiter=",").astype(np.int64) data_Mv = np.loadtxt("./genre7-9/data/d22/data.csv",delimiter=",").astype(np.int64) test_data = np.loadtxt("./genre7-9/data/d12/data.csv",delimiter=",").astype(np.int64) train_index = np.loadtxt("./genre7-9/data/d11/index.csv",delimiter=",").astype(np.int64) train_index2 = np.loadtxt("./genre7-9/data/d22/index.csv",delimiter=",").astype(np.int64) train_index3 = np.loadtxt("./genre7-9/data/d12/index.csv",delimiter=",").astype(np.int64) ''' #setting 5 user_Mu = np.loadtxt("./genre7-9/data/d22/user.csv",delimiter=",").astype(np.int64) user_Mv = np.loadtxt("./genre7-9/data/d11/user.csv",delimiter=",").astype(np.int64) test_user = np.loadtxt("./genre7-9/data/d21/user.csv",delimiter=",").astype(np.int64) item_Mu = np.loadtxt("./genre7-9/data/d22/item.csv",delimiter=",").astype(np.int64) item_Mv = np.loadtxt("./genre7-9/data/d11/item.csv",delimiter=",").astype(np.int64) test_item = np.loadtxt("./genre7-9/data/d21/item.csv",delimiter=",").astype(np.int64) data_Mu = np.loadtxt("./genre7-9/data/d22/data.csv",delimiter=",").astype(np.int64) data_Mv = np.loadtxt("./genre7-9/data/d11/data.csv",delimiter=",").astype(np.int64) test_data = np.loadtxt("./genre7-9/data/d21/data.csv",delimiter=",").astype(np.int64) train_index = np.loadtxt("./genre7-9/data/d22/index.csv",delimiter=",").astype(np.int64) train_index2 = np.loadtxt("./genre7-9/data/d11/index.csv",delimiter=",").astype(np.int64) train_index3 = np.loadtxt("./genre7-9/data/d21/index.csv",delimiter=",").astype(np.int64) for i in range(repeate): Mu_matrix = makeMatrix(data_Mu, train_index, user_Mu, item_Mu) u = learning(method, Mu_matrix, train_index, data_Mu, user_Mu, item_Mu) Mv_matrix = makeMatrix(data_Mv, train_index2, user_Mv, item_Mv) pred = learning2(method, Mv_matrix, train_index2, data_Mv, user_Mv, item_Mv, u) test_matrix = makeMatrix(test_data, train_index3, test_user, test_item) test_users = users_in_testdata(3, test_matrix, test_user) # calculating precision, recall, and nDCG using "pred" pre, rec = ev.precision(3, pred, np.array(test_matrix.todense()), test_user, test_item) dcg = ev.nDCG(3, pred, np.array(test_matrix.todense()), test_user, test_item) # print result which shows users' average, into standard output print("Process ID : " + str(os.getpid())) print("Repeat : " + str(i + 1)) #print("K-fold crossvalidation : " + str(j + 1) + "/" + str(sepalate)) print("Dataset : " + dataset) print("Mthod : " + method) print("Precision : " + str(np.mean(pre[test_users.nonzero()]))) print("Recall : " + str(np.mean(rec[test_users.nonzero()]))) print("nDCG : " + str(np.mean(dcg[test_users.nonzero()]))) print("=================================================================================================") a += np.mean(pre[test_users.nonzero()]) b += np.mean(rec[test_users.nonzero()]) c += np.mean(dcg[test_users.nonzero()]) # gavege collection, numpy_ndarray not used hereafter #del pred #del test_matrix #del train_matrix gc.collect() #np.save("result/" + dataset + "/" + method + "/Precision.npy", precision) #np.save("result/" + dataset + "/" + method + "/Recall.npy", recall) #np.save("result/" + dataset + "/" + method + "/nDCG.npy", nDCG) print("Precision AVE : " + str(a / 10)) print("Recall AVE : " + str(b / 10)) print("nDCG AVE : " + str(c / 10))
def get_stats_from_prob_mat(self, pd_prob_mat, gt_mat, thres_ary): """Get stats from prob_mat and ground truth mat. Args: pd_prob_mat: ndarray, (n_clips, n_labels) gt_prob_mat: ndarray, (n_clips, n_labels) thres_ary: list of float | 'auto' | 'no_need'. Returns: stat. """ n_lbs = len(self.lbs) stat = {} if type(thres_ary) is list: stat['thres_ary'] = thres_ary elif thres_ary == 'auto': thres_ary = self._get_best_thres_ary(pd_prob_mat, gt_mat) stat['thres_ary'] = thres_ary elif thres_ary == 'no_need': thres_ary = [0.5] * len(self.lbs) stat['thres_ary'] = ['no_need'] * len(self.lbs) else: raise Exception("thres_ary type wrong!") pd_digit_mat = self._get_digit_mat_from_thres_ary( pd_prob_mat, thres_ary) # overall stat eer = eva.eer(pd_prob_mat.flatten(), gt_mat.flatten()) auc = eva.roc_auc(pd_prob_mat.flatten(), gt_mat.flatten()) (tp, fn, fp, tn) = eva.tp_fn_fp_tn(pd_digit_mat, gt_mat, 0.5) prec = eva.precision(pd_digit_mat, gt_mat, 0.5) rec = eva.recall(pd_digit_mat, gt_mat, 0.5) f_val = eva.f_value(prec, rec) stat['overall'] = { 'tp': tp, 'fn': fn, 'fp': fp, 'tn': tn, 'precision': prec, 'recall': rec, 'f_value': f_val, 'eer': eer, 'auc': auc } # element-wise stat stat['event_wise'] = {} for k in xrange(len(self.lbs)): eer = eva.eer(pd_prob_mat[:, k], gt_mat[:, k]) auc = eva.roc_auc(pd_prob_mat[:, k], gt_mat[:, k]) (tp, fn, fp, tn) = eva.tp_fn_fp_tn(pd_digit_mat[:, k], gt_mat[:, k], 0.5) prec = eva.precision(pd_digit_mat[:, k], gt_mat[:, k], 0.5) rec = eva.recall(pd_digit_mat[:, k], gt_mat[:, k], 0.5) f_val = eva.f_value(prec, rec) stat['event_wise'][self.lbs[k]] = { 'tp': tp, 'fn': fn, 'fp': fp, 'tn': tn, 'precision': prec, 'recall': rec, 'f_value': f_val, 'eer': eer, 'auc': auc } return stat
def evaluate_boxes(dataset, all_boxes, output_dir, num_classes=11): """ :param dataset: dataset in coco format :param all_boxes: detection boxes :param output_dir: dir to save eval.json :return: res_dict = {thrs_0: {'TP': x0, 'FP': y0, 'FN': z0}, ... , thrs_N: {'TP': xN, 'FP': yN, 'FN': zN}} """ res = [] gt = get_gt(dataset) imgs = dataset.COCO.imgs num_imgs = len(imgs) thrs = [0.5, 0.3] ev_json = EvalSave(thrs) ev_json._set_dataset_name(dataset.name) res_dict = {} [res_dict.update({val: {'TP': 0, 'FP': 0, 'FN': 0}}) for val in thrs] l_limit = 2500 u_limit = 50 * 10**3 # create confusion matrix only for 10c datasets calc_cfn_matrix = True if "10c" in dataset.name else False for t in thrs: # intialize caches for different tp, fp, fn and m_iou values tp_sum = [0] * int((1/ev_json._conf_stride + 1)) fp_sum = [0] * int((1/ev_json._conf_stride + 1)) fn_sum = [0] * int((1/ev_json._conf_stride + 1)) m_iou_sum = [0] * int((1/ev_json._conf_stride + 1)) tp_sum_low = [0] * int((1/ev_json._conf_stride + 1)) fp_sum_low = [0] * int((1/ev_json._conf_stride + 1)) fn_sum_low = [0] * int((1/ev_json._conf_stride + 1)) m_iou_sum_low = [0] * int((1/ev_json._conf_stride + 1)) tp_sum_mid = [0] * int((1/ev_json._conf_stride + 1)) fp_sum_mid = [0] * int((1/ev_json._conf_stride + 1)) fn_sum_mid = [0] * int((1/ev_json._conf_stride + 1)) m_iou_sum_mid = [0] * int((1/ev_json._conf_stride + 1)) tp_sum_high = [0] * int((1/ev_json._conf_stride + 1)) fp_sum_high = [0] * int((1/ev_json._conf_stride + 1)) fn_sum_high = [0] * int((1/ev_json._conf_stride + 1)) m_iou_sum_high = [0] * int((1/ev_json._conf_stride + 1)) conf_mat = [np.zeros((num_classes, num_classes)).astype(np.uint32), np.zeros((num_classes, num_classes)).astype(np.uint32)] classification_error = [np.array([0]), np.array([0])] for im in range(num_imgs): sys.stdout.write("\rimage {:d} of {:d} - thrs = {:0.1f}".format(im+1, num_imgs, t)) # sort out empty boxes n_e_boxes = [] for idx, b in enumerate(all_boxes[1:]): if b[im].any(): for i in range(len(b[im])): n_e_boxes.append([b[im][i].tolist(), idx+1]) boxes = get_image_boxes(dataset, all_boxes, im) for box in n_e_boxes: box[0][2] = box[0][2] - box[0][0] box[0][3] = box[0][3] - box[0][1] r_end = int((1 + ev_json._conf_stride) / ev_json._conf_stride) conf = [c*ev_json._conf_stride for c in range(int(1/ev_json._conf_stride) + 1)] # as coco forces you to provide a label, empty images are detected by bounding boxes containing only a label # validation case: empty frames were left emtpy and didn't get a label if not gt[im]: gt_ev = [] else: # test case: there's no bounding box but a label to tell the network that there's no object if len(gt[im][0]) == 1: gt_ev = [] else: gt_ev = gt[im] # prepare for size dependant evaluation gt_low = [] gt_mid = [] gt_high = [] for g in gt_ev: if g[2]*g[3] <= l_limit: gt_low.append(g) elif g[2]*g[3] > u_limit: gt_high.append(g) else: gt_mid.append(g) # TODO(eomoos): change cache arrays type to numpy array so you can easily add cache to target array! # eval complete dataset if calc_cfn_matrix: tp_c, fp_c, fn_c, m_iou, max_conf = evaluation.eval_with_conf(n_e_boxes, gt_ev, t, conf, calc_conf_mat=calc_cfn_matrix, conf_mat=conf_mat, correct_pred=classification_error) else: tp_c, fp_c, fn_c, m_iou, max_conf = evaluation.eval_with_conf(n_e_boxes, gt_ev, t, conf) # eval only with tiny objects - A <= 2500 tp_c_low, fp_c_low, fn_c_low, m_iou_low, _ = evaluation.eval_with_conf(n_e_boxes, gt_low, t, conf) # eval only with medium size objects - 2500 < A < 50,000 tp_c_mid, fp_c_mid, fn_c_mid, m_iou_mid, _ = evaluation.eval_with_conf(n_e_boxes, gt_mid, t, conf) # eval only with huge objects - A >= 50000 tp_c_high, fp_c_high, fn_c_high, m_iou_high, _ = evaluation.eval_with_conf(n_e_boxes, gt_high, t, conf) # calculate sums of different size tp(conf), fp(conf), fn(conf), m_iou(conf) tp_sum = [tp_sum[i] + tp_c[i] for i in range(len(tp_c))] fp_sum = [fp_sum[i] + fp_c[i] for i in range(len(fp_c))] fn_sum = [fn_sum[i] + fn_c[i] for i in range(len(fn_c))] m_iou_sum = [m_iou_sum[i] + m_iou[i] for i in range(len(m_iou))] tp_sum_low = [tp_sum_low[i] + tp_c_low[i] for i in range(len(tp_c_low))] fn_sum_low = [fn_sum_low[i] + fn_c_low[i] for i in range(len(fn_c_low))] m_iou_sum_low = [m_iou_sum_low[i] + m_iou_low[i] for i in range(len(m_iou_low))] tp_sum_mid = [tp_sum_mid[i] + tp_c_mid[i] for i in range(len(tp_c_mid))] fn_sum_mid = [fn_sum_mid[i] + fn_c_mid[i] for i in range(len(fn_c_mid))] m_iou_sum_mid = [m_iou_sum_mid[i] + m_iou_mid[i] for i in range(len(m_iou_mid))] tp_sum_high = [tp_sum_high[i] + tp_c_high[i] for i in range(len(tp_c_high))] fn_sum_high = [fn_sum_high[i] + fn_c_high[i] for i in range(len(fn_c_high))] m_iou_sum_high = [m_iou_sum_high[i] + m_iou_high[i] for i in range(len(m_iou_high))] sys.stdout.write("\n") add_to_res_dict(res_dict, tp_sum[0], fp_sum[0], fn_sum[0], t) # calc mean iou's for different evaluation scales for i in range(len(tp_sum)): if tp_sum[i] > 0: ev_json._eval['m_iou'][t].append(m_iou_sum[i]/tp_sum[i]) else: ev_json._eval['m_iou'][t].append(0) for i in range(len(tp_sum_low)): if tp_sum_low[i] > 0: ev_json._eval['m_iou_low'][t].append(m_iou_sum_low[i]/tp_sum_low[i]) else: ev_json._eval['m_iou_low'][t].append(0) for i in range(len(tp_sum_mid)): if tp_sum_mid[i] > 0: ev_json._eval['m_iou_mid'][t].append(m_iou_sum_mid[i]/tp_sum_mid[i]) else: ev_json._eval['m_iou_mid'][t].append(0) for i in range(len(tp_sum_high)): if tp_sum_high[i] > 0: ev_json._eval['m_iou_high'][t].append(m_iou_sum_high[i]/tp_sum_high[i]) else: ev_json._eval['m_iou_high'][t].append(0) for i in range(len(tp_sum)): ev_json._eval['prec'][t].append(evaluation.precision(tp_sum[i], fp_sum[i])) ev_json._eval['rec'][t].append(evaluation.recall(tp_sum[i], fn_sum[i])) ev_json._eval['rec_low'][t].append(evaluation.recall(tp_sum_low[i], fn_sum_low[i])) ev_json._eval['rec_mid'][t].append(evaluation.recall(tp_sum_mid[i], fn_sum_mid[i])) ev_json._eval['rec_high'][t].append(evaluation.recall(tp_sum_high[i], fn_sum_high[i])) flat_conf_mat = conf_mat[0].flatten() num_tp_dets = sum(flat_conf_mat.tolist()) cls_acc = evaluation.classification_accuracy(classification_error[0][0], num_tp_dets) print(cls_acc) ev_json._eval['conf_mat'][t].append([flat_conf_mat.tolist(), num_classes]) ev_json._eval['classification_error'][t].append( [cls_acc]) flat_conf_mat = conf_mat[1].flatten() num_tp_dets = sum(flat_conf_mat.tolist()) cls_acc = evaluation.classification_accuracy(classification_error[0][0], num_tp_dets) print(cls_acc) ev_json._eval['conf_mat'][t].append([flat_conf_mat.tolist(), num_classes]) ev_json._eval['classification_error'][t].append( [cls_acc]) ev_json._max_conf = max_conf ev_json._save_as_json(output_dir) # print(np.array(ev_json._eval['conf_mat'][0.3][0][0]).reshape((num_classes, num_classes))) return res_dict
def run_scheme(scheme, descriptor_type, descriptor_param, num_clusters, clf_params, plotGraphs, PCAon, num_cols): print "Running scheme with the following parameters: " print "Scheme num: " + str(scheme) + ", BoVW: num_clusters=" + str(num_clusters) +\ "; SVM: params:" + str(clf_params) + ";\n plotGraphs=" + str(plotGraphs) +\ "; PCA_on=" + str(PCAon) start = time.time() # 1) Read the train and test files train_images_filenames = cPickle.load( open('train_images_filenames.dat', 'r')) test_images_filenames = cPickle.load(open('test_images_filenames.dat', 'r')) train_labels = cPickle.load(open('train_labels.dat', 'r')) test_labels = cPickle.load(open('test_labels.dat', 'r')) print 'Loaded ' + str( len(train_images_filenames)) + ' training images filenames\ with classes ', set(train_labels) print 'Loaded ' + str( len(test_images_filenames)) + ' testing images filenames\ with classes ', set(test_labels) # 2) Extract features (train) D, Train_descriptors, kpt_dense, pca_train, sclr_train = computeTraining_descriptors( descriptor_type, descriptor_param, train_images_filenames, train_labels, PCAon, num_cols) # 3) Reduce number of features by PCA (reducing m=128 cols) # Computed internally in computeTraining_descriptors() # 4) Compute codebook codebook = computeCodebook(num_clusters, D, descriptor_type, descriptor_param, PCAon) # 5) Get training BoVW train_VW = getBoVW_train(codebook, num_clusters, Train_descriptors) # 6) Train SVM clf, train_scaler, D_scaled = clf_train(train_VW, train_labels, clf_params) # 7) Get test BoVW test_VW = getBoVW_test(codebook, num_clusters, test_images_filenames, descriptor_type, descriptor_param, kpt_dense, PCAon, pca_train, sclr_train) # 8) Get evaluation (accuracy, f-score, graphs, etc.) predictions = clf_predict(clf, clf_params, train_scaler, test_VW, D_scaled) # Get metrics and graphs: # We need to implement our own for latter integration with the rest of the project # Accuracy, F-score (multi-class=> average? add up?) acc = accuracy(test_labels, predictions) prec = precision(test_labels, predictions) rec = recall(test_labels, predictions) f1sc = f1score(test_labels, predictions) cm = confusionMatrix(test_labels, predictions) hits, misses = HitsAndMisses(cm) print "Confusion matrix:\n" print(str(cm)) print("\n") print "Results (metrics):\n" + "Accuracy= {:04.2f}%\n" \ "Precision= {:04.2f}%\n" \ "Recall= {:04.2f}%\n" \ "F1-score= {:04.2f}%\n" \ "Hits(TP)={:d}\n" \ "Misses(FN)={:d}\n".format( 100*acc, 100*prec, 100*rec, 100*f1sc, hits, misses) print("\n") if plotGraphs: # Plot confusion matrix (and any other graph) print "Plotting confusion matrix..." plotConfusionMatrix(cm, test_labels) end = time.time() print 'Everything done in ' + str(end - start) + ' secs.'
forest = random_forest.grow_random_forest( X, y, num_trees=30, max_depth=20, max_features=max_features, min_node_size=1 ) # make predictions predictions = random_forest.forest_predict(forest, X) # calculate the numbers of true positives, false positives, true negatives, false negatives tfpns = evaluation.tfpn(predictions, y) # calculate the confusion matrix cm = evaluation.make_confusion_matrix(*tfpns, percentage=True) # calculate metrics: precision, sensitivity, false-positive-rate metrics = np.array( [evaluation.precision(cm), evaluation.sensitivity(cm), evaluation.fpr(cm)] ) print( f"{num_points} points are randomly generated in the unit cube in {dim}-dimensions.\n \ Those with the sum of coordinates >= {dim}/2 are labeled 1, \n those below are \ labeled 0." ) print("The model achieves the following in sample metrics:") print("precision:", metrics[0]) print("sensitivity:", metrics[1]) print("false-positive-rate:", metrics[2]) print('If the metrics are not 1,1,0, then there is a problem.') # if (metrics[0] == 1) & (metrics[1] == 1) & (metrics[2] == 0): # print(0) # else:
def test_pair_wise(dns=FLAGS.dns): train, test, dev = load(FLAGS.data, filter=FLAGS.clean) test = test.reindex(np.random.permutation(test.index)) q_max_sent_length = max( map(lambda x: len(x), train['question'].str.split())) a_max_sent_length = max(map(lambda x: len(x), train['answer'].str.split())) print('q_question_length:{} a_question_length:{}'.format( q_max_sent_length, a_max_sent_length)) print('train question unique:{}'.format(len(train['question'].unique()))) print('train length', len(train)) print('test length', len(test)) print('dev length', len(dev)) alphabet, embeddings = prepare([train, test, dev], dim=FLAGS.embedding_dim, is_embedding_needed=True, fresh=FLAGS.fresh) # alphabet,embeddings = prepare_300([train,test,dev]) print('alphabet:', len(alphabet)) with tf.Graph().as_default(), tf.device("/gpu:" + str(FLAGS.gpu)): # with tf.device("/cpu:0"): session_conf = tf.ConfigProto() session_conf.allow_soft_placement = FLAGS.allow_soft_placement session_conf.log_device_placement = FLAGS.log_device_placement session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) with sess.as_default(), open(precision, "w") as log: log.write(str(FLAGS.__flags) + '\n') folder = 'runs/' + timeDay + '/' + timeStamp + '/' out_dir = folder + FLAGS.data if not os.path.exists(folder): os.makedirs(folder) # train,test,dev = load("trec",filter=True) # alphabet,embeddings = prepare([train,test,dev],is_embedding_needed = True) print("start build model") cnn = QA_CNN_quantum_extend( max_input_left=q_max_sent_length, max_input_right=a_max_sent_length, batch_size=FLAGS.batch_size, vocab_size=len(alphabet), embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, dropout_keep_prob=FLAGS.dropout_keep_prob, embeddings=embeddings, l2_reg_lambda=FLAGS.l2_reg_lambda, overlap_needed=FLAGS.overlap_needed, learning_rate=FLAGS.learning_rate, trainable=FLAGS.trainable, extend_feature_dim=FLAGS.extend_feature_dim, pooling=FLAGS.pooling, position_needed=FLAGS.position_needed, conv=FLAGS.conv, margin=FLAGS.margin) cnn.build_graph() saver = tf.train.Saver(tf.global_variables(), max_to_keep=20) train_writer = tf.summary.FileWriter(log_dir + '/train', sess.graph) test_writer = tf.summary.FileWriter(log_dir + '/test') # Initialize all variables print("build over") sess.run(tf.global_variables_initializer()) print("variables_initializer") # saver.restore(sess, 'runs/20170910/20170910154937/wiki') map_max = 0.65 for i in range(FLAGS.num_epochs): datas = batch_gen_with_pair(train, alphabet, FLAGS.batch_size, q_len=q_max_sent_length, a_len=a_max_sent_length, fresh=FLAGS.fresh, overlap_dict=None) print("load data") for data in datas: feed_dict = { cnn.question: data[0], cnn.answer: data[1], cnn.answer_negative: data[2], cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, summary, step, loss, accuracy, score12, score13, see = sess.run( [ cnn.train_op, cnn.merged, cnn.global_step, cnn.loss, cnn.accuracy, cnn.score12, cnn.score13, cnn.see ], feed_dict) train_writer.add_summary(summary, i) time_str = datetime.datetime.now().isoformat() print( "{}: step {}, loss {:g}, acc {:g} ,positive {:g},negative {:g}" .format(time_str, step, loss, accuracy, np.mean(score12), np.mean(score13))) line = "{}: step {}, loss {:g}, acc {:g} ,positive {:g},negative {:g}".format( time_str, step, loss, accuracy, np.mean(score12), np.mean(score13)) # print loss if i % 1 == 0: predicted_dev = predict(sess, cnn, dev, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) map_mrr_dev = evaluation.evaluationBypandas( dev, predicted_dev) predicted_test = predict(sess, cnn, test, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) map_mrr_test = evaluation.evaluationBypandas( test, predicted_test) precise_test = evaluation.precision(test, predicted_test) print("test precise : {}".format(precise_test)) print("{}:epoch:dev map mrr {}".format(i, map_mrr_dev)) print("{}:epoch:test map mrr {}".format(i, map_mrr_test)) line = " {}:epoch: precise: {}--- map_dev{}-------map_mrr_test{}".format( i, precise_test, map_mrr_dev[0], map_mrr_test) if map_mrr_dev[0] > map_max: map_max = map_mrr_dev[0] save_path = saver.save(sess, out_dir) print("Model saved in file: ", save_path) log.write(line + '\n') log.flush() print('train over') saver.restore(sess, out_dir) predicted = predict(sess, cnn, train, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) train['predicted'] = predicted map_mrr_train = evaluation.evaluationBypandas(train, predicted) predicted_dev = predict(sess, cnn, dev, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) dev['predicted'] = predicted_dev map_mrr_dev = evaluation.evaluationBypandas(dev, predicted_dev) predicted_test = predict(sess, cnn, test, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) test['predicted'] = predicted_test map_mrr_test = evaluation.evaluationBypandas(test, predicted_test) ap = evaluation.get_ap(test, predicted_test) ap.to_csv('ap_score_qlm_wiki', header=None, sep='\t') print('map_mrr train', map_mrr_train) print('map_mrr dev', map_mrr_dev) print('map_mrr test', map_mrr_test) log.write(str(map_mrr_train) + '\n') log.write(str(map_mrr_test) + '\n') log.write(str(map_mrr_dev) + '\n')
def calculate(method): global dataset a = 0.0 b = 0.0 c = 0.0 c_pre = np.array([0., 0., 0., 0., 0., 0., 0.]) c_rec = np.array([0., 0., 0., 0., 0., 0., 0.]) c_dcg = np.array([0., 0., 0., 0., 0., 0., 0.]) set_data = sys.argv[1] setting = sys.argv[2] if setting == "1": user_list = np.loadtxt("./genre" + set_data + "/data/d11/user.csv", delimiter=",").astype(np.int64) #U1 item_list = np.loadtxt("./genre" + set_data + "/data/d11/item.csv", delimiter=",").astype(np.int64) #genre data = np.loadtxt("./genre" + set_data + "/data/d11/data.csv", delimiter=",").astype(np.int64) elif setting == "2": user_list = np.loadtxt("./genre" + set_data + "/data/d22/user.csv", delimiter=",").astype(np.int64) #U1 item_list = np.loadtxt("./genre" + set_data + "/data/d22/item.csv", delimiter=",").astype(np.int64) #genre data = np.loadtxt("./genre" + set_data + "/data/d22/data.csv", delimiter=",").astype(np.int64) elif setting == "3": user_list = np.loadtxt("./genre" + set_data + "/data/d1/user.csv", delimiter=",").astype(np.int64) #U1 item_list = np.loadtxt("./genre" + set_data + "/data/d1/item.csv", delimiter=",").astype(np.int64) #genre data = np.loadtxt("./genre" + set_data + "/data/d1/data.csv", delimiter=",").astype(np.int64) precision = np.zeros((repeate, sepalate, len(user_list))) recall = np.zeros((repeate, sepalate, len(user_list))) nDCG = np.zeros((repeate, sepalate, len(user_list))) # repeated K cross-validation for i in range(repeate): # kf : model cross validation from sikit-learn # shuffle sepalating train and test data and random state change per repeat time kf = KFold(n_splits=sepalate, random_state=i, shuffle=True) j = 0 for train_index, test_index in kf.split(data): # make train and test matrix train_matrix = makeMatrix(data, train_index, user_list, item_list) test_matrix = makeMatrix(data, test_index, user_list, item_list) test_users = users_in_testdata(3, test_matrix, user_list) # learning model and calculate predicted user-item matrix pred = learning(method, train_matrix, train_index, data, user_list, item_list) # np.save("./pred_temp.npy", pred) count_dict = collections.Counter(data[:, 1]) # calculating precision, recall, and nDCG using "pred" pre, rec, new_c_pre, new_c_rec, recom, recom2 = ev.precision( 3, pred, np.array(test_matrix.todense()), user_list, item_list, count_dict) dcg, new_c_dcg = ev.nDCG(3, pred, np.array(test_matrix.todense()), user_list, item_list) # save users' values for each criterion precision[i, j, :] = pre recall[i, j, :] = rec nDCG[i, j, :] = dcg c_pre = c_pre + new_c_pre c_rec = c_rec + new_c_rec c_dcg = c_dcg + new_c_dcg # print result which shows users' average, into standard output print("Process ID : " + str(os.getpid())) print("Repeat : " + str(i + 1)) print("K-fold crossvalidation : " + str(j + 1) + "/" + str(sepalate)) print("Dataset : " + dataset) print("Mthod : " + method) print("Precision : " + str(np.mean(pre[test_users.nonzero()]))) print("Recall : " + str(np.mean(rec[test_users.nonzero()]))) print("nDCG : " + str(np.mean(dcg[test_users.nonzero()]))) print( "=================================================================================================" ) a += np.mean(pre[test_users.nonzero()]) b += np.mean(rec[test_users.nonzero()]) c += np.mean(dcg[test_users.nonzero()]) # gavege collection, numpy_ndarray not used hereafter del pred del test_matrix del train_matrix gc.collect() j = j + 1 c_pre = c_pre / 30 c_rec = c_rec / 30 c_dcg = c_dcg / 30 print(c_pre) np.save("result/" + dataset + "/" + method + "/Precision.npy", precision) np.save("result/" + dataset + "/" + method + "/Recall.npy", recall) np.save("result/" + dataset + "/" + method + "/nDCG.npy", nDCG) np.save( "result/movie.review/" + method + "/genre" + set_data + "_set" + setting + "/recom.npy", recom) np.save( "result/movie.review/" + method + "/genre" + set_data + "_set" + setting + "/recom2.npy", recom2) np.save( "result/movie.review/" + method + "/genre" + set_data + "_set" + setting + "/c_pre.npy", c_pre) np.save( "result/movie.review/" + method + "/genre" + set_data + "_set" + setting + "/c_rec.npy", c_rec) np.save( "result/movie.review/" + method + "/genre" + set_data + "_set" + setting + "/c_dcg.npy", c_dcg) print("Precision AVE : " + str(a / 30)) print("Recall AVE : " + str(b / 30)) print("nDCG AVE : " + str(c / 30))
def calculate(method): global dataset precision_all = [] recall_all = [] ndcg_all = [] a = 0.0 b = 0.0 c = 0.0 user_Mu = np.loadtxt("./genre1-10/data/d22/user.csv", delimiter=",").astype(np.int64) user_Mv = np.loadtxt("./genre1-10/data/d11/user.csv", delimiter=",").astype(np.int64) test_user = np.loadtxt("./genre1-10/data/d21/user.csv", delimiter=",").astype(np.int64) item_Mu = np.loadtxt("./genre1-10/data/d22/item.csv", delimiter=",").astype(np.int64) item_Mv = np.loadtxt("./genre1-10/data/d11/item.csv", delimiter=",").astype(np.int64) test_item = np.loadtxt("./genre1-10/data/d21/item.csv", delimiter=",").astype(np.int64) data_Mu = np.loadtxt("./genre1-10/data/d22/data.csv", delimiter=",").astype(np.int64) data_Mv = np.loadtxt("./genre1-10/data/d11/data.csv", delimiter=",").astype(np.int64) test_data = np.loadtxt("./genre1-10/data/d21/data_s.csv", delimiter=",").astype(np.int64) train_index = np.loadtxt("./genre1-10/data/d22/index.csv", delimiter=",").astype(np.int64) train_index2 = np.loadtxt("./genre1-10/data/d11/index.csv", delimiter=",").astype(np.int64) train_index3 = np.loadtxt("./genre1-10/data/d21/index_s.csv", delimiter=",").astype(np.int64) u = np.loadtxt("./u.csv", delimiter=",") s = np.loadtxt("./s.csv", delimiter=",") vt = np.loadtxt("./vt.csv", delimiter=",") s_diag_matrix = np.diag(s) seq = (0, 1, 2, 3, 4) C_list = list(itertools.permutations(seq)) vt_new = np.zeros(vt.shape) for i in range(120): for j in range(5): vt_new[j, :] = vt[C_list[i][j], :] for i in range(1): pred = np.dot(np.dot(u, s_diag_matrix), vt_new) test_matrix = makeMatrix(test_data, train_index3, test_user, test_item) test_users = users_in_testdata(3, test_matrix, test_user) # calculating precision, recall, and nDCG using "pred" pre, rec = ev.precision(3, pred, np.array(test_matrix.todense()), test_user, test_item) dcg = ev.nDCG(3, pred, np.array(test_matrix.todense()), test_user, test_item) # print result which shows users' average, into standard output print("Process ID : " + str(os.getpid())) print("Repeat : " + str(i + 1)) #print("K-fold crossvalidation : " + str(j + 1) + "/" + str(sepalate)) print("Dataset : " + dataset) print("Mthod : " + method) print("Precision : " + str(np.mean(pre[test_users.nonzero()]))) precision_all.append(np.mean(pre[test_users.nonzero()])) print("Recall : " + str(np.mean(rec[test_users.nonzero()]))) recall_all.append(np.mean(rec[test_users.nonzero()])) print("nDCG : " + str(np.mean(dcg[test_users.nonzero()]))) ndcg_all.append(np.mean(dcg[test_users.nonzero()])) print( "=================================================================================================" ) gc.collect() np.savetxt("precision_all.csv", precision_all, delimiter=",") np.savetxt("recall_all.csv", recall_all, delimiter=",") np.savetxt("ndcg_all.csv", ndcg_all, delimiter=",")
fp = 0 tn = 0 fn = 0 f = open(file, 'r') for line in f: lines += 1 totals = line.split() # totals[0] is image name tp += int(totals[1]) fp += int(totals[2]) tn += int(totals[3]) fn += int(totals[4]) f.close() precision = evaluation.precision(tp, fp) sensitivity = evaluation.sensitivity(tp, fn) fmeasure = evaluation.fmeasure(tp, fp, fn) dicecoeff = evaluation.dicecoeff(tp, fp, fn) jaccardindex = evaluation.jaccardindex(tp, fp, fn) r = open('circle_found_results.txt', 'a') r.write('File name: ' + file + '\n') r.write('Lines: ' + str(lines) + '\n') r.write('True Positives: ' + str(tp) + '\n') r.write('False Positives: ' + str(fp) + '\n') r.write('True Negatives: ' + str(tn) + '\n') r.write('False Negatives: ' + str(fn) + '\n') r.write('Precision: ' + str(precision) + '\n') r.write('Sensitivity: ' + str(sensitivity) + '\n') r.write('F-Measure: ' + str(fmeasure) + '\n')
tp = 0 fp = 0 tn = 0 fn = 0 f = open(file, 'r') for line in f: lines += 1 totals = line.split() tp += int(totals[0]) fp += int(totals[1]) tn += int(totals[2]) fn += int(totals[3]) f.close() precision = evaluation.precision(tp, fp) sensitivity = evaluation.sensitivity(tp, fn) fmeasure = evaluation.fmeasure(tp, fp, fn) dicecoeff = evaluation.dicecoeff(tp, fp, fn) jaccardindex = evaluation.jaccardindex(tp, fp, fn) r = open('circle_found_results.txt', 'a') r.write('File name: ' + file + '\n') r.write('Lines: ' + str(lines) + '\n') r.write('True Positives: ' + str(tp) + '\n') r.write('False Positives: ' + str(fp) + '\n') r.write('True Negatives: ' + str(tn) + '\n') r.write('False Negatives: ' + str(fn) + '\n') r.write('Precision: ' + str(precision) + '\n') r.write('Sensitivity: ' + str(sensitivity) + '\n') r.write('F-Measure: ' + str(fmeasure) + '\n')
line[2], line[3] ]) predict_and_real_std.append([ med_map[uid][0], med_map[uid][1], med_map[uid][2], line[1], line[2], line[3] ]) std_thres = 5 if std_map[uid][0] > std_thres and 0 < predict_and_real_std[-1][0]: predict_and_real_std[-1][0] += 5 if std_map[uid][1] > std_thres and 0 < predict_and_real_std[-1][1]: predict_and_real_std[-1][1] += 5 if std_map[uid][2] > std_thres and 0 < predict_and_real_std[-1][2]: predict_and_real_std[-1][2] += 5 predict_and_real_0_baseline.append( [0, 0, 0, line[1], line[2], line[3]]) # print(predict_and_real_med[-1]) print "Median predict: {:.2f}%".format(precision(predict_and_real_med)) print "Average predict: {:.2f}%".format(precision(predict_and_real_avg)) print "Median and STD predict: {:.2f}%".format( precision(predict_and_real_std)) print "Zero baseline: {:.2f}%".format( precision(predict_and_real_0_baseline)) cursor.close() cnx.close() except mysql.connector.Error as err: print(err.msg)
def test_point_wise(): train, dev, test = load(FLAGS.data, filter=FLAGS.clean) #wiki # train, test, dev = load(FLAGS.data, filter=FLAGS.clean) #trec q_max_sent_length = max( map(lambda x: len(x), train['question'].str.split())) a_max_sent_length = max(map(lambda x: len(x), train['answer'].str.split())) print(q_max_sent_length) print(a_max_sent_length) print(len(train)) print('train question unique:{}'.format(len(train['question'].unique()))) print('train length', len(train)) print('test length', len(test)) print('dev length', len(dev)) alphabet, embeddings, embeddings_complex = prepare( [train, test, dev], dim=FLAGS.embedding_dim, is_embedding_needed=True, fresh=True) print(embeddings_complex) print('alphabet:', len(alphabet)) with tf.Graph().as_default(): with tf.device("/gpu:0"): # session_conf = tf.ConfigProto( # allow_soft_placement=FLAGS.allow_soft_placement, # log_device_placement=FLAGS.log_device_placement) session_conf = tf.ConfigProto() session_conf.allow_soft_placement = FLAGS.allow_soft_placement session_conf.log_device_placement = FLAGS.log_device_placement session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) with sess.as_default(), open(precision, "w") as log: log.write(str(FLAGS.__flags) + '\n') # train,test,dev = load("trec",filter=True) # alphabet,embeddings = prepare([train,test,dev],is_embedding_needed = True) cnn = QA_quantum(max_input_left=q_max_sent_length, max_input_right=a_max_sent_length, vocab_size=len(alphabet), embedding_size=FLAGS.embedding_dim, batch_size=FLAGS.batch_size, embeddings=embeddings, embeddings_complex=embeddings_complex, dropout_keep_prob=FLAGS.dropout_keep_prob, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, is_Embedding_Needed=True, trainable=FLAGS.trainable, overlap_needed=FLAGS.overlap_needed, position_needed=FLAGS.position_needed, pooling=FLAGS.pooling, hidden_num=FLAGS.hidden_num, extend_feature_dim=FLAGS.extend_feature_dim) cnn.build_graph() # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) starter_learning_rate = FLAGS.learning_rate learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 100, 0.96) optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate) # optimizer = tf.train.GradientDescentOptimizer(learning_rate) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) saver = tf.train.Saver(tf.global_variables(), max_to_keep=20) merged = tf.summary.merge_all() writer = tf.summary.FileWriter( "logs_NNQLM2_embedding_xiangwei_uniform/", sess.graph) sess.run(tf.global_variables_initializer()) map_max = 0.65 now = int(time.time()) timeArray = time.localtime(now) timeStamp = time.strftime("%Y%m%d%H%M%S", timeArray) timeDay = time.strftime("%Y%m%d", timeArray) print(timeStamp) for i in range(FLAGS.num_epochs): d = get_overlap_dict(train, alphabet, q_len=q_max_sent_length, a_len=a_max_sent_length) datas = batch_gen_with_point_wise(train, alphabet, FLAGS.batch_size, overlap_dict=d, q_len=q_max_sent_length, a_len=a_max_sent_length) for data in datas: feed_dict = { cnn.question: data[0], cnn.answer: data[1], cnn.input_y: data[2], cnn.q_position: data[3], cnn.a_position: data[4], cnn.overlap: data[5], cnn.q_overlap: data[6], cnn.a_overlap: data[7] } _, step, loss, accuracy, pred, scores, input_y, position = sess.run( [ train_op, global_step, cnn.loss, cnn.accuracy, cnn.predictions, cnn.scores, cnn.input_y, cnn.embedding_W_pos ], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g} ".format( time_str, step, loss, accuracy)) now = int(time.time()) timeArray = time.localtime(now) timeStamp = time.strftime("%Y%m%d%H%M%S", timeArray) timeDay = time.strftime("%Y%m%d", timeArray) print(timeStamp) predicted = predict(sess, cnn, train, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) predicted_label = np.argmax(predicted, 1) map_mrr_train = evaluation.evaluationBypandas( train, predicted[:, -1]) predicted_test = predict(sess, cnn, test, alphabet, FLAGS.batch_size, q_max_sent_length, a_max_sent_length) predicted_label = np.argmax(predicted_test, 1) p1 = evaluation.precision(test, predicted_test[:, -1]) map_mrr_test = evaluation.evaluationBypandas( test, predicted_test[:, -1]) if map_mrr_test[0] > map_max: map_max = map_mrr_test[0] timeStamp = time.strftime("%Y%m%d%H%M%S", time.localtime(int(time.time()))) folder = 'runs/' + timeDay out_dir = folder + '/' + timeStamp + \ '__' + FLAGS.data + str(map_mrr_test[0]) if not os.path.exists(folder): os.makedirs(folder) #save_path = saver.save(sess, out_dir) print("{}:train epoch:map mrr {}".format(i, map_mrr_train)) print("{}:test epoch:map mrr {}".format(i, map_mrr_test)) print("{}:test epoch: p1 {}".format(i, p1)) line1 = " {}:epoch: map_train{}".format(i, map_mrr_train) line2 = " {}:epoch: map_test{}".format(i, map_mrr_test) line3 = " {}:epoch: p1{}".format(i, p1) log.write(line1 + '\n' + line2 + '\n' + line3 + '\n') log.flush() log.close()
print print "Some documents couldn't be predicted, then they was assigned with None and will not be evaluated" positive_docs_non_predicted = 0 list_of_true_negative_documents = [] for tn in corpora.negatives[:num_of_documents]: if tn.predicted_polarity: list_of_true_negative_documents.append(tn) else: positive_docs_non_predicted += 1 negative_docs_non_predicted = 0 list_of_true_positive_documents = [] for tp in corpora.positives[:num_of_documents]: if tp.predicted_polarity: list_of_true_positive_documents.append(tp) else: negative_docs_non_predicted += 1 print "Positive docs non predicted: " + str(positive_docs_non_predicted) print "Negative docs non predicted: " + str(negative_docs_non_predicted) print print "Precision" print str(eval.precision(len(corpora.positives[:num_of_documents]), list_of_true_negative_documents, ref=0.5) * decimal.Decimal(100)) + ' %' print "Recall" print str(eval.recall(len(corpora.positives[:num_of_documents]), list_of_true_positive_documents, ref=0.5) * decimal.Decimal(100)) + ' %' print "Accuracy" print str(eval.accuracy(len(corpora.positives), len(corpora.negatives), list_of_true_positive_documents + list_of_true_negative_documents, ref=0.5) * decimal.Decimal(100)) + ' %'
num_of_documents = args.qDocs for p_doc in list(enumerate(corpora.positives[:num_of_documents])): print "extracting ngrams from positive documents" print p_doc[0] pp.extract_ngrams(p_doc[1], stopwords=args.stopwords) clear() for n_doc in list(enumerate(corpora.negatives[:num_of_documents])): print "extracting ngrams from negative documents" print n_doc[0] pp.extract_ngrams(n_doc[1], stopwords=args.stopwords) clear() print "____________________CLASSIFICATION STAGE____________________" all_documents = corpora.positives[:num_of_documents] + corpora.negatives[:num_of_documents] classifier = classification.OhanaBrendan(all_documents) classifier.rule = args.tags classifier.term_counting() print "____________________EVALUATION STAGE____________________" print args print print "Precision" print str(eval.precision(len(corpora.positives[:num_of_documents]), corpora.negatives[:num_of_documents]) * decimal.Decimal(100)) + ' %' print "Recall" print str(eval.recall(len(corpora.positives[:num_of_documents]), corpora.positives[:num_of_documents]) * decimal.Decimal(100)) + ' %' print "Accuracy" print str(eval.accuracy(len(corpora.positives), len(corpora.negatives), all_documents) * decimal.Decimal(100)) + ' %'
print c_regr.coef_ print l_regr.coef_ # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % f_regr.score(X_test, f_y_test)) print('Variance score: %.2f' % c_regr.score(X_test, c_y_test)) print('Variance score: %.2f' % l_regr.score(X_test, l_y_test)) for i in range(0, len(y_predict)): if y_predict[i] == 0: f_y_test.append(y_test[i][0]) c_y_test.append(y_test[i][1]) l_y_test.append(y_test[i][2]) f_y_predict.append(0) c_y_predict.append(0) l_y_predict.append(0) predict_and_real = [] for i in range(0, len(y_predict)): predict_and_real.append([ f_y_predict[i], c_y_predict[i], l_y_predict[i], f_y_test[i], c_y_test[i], l_y_test[i] ]) predict_and_real0 = [] for i in range(0, len(y_predict)): predict_and_real0.append([0, 0, 0, f_y_test[i], c_y_test[i], l_y_test[i]]) print(len(y_predict)) print "Predict: {:.2f}%".format(precision(predict_and_real)) print "Zero baseline: {:.2f}%".format(precision(predict_and_real0))
def main(blocking_fn, classification_fn, threshold, minthresh, weightvec, blocking_attrs, func_list, save=False): # ******** In lab 3, explore different attribute sets for blocking ************ # The list of attributes to use for blocking (all must occur in the above # attribute lists) blocking_attrA_list = blocking_attrs blocking_attrB_list = blocking_attrs # ******** In lab 4, explore different comparison functions for different **** # ******** attributes **** # The list of tuples (comparison function, attribute number in record A, # attribute number in record B) # exact_comp_funct_list = [ (comparison.exact_comp, 1, 1), # First name (comparison.exact_comp, 2, 2), # Middle name (comparison.exact_comp, 3, 3), # Last name (comparison.exact_comp, 8, 8), # Suburb (comparison.exact_comp, 10, 10), # State ] approx_comp_funct_list = [ (func_list[0], 1, 1), # First name (func_list[1], 2, 2), # Middle name (func_list[2], 3, 3), # Last name (func_list[3], 7, 7), # Address (func_list[4], 8, 8), # Suburb (func_list[5], 10, 10), # State ] # ============================================================================= # # Step 1: Load the two datasets from CSV files start_time = time.time() recA_dict = loadDataset.load_data_set(datasetA_name, rec_idA_col, \ attrA_list, headerA_line) recB_dict = loadDataset.load_data_set(datasetB_name, rec_idB_col, \ attrB_list, headerB_line) # Load data set of true matching pairs # true_match_set = loadDataset.load_truth_data(truthfile_name) loading_time = time.time() - start_time # ----------------------------------------------------------------------------- # Step 2: Block the datasets def genericBlock(block_function='none', recA_dict=recA_dict, recB_dict=recB_dict, blocking_attrA_list=blocking_attrA_list, blocking_attrB_list=blocking_attrB_list): start_time = time.time() # Select one blocking technique if block_function == 'none': # No blocking (all records in one block) # resultA = blocking.noBlocking(recA_dict) resultB = blocking.noBlocking(recB_dict) if block_function == 'attr': # Simple attribute-based blocking # resultA = blocking.simpleBlocking(recA_dict, blocking_attrA_list) resultB = blocking.simpleBlocking(recB_dict, blocking_attrB_list) if block_function == 'soundex': # Phonetic (Soundex) based blocking # resultA = blocking.phoneticBlocking(recA_dict, blocking_attrA_list) resultB = blocking.phoneticBlocking(recB_dict, blocking_attrB_list) if block_function == 'slk': # Statistical linkage key (SLK-581) based blocking # fam_name_attr_ind = 3 giv_name_attr_ind = 1 dob_attr_ind = 6 gender_attr_ind = 4 resultA = blocking.slkBlocking(recA_dict, fam_name_attr_ind, \ giv_name_attr_ind, dob_attr_ind, \ gender_attr_ind) resultB = blocking.slkBlocking(recB_dict, fam_name_attr_ind, \ giv_name_attr_ind, dob_attr_ind, \ gender_attr_ind) block_time = time.time() - start_time # Print blocking statistics # # blocking.printBlockStatistics(resultA, resultB) return resultA, resultB, block_time blockA_dict, blockB_dict, blocking_time = genericBlock( block_function=blocking_fn) # ----------------------------------------------------------------------------- # Step 3: Compare the candidate pairs start_time = time.time() sim_vec_dict = comparison.compareBlocks(blockA_dict, blockB_dict, \ recA_dict, recB_dict, \ approx_comp_funct_list) comparison_time = time.time() - start_time # ----------------------------------------------------------------------------- # Step 4: Classify the candidate pairs def genericClassification(classification_function='exact', sim_vec_dict=sim_vec_dict, sim_threshold=threshold, min_sim_threshold=minthresh, weight_vec=weightvec, true_match_set=true_match_set): start_time = time.time() if classification_function == 'exact': # Exact matching based classification class_match_set1, class_nonmatch_set1 = \ classification.exactClassify(sim_vec_dict) if classification_function == 'simthresh': # Similarity threshold based classification # class_match_set1, class_nonmatch_set1 = \ classification.thresholdClassify(sim_vec_dict, sim_threshold) if classification_function == 'minsim': # Minimum similarity threshold based classification # class_match_set1, class_nonmatch_set1 = \ classification.minThresholdClassify(sim_vec_dict, min_sim_threshold) if classification_function == 'weightsim': # Weighted similarity threshold based classification # # weight_vec = [1.0] * len(approx_comp_funct_list) # Lower weights for middle name and state # # weight_vec = [2.0, 1.0, 2.0, 2.0, 2.0, 1.0] class_match_set1, class_nonmatch_set1 = \ classification.weightedSimilarityClassify(sim_vec_dict, weight_vec, sim_threshold) if classification_function == 'dt': # A supervised decision tree classifier # class_match_set1, class_nonmatch_set1 = \ classification.supervisedMLClassify(sim_vec_dict, true_match_set) class_time = time.time() - start_time return class_match_set1, class_nonmatch_set1, class_time threshold = minthresh class_match_set, class_nonmatch_set, classification_time = genericClassification( classification_fn) # ----------------------------------------------------------------------------- # Step 5: Evaluate the classification # Initialise dictionary of results dict = {} # Get the number of record pairs compared # num_comparisons = len(sim_vec_dict) # Get the number of total record pairs to compared if no blocking used # all_comparisons = len(recA_dict) * len(recB_dict) # Get the list of identifiers of the compared record pairs # cand_rec_id_pair_list = sim_vec_dict.keys() # Blocking evaluation # rr = evaluation.reduction_ratio(num_comparisons, all_comparisons) pc = evaluation.pairs_completeness(cand_rec_id_pair_list, true_match_set) pq = evaluation.pairs_quality(cand_rec_id_pair_list, true_match_set) # Linkage evaluation # linkage_result = evaluation.confusion_matrix(class_match_set, class_nonmatch_set, true_match_set, all_comparisons) accuracy = evaluation.accuracy(linkage_result) precision = evaluation.precision(linkage_result) recall = evaluation.recall(linkage_result) fmeasure = evaluation.fmeasure(linkage_result) # print('Linkage evaluation:') # print(' Accuracy: %.3f' % (accuracy)) # print(' Precision: %.3f' % (precision)) # print(' Recall: %.3f' % (recall)) # print(' F-measure: %.3f' % (fmeasure)) # print('') linkage_time = loading_time + blocking_time + comparison_time + \ classification_time # print('Total runtime required for linkage: %.3f sec' % (linkage_time)) # Export blocking metrics dict['blocking_fn'] = blocking_fn dict['classification_fn'] = classification_fn dict['threshold'] = threshold dict['min_thresh'] = minthresh dict['weight_vec'] = weightvec dict['blocking_attrs'] = blocking_attrs dict['comp_funcs'] = func_list dict['num_comparisons'] = num_comparisons dict['all_comparisons'] = all_comparisons # dict['cand_rec_id_pair_list'] = cand_rec_id_pair_list dict['rr'] = rr dict['pc'] = pc dict['pq'] = pq dict['blocking_time'] = blocking_time # dict['linkage_result'] = linkage_result dict['accuracy'] = accuracy dict['precision'] = precision dict['recall'] = recall dict['fmeasure'] = fmeasure dict['linkage_time'] = linkage_time # Save results if save: saveLinkResult.save_linkage_set('final_results.txt', class_match_set) # Return results return dict