def ap_y_classwise(Y_target, Y_score): """ Y_target: list of lists. {0, 1} real labels Y_score: list of lists. real values prediction values """ # Y_target = np.squeeze(np.array(Y_target)) # Y_score = np.squeeze(np.array(Y_score)) Y_target = np.array(Y_target) Y_score = np.array(Y_score) ap_list = ap(Y_target, Y_score, average=None) return ap_list
def computeAUROC (dataGT, dataPRED, classCount): outAUROC = [] outAUPRC = [] outAP = [] datanpGT = dataGT.cpu().numpy() datanpPRED = dataPRED.cpu().numpy() for i in range(classCount): outAUROC.append(roc_auc_score(datanpGT[:, i], datanpPRED[:, i], average = 'weighted')) outP, outR, _ = prc(datanpGT[:, i], datanpPRED[:, i]) outAUPRC.append(auc(outR, outP, False)) outAP.append(ap(datanpGT[:, i], datanpPRED[:, i])) return outAUROC, outAUPRC, outAP
epoch_loss = running_loss / dataset_sizes[phase] if phase == 'train': last_train_loss = epoch_loss else: with open("results/log_train", 'a') as logfile: logwriter = csv.writer(logfile, delimiter=',') if (epoch == 1): logwriter.writerow([ "epoch", "train_loss", "test_loss", "test_acc_vw", "test_ap_vw" ]) logwriter.writerow([ epoch, last_train_loss, epoch_loss, acc / dataset_sizes[phase], ap(y_label, y_pred) ]) print("test set accuracy visual world: ", acc / dataset_sizes[phase]) print("average_precision_score visual world: ", ap(y_label, y_pred)) print(phase + ' epoch {}:loss {:.4f} with data size {}'.format( epoch, epoch_loss, dataset_sizes[phase])) total_done += batch_size if (total_done % (100 * batch_size) == 0): print("completed " + str(total_done) + " so far in epoch") time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
assert dcg_score([5, 3, 2], [2, 1, 0]) == dcg_from_ranking([5, 3, 2], [0, 1, 2]) assert dcg_score([1, 3, 2], [2, 1, 0]) == dcg_from_ranking([1, 3, 2], [0, 1, 2]) assert dcg_score([1, 3, 2], [0, 2, 1]) == dcg_from_ranking([1, 3, 2], [1, 2, 0]) assert ndcg_score([1, 3, 2], [2, 1, 0]) == ndcg_from_ranking([1, 3, 2], [0, 1, 2]) assert dcg_score([5, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([5, 3, 2], [0, 1]) assert dcg_score([1, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([1, 3, 2], [0, 1]) assert dcg_score([1, 3, 2], [0, 2, 1], k=2) == dcg_from_ranking([1, 3, 2], [1, 2]) assert ndcg_score([1, 3, 2], [2, 1, 0], k=2) == \ ndcg_from_ranking([1, 3, 2], [0, 1]) # Precision assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=2) == 1.0 assert ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=2) == 0.5 assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=3) == \ ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=3) # Average precision from sklearn.metrics import average_precision_score as ap assert average_precision_score([1, 1, 0], [3, 2, 1]) == ap([1, 1, 0], [3, 2, 1]) assert average_precision_score([1, 1, 0], [3, 1, 0]) == ap([1, 1, 0], [3, 1, 0])
for k in ks: f = open(os.path.join(result_dir, "aps." + str(k) + ".csv"), "w") f.close() pb_widgets = [ progressbar.Percentage(), ' ', progressbar.Counter(), ' ', progressbar.Bar(marker='*'), ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed(), ] bar = progressbar.ProgressBar(maxval=len(keys), \ widgets=pb_widgets) bar.start() for i, imageid in enumerate(keys): bar.update(i) p_short, gt_short = get_gt(imageid, "es-short", d_short, code_len=64) p_twostage, gt_twostage = get_gt(imageid, "es-twostage", d_twostage, code_len=256) p_long, gt_long = get_gt(imageid, "es-long", d_long, code_len=256) for k in ks: with open(os.path.join(result_dir, "aps." + str(k) + ".csv"), "a") as f: ap_short = ap(gt_short[:k], p_short[:k]) if 1 in gt_short[:k] else 0 ap_twostage = ap(gt_twostage[:k], p_twostage[:k]) if 1 in gt_twostage[:k] else 0 ap_long = ap(gt_long[:k], p_long[:k]) if 1 in gt_long[:k] else 0 csv_writer = csv.writer(f) csv_writer.writerow([imageid, ap_short, ap_twostage, ap_long]) bar.finish()
epoch_loss = running_loss / dataset_sizes[phase] if phase == 'train': last_train_loss = epoch_loss else: with open("results/baseline_holdout_log_train", 'a') as logfile: logwriter = csv.writer(logfile, delimiter=',') if (epoch == 1): logwriter.writerow([ "epoch", "train_loss", "test_loss", "test_acc", "test_ap" ]) logwriter.writerow([ epoch, last_train_loss, epoch_loss, acc / dataset_sizes[phase], ap(y_label, y_pred) ]) print("test set accuracy: ", acc / dataset_sizes[phase]) print("average_precision_score: ", ap(y_label, y_pred)) print(phase + ' epoch {}:loss {:.4f} with data size {}'.format( epoch, epoch_loss, dataset_sizes[phase])) total_done += batch_size if (total_done % (100 * batch_size) == 0): print("completed " + str(total_done) + " so far in epoch") time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
epoch_loss = running_loss / dataset_sizes[phase] if phase == 'train': last_train_loss = epoch_loss else: with open("results/log_train", 'a') as logfile: logwriter = csv.writer(logfile, delimiter=',') if (epoch == 1): logwriter.writerow([ "epoch", "train_loss", "test_loss", "test_acc_vw", "test_acc_bias", "test_ap_vw", "test_ap_bias" ]) logwriter.writerow([ epoch, last_train_loss, epoch_loss, acc1 / dataset_sizes[phase], acc2 / dataset_sizes[phase], ap(y_label, y_pred1), ap(y_label, y_pred2) ]) print("test set accuracy visual world: ", acc1 / dataset_sizes[phase]) print("test set accuracy bias dataset: ", acc2 / dataset_sizes[phase]) print("average_precision_score visual world: ", ap(y_label, y_pred1)) print("average_precision_score bias dataset: ", ap(y_label, y_pred2)) print(phase + ' epoch {}:loss {:.4f} with data size {}'.format( epoch, epoch_loss, dataset_sizes[phase])) total_done += batch_size
def average_precision_score(results): y_true = np.array([0 if l == "diff_author" else 1 for l, _ in results]) scores = np.array([score for _, score in results]) scores = 1 - scores # highest similarity highest in rank return ap(y_true, scores)
assert ndcg_score([2, 3, 5], [0, 1, 2], k=2) == 1.0 assert ndcg_from_ranking([5, 3, 2], [0, 1]) == 1.0 # Check that sample order is irrelevant assert dcg_score([5, 3, 2], [2, 1, 0]) == dcg_score([2, 3, 5], [0, 1, 2]) assert dcg_score([5, 3, 2], [2, 1, 0], k=2) == dcg_score([2, 3, 5], [0, 1, 2], k=2) # Check equivalence between two interfaces. assert dcg_score([5, 3, 2], [2, 1, 0]) == dcg_from_ranking([5, 3, 2], [0, 1, 2]) assert dcg_score([1, 3, 2], [2, 1, 0]) == dcg_from_ranking([1, 3, 2], [0, 1, 2]) assert dcg_score([1, 3, 2], [0, 2, 1]) == dcg_from_ranking([1, 3, 2], [1, 2, 0]) assert ndcg_score([1, 3, 2], [2, 1, 0]) == ndcg_from_ranking([1, 3, 2], [0, 1, 2]) assert dcg_score([5, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([5, 3, 2], [0, 1]) assert dcg_score([1, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([1, 3, 2], [0, 1]) assert dcg_score([1, 3, 2], [0, 2, 1], k=2) == dcg_from_ranking([1, 3, 2], [1, 2]) assert ndcg_score([1, 3, 2], [2, 1, 0], k=2) == \ ndcg_from_ranking([1, 3, 2], [0, 1]) # Precision assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=2) == 1.0 assert ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=2) == 0.5 assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=3) == \ ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=3) # Average precision from sklearn.metrics import average_precision_score as ap assert average_precision_score([1, 1, 0], [3, 2, 1]) == ap([1, 1, 0], [3, 2, 1]) assert average_precision_score([1, 1, 0], [3, 1, 0]) == ap([1, 1, 0], [3, 1, 0])
y_pred.append(outputs[0].item()) y_label.append(labels[0].item()) running_loss += loss.data * batch_size #print ("training batch loss: ", loss.data) epoch_loss = running_loss / dataset_sizes[phase] if phase == 'train': last_train_loss = epoch_loss else: with open("results/log_train", 'a') as logfile: logwriter = csv.writer(logfile, delimiter=',') if(epoch == 1): logwriter.writerow(["epoch", "train_loss", "test_loss", "test_acc", "test_ap"]) logwriter.writerow([epoch, last_train_loss, epoch_loss, acc / dataset_sizes[phase], ap(y_label, y_pred)]) print ("test set accuracy: ", acc / dataset_sizes[phase]) print ("average_precision_score: ", ap(y_label, y_pred)) print(phase + ' epoch {}:loss {:.4f} with data size {}'.format( epoch, epoch_loss, dataset_sizes[phase])) total_done += batch_size if(total_done % (100 * batch_size) == 0): print("completed " + str(total_done) + " so far in epoch") time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format(