def ap_y_classwise(Y_target, Y_score):
"""
Y_target: list of lists. {0, 1}
real labels
Y_score: list of lists. real values
prediction values
"""
# Y_target = np.squeeze(np.array(Y_target))
# Y_score = np.squeeze(np.array(Y_score))
Y_target = np.array(Y_target)
Y_score = np.array(Y_score)
ap_list = ap(Y_target, Y_score, average=None)
return ap_list
def computeAUROC (dataGT, dataPRED, classCount):
outAUROC = []
outAUPRC = []
outAP = []
datanpGT = dataGT.cpu().numpy()
datanpPRED = dataPRED.cpu().numpy()
for i in range(classCount):
outAUROC.append(roc_auc_score(datanpGT[:, i], datanpPRED[:, i], average = 'weighted'))
outP, outR, _ = prc(datanpGT[:, i], datanpPRED[:, i])
outAUPRC.append(auc(outR, outP, False))
outAP.append(ap(datanpGT[:, i], datanpPRED[:, i]))
return outAUROC, outAUPRC, outAP
epoch_loss = running_loss / dataset_sizes[phase]
if phase == 'train':
last_train_loss = epoch_loss
else:
with open("results/log_train", 'a') as logfile:
logwriter = csv.writer(logfile, delimiter=',')
if (epoch == 1):
logwriter.writerow([
"epoch", "train_loss", "test_loss", "test_acc_vw",
"test_ap_vw"
])
logwriter.writerow([
epoch, last_train_loss, epoch_loss,
acc / dataset_sizes[phase],
ap(y_label, y_pred)
])
print("test set accuracy visual world: ",
acc / dataset_sizes[phase])
print("average_precision_score visual world: ",
ap(y_label, y_pred))
print(phase + ' epoch {}:loss {:.4f} with data size {}'.format(
epoch, epoch_loss, dataset_sizes[phase]))
total_done += batch_size
if (total_done % (100 * batch_size) == 0):
print("completed " + str(total_done) + " so far in epoch")
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
assert dcg_score([5, 3, 2], [2, 1, 0]) == dcg_from_ranking([5, 3, 2],
[0, 1, 2])
assert dcg_score([1, 3, 2], [2, 1, 0]) == dcg_from_ranking([1, 3, 2],
[0, 1, 2])
assert dcg_score([1, 3, 2], [0, 2, 1]) == dcg_from_ranking([1, 3, 2],
[1, 2, 0])
assert ndcg_score([1, 3, 2], [2, 1, 0]) == ndcg_from_ranking([1, 3, 2],
[0, 1, 2])
assert dcg_score([5, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([5, 3, 2],
[0, 1])
assert dcg_score([1, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([1, 3, 2],
[0, 1])
assert dcg_score([1, 3, 2], [0, 2, 1], k=2) == dcg_from_ranking([1, 3, 2],
[1, 2])
assert ndcg_score([1, 3, 2], [2, 1, 0], k=2) == \
ndcg_from_ranking([1, 3, 2], [0, 1])
# Precision
assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=2) == 1.0
assert ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=2) == 0.5
assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=3) == \
ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=3)
# Average precision
from sklearn.metrics import average_precision_score as ap
assert average_precision_score([1, 1, 0], [3, 2, 1]) == ap([1, 1, 0],
[3, 2, 1])
assert average_precision_score([1, 1, 0], [3, 1, 0]) == ap([1, 1, 0],
[3, 1, 0])
for k in ks:
f = open(os.path.join(result_dir, "aps." + str(k) + ".csv"), "w")
f.close()
pb_widgets = [
progressbar.Percentage(),
' ', progressbar.Counter(),
' ', progressbar.Bar(marker='*'),
' ', progressbar.ETA(),
' ', progressbar.FileTransferSpeed(),
]
bar = progressbar.ProgressBar(maxval=len(keys), \
widgets=pb_widgets)
bar.start()
for i, imageid in enumerate(keys):
bar.update(i)
p_short, gt_short = get_gt(imageid, "es-short", d_short, code_len=64)
p_twostage, gt_twostage = get_gt(imageid, "es-twostage", d_twostage, code_len=256)
p_long, gt_long = get_gt(imageid, "es-long", d_long, code_len=256)
for k in ks:
with open(os.path.join(result_dir, "aps." + str(k) + ".csv"), "a") as f:
ap_short = ap(gt_short[:k], p_short[:k]) if 1 in gt_short[:k] else 0
ap_twostage = ap(gt_twostage[:k], p_twostage[:k]) if 1 in gt_twostage[:k] else 0
ap_long = ap(gt_long[:k], p_long[:k]) if 1 in gt_long[:k] else 0
csv_writer = csv.writer(f)
csv_writer.writerow([imageid, ap_short, ap_twostage, ap_long])
bar.finish()
epoch_loss = running_loss / dataset_sizes[phase]
if phase == 'train':
last_train_loss = epoch_loss
else:
with open("results/baseline_holdout_log_train", 'a') as logfile:
logwriter = csv.writer(logfile, delimiter=',')
if (epoch == 1):
logwriter.writerow([
"epoch", "train_loss", "test_loss", "test_acc",
"test_ap"
])
logwriter.writerow([
epoch, last_train_loss, epoch_loss,
acc / dataset_sizes[phase],
ap(y_label, y_pred)
])
print("test set accuracy: ", acc / dataset_sizes[phase])
print("average_precision_score: ", ap(y_label, y_pred))
print(phase + ' epoch {}:loss {:.4f} with data size {}'.format(
epoch, epoch_loss, dataset_sizes[phase]))
total_done += batch_size
if (total_done % (100 * batch_size) == 0):
print("completed " + str(total_done) + " so far in epoch")
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
time_elapsed % 60))
epoch_loss = running_loss / dataset_sizes[phase]
if phase == 'train':
last_train_loss = epoch_loss
else:
with open("results/log_train", 'a') as logfile:
logwriter = csv.writer(logfile, delimiter=',')
if (epoch == 1):
logwriter.writerow([
"epoch", "train_loss", "test_loss", "test_acc_vw",
"test_acc_bias", "test_ap_vw", "test_ap_bias"
])
logwriter.writerow([
epoch, last_train_loss, epoch_loss,
acc1 / dataset_sizes[phase], acc2 / dataset_sizes[phase],
ap(y_label, y_pred1),
ap(y_label, y_pred2)
])
print("test set accuracy visual world: ",
acc1 / dataset_sizes[phase])
print("test set accuracy bias dataset: ",
acc2 / dataset_sizes[phase])
print("average_precision_score visual world: ",
ap(y_label, y_pred1))
print("average_precision_score bias dataset: ",
ap(y_label, y_pred2))
print(phase + ' epoch {}:loss {:.4f} with data size {}'.format(
epoch, epoch_loss, dataset_sizes[phase]))
total_done += batch_size
def average_precision_score(results):
y_true = np.array([0 if l == "diff_author" else 1 for l, _ in results])
scores = np.array([score for _, score in results])
scores = 1 - scores # highest similarity highest in rank
return ap(y_true, scores)
assert ndcg_score([2, 3, 5], [0, 1, 2], k=2) == 1.0
assert ndcg_from_ranking([5, 3, 2], [0, 1]) == 1.0
# Check that sample order is irrelevant
assert dcg_score([5, 3, 2], [2, 1, 0]) == dcg_score([2, 3, 5], [0, 1, 2])
assert dcg_score([5, 3, 2], [2, 1, 0], k=2) == dcg_score([2, 3, 5], [0, 1, 2], k=2)
# Check equivalence between two interfaces.
assert dcg_score([5, 3, 2], [2, 1, 0]) == dcg_from_ranking([5, 3, 2], [0, 1, 2])
assert dcg_score([1, 3, 2], [2, 1, 0]) == dcg_from_ranking([1, 3, 2], [0, 1, 2])
assert dcg_score([1, 3, 2], [0, 2, 1]) == dcg_from_ranking([1, 3, 2], [1, 2, 0])
assert ndcg_score([1, 3, 2], [2, 1, 0]) == ndcg_from_ranking([1, 3, 2], [0, 1, 2])
assert dcg_score([5, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([5, 3, 2], [0, 1])
assert dcg_score([1, 3, 2], [2, 1, 0], k=2) == dcg_from_ranking([1, 3, 2], [0, 1])
assert dcg_score([1, 3, 2], [0, 2, 1], k=2) == dcg_from_ranking([1, 3, 2], [1, 2])
assert ndcg_score([1, 3, 2], [2, 1, 0], k=2) == \
ndcg_from_ranking([1, 3, 2], [0, 1])
# Precision
assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=2) == 1.0
assert ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=2) == 0.5
assert ranking_precision_score([1, 1, 0], [3, 2, 1], k=3) == \
ranking_precision_score([1, 1, 0], [1, 0, 0.5], k=3)
# Average precision
from sklearn.metrics import average_precision_score as ap
assert average_precision_score([1, 1, 0], [3, 2, 1]) == ap([1, 1, 0], [3, 2, 1])
assert average_precision_score([1, 1, 0], [3, 1, 0]) == ap([1, 1, 0], [3, 1, 0])
def average_precision_score(results):
y_true = np.array([0 if l == "diff_author" else 1 for l, _ in results])
scores = np.array([score for _, score in results])
scores = 1 - scores # highest similarity highest in rank
return ap(y_true, scores)
y_pred.append(outputs[0].item())
y_label.append(labels[0].item())
running_loss += loss.data * batch_size
#print ("training batch loss: ", loss.data)
epoch_loss = running_loss / dataset_sizes[phase]
if phase == 'train':
last_train_loss = epoch_loss
else:
with open("results/log_train", 'a') as logfile:
logwriter = csv.writer(logfile, delimiter=',')
if(epoch == 1):
logwriter.writerow(["epoch", "train_loss", "test_loss", "test_acc", "test_ap"])
logwriter.writerow([epoch, last_train_loss, epoch_loss, acc / dataset_sizes[phase], ap(y_label, y_pred)])
print ("test set accuracy: ", acc / dataset_sizes[phase])
print ("average_precision_score: ", ap(y_label, y_pred))
print(phase + ' epoch {}:loss {:.4f} with data size {}'.format(
epoch, epoch_loss, dataset_sizes[phase]))
total_done += batch_size
if(total_done % (100 * batch_size) == 0):
print("completed " + str(total_done) + " so far in epoch")
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
