def calculate_metrics(df):
""" Calculates metrics at different k (1 to 10 + 20,30,40,50)"""
#print(df.columns)
klist = list(range(1, 11))
klist.extend([20, 30, 40, 50, 100, 200, 300, 500])
print(klist)
# 14 x 3 x 4 columns added for each
for k in tqdm(klist):
df['average_precision_p2v_{}'.format(k)] = df['p2v_binary'].apply(lambda x: average_precision(x, k))
df['recall_p2v_{}'.format(k)] = df[['p2v_binary', 'ground_truth']].apply(
lambda x: recall_at_k(x.p2v_binary, x.ground_truth, k), axis=1)
df['reciprocal_rank_p2v_{}'.format(k)] = df['p2v_binary'].apply(lambda x: reciprocal_rank(x, k))
df['ndcg_p2v_{}'.format(k)] = df['p2v_binary'].apply(lambda x: ndcg(x, k))
df['average_precision_d2v_{}'.format(k)] = df['d2v_binary'].apply(lambda x: average_precision(x, k))
df['recall_d2v_{}'.format(k)] = df[['d2v_binary', 'ground_truth']].apply(
lambda x: recall_at_k(x.d2v_binary, x.ground_truth, k), axis=1)
df['reciprocal_rank_D2v_{}'.format(k)] = df['d2v_binary'].apply(lambda x: reciprocal_rank(x, k))
df['ndcg_d2v_{}'.format(k)] = df['d2v_binary'].apply(lambda x: ndcg(x, k))
df.to_pickle('/home/ashwath/Programs/MAGCS/Pickles/paperwisemetrics_mag50_d2v_p2v_may23_df.pickle')
print("METRICS CALCULATED, time to calculate the means")
# Get the mean of all the index columns
df = df.drop(['p2v_recommendations', 'p2v_binary', 'd2v_recommendations', 'd2v_binary', 'ground_truth'], axis=1)
mean_series = df.mean()
mean_series.to_csv('/home/ashwath/Programs/MAGCS/Evaluation/meanmetrics_mag50_d2v_p2v_may21.tsv', sep='\t', index=True, header=False)
print("C'est fini.")
def evaluate(self):
# Array of size the same as the number of labels
ap = np.zeros((self.num_labels, ))
baseline_ap = np.zeros((self.num_labels, ))
size_counter = 0
print("# of batches: ", len(self.dataloader))
ground_truth = np.empty((0, self.num_labels))
predictions = np.empty((0, self.num_labels))
# Switch to evaluation mode
self.model.eval()
with tqdm(total=len(self.dataloader)) as progress_bar:
for i, (inp, target) in enumerate(self.dataloader):
progress_bar.update(1)
# Load CPU version of target as numpy array
gts = target.numpy()
input_var = inp.cuda()
# compute output
output = self.model(input_var)
ests = torch.sigmoid(output).data.cpu().numpy()
predictions = np.vstack((predictions, ests))
ground_truth = np.vstack((ground_truth, gts))
size_counter += ests.shape[0]
for dim in range(self.num_labels):
# rescale ground truth to [-1, 1]
gt = 2 * ground_truth[:, dim] - 1
est = predictions[:, dim]
est_base = np.zeros(est.shape)
ap_score = average_precision(gt, est)
base_ap_score = average_precision(gt, est_base)
ap[dim] = ap_score
baseline_ap[dim] = base_ap_score
# for i in range(self.num_labels):
# print(ap[i])
ap_scores = {
'ap': ap,
'baseline_ap': baseline_ap,
'predictions': predictions,
'ground_truth': ground_truth
}
print('*** mAP and Baseline AP scores ***')
print(np.mean([a if not np.isnan(a) else 0 for a in ap]))
print(np.mean([a if not np.isnan(a) else 0 for a in baseline_ap]))
joblib.dump(ap_scores,
osp.join("{0}_ap_scores.jbl".format(self.name)),
compress=6)
def evaluate(path):
queries = read_dataset('queries.csv')
targets = read_dataset('targets.csv')
freqs = freq_count(targets)
results = load_results(path, queries, targets)
cutoff = 1000
precisions = []
recalls = []
f1scores = []
aps = []
gains = []
nnt1s = []
nnt2s = []
for (queried, retrieved) in results:
x = categories_to_rel(queried, retrieved)[:cutoff]
p = precision(x)
r = recall(x, freqs[queried[0]])
f = f1score(x, freqs[queried[0]])
g = ndcg(x)
ap = average_precision(x, freqs[queried[0]])
t1 = nnt1(x, freqs[queried[0]])
t2 = nnt2(x, freqs[queried[0]])
precisions.append(p)
recalls.append(r)
f1scores.append(f)
gains.append(g)
aps.append(ap)
nnt1s.append(t1)
nnt2s.append(t2)
print('precision:', p)
print('recall:', r)
print('F1 score:', f)
print('average precision:', ap)
print('NDCG:', g)
print('nearest neighbor:', t1, t2)
def run(training, validation, k, config):
texts_by_motifs = defaultdict(list)
motifs_in_docs = defaultdict(list)
# construct the bigdocuments
for i, (source, motifs, text) in enumerate(training):
for motif in motifs:
if motif != 'DUMMY':
motifs_in_docs[motif].append(i)
texts_by_motifs[motif].extend(text)
labels, texts = zip(*texts_by_motifs.items())
indexer = Indexer()
for label, text in zip(labels, texts):
indexer.add(text, label)
isError, OneError, nDocs = 0, 0, 0
margins, AP = [], []
for j, (source, motifs, text) in enumerate(validation):
nDocs += 1
scores = list(indexer.predict_proba(
text, config.getfloat('bm25', 'k1'), config.getfloat('bm25', 'b')))
preds = sorted(scores, key=lambda i: i[1], reverse=True)
preds = [label for label,score in preds]
refs = set(motifs)
ap = average_precision(preds, refs)
AP.append(ap)
isError += is_error(ap)
OneError += one_error(preds, refs)
margins.append(margin(preds, refs))
return isError, OneError, nDocs, margins, AP
def evaluate(path):
cad = read_dataset('cad.csv')
rgbd = read_dataset('rgbd.csv')
freqs = freq_count(cad)
results = load_results(path, rgbd, cad)
mP = 0.0
mR = 0.0
mF = 0.0
mAP = 0.0
mNDCG = 0.0
mNNT1 = 0.0
mNNT2 = 0.0
for (queried, retrieved) in results:
f = freqs[queried[0]]
x = categories_to_rel(queried, retrieved)[:f]
# Sum up the retrieval scores
mP += precision(x)
mR += recall(x, f)
mF += f1score(x, f)
mNDCG += ndcg(x)
mAP += average_precision(x, f)
mNNT1 += nnt1(x, f)
mNNT2 += nnt2(x, f)
n = len(results)
print('num queries:', n)
print('mean precision:', mP / n)
print('mean recall:', mR / n)
print('mean F1:', mF / n)
print('mean AP:', mAP / n)
print('mean NDCG: ', mNDCG / n)
print('mean NNT1: ', mNNT1 / n)
print('mean NNT2: ', mNNT2 / n)
# Plot PR-curve
cutoff = 1000
mean_precisions = np.zeros(cutoff, np.float64)
mean_recalls = np.zeros(cutoff, np.float64)
for (queried, retrieved) in results:
x = categories_to_rel(queried, retrieved)[:cutoff]
x = np.pad(x, (0, cutoff - len(x)), 'constant', constant_values=(0))
precisions = []
recalls = []
for k, _ in enumerate(x):
p = precision(x[:k + 1])
r = recall(x[:k + 1], freqs[queried[0]])
precisions.append(p)
recalls.append(r)
mean_precisions += precisions
mean_recalls += recalls
mean_precisions /= len(results)
mean_recalls /= len(results)
plt.plot(mean_recalls, mean_precisions)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.axis([0, 1, 0, 1.05])
plt.show()
def run(training, validation, k, config=None):
isError, OneError, nDocs = 0, 0, 0
margins, AP = [], []
class_index = Index()
traindocs, train_X, train_y = zip(*load_data(training, class_index))
testdocs, test_X, test_y = zip(*load_data(validation, class_index))
n_iter = np.ceil(10**6 / len(traindocs))
clf = SGDClassifier(alpha=.000001, loss='log', n_iter=50, penalty='elasticnet')
#clf = MultinomialNB(alpha=0.000001)
classifier = Pipeline([
('vectorizer', CountVectorizer(min_df=1, max_df=1.0, analyzer=lambda t: t)),
('tfidf', TfidfTransformer(norm='l2')),
('clf', OneVsRestClassifier(clf, n_jobs=-1))])
classifier.fit(train_X, train_y)
predictions = classifier.predict_proba(test_X)
for j, prediction in enumerate(predictions):
nDocs += 1
refs = np.zeros(len(prediction))
refs[list(test_y[j])] = 1
preds = sorted(range(len(prediction)), key=lambda i: prediction[i], reverse=True)
refs = set(test_y[j])
ap = average_precision(preds, refs)
AP.append(ap)
isError += is_error(ap)
OneError += one_error(preds, refs)
margins.append(margin(preds, refs))
return isError, OneError, nDocs, margins, AP
def run(training, validation, k, config):
texts_by_motifs = defaultdict(list)
motifs_in_docs = defaultdict(list)
# construct the bigdocuments
for i, (source, motifs, text) in enumerate(training):
for motif in motifs:
if motif != 'DUMMY':
motifs_in_docs[motif].append(i)
texts_by_motifs[motif].extend(text)
labels, texts = zip(*texts_by_motifs.items())
indexer = Indexer()
for label, text in zip(labels, texts):
indexer.add(text, label)
isError, OneError, nDocs = 0, 0, 0
margins, AP = [], []
for j, (source, motifs, text) in enumerate(validation):
nDocs += 1
scores = list(
indexer.predict_proba(text, config.getfloat('bm25', 'k1'),
config.getfloat('bm25', 'b')))
preds = sorted(scores, key=lambda i: i[1], reverse=True)
preds = [label for label, score in preds]
refs = set(motifs)
ap = average_precision(preds, refs)
AP.append(ap)
isError += is_error(ap)
OneError += one_error(preds, refs)
margins.append(margin(preds, refs))
return isError, OneError, nDocs, margins, AP
def evaluate(path):
queries = read_dataset('queries.csv')
targets = read_dataset('targets.csv')
freqs = freq_count(targets)
results = load_results(path, queries, targets)
cutoff = 1000
precisions = []
recalls = []
f1scores = []
aps = []
gains = []
nnt1s = []
nnt2s = []
for (queried, retrieved) in results:
x = categories_to_rel(queried, retrieved)[:cutoff]
p = precision(x)
r = recall(x, freqs[queried[0]])
f = f1score(x, freqs[queried[0]])
g = ndcg(x)
ap = average_precision(x, freqs[queried[0]])
t1 = nnt1(x, freqs[queried[0]])
t2 = nnt2(x, freqs[queried[0]])
precisions.append(p)
recalls.append(r)
f1scores.append(f)
gains.append(g)
aps.append(ap)
nnt1s.append(t1)
nnt2s.append(t2)
print('mean precision:', numpy.mean(precisions))
print('mean recall:', numpy.mean(recalls))
print('mean F1 score:', numpy.mean(f1scores))
print('mAP:', numpy.mean(aps))
print('mean NDCG:', numpy.mean(gains))
print('mean nearest neighbor:', numpy.mean(nnt1s), numpy.mean(nnt2s))
# plot precision-recall curve
mean_precisions = numpy.zeros(cutoff, numpy.float64)
mean_recalls = numpy.zeros(cutoff, numpy.float64)
for (queried, retrieved) in results:
x = categories_to_rel(queried, retrieved)[:cutoff]
x = numpy.pad(x, (0, cutoff - len(x)), 'constant', constant_values=(0))
precisions = []
recalls = []
for k, _ in enumerate(x):
p = precision(x[:k + 1])
r = recall(x[:k + 1], freqs[queried[0]])
precisions.append(p)
recalls.append(r)
mean_precisions += precisions
mean_recalls += recalls
mean_precisions /= len(results)
mean_recalls /= len(results)
plt.plot(mean_recalls, mean_precisions)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.axis([0, 1, 0, 1.05])
plt.show()
def evaluate_gpu(train_dataset,
test_dataset,
model,
thresh,
cmc_rank,
restart=False): # con 1000 persone non bastano 16gb VRAM
cuda = torch.cuda.is_available()
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=32,
shuffle=False,
**kwargs)
train_emb, train_lab = extract_embeddings_gpu(train_loader, model, cuda)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=32,
shuffle=False,
**kwargs)
test_emb, test_lab = extract_embeddings_gpu(test_loader, model, cuda)
train_emb = train_emb.half()
test_emb = test_emb.half()
dists = -2 * torch.mm(test_emb, torch.t(train_emb)) + torch.sum(
torch.pow(train_emb, 2), dim=1) + torch.sum(torch.pow(test_emb, 2),
dim=1).view(-1, 1)
dists = dists.cpu().numpy()
rank_list = []
dist_list = []
match_list = []
ap_list = []
start = 0
tot = len(test_lab)
for i in range(start, tot):
dist_vec = np.array(dists[i], dtype=float)
pred_labels = train_lab[dist_vec.flatten().argsort()]
ap = average_precision(test_lab[i], pred_labels)
pred_labels = pred_labels[:cmc_rank]
dist_vec = np.sort(dist_vec.flatten())
rank = 0
for k in range(len(pred_labels) - 1, -1, -1):
if pred_labels[k] == test_lab[i]:
rank = k + 1
rank_list.append(rank)
dist_list.append(dist_vec[0])
match_list.append(pred_labels[0])
ap_list.append(ap)
print('\r Test {} of {}'.format(i + 1, tot), end="")
print("")
print("mAP: {}%".format(np.mean(ap_list) * 100))
cmc_score(rank_list, rank_max=cmc_rank)
open_set_scores(match_list, dist_list, test_lab, thresh)
def run(training, validation, k, config):
norm = config.get('tfidf', 'norm')
smooth_idf = config.getboolean('tfidf', 'smooth_idf')
bigdoc = config.getboolean('NB', 'bigdoc')
clf = config.get('system', 'system')
if clf == 'NB':
clf = MultinomialNB(alpha=config.getfloat('NB', 'alpha'))
if not bigdoc:
clf = OneVsRestClassifier(clf, n_jobs=-1)
elif clf == 'KNN':
clf = KNeighborsClassifier(n_neighbors=10, weights='distance')
if not bigdoc:
clf = OneVsRestClassifier(clf)
elif clf == 'SVC':
clf = LinearSVC(loss='l2', penalty="l2", dual=False, tol=1e-3)
if not bigdoc:
clf = OneVsRestClassifier(clf)
elif clf == 'dtree':
clf = DecisionTreeClassifier()
else:
clf = OneVsRestClassifier(
SGDClassifier(alpha=config.getfloat('sgd', 'alpha'),
loss=config.get('sgd', 'loss'),
n_iter=config.getint('sgd', 'iterations'),
penalty=config.get('sgd', 'penalty')), n_jobs=-1)
classifier = Pipeline([
('vectorizer', CountVectorizer(min_df=1, max_df=1, analyzer=lambda t: t)),
('tfidf', TfidfTransformer(norm=norm, smooth_idf=smooth_idf)),
('clf', clf)])
if bigdoc:
(train_y, train_X), class_index = construct_bigdocuments(training)
_, test_y, test_X = zip(*validation)
test_y = [set(class_index[l] for l in ls) for ls in test_y]
else:
class_index = Index()
_, train_X, train_y = zip(*load_data(training, class_index))
_, test_X, test_y = zip(*load_data(validation, class_index))
classifier.fit(train_X, train_y)
isError, OneError, nDocs = 0, 0, 0
margins, AP = [], []
predictions = classifier.predict_proba(test_X)
for j, prediction in enumerate(predictions):
nDocs += 1
preds = sorted(range(len(prediction)), key=lambda i: prediction[i], reverse=True)
refs = test_y[j]
ap = average_precision(preds, refs)
AP.append(ap)
isError += is_error(ap)
OneError += one_error(preds, refs)
margins.append(margin(preds, refs))
return isError, OneError, nDocs, margins, AP
def run(training, validation, k, config):
norm = config.get('tfidf', 'norm')
smooth_idf = config.getBoolean('tfidf', 'smooth_idf')
bigdoc = False
clf = config.get('system', 'system')
if clf == 'NB':
alpha = config.getFloat('NB', 'alpha')
if config.getBoolean('NB', 'bigdoc'):
bigdoc = True
clf = MultinomialNB(alpha=alpha)
else:
clf = OneVsRestClassifier(BernoulliNB(alpha=alpha))
else:
clf = SGDClassifier(alpha=config.getFloat('sgd', 'alpha'),
loss=config.get('sgd', 'loss'),
n_iter=config.getInt('sgd', 'iterations'),
penalty=config.get('sgd', 'penalty'))
classifier = Pipeline([('vectorizer',
CountVectorizer(min_df=1,
max_df=1.0,
analyzer=lambda t: t)),
('tfidf',
TfidfTransformer(norm=norm,
smooth_idf=smooth_idf)),
('clf', clf)])
if bigdoc:
(train_y, train_X), class_index = construct_bigdocuments(training)
_, test_y, test_X = zip(*validation)
test_y = [tuple(class_index[l] for l in ls) for ls in test_y]
else:
class_index = Index()
_, train_X, train_y = zip(*load_data(training, class_index))
_, test_X, test_y = zip(*load_data(validation, class_index))
classifier.fit(train_X, train_y)
isError, OneError, nDocs = 0, 0, 0
margins, AP = [], []
predictions = classifier.predict_proba(test_X)
for j, prediction in enumerate(predictions):
nDocs += 1
preds = sorted(range(len(prediction)),
key=lambda i: prediction[i],
reverse=True)
refs = set(labelings[j])
ap = average_precision(preds, refs)
AP.append(ap)
isError += is_error(ap)
OneError += one_error(preds, refs)
margins.append(margin(preds, refs))
return isError, OneError, nDocs, margins, AP
def run(training, validation, k, config):
ground_truth = {}
ROOTDIR = config.get('filepaths', 'corpus')
alpha, beta = config.get('llda', 'alpha'), config.get('llda', 'beta')
iterations = config.get('llda', 'iterations')
with open(ROOTDIR + 'training-%s.tmp' % k, 'w') as training_out:
writer = csv.writer(training_out, quoting=csv.QUOTE_MINIMAL)
for (source, motifs, text) in training:
motifs = r' '.join(motifs) + ' DUMMY'
writer.writerow([source, motifs, ' '.join(text)])
with open(ROOTDIR + 'testing-%s.tmp' % k, 'w') as testing_out:
writer = csv.writer(testing_out, quoting=csv.QUOTE_MINIMAL)
for (source, motifs, text) in validation:
ground_truth[source] = motifs
writer.writerow([source, r' '.join(motifs), ' '.join(text)])
# train LLDA
with open(os.devnull, 'w') as null:
subprocess.call('java -Xmx2000mb -jar tmt-0.4.0.jar llda-train.scala %s %s %s %s' %
(ROOTDIR + 'training-%s.tmp' % k, alpha, beta, iterations),
stdout=null, stderr=null, shell=True)
# retrieve the model path
modelpath = open(ROOTDIR + 'training-%s.tmp.config' % k).read().strip()
# preform inference on led-out dataset using trained model
with open(os.devnull, 'w') as null:
subprocess.call('java -Xmx2000mb -jar tmt-0.4.0.jar llda-test.scala %s %s' %
(modelpath, (ROOTDIR + 'testing-%s.tmp' % k)),
stdout=sys.stdout, stderr=sys.stderr, shell=True)
# evaluation starts here!
isError, oneError, nDocs = 0, 0, 0
AP, margins = [], []
label_file = '/%05d/label-index.txt' % config.getint('llda', 'iterations')
topicIndex = [topic.strip() for topic in open(modelpath + label_file)]
reader = csv.reader(open(modelpath + '/testing-%s.tmp-document-topic-distributuions.csv' % k))
for row in reader:
nDocs += 1
idnumber, topics = row[0], [float(score) for score in row[1:]]
topics = sorted([(topicIndex[i], score) for i, score in enumerate(topics)],
key=lambda i: i[1], reverse=True)
preds = [topic for topic, _ in topics if topic != 'DUMMY']
refs = ground_truth[idnumber]
ap = average_precision(preds, refs)
isError += is_error(ap)
oneError += one_error(preds, refs)
margins.append(margin(preds, refs))
AP.append(ap)
return isError, oneError, nDocs, margins, AP
def run(training, validation, k, config):
norm = config.get('tfidf', 'norm')
smooth_idf = config.getBoolean('tfidf', 'smooth_idf')
bigdoc = False
clf = config.get('system', 'system')
if clf == 'NB':
alpha=config.getFloat('NB', 'alpha')
if config.getBoolean('NB', 'bigdoc'):
bigdoc = True
clf = MultinomialNB(alpha=alpha)
else:
clf = OneVsRestClassifier(BernoulliNB(alpha=alpha))
else:
clf = SGDClassifier(alpha=config.getFloat('sgd', 'alpha'),
loss=config.get('sgd', 'loss'),
n_iter=config.getInt('sgd', 'iterations'),
penalty=config.get('sgd', 'penalty'))
classifier = Pipeline([
('vectorizer', CountVectorizer(min_df=1, max_df=1.0, analyzer=lambda t: t)),
('tfidf', TfidfTransformer(norm=norm, smooth_idf=smooth_idf)),
('clf', clf)])
if bigdoc:
(train_y, train_X), class_index = construct_bigdocuments(training)
_, test_y, test_X = zip(*validation)
test_y = [tuple(class_index[l] for l in ls) for ls in test_y]
else:
class_index = Index()
_, train_X, train_y = zip(*load_data(training, class_index))
_, test_X, test_y = zip(*load_data(validation, class_index))
classifier.fit(train_X, train_y)
isError, OneError, nDocs = 0, 0, 0
margins, AP = [], []
predictions = classifier.predict_proba(test_X)
for j, prediction in enumerate(predictions):
nDocs += 1
preds = sorted(range(len(prediction)), key=lambda i: prediction[i], reverse=True)
refs = set(labelings[j])
ap = average_precision(preds, refs)
AP.append(ap)
isError += is_error(ap)
OneError += one_error(preds, refs)
margins.append(margin(preds, refs))
return isError, OneError, nDocs, margins, AP
def get_performance(user_pos_test, r, auc, Ks):
precision, recall, ndcg, hit_ratio = [], [], [], []
for K in Ks:
precision.append(metrics.precision_at_k(r, K))
recall.append(metrics.recall_at_k(r, K, len(user_pos_test)))
ndcg.append(metrics.ndcg_at_k(r, K))
# hit_ratio.append(metrics.hit_at_k(r, K))
hit_ratio.append(metrics.average_precision(r, K))
return {
'recall': np.array(recall),
'precision': np.array(precision),
'ndcg': np.array(ndcg),
'hit_ratio': np.array(hit_ratio),
'auc': auc
}
def validate(batch, iter_no):
# x1 = tr.tensor(batch['img_1']).permute([0, 3, 1, 2])
# x2 = tr.tensor(batch['img_2']).permute([0, 3, 1, 2])
# p1 = tr.tensor(batch['pos_1'])
# p2 = tr.tensor(batch['pos_2'])
x = tr.tensor(batch['image']).permute([0, 3, 1, 2]).cuda()
gt_vox = tr.tensor(batch['vox']).cuda()
p = tr.tensor(batch['pose']).cuda()
pred_vox = gan.predict_voxel(x)
t05_iou = metrics.iou_t(gt_vox, pred_vox, threshold=0.5).mean()
t04_iou = metrics.iou_t(gt_vox, pred_vox, threshold=0.4).mean()
max_iou = metrics.maxIoU(gt_vox, pred_vox)
avg_precision = metrics.average_precision(gt_vox, pred_vox)
val_writer.add_scalar('t05_iou', t05_iou, iter_no)
val_writer.add_scalar('t04_iou', t04_iou, iter_no)
val_writer.add_scalar('max_iou', max_iou, iter_no)
val_writer.add_scalar('avg_precision', avg_precision, iter_no)
def classification(test_loader, model, cmc_rank, n_classes):
cuda = torch.cuda.is_available()
test_lab = []
test_scores = np.array([], dtype=np.int64).reshape(0, n_classes)
rank_list = []
match_list = []
ap_list = []
with torch.no_grad():
for data in test_loader:
images, labels = data
if cuda:
images = images.cuda()
outputs = model(images)
test_lab = np.append(test_lab, labels.cpu().numpy())
test_scores = np.concatenate(
(test_scores, outputs.data.cpu().numpy()), axis=0)
tot = len(test_lab)
for i in range(0, tot):
pred_labels = np.argsort(-test_scores[i])
ap = average_precision(test_lab[i], pred_labels)
pred_labels = pred_labels[:cmc_rank]
rank = 0
for k in range(len(pred_labels) - 1, -1,
-1): # pre-ranking of query vector
if pred_labels[k] == test_lab[i]:
rank = k + 1
rank_list.append(rank)
match_list.append(pred_labels[0])
ap_list.append(ap)
print('\r Test {} of {}'.format(i + 1, tot), end="")
print("")
print("mAP: {}%".format(np.mean(ap_list) * 100))
cmc_score(rank_list, rank_max=cmc_rank)
len(trainset) - int(len(trainset) * 0.9)))
trainloader = DataLoader(trainset,
batch_size=256,
shuffle=True,
num_workers=0)
validationloader = DataLoader(validationset,
batch_size=256,
shuffle=True,
num_workers=0)
model = give_me_resnet(pretrained=True)
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters())
for alpha in alpha_list:
for epoch in range(epochs):
total_loss = 0
for x, y in trainloader:
optimizer.zero_grad()
output, embeddings = model(x.to(device))
loss = margin_loss(embeddings, y.to(device), alpha)
loss.backward()
optimizer.step()
total_loss += loss.item()
print('Epoch {} \t Loss: {}'.format(epoch, total_loss))
train_X, train_Y = get_embeddings(model, trainloader, device)
valid_X, valid_Y = get_embeddings(model, validationloader, device)
pred_class = euclidean_knn(valid_X, train_X, train_Y)
mAP = average_precision(valid_Y.reshape(-1, 1), pred_class)
results[str(alpha)].append(mAP.item())
def evaluate(train_dataset,
test_dataset,
model,
thresh,
cmc_rank,
restart=False):
"""
:param train_dataset: train dataset object
:param test_dataset: test dataset object
:param model: model object
:param restart: resume evaluation from a pickle dump if True
:param thresh: discard distance for ttr,ftr metrics
:param cmc_rank: max cmc rank
:return:
"""
cuda = torch.cuda.is_available()
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=32,
shuffle=False,
**kwargs)
train_emb, train_lab = extract_embeddings(train_loader, model, cuda)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=32,
shuffle=False,
**kwargs)
test_emb, test_lab = extract_embeddings(test_loader, model, cuda)
rank_list = [] # contains ranks of every test/query vector
dist_list = [
] # a single element is the shortest distance found between the vector query and all the train vectors
match_list = [
] # contains the predicted labels of the query vectors, rank1 match
ap_list = [] # contains the average precision of the query vectors
start = 0
tot = len(test_lab)
if restart:
with open('dump.pkl', 'rb') as f:
data = pickle.load(f) # backup loading
rank_list = data[0]
dist_list = data[1]
match_list = data[2]
ap_list = data[3]
start = len(rank_list) - 1
for i in range(start, tot):
# single query preparation for matrix calculation, more efficient than multiplying all queries
query_vec = np.reshape(test_emb[i], [1, 1000])
# distance calculation between query vector e training/gallery feature vector
dist_vec = -2 * np.dot(query_vec, train_emb.T) + np.sum(
train_emb**2, axis=1) + np.sum(query_vec**2, axis=1)[:, np.newaxis]
pred_labels = train_lab[dist_vec.flatten().argsort()]
ap = average_precision(test_lab[i], pred_labels)
pred_labels = pred_labels[:
cmc_rank] # contains the predicted labels ordered according to their distance
dist_vec = np.sort(
dist_vec.flatten()) # distance vector ordered by the smallest
rank = 0
for k in range(len(pred_labels) - 1, -1,
-1): # pre-ranking of query vector
if pred_labels[k] == test_lab[i]:
rank = k + 1
rank_list.append(rank)
dist_list.append(dist_vec[0])
match_list.append(pred_labels[0])
ap_list.append(ap)
print('\r Test {} of {}'.format(i + 1, tot), end="")
if (i % 1000) == 0:
with open('dump.pkl', 'wb') as f:
data = [rank_list, dist_list, match_list, ap_list]
pickle.dump(data, f) # backup saving
print("")
print("mAP: {}%".format(np.mean(ap_list) * 100))
cmc_score(rank_list, rank_max=cmc_rank)
open_set_scores(match_list, dist_list, test_lab, thresh)
def evaluate_vram_opt(train_dataset,
test_dataset,
model,
thresh,
cmc_rank,
restart=False):
cuda = torch.cuda.is_available()
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=32,
shuffle=False,
**kwargs)
train_emb, train_lab = extract_embeddings_gpu(train_loader, model, cuda)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=32,
shuffle=False,
**kwargs)
test_emb, test_lab = extract_embeddings_gpu(test_loader, model, cuda)
rank_list = []
dist_list = []
match_list = []
ap_list = []
start = 0
tot = len(test_lab)
if restart:
with open('dump.pkl', 'rb') as f:
data = pickle.load(f)
rank_list = data[0]
dist_list = data[1]
match_list = data[2]
ap_list = data[3]
start = len(rank_list) - 1
for i in range(start, tot):
# single query preparation for matrix calculation, more efficient than multiplying all queries
query_vec = test_emb[i].view(1, 1000)
# distance calculation between query vector e training/gallery feature vector
dist_vec = -2 * torch.mm(query_vec, torch.t(train_emb)) + torch.sum(
torch.pow(train_emb, 2), dim=1) + torch.sum(
torch.pow(query_vec, 2), dim=1).view(-1, 1)
dist_vec = dist_vec.cpu().numpy()
pred_labels = train_lab[dist_vec.flatten().argsort()]
ap = average_precision(test_lab[i], pred_labels)
pred_labels = pred_labels[:cmc_rank]
dist_vec = np.sort(dist_vec.flatten())
rank = 0
for k in range(len(pred_labels) - 1, -1,
-1): # pre-ranking dell'attuale query vector
if pred_labels[k] == test_lab[i]:
rank = k + 1
rank_list.append(rank)
dist_list.append(dist_vec[0])
match_list.append(pred_labels[0])
ap_list.append(ap)
print('\r Test {} of {}'.format(i + 1, tot), end="")
if (i % 1000) == 0:
with open('dump.pkl', 'wb') as f:
data = [rank_list, dist_list, match_list, ap_list]
pickle.dump(data, f)
print("")
print("mAP: {}%".format(np.mean(ap_list) * 100))
cmc_score(rank_list, rank_max=cmc_rank)
open_set_scores(match_list, dist_list, test_lab, thresh)
criterion = torch.nn.CrossEntropyLoss()
alpha = 1.0
epochs = 100
for epoch in range(epochs):
total_loss = 0
for x, y in trainloader:
optimizer.zero_grad()
output, embeddings = model(x.to(device))
loss, _ = margin_hard_loss(embeddings, y.to(device), alpha)
loss.backward()
optimizer.step()
total_loss += loss.item()
print('Epoch {} \t Loss: {:.3f}'.format(epoch, total_loss))
train_X, train_Y = get_embeddings(model, trainloader, device)
test_X, test_Y = get_embeddings(model, testloader, device)
reduced_umap = umap.UMAP().fit_transform(test_X.numpy())
reduced_tsne = TSNE(n_components=2).fit_transform(test_X.numpy())
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
ax1.scatter(reduced_umap[:, 0], reduced_umap[:, 1], c=test_Y)
ax2.scatter(reduced_tsne[:, 0], reduced_tsne[:, 1], c=test_Y)
plt.tight_layout()
plt.savefig(results_path+'scatter.pdf', format='pdf')
plt.close()
pred_class = euclidean_knn(test_X, train_X, train_Y)
mAP = average_precision(test_Y.reshape(-1, 1), pred_class)
def main():
args = arguments.parse()
feat_extractor = FeatureExtractor().cuda()
svms = [SVM() for _ in range(args.n_attrs)]
# Dataloader code
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_transforms = transforms.Compose([
transforms.Resize((224, 224)),
# transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
val_transforms = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize,
])
logger.info("Setting up training data")
train_loader = data.DataLoader(COCOAttributes(
args.attributes,
args.train_ann,
train=True,
split='train2014',
transforms=train_transforms,
dataset_root=args.dataset_root),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
logger.info("Setting up validation data")
val_loader = data.DataLoader(COCOAttributes(
args.attributes,
args.val_ann,
train=False,
split='val2014',
transforms=val_transforms,
dataset_root=args.dataset_root),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
logger.info("Beginning training...")
feats, targets = get_features(feat_extractor,
train_loader,
n_attrs=args.n_attrs,
split='train')
if args.checkpoint:
[svms, _] = joblib.load(args.checkpoint)
else:
for i in range(args.n_attrs):
print("Training for attribute", i)
# using [0, 1] or [-1, 1] doesn't really make a difference
svms[i].train(feats, targets[:, i])
print()
logger.info('Finished Training')
logger.info("Running evaluation")
feats, targets = get_features(feat_extractor,
val_loader,
n_attrs=args.n_attrs,
split='val')
ap_scores = []
for i in range(args.n_attrs):
est = svms[i].test(feats)
ap_score = average_precision(2 * targets[:, i] - 1, est)
print("AP score for {0}".format(i), ap_score)
ap_scores.append(ap_score)
print("mean AP", sum(ap_scores) / args.n_attrs)
if not args.checkpoint:
logger.info("Saving models and AP scores")
joblib.dump([svms, ap_scores], "svm_baseline.jbl")