def run_validation(val_predicted_labels, tst_X_Y_val, tst_exact_remove,
tst_X_Y_trn, inv_prop):
data = []
indptr = [0]
indices = []
for i in range(val_predicted_labels.shape[0]):
_indices1 = val_predicted_labels.indices[
val_predicted_labels.indptr[i]:val_predicted_labels.indptr[i + 1]]
_vals1 = val_predicted_labels.data[
val_predicted_labels.indptr[i]:val_predicted_labels.indptr[i + 1]]
_indices, _vals = [], []
for _ind, _val in zip(_indices1, _vals1):
if (_ind not in tst_exact_remove[i]) and (
_ind not in tst_X_Y_trn.
indices[tst_X_Y_trn.indptr[i]:tst_X_Y_trn.indptr[i + 1]]):
_indices.append(_ind)
_vals.append(_val)
indices += list(_indices)
data += list(_vals)
indptr.append(len(indices))
_pred = csr_matrix((data, indices, indptr),
shape=(val_predicted_labels.shape))
print(tst_X_Y_val.shape, _pred.shape)
acc = xc_metrics.Metrics(tst_X_Y_val, inv_psp=inv_prop)
acc = acc.eval(_pred, 5)
_recall = recall(tst_X_Y_val, _pred, 100)
return (acc, _recall), _pred
def compute_metrics(model, X_feat, Y_labels, wts, batch_size=50000):
'''
Compute P@k, nDCG@k and their propensity scored versions PSP@k and PSnDCG@k, respectively. k=1,...,5
'''
metrics_all = xc_metrics.Metrics(Y_labels, wts)
if Y_labels.shape[0] > 20000 or Y_labels.shape[1] > 20000:
count = 0
dataset_tf = tf.data.Dataset.from_tensor_slices(X_feat).batch(
batch_size)
mtr_scores = lil_matrix((Y_labels.shape[0], Y_labels.shape[1]),
dtype=np.float64)
start_time = time.time()
for batch_X in dataset_tf:
mtr_scores_tmp = model.predict_scores(batch_X).numpy()
for n_test in range(batch_X.shape[0]):
Prob_scores_n_test = mtr_scores_tmp[n_test]
top5_indices = np.argpartition(Prob_scores_n_test, -5)[-5:]
top5_indices.sort()
mtr_scores[count + n_test,
top5_indices] = Prob_scores_n_test[top5_indices]
count += batch_X.shape[0]
pred_time = time.time() - start_time
mtr_scores = mtr_scores.tocsr()
else:
start_time = time.time()
mtr_scores = model.predict_scores(X_feat).numpy()
pred_time = time.time() - start_time
all_metrics = [100 * elem for elem in metrics_all.eval(mtr_scores)]
return all_metrics[0], all_metrics[1], all_metrics[2], all_metrics[
3], pred_time
def evaluate_scores(gt: torch.Tensor, scores: torch.Tensor, model_cfg):
num_ins, num_labels = gt.shape
if gt.is_sparse:
gt_np = gt.coalesce()
gt_np = scipy.sparse.coo_matrix(
(gt_np.values().cpu().numpy(), gt_np.indices().cpu().numpy()),
shape=(num_ins, num_labels))
else:
gt_np = scipy.sparse.coo_matrix(gt.cpu().numpy(),
shape=(num_ins, num_labels))
if isinstance(scores, torch.Tensor):
scores_np = scores.numpy()
else:
scores_np = scores
inv_propen = xc_metrics.compute_inv_propesity(gt_np, model_cfg["ps_A"],
model_cfg["ps_B"])
acc = xc_metrics.Metrics(true_labels=gt_np,
inv_psp=inv_propen,
remove_invalid=False)
map_meter = meter.mAPMeter()
# map meter requires tensor
gt_dense = gt if not gt.is_sparse else gt.to_dense()
map_meter.add(scores, gt_dense)
prec, ndcg, PSprec, PSnDCG = acc.eval(scores_np, model_cfg["at_k"])
d = {
"prec": prec,
"ndcg": ndcg,
"psp": PSprec,
"psn": PSnDCG,
"mAP": [map_meter.value()]
}
return d
def _evaluate(self, true_labels, predicted_labels):
"""
Evaluate predicted matrix
"""
pmat = predicted_labels.tocsr()
acc = xc.Metrics(true_labels)
rec = xc.recall(pmat, true_labels, self.beam)
_p, _n = acc.eval(predicted_labels.tocsr(), self.beam)
return _p, _n, rec
def evaluate(self, train_labels, test_labels, predictionScore, ps_A, ps_B,
eval_file_path):
inv_propen = xc_metrics.compute_inv_propesity(train_labels, ps_A, ps_B)
acc = xc_metrics.Metrics(true_labels=test_labels, inv_psp=inv_propen)
args = acc.eval(predictionScore, 5)
strs = ['P@K', 'nDCG@K', 'PSP@K', 'PSnDCG@K']
msg = ''
for i in range(4):
msg += strs[i] + str(args[i][[0, 2, 4]]) + '\n'
print(msg)
if eval_file_path != None:
with open(eval_file_path) as f:
f.write(msg)
def main(targets_file, train_file, predictions_file, A, B):
"""
Args:
targets_file: test labels
train_file: train labels (to compute prop)
prediction_file: predicted labels
A: int: to compute propensity
B: int: to compute propensity
"""
true_labels = data_utils.read_sparse_file(targets_file)
predicted_labels = data_utils.read_sparse_file(predictions_file)
inv_psp = compute_inv_propensity(train_file, A, B)
acc = xc_metrics.Metrics(true_labels=true_labels, inv_psp=inv_psp)
args = acc.eval(predicted_labels, 5)
print(xc_metrics.format(*args))
def main(targets_label_file, train_label_file, predictions_file, A, B, docs, lbls):
true_labels = _remove_overlap(
data_utils.read_sparse_file(
targets_label_file, force_header=True).tolil(),
docs, lbls)
trn_labels = data_utils.read_sparse_file(
train_label_file, force_header=True)
inv_propen = xc_metrics.compute_inv_propesity(trn_labels, A=A, B=B)
acc = xc_metrics.Metrics(
true_labels, inv_psp=inv_propen, remove_invalid=False)
predicted_labels = _remove_overlap(
load_npz(predictions_file+'.npz').tolil(),
docs, lbls)
rec = xc_metrics.recall(predicted_labels, true_labels, k=20)[-1]*100
print("R@20=%0.2f" % (rec))
args = acc.eval(predicted_labels, 5)
print(xc_metrics.format(*args))
def main(tst_label_fname, trn_label_fname, filter_fname, pred_fname, A, B,
betas, top_k, save):
true_labels = data_utils.read_sparse_file(tst_label_fname)
trn_labels = data_utils.read_sparse_file(trn_label_fname)
inv_propen = xc_metrics.compute_inv_propesity(trn_labels, A, B)
mapping = get_filter_map(filter_fname)
acc = xc_metrics.Metrics(true_labels, inv_psp=inv_propen)
root = os.path.dirname(pred_fname)
ans = ""
if isinstance(betas, list) and betas[0] != -1:
knn = filter_predictions(load_npz(pred_fname + '_knn.npz'), mapping)
clf = filter_predictions(load_npz(pred_fname + '_clf.npz'), mapping)
args = acc.eval(clf, 5)
ans = f"classifier\n{xc_metrics.format(*args)}"
args = acc.eval(knn, 5)
ans = ans + f"\nshortlist\n{xc_metrics.format(*args)}"
clf = retain_topk(clf, k=top_k)
knn = retain_topk(knn, k=top_k)
clf = normalize(sigmoid(clf), norm='max')
knn = normalize(sigmoid(knn), norm='max')
for beta in betas:
predicted_labels = beta * clf + (1 - beta) * knn
args = acc.eval(predicted_labels, 5)
ans = ans + f"\nbeta: {beta:.2f}\n{xc_metrics.format(*args)}"
if save:
fname = os.path.join(root, f"score_{beta:.2f}.npz")
save_npz(fname,
retain_topk(predicted_labels, k=top_k),
compressed=False)
else:
predicted_labels = filter_predictions(
sigmoid(load_npz(pred_fname + '.npz')), mapping)
args = acc.eval(predicted_labels, 5)
ans = xc_metrics.format(*args)
if save:
print("Saving predictions..")
fname = os.path.join(root, "score.npz")
save_npz(fname,
retain_topk(predicted_labels, k=top_k),
compressed=False)
line = "-" * 30
print(f"\n{line}\n{ans}\n{line}")
return ans
def main(tst_label_fname, trn_label_fname, pred_fname,
A, B, save, *args, **kwargs):
true_labels = data_utils.read_sparse_file(tst_label_fname)
trn_labels = data_utils.read_sparse_file(trn_label_fname)
inv_propen = xc_metrics.compute_inv_propesity(trn_labels, A, B)
acc = xc_metrics.Metrics(true_labels, inv_psp=inv_propen)
root = os.path.dirname(pred_fname[-1])
predicted_labels = read_files(pred_fname)
ens_predicted_labels = merge(predicted_labels)
ans = ""
for idx, pred in enumerate(predicted_labels):
args = acc.eval(pred, 5)
ans = ans + f"learner: {idx}\n{xc_metrics.format(*args)}\n"
args = acc.eval(ens_predicted_labels, 5)
ans = ans + f"Ensemble\n{xc_metrics.format(*args)}"
if save:
print("Saving predictions..")
fname = os.path.join(root, "score.npz")
save_npz(fname, ens_predicted_labels, compressed=False)
line = "-"*30
print(f"\n{line}\n{ans}\n{line}")
return ans
def _evaluate(self, true_labels, predicted_labels):
acc = xc_metrics.Metrics(true_labels)
acc = acc.eval(predicted_labels.tocsr(), 5)
return acc
