def test_one_user(x):
# user u's ratings for user u
rating = x[0]
#uid
u = x[1]
#user u's items in the training set
item_num = x[2]
data_generator = data_generator_all[x[3]]
training_items = data_generator.train_items[u]
#user u's items in the test set
user_pos_test = data_generator.test_set[u]
all_items = set(range(item_num))
test_items = list(all_items - set(training_items))
item_score = []
for i in test_items:
item_score.append((i, rating[i]))
item_score = sorted(item_score, key=lambda x: x[1])
item_score.reverse()
item_sort = [x[0] for x in item_score]
r = []
for i in item_sort:
if i in user_pos_test:
r.append(1)
else:
r.append(0)
recall_20 = ut.recall_at_k(r, 20, len(user_pos_test))
recall_40 = ut.recall_at_k(r, 40, len(user_pos_test))
recall_60 = ut.recall_at_k(r, 60, len(user_pos_test))
recall_80 = ut.recall_at_k(r, 80, len(user_pos_test))
recall_100 = ut.recall_at_k(r, 100, len(user_pos_test))
ap_20 = ut.average_precision(r, 20)
ap_40 = ut.average_precision(r, 40)
ap_60 = ut.average_precision(r, 60)
ap_80 = ut.average_precision(r, 80)
ap_100 = ut.average_precision(r, 100)
return np.array([
recall_20, recall_40, recall_60, recall_80, recall_100, ap_20, ap_40,
ap_60, ap_80, ap_100
])
def score(self, Kyx, cy):
""" Returns the mean average precision score. """
cy = self.lb.transform(cy)
average_precisions = np.zeros(self.nr_classes)
preds = []
for ii in xrange(self.nr_classes):
true_labels = cy[:, ii]
predicted_values = self.clf[ii].predict_proba(Kyx)[:, 1]
preds.append(predicted_values)
average_precisions[ii] = average_precision(true_labels,
predicted_values)
#return np.mean(average_precisions) * 100
#return preds
return average_precisions
def score(self, Kyx, cy):
""" Returns the mean average precision score. """
cy = self.lb.transform(cy)
average_precisions = np.zeros(self.nr_classes)
preds = []
for ii in xrange(self.nr_classes):
true_labels = cy[:, ii]
predicted_values = self.clf[ii].predict_proba(Kyx)[:, 1]
preds.append(predicted_values)
average_precisions[ii] = average_precision(
true_labels, predicted_values)
#return np.mean(average_precisions) * 100
#return preds
return average_precisions
def _score_one_vs_rest_sklearn(self, Kyx, cy):
""" Returns the mean average precision score. """
ap_scores = np.zeros(self.nr_classes - 1)
for ii in xrange(self.nr_classes - 1):
# for ii in xrange(1):
good_idxs = (cy == ii) | (cy == self.null_class_idx)
K_good_idxs = np.ix_(good_idxs, self.cx_idxs[ii])
# Get a +1, -1 vector of labels.
cy_ = map(lambda label: +1 if label == ii else -1, cy[good_idxs])
# Predict.
predicted_values = self.clf[ii].predict_proba(Kyx[K_good_idxs])[:, 1]
ap_scores[ii] = average_precision(cy_, predicted_values)
# print "%d sklearn %2.3f" % (ii, 100 * ap_scores[ii])
print "Class #%d, test score: %2.3f" % (ii, 100 * ap_scores[ii])
return np.mean(ap_scores) * 100
def _score_one_vs_rest_sklearn(self, Kyx, cy):
""" Returns the mean average precision score. """
ap_scores = np.zeros(self.nr_classes - 1)
for ii in xrange(self.nr_classes - 1):
#for ii in xrange(1):
good_idxs = (cy == ii) | (cy == self.null_class_idx)
K_good_idxs = np.ix_(good_idxs, self.cx_idxs[ii])
# Get a +1, -1 vector of labels.
cy_ = map(lambda label: +1 if label == ii else -1, cy[good_idxs])
# Predict.
predicted_values = self.clf[ii].predict_proba(Kyx[K_good_idxs])[:,
1]
ap_scores[ii] = average_precision(cy_, predicted_values)
#print "%d sklearn %2.3f" % (ii, 100 * ap_scores[ii])
print 'Class #%d, test score: %2.3f' % (ii, 100 * ap_scores[ii])
return np.mean(ap_scores) * 100
def _get_average_precisions(latent_model, latent_space, x_test, y_test):
average_precisions = np.zeros(x_test.shape[0])
for i in range(x_test.shape[0]):
if i % 100 == 0:
print('precisions_done_calculating{}'.format(i))
num = i
num_retrievable = (np.argmax(y_test[num]) == \
np.argmax(y_test, axis=1)).sum()
# latent_object = latent_model.predict(x_test[num:num+1])
latent_object = latent_space[num: num+1]
sims, latent_indices = query_latent_space(latent_object,
latent_space,
x_test.shape[0])
ranked_relevant = np.argmax(y_test[num]) ==\
np.argmax(y_test[latent_indices], axis=1)
average_precisions[i] = average_precision(ranked_relevant, num_retrievable)
return average_precisions
def score(self, Kyx, cy):
""" Returns the mean average precision score. """
ap_scores = np.zeros(self.nr_classes - 1)
for ii, class_idx in enumerate(xrange(1, 16)):
good_idxs = (cy == class_idx) | (cy == self.null_class_idx)
K_good_idxs = np.ix_(good_idxs, self.cx_idxs[ii])
# Get a +1, -1 vector of labels.
cy_ = map(
lambda label: +1 if label == class_idx else -1,
cy[good_idxs])
# Predict.
predicted_values = self.clf[ii].predict_proba(Kyx[K_good_idxs])[:, 1]
ap_scores[ii] = average_precision(
cy_, predicted_values)
#print "%d sklearn %2.3f" % (ii, 100 * ap_scores[ii])
#print 'Class #%d, test score: %2.3f' % (
# class_idx, 100 * ap_scores[ii])
print '%d\t%2.3f' % (
class_idx, 100 * ap_scores[ii])
return np.mean(ap_scores) * 100
def validation(self, max_validation_batches, t):
print("=" * 80)
print("Validation:")
num_batches = 0
ap = 0
iou_t4 = 0
iou_t5 = 0
ts = np.arange(0., 1., 1e-1)
iou_max = np.zeros(ts.shape)
pbar = progressbar.ProgressBar(maxval=progressbar.UnknownLength)
self.sess.run(self.val_init)
try:
while True:
vox_p, vox_1 = self.sess.run(
[self.val_pass_vox, self.val_vox],
feed_dict={self.is_training: False})
ious = []
for t in ts:
iou = iou_t(vox_1, vox_p, threshold=t).mean()
ious.append(iou)
iou_max += np.array(ious)
ap += average_precision(vox_1, vox_p)
iou_t4 += iou_t(vox_1, vox_p, threshold=0.4).mean()
iou_t5 += iou_t(vox_1, vox_p, threshold=0.5).mean()
num_batches += 1
pbar.update(num_batches)
if num_batches == max_validation_batches:
break
except tf.errors.OutOfRangeError:
print("End of validation dataset")
# # pos_acc /= float(num_batches)
iou_t4 /= float(num_batches)
iou_t5 /= float(num_batches)
ap /= float(num_batches)
iou_max /= float(num_batches)
iou_thr = iou_max.argmax() * (1 / float(len(iou_max)))
iou_max = iou_max.max()
print("Performance (this pass)")
print("\tAP:%.5f\tMaxIoU:%.5f\tIoU(t>0.4):%.5f\tIoU(t>0.5):%.5f"\
%(ap, iou_max, iou_t4, iou_t5))
if ap > self.max_ap:
self.max_ap = ap
print("New best avg prc:%.5f" % self.max_ap)
self._save_model(self.t,
saver=self.best_avg_prc_saver,
prefix="model-best-avgprc")
if iou_max > self.max_iou_max:
self.max_iou_max = iou_max
self.max_iou_thr = iou_thr
print("New best max iou:%.5f, %.5f" %
(self.max_iou_max, self.max_iou_thr))
self._save_model(self.t,
saver=self.best_max_iou_saver,
prefix="model-best-maxiou")
if iou_t4 > self.max_iou_t4:
self.max_iou_t4 = iou_t4
print("New best IoU(t04):%.5f" % self.max_iou_t4)
self._save_model(self.t,
saver=self.best_iou_t04_saver,
prefix="model-best-iout04")
if iou_t5 > self.max_iou_t5:
self.max_iou_t5 = iou_t5
print("New best IoU(t05):%.5f" % self.max_iou_t5)
self._save_model(self.t,
saver=self.best_iou_t05_saver,
prefix="model-best-iout05")
print("Performance (best pass)")
print("\tAP:%.5f\tMaxIoU:%.5f\tIoU(t>0.4):%.5f\tIoU(t>0.5):%.5f"\
%(self.max_ap, self.max_iou_max, self.max_iou_t4, self.max_iou_t5))
val_summary = self.sess.run(self.val_summary,
feed_dict={
self.val_max_iou: iou_max,
self.val_t04_iou: iou_t4,
self.val_t05_iou: iou_t5,
self.val_avg_prc: ap,
self.val_iou_thr: iou_thr,
self.val_max_iou_best:
self.max_iou_max,
self.val_t04_iou_best: self.max_iou_t4,
self.val_t05_iou_best: self.max_iou_t5,
self.val_avg_prc_best: self.max_ap,
self.val_iou_thr_best: self.max_iou_thr
})
self.log_writer.add_summary(val_summary, self.t)
print("=" * 80)
return ap, iou_max, iou_t4, iou_t5
def predict(self, mode):
"""function for prediction based mode
Args:
mode (str): validation when mode is set to "valid", otherwise testing
Returns:
None
"""
result = np.array([0.] * 15)
pool = multiprocessing.Pool(cores)
# all users needed to test
if mode == 'valid':
test_users = list(self.data.valid_set.keys())
else:
test_users = list(self.data.test_set.keys())
test_user_num = len(test_users)
for u in test_users:
users = [u] * self.batch_size
user_pos_test = self.data.test_set[
u] if mode == 'test' else self.data.valid_set[u]
neg_items = set(range(self.data.n_items)) - set(
self.data.train_items[u])
if u in self.data.valid_set:
neg_items = neg_items - set(self.data.valid_set[u])
if u in self.data.test_set:
neg_items = neg_items - set(self.data.test_set[u])
neg_items = rd.sample(neg_items, self.batch_size - 1)
items_to_test = neg_items + user_pos_test
ratings = self.sess.run(self.distance, {
self.users: users,
self.pos_items: items_to_test
})
item_score = [(items_to_test[i], ratings[i])
for i in range(len(items_to_test))]
item_score = sorted(item_score, key=lambda x: x[1])
#item_score.reverse()
item_sort = [x[0] for x in item_score]
r = []
for i in item_sort:
if i in user_pos_test:
r.append(1)
else:
r.append(0)
hr_1 = ut.hr_at_k(r, 1)
hr_3 = ut.hr_at_k(r, 3)
hr_5 = ut.hr_at_k(r, 5)
hr_7 = ut.hr_at_k(r, 7)
hr_10 = ut.hr_at_k(r, 10)
ap_1 = ut.average_precision(r, 1)
ap_3 = ut.average_precision(r, 3)
ap_5 = ut.average_precision(r, 5)
ap_7 = ut.average_precision(r, 7)
ap_10 = ut.average_precision(r, 10)
ndcg_1 = ut.ndcg_at_k(r, 1)
ndcg_3 = ut.ndcg_at_k(r, 3)
ndcg_5 = ut.ndcg_at_k(r, 5)
ndcg_7 = ut.ndcg_at_k(r, 7)
ndcg_10 = ut.ndcg_at_k(r, 10)
ret = np.array([
hr_1, hr_3, hr_5, hr_7, hr_10, ap_1, ap_3, ap_5, ap_7, ap_10,
ndcg_1, ndcg_3, ndcg_5, ndcg_7, ndcg_10
])
result += ret
ret = result / test_user_num
hr_1, hr_3, hr_5, hr_7, hr_10 = ret[0], ret[1], ret[2], ret[3], ret[4]
map_1, map_3, map_5, map_7, map_10 = ret[5], ret[6], ret[7], ret[
8], ret[9]
ndcg_1, ndcg_3, ndcg_5, ndcg_7, ndcg_10 = ret[10], ret[11], ret[
12], ret[13], ret[14]
print('Test:') if mode == 'test' else print('Valid:')
print('hr@1 %f hr@3 %f hr@5 %f hr@7 %f hr@10 %f' %
(hr_1, hr_3, hr_5, hr_7, hr_10))
print('MAP@1 %f MAP@3 %f MAP@5 %f MAP@7 %f MAP@10 %f' %
(map_1, map_3, map_5, map_7, map_10))
print('ndcg@1 %f ndcg@3 %f ndcg@5 %f ndcg@7 %f ndcg@10 %f' %
(ndcg_1, ndcg_3, ndcg_5, ndcg_7, ndcg_10))
