def test_ndcg(problem_for_pred):
# ties don't matter here because it doesn't change the two highest predictions
y_true, y_pred, _ties = problem_for_pred
# We have:
# y_true = [0, 1, 2, 3, 4]
# y_pred = [0, 2, 1, 2, 3]
# Inverted (with max_rank = 4) that is
# y_true_inv = [4, 3, 2, 1, 0]
# y_pred_inv = [4, 2, 3, 2, 1]
# And normalized to [0, 1] this gives us the relevance:
# rel_true = [1, 3/4, 1/2, 1/4, 0]
# rel_pred = [1, 1/2, 3/4, 1/2, 1/4]
# With this we can first compute the ideal dcg, considering only the first
# k=2 elements (all logs are base 2, equality is approximate):
idcg = (2**1 - 1) / np.log2(2) + (2**(3 / 4) - 1) / np.log2(3) # = 1.43
# And the dcg of the predictions at the same positions as the elements we
# considered for the idcg (i.e. the "true" best elements):
dcg = (2**1 - 1) / np.log2(2) + (2**(1 / 2) - 1) / np.log2(3) # = 1.26
# Now the gain is:
expected_gain = dcg / idcg # = 0.882
ndcg = make_ndcg_at_k_loss(k=2)
real_gain = K.eval(ndcg(y_true, y_pred))
assert_almost_equal(actual=real_gain, desired=expected_gain, decimal=5)
def test_ndcg(problem_for_pred):
y_true, y_pred, ties = problem_for_pred
ndcg = make_ndcg_at_k_loss(k=2)
gain = ndcg(y_true, y_pred)
real_gain = K.eval(gain)
expected_dcg = 15. + 3. / np.log2(3.)
expected_idcg = 15. + 7. / np.log2(3.)
assert_almost_equal(actual=real_gain,
desired=np.array([[expected_dcg / expected_idcg]]),
decimal=5)
f.create_dataset('scores', data=s_pred)
f.close()
logger.info("Saved predictions at: {}".format(pred_file))
results = {
'job_id': str(current_job_id),
'cluster_id': str(cluster_id)
}
for name, evaluation_metric in lp_metric_dict[
learning_problem].items():
predictions = s_pred
if evaluation_metric in metrics_on_predictions:
logger.info("Metric on predictions")
predictions = y_pred
if "NDCG" in name:
evaluation_metric = make_ndcg_at_k_loss(k=n_objects)
predictions = y_pred
if "CategoricalTopK{}" == name:
k = int(n_objects / 2) + 1
evaluation_metric = topk_categorical_accuracy_np(k=k)
name = name.format(k)
metric_loss = get_mean_loss(evaluation_metric, Y_test,
predictions)
logger.info(ERROR_OUTPUT_STRING % (name, metric_loss))
if np.isnan(metric_loss):
results[name] = "\'Infinity\'"
else:
results[name] = "{0:.4f}".format(metric_loss)
dbConnector.insert_results(experiment_schema=experiment_schema,
experiment_table=experiment_table,
results=results)
