def test_ndcg(problem_for_pred): # ties don't matter here because it doesn't change the two highest predictions y_true, y_pred, _ties = problem_for_pred # We have: # y_true = [0, 1, 2, 3, 4] # y_pred = [0, 2, 1, 2, 3] # Inverted (with max_rank = 4) that is # y_true_inv = [4, 3, 2, 1, 0] # y_pred_inv = [4, 2, 3, 2, 1] # And normalized to [0, 1] this gives us the relevance: # rel_true = [1, 3/4, 1/2, 1/4, 0] # rel_pred = [1, 1/2, 3/4, 1/2, 1/4] # With this we can first compute the ideal dcg, considering only the first # k=2 elements (all logs are base 2, equality is approximate): idcg = (2**1 - 1) / np.log2(2) + (2**(3 / 4) - 1) / np.log2(3) # = 1.43 # And the dcg of the predictions at the same positions as the elements we # considered for the idcg (i.e. the "true" best elements): dcg = (2**1 - 1) / np.log2(2) + (2**(1 / 2) - 1) / np.log2(3) # = 1.26 # Now the gain is: expected_gain = dcg / idcg # = 0.882 ndcg = make_ndcg_at_k_loss(k=2) real_gain = K.eval(ndcg(y_true, y_pred)) assert_almost_equal(actual=real_gain, desired=expected_gain, decimal=5)
def test_ndcg(problem_for_pred): y_true, y_pred, ties = problem_for_pred ndcg = make_ndcg_at_k_loss(k=2) gain = ndcg(y_true, y_pred) real_gain = K.eval(gain) expected_dcg = 15. + 3. / np.log2(3.) expected_idcg = 15. + 7. / np.log2(3.) assert_almost_equal(actual=real_gain, desired=np.array([[expected_dcg / expected_idcg]]), decimal=5)
f.create_dataset('scores', data=s_pred) f.close() logger.info("Saved predictions at: {}".format(pred_file)) results = { 'job_id': str(current_job_id), 'cluster_id': str(cluster_id) } for name, evaluation_metric in lp_metric_dict[ learning_problem].items(): predictions = s_pred if evaluation_metric in metrics_on_predictions: logger.info("Metric on predictions") predictions = y_pred if "NDCG" in name: evaluation_metric = make_ndcg_at_k_loss(k=n_objects) predictions = y_pred if "CategoricalTopK{}" == name: k = int(n_objects / 2) + 1 evaluation_metric = topk_categorical_accuracy_np(k=k) name = name.format(k) metric_loss = get_mean_loss(evaluation_metric, Y_test, predictions) logger.info(ERROR_OUTPUT_STRING % (name, metric_loss)) if np.isnan(metric_loss): results[name] = "\'Infinity\'" else: results[name] = "{0:.4f}".format(metric_loss) dbConnector.insert_results(experiment_schema=experiment_schema, experiment_table=experiment_table, results=results)