def smooth_rank_loss(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) mask = K.cast(K.greater(y_true[:, None] - y_true[:, :, None], 0), dtype='float32') exped = K.exp(y_pred[:, None] - y_pred[:, :, None]) result = K.sum(exped * mask, axis=[1, 2]) return result / K.sum(mask, axis=(1, 2))
def zero_one_accuracy(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) n_instances, n_objects = get_instances_objects(y_true) equal_ranks = K.cast(K.all(K.equal(y_pred, y_true), axis=1), dtype="float32") denominator = K.cast(n_instances, dtype="float32") zero_one_loss = K.sum(equal_ranks) / denominator return zero_one_loss
def hinged_rank_loss(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) mask = K.cast(K.greater(y_true[:, None] - y_true[:, :, None], 0), dtype='float32') diff = y_pred[:, :, None] - y_pred[:, None] hinge = K.maximum(mask * (1 - diff), 0) n = K.sum(mask, axis=(1, 2)) return K.sum(hinge, axis=(1, 2)) / n
def test_tensorify(): a = np.array([1.0, 2.0]) out = tensorify(a) assert isinstance(out, tf.Tensor) b = K.zeros((5, 3)) out = tensorify(b) assert b == out
def zero_one_accuracy_for_scores(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) n_instances, n_objects = get_instances_objects(y_true) predicted_rankings = scores_to_rankings(n_objects, y_pred) y_true = K.cast(y_true, dtype='float32') equal_ranks = K.cast(K.all(K.equal(predicted_rankings, y_true), axis=1), dtype='float32') denominator = K.cast(n_instances, dtype='float32') zero_one_loss = K.sum(equal_ranks) / denominator return zero_one_loss
def spearman_correlation_for_scores(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) n_instances, n_objects = get_instances_objects(y_true) predicted_rankings = scores_to_rankings(n_objects, y_pred) y_true = K.cast(y_true, dtype='float32') sum_of_squared_distances = tf.constant(0.0) for i in np.arange(K.int_shape(y_pred)[1]): objects_pred = predicted_rankings[:, i] objects_true = y_true[:, i] t = (objects_pred - objects_true)**2 sum_of_squared_distances = sum_of_squared_distances + tf.reduce_sum(t) denominator = K.cast(n_objects * (n_objects**2 - 1) * n_instances, dtype='float32') spearman_correlation = 1 - (6 * sum_of_squared_distances) / denominator return spearman_correlation
def ndcg(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) max_rank = K.max(y_true) def rank_to_relevance(rank): # Convert a rank to a relevance, which is (somewhat arbitrarily) # defined to be inversely proportional to the rank and normalized # to [0, 1]. Other conversion functions are possible. normalized_inverse = (max_rank - rank) / max_rank # define the relevance as 2**a return K.pow(2.0, normalized_inverse) - 1.0 relevance_true = rank_to_relevance(y_true) relevance_pred = rank_to_relevance(y_pred) # Calculate ideal dcg: most_relevant_items, most_relevant_idx = tf.math.top_k( relevance_true, k) # arange starts at 0, but ranks start at 1 and the log term starts at 2 log_term = K.log(K.arange(k, dtype="float32") + 2.0) # keras only natively supports the natural logarithm, have to switch base log2_term = log_term / K.log(2.0) idcg = K.sum(most_relevant_items / log2_term, axis=-1, keepdims=True) # Calculate actual dcg: # The index of the row of every element in toppred_ind, i.e. # [[0, 0], # [1, 1]] row_ind = K.cumsum(K.ones_like(most_relevant_idx, dtype="int32"), axis=0) - 1 # Indices of the k truly most relevant items, sorted by relevance. We # want to sort the predictions based on those indices, since that is # what we're trying to match. full_indices = K.stack([row_ind, most_relevant_idx], axis=-1) # Predicted relevances for the items that *should* have the top k # slots, ordered by the relevance rank they *should* have (so the # log2_term from the true predictions still has the right oder) top_k_preds = tf.gather_nd(relevance_pred, full_indices) weighted = top_k_preds / log2_term dcg = K.sum(weighted, axis=-1, keepdims=True) gain = dcg / idcg return gain
def make_smooth_ndcg_loss(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) n_objects = K.max(y_true) + 1. y_true_f = K.cast(y_true, 'float32') relevance = n_objects - y_true_f - 1. log_term = K.log(relevance + 2.) / K.log(2.) exp_relevance = K.pow(2., relevance) - 1. gains = exp_relevance / log_term # Calculate ideal dcg: idcg = K.sum(gains, axis=-1) # Calculate smoothed dcg: exped = K.exp(y_pred) exped = exped / K.sum(exped, axis=-1, keepdims=True) # toppred, toppred_ind = tf.nn.top_k(gains * exped, k) return 1 - K.sum(exped * gains, axis=-1) / idcg
def zero_one_rank_loss_for_scores_ties(y_true, s_pred): y_true, s_pred = tensorify(y_true), tensorify(s_pred) n_objects = K.cast(K.max(y_true) + 1, dtype='float32') mask = K.greater(y_true[:, None] - y_true[:, :, None], 0) mask2 = K.greater(s_pred[:, None] - s_pred[:, :, None], 0) mask3 = K.equal(s_pred[:, None] - s_pred[:, :, None], 0) # Calculate Transpositions transpositions = tf.logical_and(mask, mask2) transpositions = K.sum(K.cast(transpositions, dtype='float32'), axis=[1, 2]) transpositions += (K.sum(K.cast(mask3, dtype='float32'), axis=[1, 2]) - n_objects) / 4. denominator = n_objects * (n_objects - 1.) / 2. result = transpositions / denominator return K.mean(result)
def zero_one_rank_loss(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) mask = K.greater(y_true[:, None] - y_true[:, :, None], 0) # Count the number of mistakes (here position difference less than 0) mask2 = K.less(y_pred[:, None] - y_pred[:, :, None], 0) mask3 = K.equal(y_pred[:, None] - y_pred[:, :, None], 0) # Calculate Transpositions transpositions = tf.logical_and(mask, mask2) transpositions = K.sum(K.cast(transpositions, dtype='float32'), axis=[1, 2]) n_objects = K.max(y_true) + 1 transpositions += (K.sum(K.cast(mask3, dtype='float32'), axis=[1, 2]) - n_objects) / 4. denominator = K.cast((n_objects * (n_objects - 1.)) / 2., dtype='float32') result = transpositions / denominator return K.mean(result)
def _predict_scores_fixed(self, X, **kwargs): self.logger.info("Predicting scores") n_instances, n_objects, n_features = tensorify(X).get_shape().as_list() y = [] for i in range(n_instances): y.append(np.arange(n_objects)) y = np.array(y) X = self.one_hot_encoder_labels(X, y) return FATENetwork._predict_scores_fixed(self, X, **kwargs)
def ndcg(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) n_objects = K.cast(K.int_shape(y_pred)[1], 'float32') relevance = K.pow(2., ((n_objects - y_true) * 60) / n_objects) - 1. relevance_pred = K.pow(2., ((n_objects - y_pred) * 60) / n_objects) - 1. # Calculate ideal dcg: toprel, toprel_ind = tf.nn.top_k(relevance, k) log_term = K.log(K.arange(k, dtype='float32') + 2.) / K.log(2.) idcg = K.sum(toprel / log_term, axis=-1, keepdims=True) # Calculate actual dcg: toppred, toppred_ind = tf.nn.top_k(relevance_pred, k) row_ind = K.cumsum(K.ones_like(toppred_ind), axis=0) - 1 ind = K.stack([row_ind, toppred_ind], axis=-1) pred_rel = K.sum(tf.gather_nd(relevance, ind) / log_term, axis=-1, keepdims=True) gain = pred_rel / idcg return gain
def relevance_gain(grading, max_grade): """Maps a ranking (0 to `max_grade`, lower is better) to its gain. The gain is defined similar to the Discounted Cumulative Gain (DCG) metric. The value is always in [0, 1]. Therefore, it can be interpreted as a probability. Parameters ---------- max_grade: float The highest achievable grade. grading: float A grading. Higher gradings are assumed to be better. 0 <= grading <= max_grade must always hold. Tests ----- >>> y_true = [0, 3, 2, 1] # (A > D > C > B) >>> K.eval(relevance_gain(y_true, max_grade=3)).tolist() [0.875, 0.0, 0.125, 0.375] """ grading = tensorify(grading) inverse_grading = -grading + tf.cast(max_grade, grading.dtype) return (2**inverse_grading - 1) / (2**tf.cast(max_grade, tf.float32))
def topk_acc(y_true, y_pred): y_true, y_pred = tensorify(y_true), tensorify(y_pred) acc = tf.nn.in_top_k(y_pred, tf.argmax(y_true, axis=-1), k=k) acc = K.cast(acc, dtype='float32') return acc
def err(y_true, y_pred, utility_function=None, probability_mapping=None): """Computes the Expected Reciprocal Rank or any Cascade Metric. ERR[1] is the cascade metric with the reciprocal rank as the utility function. Parameters ---------- y_true: list of int The "ground truth" ranking. In this case, this does not need to actually be a ranking. It can be any 2 dimensional array whose elements can be transformed to probabilities by the `probability_mapping`. y_pred: list of int The predicted ranking that is to be evaluated. probability_mapping: list of int -> list of float A function that maps the elements of `y_true` to probabilities. Those values are then interpreted as the probability that the corresponding object satisfies the user's need. If `None` is specified, the `relevance_gain` function with `max_grade` set to the highest grade occurring in the grading is used. utility_function: int -> float A function that maps a rank (0 being the highest) to its "utility". If `None` is specified, this is defined as the reciprocal of the rank (resulting in the ERR metric). If a different utility is specified, this function can compute any cascade metric. Corresponds to the function represented by :math:`\\phi` in [1]. This will usually be a monotonically decreasing function, since the user is more likely to examine the first few results and therefore more likely to derive utility from them. Examples -------- First, let's keep the default values and evaluate a ranking: >>> y_true = [ ... [0, 3, 2, 1], ... [2, 3, 1, 0], ... ] >>> y_pred = [ ... [0, 3, 1, 2], ... [1, 2, 3, 0], ... ] >>> K.eval(err(y_true, y_pred)) 0.6365559895833333 Instead of relying on the relevance gain, we can also explicitly specify our own probabilities: >>> y_true = [ ... [0.3, 0.6, 0.05, 0.05], ... [0.1, 0.1, 0.1, 0.7], ... ] >>> y_pred = [ ... [0, 3, 1, 2], ... [1, 2, 3, 0], ... ] Now `y_true[i, j]` is the probability that object `j` in instance `i` satisfies the user's need. To use this probabilities unchanged, we need to override the probability mapping with the identity: >>> probability_mapping = lambda x: x Let us further specify that the rank utilities decrease in an exponential manner, e.g. every rank is only half as "valuable" as its predecessor: >>> utility_function = lambda r: 1/2**(r - 1) # start with 2**0 = 1 We can now evaluate the metric: >>> K.eval(err( ... y_true, ... y_pred, ... probability_mapping=probability_mapping, ... utility_function=utility_function, ... )) 0.3543499991945922 The resulting metric is technically no longer an expected reciprocal rank, since the utility is not given by the reciprocal of the rank. It is a different version of a cascade metric. The original paper [1] called it an abandonment cascade (with gamma = 1/2), so let us define a new name for it: >>> from functools import partial >>> abandonment_cascade_half = partial( ... err, ... probability_mapping=probability_mapping, ... utility_function=utility_function, ... ) >>> K.eval(abandonment_cascade_half(y_true, y_pred)) 0.3543499991945922 References ---------- [1] Chapelle, Olivier, et al. "Expected reciprocal rank for graded relevance." Proceedings of the 18th ACM conference on Information and knowledge management. ACM, 2009. http://olivier.chapelle.cc/pub/err.pdf """ if probability_mapping is None: max_grade = tf.reduce_max(y_true) probability_mapping = partial(relevance_gain, max_grade=max_grade) if utility_function is None: def reciprocal_rank(rank): return 1 / rank utility_function = reciprocal_rank y_true, y_pred = tensorify(y_true), tensorify(y_pred) ninstances = tf.shape(y_pred)[0] nobjects = tf.shape(y_pred)[1] # Using y_true and the probability mapping, we can derive the # probability that each object satisfies the users need (we need to # map over the flattened array and then restore the shape): satisfied_probs = tf.reshape( tf.map_fn(probability_mapping, tf.reshape(y_true, (-1, ))), tf.shape(y_true)) # sort satisfied probabilities according to the predicted ranking rows = tf.range(0, ninstances) rows_cast = tf.broadcast_to(tf.reshape(rows, (-1, 1)), tf.shape(y_pred)) full_indices = tf.stack([rows_cast, tf.cast(y_pred, tf.int32)], axis=2) satisfied_at_rank = tf.gather_nd(satisfied_probs, full_indices) not_satisfied_n_times = tf.cumprod(1 - satisfied_at_rank, axis=1, exclusive=True) # And from the positions predicted in y_pred we can further derive # the utilities of each object given their position: utilities = tf.map_fn( utility_function, tf.range(1, nobjects + 1), dtype=tf.float64, ) discount_at_rank = tf.cast(not_satisfied_n_times, tf.float64) * tf.reshape( utilities, (1, -1)) discounted_document_values = (tf.cast(satisfied_at_rank, tf.float64) * discount_at_rank) results = tf.reduce_sum(discounted_document_values, axis=1) return K.mean(results)