def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope('head'): logits = head_lib._check_logits(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits,)): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ '': export_output.ClassificationOutput(scores=probabilities) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) # Averages loss over classes. per_example_loss = math_ops.reduce_mean( unweighted_loss, axis=-1, keep_dims=True) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( per_example_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, probabilities=probabilities, weights=weights, per_example_loss=per_example_loss)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access training_loss) summary.scalar( head_lib._summary_key( # pylint:disable=protected-access self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def wasserstein_discriminator_loss( discriminator_real_outputs, discriminator_gen_outputs, real_weights=1.0, generated_weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Wasserstein discriminator loss for GANs. See `Wasserstein GAN` (https://arxiv.org/abs/1701.07875) for more details. Args: discriminator_real_outputs: Discriminator output on real data. discriminator_gen_outputs: Discriminator output on generated data. Expected to be in the range of (-inf, inf). real_weights: Optional `Tensor` whose rank is either 0, or the same rank as `discriminator_real_outputs`, and must be broadcastable to `discriminator_real_outputs` (i.e., all dimensions must be either `1`, or the same as the corresponding dimension). generated_weights: Same as `real_weights`, but for `discriminator_gen_outputs`. scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.compat.v1.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. """ with ops.name_scope(scope, 'discriminator_wasserstein_loss', (discriminator_real_outputs, discriminator_gen_outputs, real_weights, generated_weights)) as scope: discriminator_real_outputs = _to_float(discriminator_real_outputs) discriminator_gen_outputs = _to_float(discriminator_gen_outputs) discriminator_real_outputs.shape.assert_is_compatible_with( discriminator_gen_outputs.shape) loss_on_generated = losses.compute_weighted_loss( discriminator_gen_outputs, generated_weights, scope, loss_collection=None, reduction=reduction) loss_on_real = losses.compute_weighted_loss( discriminator_real_outputs, real_weights, scope, loss_collection=None, reduction=reduction) loss = loss_on_generated - loss_on_real util.add_loss(loss, loss_collection) if add_summaries: summary.scalar('discriminator_gen_wass_loss', loss_on_generated) summary.scalar('discriminator_real_wass_loss', loss_on_real) summary.scalar('discriminator_wass_loss', loss) return loss
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with variable_scope.variable_scope( None, default_name='regression_head', values=(tuple(six.itervalues(features)) + (labels, logits))): # Predict. logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={'': export_output.RegressionOutput(value=logits)}) # Eval. labels = _check_labels(_maybe_expand_dim(math_ops.to_float(labels)), self._logits_dimension) unweighted_loss = losses.mean_squared_error( labels=labels, predictions=logits, reduction=losses.Reduction.NONE) weights = ( 1. if (self._weight_feature_key is None) else features[self._weight_feature_key]) weights = _maybe_expand_dim(math_ops.to_float(weights, name='weights')) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. eval_metric_ops = { metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean( unweighted_loss, weights=weights) } return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') logging_ops.scalar_summary(metric_keys.MetricKeys.LOSS, training_loss) logging_ops.scalar_summary( metric_keys.MetricKeys.LOSS_MEAN, losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope('head'): logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: regression_output = export_output.RegressionOutput(value=logits) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: regression_output, _REGRESS_SERVING_KEY: regression_output, _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, _ = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. eval_metric_ops = { metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean( unweighted_loss, weights=weights) } return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def wasserstein_discriminator_loss( discriminator_real_outputs, discriminator_gen_outputs, real_weights=1.0, generated_weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Wasserstein discriminator loss for GANs. See `Wasserstein GAN` (https://arxiv.org/abs/1701.07875) for more details. Args: discriminator_real_outputs: Discriminator output on real data. discriminator_gen_outputs: Discriminator output on generated data. Expected to be in the range of (-inf, inf). real_weights: A scalar or a `Tensor` of size [batch_size, K] used to rescale the real loss. generated_weights: A scalar or a `Tensor` of size [batch_size, K] used to rescale the generated loss. scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. """ with ops.name_scope(scope, 'discriminator_wasserstein_loss', ( discriminator_real_outputs, discriminator_gen_outputs, real_weights, generated_weights)) as scope: discriminator_real_outputs = math_ops.to_float(discriminator_real_outputs) discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs) discriminator_real_outputs.shape.assert_is_compatible_with( discriminator_gen_outputs.shape) loss_on_generated = losses.compute_weighted_loss( discriminator_gen_outputs, generated_weights, scope, loss_collection=None, reduction=reduction) loss_on_real = losses.compute_weighted_loss( discriminator_real_outputs, real_weights, scope, loss_collection=None, reduction=reduction) loss = loss_on_generated - loss_on_real util.add_loss(loss, loss_collection) if add_summaries: summary.scalar('discriminator_gen_wass_loss', loss_on_generated) summary.scalar('discriminator_real_wass_loss', loss_on_real) summary.scalar('discriminator_wass_loss', loss) return loss
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope('head'): logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={'': export_output.RegressionOutput(value=logits)}) # Eval. labels = _check_labels(_maybe_expand_dim(math_ops.to_float(labels)), self._logits_dimension) unweighted_loss = losses.mean_squared_error( labels=labels, predictions=logits, reduction=losses.Reduction.NONE) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. eval_metric_ops = { metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean( unweighted_loss, weights=weights) } return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar(metric_keys.MetricKeys.LOSS, training_loss) summary.scalar(metric_keys.MetricKeys.LOSS_MEAN, losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_loss(self, features, mode, logits, labels): """See `Head`.""" del mode # Unused for this head. processed_labels = self._process_labels(labels) if self._loss_fn: unweighted_loss = _call_loss_fn( loss_fn=self._loss_fn, labels=processed_labels, logits=logits, features=features) else: unweighted_loss = losses.sigmoid_cross_entropy( multi_class_labels=processed_labels, logits=logits, reduction=losses.Reduction.NONE) # Averages loss over classes. unweighted_loss = math_ops.reduce_mean( unweighted_loss, axis=-1, keep_dims=True) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access, weighted_sum_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) # _weights() can return 1. example_weight_sum = math_ops.reduce_sum( weights * array_ops.ones_like(unweighted_loss)) return head_lib.LossSpec( weighted_sum_loss=weighted_sum_loss, example_weight_sum=example_weight_sum, processed_labels=processed_labels)
def create_loss(self, features, mode, logits, labels): """See `Head`.""" del mode # Unused for this head. logits = ops.convert_to_tensor(logits) labels = _check_dense_labels_match_logits_and_reshape( labels=labels, logits=logits, expected_labels_dimension=1) if self._label_vocabulary is not None: labels = lookup_ops.index_table_from_tensor( vocabulary_list=tuple(self._label_vocabulary), name='class_id_lookup').lookup(labels) labels = math_ops.to_float(labels) labels = _assert_range(labels, 2) unweighted_loss = nn.sigmoid_cross_entropy_with_logits( labels=labels, logits=logits) weights = _get_weights_and_check_match_logits( features=features, weight_column=self._weight_column, logits=logits) weighted_sum_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) # _weights() can return 1. example_weight_sum = math_ops.reduce_sum( weights * array_ops.ones_like(unweighted_loss)) return LossSpec( weighted_sum_loss=weighted_sum_loss, example_weight_sum=example_weight_sum, processed_labels=labels)
def create_loss(self, features, mode, logits, labels): """See `Head`.""" del mode # Unused for this head. logits = ops.convert_to_tensor(logits) processed_labels = self._process_labels(labels) processed_labels = head_lib._check_dense_labels_match_logits_and_reshape( # pylint:disable=protected-access labels=processed_labels, logits=logits, expected_labels_dimension=self.logits_dimension) if self._loss_fn: unweighted_loss = head_lib._call_loss_fn( # pylint:disable=protected-access loss_fn=self._loss_fn, labels=processed_labels, logits=logits, features=features, expected_loss_dim=1) else: unweighted_loss = losses.sigmoid_cross_entropy( multi_class_labels=processed_labels, logits=logits, reduction=losses.Reduction.NONE) # Averages loss over classes. unweighted_loss = math_ops.reduce_mean( unweighted_loss, axis=-1, keepdims=True) weights = head_lib._get_weights_and_check_match_logits( # pylint:disable=protected-access, features=features, weight_column=self._weight_column, logits=logits) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=self._loss_reduction) return head_lib.LossSpec( training_loss=training_loss, unreduced_loss=unweighted_loss, weights=weights, processed_labels=processed_labels)
def mutual_information_penalty( structured_generator_inputs, predicted_distributions, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Returns a penalty on the mutual information in an InfoGAN model. This loss comes from an InfoGAN paper https://arxiv.org/abs/1606.03657. Args: structured_generator_inputs: A list of Tensors representing the random noise that must have high mutual information with the generator output. List length should match `predicted_distributions`. predicted_distributions: A list of `tfp.distributions.Distribution`s. Predicted by the recognizer, and used to evaluate the likelihood of the structured noise. List length should match `structured_generator_inputs`. weights: Optional `Tensor` whose rank is either 0, or the same dimensions as `structured_generator_inputs`. scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.compat.v1.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A scalar Tensor representing the mutual information loss. """ _validate_information_penalty_inputs(structured_generator_inputs, predicted_distributions) with ops.name_scope(scope, 'mutual_information_loss') as scope: # Calculate the negative log-likelihood of the reconstructed noise. log_probs = [ math_ops.reduce_mean(dist.log_prob(noise)) for dist, noise in zip( predicted_distributions, structured_generator_inputs) ] loss = -1 * losses.compute_weighted_loss( log_probs, weights, scope, loss_collection=loss_collection, reduction=reduction) if add_summaries: summary.scalar('mutual_information_penalty', loss) return loss
def least_squares_generator_loss( discriminator_gen_outputs, real_label=1, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Least squares generator loss. This loss comes from `Least Squares Generative Adversarial Networks` (https://arxiv.org/abs/1611.04076). L = 1/2 * (D(G(z)) - `real_label`) ** 2 where D(y) are discriminator logits. Args: discriminator_gen_outputs: Discriminator output on generated data. Expected to be in the range of (-inf, inf). real_label: The value that the generator is trying to get the discriminator to output on generated data. weights: Optional `Tensor` whose rank is either 0, or the same rank as `discriminator_gen_outputs`, and must be broadcastable to `discriminator_gen_outputs` (i.e., all dimensions must be either `1`, or the same as the corresponding dimension). scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. """ with ops.name_scope(scope, 'lsq_generator_loss', (discriminator_gen_outputs, real_label)) as scope: discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs) loss = math_ops.squared_difference( discriminator_gen_outputs, real_label) / 2.0 loss = losses.compute_weighted_loss( loss, weights, scope, loss_collection, reduction) if add_summaries: summary.scalar('generator_lsq_loss', loss) return loss
def create_loss(self, features, mode, logits, labels): """See `Head`.""" del mode # Unused for this head. label_ids = self._label_ids(_check_and_reshape_dense_labels(labels, 1)) unweighted_loss = losses.sparse_softmax_cross_entropy( labels=label_ids, logits=logits, reduction=losses.Reduction.NONE) # Restore the squeezed dim, so unweighted_loss matches the weights shape. unweighted_loss = array_ops.expand_dims(unweighted_loss, axis=(1,)) weights = _weights(features, self._weight_column) weighted_sum_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) # _weights() can return 1. example_weight_sum = math_ops.reduce_sum( weights * array_ops.ones_like(unweighted_loss)) return LossSpec( weighted_sum_loss=weighted_sum_loss, example_weight_sum=example_weight_sum, processed_labels=label_ids)
def create_loss(self, features, mode, logits, labels): """See `Head`.""" del mode # Unused for this head. logits = ops.convert_to_tensor(logits) labels = _check_dense_labels_match_logits_and_reshape( labels=labels, logits=logits, expected_labels_dimension=self._logits_dimension) labels = math_ops.to_float(labels) unweighted_loss = losses.mean_squared_error( labels=labels, predictions=logits, reduction=losses.Reduction.NONE) weights = _get_weights_and_check_match_logits( features=features, weight_column=self._weight_column, logits=logits, allow_per_logit_weights=True) weighted_sum_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) # _weights() can return 1. example_weight_sum = math_ops.reduce_sum( weights * array_ops.ones_like(unweighted_loss)) return LossSpec( weighted_sum_loss=weighted_sum_loss, example_weight_sum=example_weight_sum, processed_labels=labels)
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits,)): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = head_lib._classification_output( # pylint:disable=protected-access scores=probabilities, n_classes=self._n_classes, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access export_output.PredictOutput(predictions)) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, probabilities=probabilities, weights=weights, unweighted_loss=unweighted_loss)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access training_loss) summary.scalar( head_lib._summary_key( # pylint:disable=protected-access self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope('head'): with ops.name_scope(None, 'predictions', (logits,)): pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') scores = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES) class_ids = array_ops.reshape( math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, pred_keys.PROBABILITIES: scores, pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: batch_size = array_ops.shape(logistic)[0] export_class_list = self._label_vocabulary if not export_class_list: export_class_list = string_ops.as_string([0, 1]) export_output_classes = array_ops.tile( input=array_ops.expand_dims(input=export_class_list, axis=0), multiples=[batch_size, 1]) classifier_output = export_output.ClassificationOutput( scores=scores, # `ClassificationOutput` requires string classes. classes=export_output_classes) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ '': classifier_output, # to be same as other heads. 'classification': classifier_output, # to be called by name. _DEFAULT_SERVING_KEY: classifier_output, # default 'regression': export_output.RegressionOutput(value=logistic) }) # Eval. labels = _check_labels(_maybe_expand_dim(labels), self.logits_dimension) if self._label_vocabulary is not None: labels = lookup_ops.index_table_from_tensor( vocabulary_list=tuple(self._label_vocabulary), name='class_id_lookup').lookup(labels) labels = math_ops.to_float(labels) labels = _assert_range(labels, 2) unweighted_loss = nn.sigmoid_cross_entropy_with_logits( labels=labels, logits=logits, name='loss') weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=labels, logits=logits, logistic=logistic, scores=scores, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar(metric_keys.MetricKeys.LOSS, training_loss) summary.scalar(metric_keys.MetricKeys.LOSS_MEAN, losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits, )): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = head_lib._classification_output( # pylint:disable=protected-access scores=probabilities, n_classes=self._n_classes, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access export_output.PredictOutput(predictions)) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, probabilities=probabilities, weights=weights, unweighted_loss=unweighted_loss)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access training_loss) summary.scalar( head_lib._summary_key( # pylint:disable=protected-access self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss(unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope('head'): logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: regression_output = export_output.RegressionOutput( value=logits) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: regression_output, _REGRESS_SERVING_KEY: regression_output, _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, _ = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. eval_metric_ops = { metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean(unweighted_loss, weights=weights) } return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss(unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope('head'): logits = _check_logits(logits, self.logits_dimension) # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits, )): # class_ids's shape is [batch_size] class_ids = math_ops.argmax(logits, 1, name=pred_keys.CLASS_IDS) class_ids = array_ops.expand_dims(class_ids, axis=(1, )) if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') probabilities = nn.softmax(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, # Expand to [batch_size, 1] pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: batch_size = array_ops.shape(probabilities)[0] export_class_list = self._label_vocabulary if not export_class_list: export_class_list = string_ops.as_string( math_ops.range(self._n_classes)) export_output_classes = array_ops.tile( input=array_ops.expand_dims(input=export_class_list, axis=0), multiples=[batch_size, 1]) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ '': export_output.ClassificationOutput( scores=probabilities, # `ClassificationOutput` requires string classes. classes=export_output_classes) }) # Eval. unweighted_loss, label_ids = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, probabilities=probabilities, logits=logits, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar(metric_keys.MetricKeys.LOSS, training_loss) summary.scalar( metric_keys.MetricKeys.LOSS_MEAN, losses.compute_weighted_loss(unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def wasserstein_gradient_penalty( real_data, generated_data, generator_inputs, discriminator_fn, discriminator_scope, epsilon=1e-10, target=1.0, one_sided=False, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """The gradient penalty for the Wasserstein discriminator loss. See `Improved Training of Wasserstein GANs` (https://arxiv.org/abs/1704.00028) for more details. Args: real_data: Real data. generated_data: Output of the generator. generator_inputs: Exact argument to pass to the generator, which is used as optional conditioning to the discriminator. discriminator_fn: A discriminator function that conforms to TFGAN API. discriminator_scope: If not `None`, reuse discriminators from this scope. epsilon: A small positive number added for numerical stability when computing the gradient norm. target: Optional Python number or `Tensor` indicating the target value of gradient norm. Defaults to 1.0. one_sided: If `True`, penalty proposed in https://arxiv.org/abs/1709.08894 is used. Defaults to `False`. weights: Optional `Tensor` whose rank is either 0, or the same rank as `real_data` and `generated_data`, and must be broadcastable to them (i.e., all dimensions must be either `1`, or the same as the corresponding dimension). scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. Raises: ValueError: If the rank of data Tensors is unknown. """ with ops.name_scope(scope, 'wasserstein_gradient_penalty', (real_data, generated_data)) as scope: real_data = ops.convert_to_tensor(real_data) generated_data = ops.convert_to_tensor(generated_data) if real_data.shape.ndims is None: raise ValueError('`real_data` can\'t have unknown rank.') if generated_data.shape.ndims is None: raise ValueError('`generated_data` can\'t have unknown rank.') differences = generated_data - real_data batch_size = differences.shape.dims[0].value or array_ops.shape( differences)[0] alpha_shape = [batch_size] + [1] * (differences.shape.ndims - 1) alpha = random_ops.random_uniform(shape=alpha_shape) interpolates = real_data + (alpha * differences) with ops.name_scope(None): # Clear scope so update ops are added properly. # Reuse variables if variables already exists. with variable_scope.variable_scope(discriminator_scope, 'gpenalty_dscope', reuse=variable_scope.AUTO_REUSE): disc_interpolates = discriminator_fn(interpolates, generator_inputs) if isinstance(disc_interpolates, tuple): # ACGAN case: disc outputs more than one tensor disc_interpolates = disc_interpolates[0] gradients = gradients_impl.gradients(disc_interpolates, interpolates)[0] gradient_squares = math_ops.reduce_sum( math_ops.square(gradients), axis=list(range(1, gradients.shape.ndims))) # Propagate shape information, if possible. if isinstance(batch_size, int): gradient_squares.set_shape([ batch_size] + gradient_squares.shape.as_list()[1:]) # For numerical stability, add epsilon to the sum before taking the square # root. Note tf.norm does not add epsilon. slopes = math_ops.sqrt(gradient_squares + epsilon) penalties = slopes / target - 1.0 if one_sided: penalties = math_ops.maximum(0., penalties) penalties_squared = math_ops.square(penalties) penalty = losses.compute_weighted_loss( penalties_squared, weights, scope=scope, loss_collection=loss_collection, reduction=reduction) if add_summaries: summary.scalar('gradient_penalty_loss', penalty) return penalty
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with variable_scope.variable_scope( None, default_name='binary_logistic_head', values=(tuple(six.itervalues(features)) + (labels, logits))): # Predict. pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') scores = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES) classes = array_ops.reshape( math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, pred_keys.PROBABILITIES: scores, pred_keys.CLASS_IDS: classes } if mode == model_fn.ModeKeys.PREDICT: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={'': export_output.ClassificationOutput( scores=scores, # `ClassificationOutput` requires string classes. # TODO(ptucker): Support label_keys. classes=string_ops.as_string(classes, name='str_classes'))}) # Eval. labels = _check_labels(math_ops.to_float(labels), self.logits_dimension) unweighted_loss = nn.sigmoid_cross_entropy_with_logits( labels=labels, logits=logits, name='loss') weights = ( 1. if (self._weight_feature_key is None) else features[self._weight_feature_key]) weights = math_ops.to_float(weights, name='weights') training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=labels, logits=logits, logistic=logistic, scores=scores, classes=classes, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') logging_ops.scalar_summary(metric_keys.MetricKeys.LOSS, training_loss) logging_ops.scalar_summary( metric_keys.MetricKeys.LOSS_MEAN, losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def sparse_multiclass_hinge_loss( labels, logits, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS): """Adds Ops for computing the multiclass hinge loss. The implementation is based on the following paper: On the Algorithmic Implementation of Multiclass Kernel-based Vector Machines by Crammer and Singer. link: http://jmlr.csail.mit.edu/papers/volume2/crammer01a/crammer01a.pdf This is a generalization of standard (binary) hinge loss. For a given instance with correct label c*, the loss is given by: $$loss = max_{c != c*} logits_c - logits_{c*} + 1.$$ or equivalently $$loss = max_c { logits_c - logits_{c*} + I_{c != c*} }$$ where \\(I_{c != c*} = 1\ \text{if}\ c != c*\\) and 0 otherwise. Args: labels: `Tensor` of shape [batch_size] or [batch_size, 1]. Corresponds to the ground truth. Each entry must be an index in `[0, num_classes)`. logits: `Tensor` of shape [batch_size, num_classes] corresponding to the unscaled logits. Its dtype should be either `float32` or `float64`. weights: Optional (python) scalar or `Tensor`. If a non-scalar `Tensor`, its rank should be either 1 ([batch_size]) or 2 ([batch_size, 1]). scope: The scope for the operations performed in computing the loss. loss_collection: collection to which the loss will be added. reduction: Type of reduction to apply to loss. Returns: Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same shape as `labels`; otherwise, it is a scalar. Raises: ValueError: If `logits`, `labels` or `weights` have invalid or inconsistent shapes. ValueError: If `labels` tensor has invalid dtype. """ with ops.name_scope(scope, 'sparse_multiclass_hinge_loss', (logits, labels)) as scope: # Check logits Tensor has valid rank. logits_rank = logits.get_shape().ndims if logits_rank != 2: raise ValueError( 'logits should have rank 2 ([batch_size, num_classes]). Given rank is' ' {}'.format(logits_rank)) logits_shape = array_ops.shape(logits) batch_size, num_classes = logits_shape[0], logits_shape[1] logits = math_ops.to_float(logits) # Check labels have valid type. if labels.dtype != dtypes.int32 and labels.dtype != dtypes.int64: raise ValueError( 'Invalid dtype for labels: {}. Acceptable dtypes: int32 and int64'. format(labels.dtype)) # Check labels and weights have valid ranks and are consistent. labels_rank = labels.get_shape().ndims if labels_rank not in [1, 2]: raise ValueError( 'labels should have rank 1 ([batch_size]) or 2 ([batch_size, 1]). ' 'Given rank is {}'.format(labels_rank)) with ops.control_dependencies([ check_ops.assert_less(labels, math_ops.cast(num_classes, labels.dtype)) ]): labels = array_ops.reshape(labels, shape=[-1]) weights = ops.convert_to_tensor(weights) weights_rank = weights.get_shape().ndims if weights_rank not in [0, 1, 2]: raise ValueError( 'non-scalar weights should have rank 1 ([batch_size]) or 2 ' '([batch_size, 1]). Given rank is {}'.format(labels_rank)) if weights_rank > 0: weights = array_ops.reshape(weights, shape=[-1]) # Check weights and labels have the same number of elements. weights.get_shape().assert_is_compatible_with(labels.get_shape()) # Compute the logits tensor corresponding to the correct class per instance. example_indices = array_ops.reshape( math_ops.range(batch_size), shape=[batch_size, 1]) indices = array_ops.concat( [ example_indices, array_ops.reshape( math_ops.cast(labels, example_indices.dtype), shape=[batch_size, 1]) ], axis=1) label_logits = array_ops.reshape( array_ops.gather_nd(params=logits, indices=indices), shape=[batch_size, 1]) one_cold_labels = array_ops.one_hot( indices=labels, depth=num_classes, on_value=0.0, off_value=1.0) margin = logits - label_logits + one_cold_labels margin = nn_ops.relu(margin) loss = math_ops.reduce_max(margin, axis=1) return losses.compute_weighted_loss( loss, weights, scope, loss_collection, reduction=reduction)
def wasserstein_discriminator_loss( discriminator_real_outputs, discriminator_gen_outputs, real_weights=1.0, generated_weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Wasserstein discriminator loss for GANs. See `Wasserstein GAN` (https://arxiv.org/abs/1701.07875) for more details. Args: discriminator_real_outputs: Discriminator output on real data. discriminator_gen_outputs: Discriminator output on generated data. Expected to be in the range of (-inf, inf). real_weights: Optional `Tensor` whose rank is either 0, or the same rank as `discriminator_real_outputs`, and must be broadcastable to `discriminator_real_outputs` (i.e., all dimensions must be either `1`, or the same as the corresponding dimension). generated_weights: Same as `real_weights`, but for `discriminator_gen_outputs`. scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. """ with ops.name_scope(scope, 'discriminator_wasserstein_loss', (discriminator_real_outputs, discriminator_gen_outputs, real_weights, generated_weights)) as scope: discriminator_real_outputs = math_ops.to_float( discriminator_real_outputs) discriminator_gen_outputs = math_ops.to_float( discriminator_gen_outputs) discriminator_real_outputs.shape.assert_is_compatible_with( discriminator_gen_outputs.shape) loss_on_generated = losses.compute_weighted_loss( discriminator_gen_outputs, generated_weights, scope, loss_collection=None, reduction=reduction) loss_on_real = losses.compute_weighted_loss(discriminator_real_outputs, real_weights, scope, loss_collection=None, reduction=reduction) loss = loss_on_generated - loss_on_real util.add_loss(loss, loss_collection) if add_summaries: summary.scalar('discriminator_gen_wass_loss', loss_on_generated) summary.scalar('discriminator_real_wass_loss', loss_on_real) summary.scalar('discriminator_wass_loss', loss) return loss
def wasserstein_gradient_penalty( real_data, generated_data, generator_inputs, discriminator_fn, discriminator_scope, epsilon=1e-10, target=1.0, one_sided=False, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """The gradient penalty for the Wasserstein discriminator loss. See `Improved Training of Wasserstein GANs` (https://arxiv.org/abs/1704.00028) for more details. Args: real_data: Real data. generated_data: Output of the generator. generator_inputs: Exact argument to pass to the generator, which is used as optional conditioning to the discriminator. discriminator_fn: A discriminator function that conforms to TFGAN API. discriminator_scope: If not `None`, reuse discriminators from this scope. epsilon: A small positive number added for numerical stability when computing the gradient norm. target: Optional Python number or `Tensor` indicating the target value of gradient norm. Defaults to 1.0. one_sided: If `True`, penalty proposed in https://arxiv.org/abs/1709.08894 is used. Defaults to `False`. weights: Optional `Tensor` whose rank is either 0, or the same rank as `real_data` and `generated_data`, and must be broadcastable to them (i.e., all dimensions must be either `1`, or the same as the corresponding dimension). scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. Raises: ValueError: If the rank of data Tensors is unknown. """ with ops.name_scope(scope, 'wasserstein_gradient_penalty', (real_data, generated_data)) as scope: real_data = ops.convert_to_tensor(real_data) generated_data = ops.convert_to_tensor(generated_data) if real_data.shape.ndims is None: raise ValueError('`real_data` can\'t have unknown rank.') if generated_data.shape.ndims is None: raise ValueError('`generated_data` can\'t have unknown rank.') differences = generated_data - real_data batch_size = differences.shape.dims[0].value or array_ops.shape( differences)[0] alpha_shape = [batch_size] + [1] * (differences.shape.ndims - 1) alpha = random_ops.random_uniform(shape=alpha_shape) interpolates = real_data + (alpha * differences) with ops.name_scope( None): # Clear scope so update ops are added properly. # Reuse variables if variables already exists. with variable_scope.variable_scope( discriminator_scope, 'gpenalty_dscope', reuse=variable_scope.AUTO_REUSE): disc_interpolates = discriminator_fn(interpolates, generator_inputs) if isinstance(disc_interpolates, tuple): # ACGAN case: disc outputs more than one tensor disc_interpolates = disc_interpolates[0] gradients = gradients_impl.gradients(disc_interpolates, interpolates)[0] gradient_squares = math_ops.reduce_sum( math_ops.square(gradients), axis=list(range(1, gradients.shape.ndims))) # Propagate shape information, if possible. if isinstance(batch_size, int): gradient_squares.set_shape([batch_size] + gradient_squares.shape.as_list()[1:]) # For numerical stability, add epsilon to the sum before taking the square # root. Note tf.norm does not add epsilon. slopes = math_ops.sqrt(gradient_squares + epsilon) penalties = slopes / target - 1.0 if one_sided: penalties = math_ops.maximum(0., penalties) penalties_squared = math_ops.square(penalties) penalty = losses.compute_weighted_loss(penalties_squared, weights, scope=scope, loss_collection=loss_collection, reduction=reduction) if add_summaries: summary.scalar('gradient_penalty_loss', penalty) return penalty
def least_squares_discriminator_loss( discriminator_real_outputs, discriminator_gen_outputs, real_label=1, fake_label=0, real_weights=1.0, generated_weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Least squares discriminator loss. This loss comes from `Least Squares Generative Adversarial Networks` (https://arxiv.org/abs/1611.04076). L = 1/2 * (D(x) - `real`) ** 2 + 1/2 * (D(G(z)) - `fake_label`) ** 2 where D(y) are discriminator logits. Args: discriminator_real_outputs: Discriminator output on real data. discriminator_gen_outputs: Discriminator output on generated data. Expected to be in the range of (-inf, inf). real_label: The value that the discriminator tries to output for real data. fake_label: The value that the discriminator tries to output for fake data. real_weights: Optional `Tensor` whose rank is either 0, or the same rank as `discriminator_real_outputs`, and must be broadcastable to `discriminator_real_outputs` (i.e., all dimensions must be either `1`, or the same as the corresponding dimension). generated_weights: Same as `real_weights`, but for `discriminator_gen_outputs`. scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. """ with ops.name_scope(scope, 'lsq_discriminator_loss', (discriminator_gen_outputs, real_label)) as scope: discriminator_real_outputs = math_ops.to_float( discriminator_real_outputs) discriminator_gen_outputs = math_ops.to_float( discriminator_gen_outputs) discriminator_real_outputs.shape.assert_is_compatible_with( discriminator_gen_outputs.shape) real_losses = math_ops.squared_difference(discriminator_real_outputs, real_label) / 2.0 fake_losses = math_ops.squared_difference(discriminator_gen_outputs, fake_label) / 2.0 loss_on_real = losses.compute_weighted_loss(real_losses, real_weights, scope, loss_collection=None, reduction=reduction) loss_on_generated = losses.compute_weighted_loss(fake_losses, generated_weights, scope, loss_collection=None, reduction=reduction) loss = loss_on_real + loss_on_generated util.add_loss(loss, loss_collection) if add_summaries: summary.scalar('discriminator_gen_lsq_loss', loss_on_generated) summary.scalar('discriminator_real_lsq_loss', loss_on_real) summary.scalar('discriminator_lsq_loss', loss) return loss
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope(self._name, 'head'): with ops.name_scope(None, 'predictions', (logits,)): pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') probabilities = nn.softmax( two_class_logits, name=pred_keys.PROBABILITIES) class_ids = array_ops.reshape( math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, pred_keys.PROBABILITIES: probabilities, pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = _classification_output( scores=probabilities, n_classes=2, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, _CLASSIFY_SERVING_KEY: classifier_output, _REGRESS_SERVING_KEY: export_output.RegressionOutput( value=logistic), _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, logits=logits, logistic=logistic, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope('head'): with ops.name_scope(None, 'predictions', (logits,)): pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') scores = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES) class_ids = array_ops.reshape( math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, pred_keys.PROBABILITIES: scores, pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: batch_size = array_ops.shape(logistic)[0] export_class_list = self._label_vocabulary if not export_class_list: export_class_list = string_ops.as_string([0, 1]) export_output_classes = array_ops.tile( input=array_ops.expand_dims(input=export_class_list, axis=0), multiples=[batch_size, 1]) classifier_output = export_output.ClassificationOutput( scores=scores, # `ClassificationOutput` requires string classes. classes=export_output_classes) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, _CLASSIFY_SERVING_KEY: classifier_output, _REGRESS_SERVING_KEY: export_output.RegressionOutput( value=logistic), _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, logits=logits, logistic=logistic, scores=scores, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope('head'): logits = head_lib._check_logits(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits, )): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ '': export_output.ClassificationOutput( scores=probabilities) }) # Eval. unweighted_loss, _ = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) # Averages loss over classes. per_example_loss = math_ops.reduce_mean(unweighted_loss, axis=-1, keep_dims=True) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( per_example_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=labels, probabilities=probabilities, weights=weights, per_example_loss=per_example_loss)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._head_name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access training_loss) summary.scalar( head_lib._summary_key( # pylint:disable=protected-access self._head_name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss(unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with variable_scope.variable_scope( None, default_name='multi_class_head', values=(tuple(six.itervalues(features)) + (labels, logits))): logits = _check_logits(logits, self.logits_dimension) # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits,)): # class_ids's shape is [batch_size] class_ids = math_ops.argmax(logits, 1, name=pred_keys.CLASS_IDS) class_ids = array_ops.expand_dims(class_ids, axis=(1,)) if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') probabilities = nn.softmax(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, # Expand to [batch_size, 1] pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: batch_size = array_ops.shape(probabilities)[0] export_class_list = self._label_vocabulary if not export_class_list: export_class_list = string_ops.as_string( math_ops.range(self._n_classes)) export_output_classes = array_ops.tile( input=array_ops.expand_dims(input=export_class_list, axis=0), multiples=[batch_size, 1]) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ '': export_output.ClassificationOutput( scores=probabilities, # `ClassificationOutput` requires string classes. classes=export_output_classes) }) # Eval. label_ids = self._label_ids(_check_labels(labels, 1)) unweighted_loss = losses.sparse_softmax_cross_entropy( labels=label_ids, logits=logits, reduction=losses.Reduction.NONE) # Restore the squeezed dim, so unweighted_loss matches the weights shape. unweighted_loss = array_ops.expand_dims(unweighted_loss, axis=(1,)) weights = ( 1. if (self._weight_feature_key is None) else features[self._weight_feature_key]) weights = math_ops.to_float(weights, name='weights') training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, probabilities=probabilities, logits=logits, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') logging_ops.scalar_summary(metric_keys.MetricKeys.LOSS, training_loss) logging_ops.scalar_summary( metric_keys.MetricKeys.LOSS_MEAN, losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def sparse_multiclass_hinge_loss( labels, logits, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS): """Adds Ops for computing the multiclass hinge loss. The implementation is based on the following paper: On the Algorithmic Implementation of Multiclass Kernel-based Vector Machines by Crammer and Singer. link: http://jmlr.csail.mit.edu/papers/volume2/crammer01a/crammer01a.pdf This is a generalization of standard (binary) hinge loss. For a given instance with correct label c*, the loss is given by: $$loss = max_{c != c*} logits_c - logits_{c*} + 1.$$ or equivalently $$loss = max_c { logits_c - logits_{c*} + I_{c != c*} }$$ where \\(I_{c != c*} = 1\ \text{if}\ c != c*\\) and 0 otherwise. Args: labels: `Tensor` of shape [batch_size] or [batch_size, 1]. Corresponds to the ground truth. Each entry must be an index in `[0, num_classes)`. logits: `Tensor` of shape [batch_size, num_classes] corresponding to the unscaled logits. Its dtype should be either `float32` or `float64`. weights: Optional (python) scalar or `Tensor`. If a non-scalar `Tensor`, its rank should be either 1 ([batch_size]) or 2 ([batch_size, 1]). scope: The scope for the operations performed in computing the loss. loss_collection: collection to which the loss will be added. reduction: Type of reduction to apply to loss. Returns: Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same shape as `labels`; otherwise, it is a scalar. Raises: ValueError: If `logits`, `labels` or `weights` have invalid or inconsistent shapes. ValueError: If `labels` tensor has invalid dtype. """ with ops.name_scope(scope, 'sparse_multiclass_hinge_loss', (logits, labels)) as scope: # Check logits Tensor has valid rank. logits_rank = logits.get_shape().ndims if logits_rank != 2: raise ValueError( 'logits should have rank 2 ([batch_size, num_classes]). Given rank is' ' {}'.format(logits_rank)) logits_shape = array_ops.shape(logits) batch_size, num_classes = logits_shape[0], logits_shape[1] logits = math_ops.to_float(logits) # Check labels have valid type. if labels.dtype != dtypes.int32 and labels.dtype != dtypes.int64: raise ValueError( 'Invalid dtype for labels: {}. Acceptable dtypes: int32 and int64' .format(labels.dtype)) # Check labels and weights have valid ranks and are consistent. labels_rank = labels.get_shape().ndims if labels_rank not in [1, 2]: raise ValueError( 'labels should have rank 1 ([batch_size]) or 2 ([batch_size, 1]). ' 'Given rank is {}'.format(labels_rank)) with ops.control_dependencies([ check_ops.assert_less(labels, math_ops.cast(num_classes, labels.dtype)) ]): labels = array_ops.reshape(labels, shape=[-1]) weights = ops.convert_to_tensor(weights) weights_rank = weights.get_shape().ndims if weights_rank not in [0, 1, 2]: raise ValueError( 'non-scalar weights should have rank 1 ([batch_size]) or 2 ' '([batch_size, 1]). Given rank is {}'.format(labels_rank)) if weights_rank > 0: weights = array_ops.reshape(weights, shape=[-1]) # Check weights and labels have the same number of elements. weights.get_shape().assert_is_compatible_with(labels.get_shape()) # Compute the logits tensor corresponding to the correct class per instance. example_indices = array_ops.reshape(math_ops.range(batch_size), shape=[batch_size, 1]) indices = array_ops.concat([ example_indices, array_ops.reshape(math_ops.cast(labels, example_indices.dtype), shape=[batch_size, 1]) ], axis=1) label_logits = array_ops.reshape(array_ops.gather_nd(params=logits, indices=indices), shape=[batch_size, 1]) one_cold_labels = array_ops.one_hot(indices=labels, depth=num_classes, on_value=0.0, off_value=1.0) margin = logits - label_logits + one_cold_labels margin = nn_ops.relu(margin) loss = math_ops.reduce_max(margin, axis=1) return losses.compute_weighted_loss(loss, weights, scope, loss_collection, reduction=reduction)
def computation(x): return losses.compute_weighted_loss( x, weights=array_ops.ones_like(x))
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope('head'): logits = _check_logits(logits, self.logits_dimension) # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits,)): # class_ids's shape is [batch_size] class_ids = math_ops.argmax(logits, 1, name=pred_keys.CLASS_IDS) class_ids = array_ops.expand_dims(class_ids, axis=(1,)) if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') probabilities = nn.softmax(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, # Expand to [batch_size, 1] pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: batch_size = array_ops.shape(probabilities)[0] export_class_list = self._label_vocabulary if not export_class_list: export_class_list = string_ops.as_string( math_ops.range(self._n_classes)) export_output_classes = array_ops.tile( input=array_ops.expand_dims(input=export_class_list, axis=0), multiples=[batch_size, 1]) classifier_output = export_output.ClassificationOutput( scores=probabilities, # `ClassificationOutput` requires string classes. classes=export_output_classes) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, _CLASSIFY_SERVING_KEY: classifier_output, _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, label_ids = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, probabilities=probabilities, logits=logits, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def least_squares_discriminator_loss( discriminator_real_outputs, discriminator_gen_outputs, real_label=1, fake_label=0, real_weights=1.0, generated_weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): """Least squares generator loss. This loss comes from `Least Squares Generative Adversarial Networks` (https://arxiv.org/abs/1611.04076). L = 1/2 * (D(x) - `real`) ** 2 + 1/2 * (D(G(z)) - `fake_label`) ** 2 where D(y) are discriminator logits. Args: discriminator_real_outputs: Discriminator output on real data. discriminator_gen_outputs: Discriminator output on generated data. Expected to be in the range of (-inf, inf). real_label: The value that the discriminator tries to output for real data. fake_label: The value that the discriminator tries to output for fake data. real_weights: A scalar or a `Tensor` of size [batch_size, K] used to rescale the real loss. generated_weights: A scalar or a `Tensor` of size [batch_size, K] used to rescale the generated loss. scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: A `tf.losses.Reduction` to apply to loss. add_summaries: Whether or not to add summaries for the loss. Returns: A loss Tensor. The shape depends on `reduction`. """ with ops.name_scope(scope, 'lsq_discriminator_loss', (discriminator_gen_outputs, real_label)) as scope: discriminator_real_outputs = math_ops.to_float(discriminator_real_outputs) discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs) discriminator_real_outputs.shape.assert_is_compatible_with( discriminator_gen_outputs.shape) real_losses = math_ops.squared_difference( discriminator_real_outputs, real_label) / 2.0 fake_losses = math_ops.squared_difference( discriminator_gen_outputs, fake_label) / 2.0 loss_on_real = losses.compute_weighted_loss( real_losses, real_weights, scope, loss_collection=None, reduction=reduction) loss_on_generated = losses.compute_weighted_loss( fake_losses, generated_weights, scope, loss_collection=None, reduction=reduction) loss = loss_on_real + loss_on_generated util.add_loss(loss, loss_collection) if add_summaries: summary.scalar('discriminator_gen_lsq_loss', loss_on_generated) summary.scalar('discriminator_real_lsq_loss', loss_on_real) summary.scalar('discriminator_lsq_loss', loss) return loss
def _list_mle_loss(labels, logits, weights=None, lambda_weight=None, reduction=core_losses.Reduction.SUM_BY_NONZERO_WEIGHTS, name=None, seed=None): """Computes the ListMLE loss [Xia et al. 2008] for a list. Given the labels of graded relevance l_i and the logits s_i, we calculate the ListMLE loss for the given list. The `lambda_weight` re-weights examples based on l_i and r_i. The recommended weighting scheme is the formulation presented in the "Position-Aware ListMLE" paper (Lan et al.) and available using create_p_list_mle_lambda_weight() factory function above. Args: labels: A `Tensor` of the same shape as `logits` representing graded relevance. logits: A `Tensor` with shape [batch_size, list_size]. Each value is the ranking score of the corresponding item. weights: A scalar, a `Tensor` with shape [batch_size, 1] for list-wise weights, or a `Tensor` with shape [batch_size, list_size] for item-wise weights. lambda_weight: A `DCGLambdaWeight` instance. reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to reduce training loss over batch. name: A string used as the name for this loss. seed: A randomization seed used when shuffling ground truth permutations. Returns: An op for the ListMLE loss. """ with ops.name_scope(name, 'list_mle_loss', (labels, logits, weights)): is_label_valid = utils.is_label_valid(labels) # Reset the invalid labels to 0 and reset the invalid logits to a logit with # ~= 0 contribution. labels = array_ops.where(is_label_valid, labels, array_ops.zeros_like(labels)) logits = array_ops.where( is_label_valid, logits, math_ops.log(_EPSILON) * array_ops.ones_like(logits)) weights = 1.0 if weights is None else ops.convert_to_tensor(weights) weights = array_ops.squeeze(weights) # Shuffle labels and logits to add randomness to sort. shuffled_indices = utils.shuffle_valid_indices(is_label_valid, seed) shuffled_labels = array_ops.gather_nd(labels, shuffled_indices) shuffled_logits = array_ops.gather_nd(logits, shuffled_indices) sorted_labels, sorted_logits = utils.sort_by_scores( shuffled_labels, [shuffled_labels, shuffled_logits]) raw_max = math_ops.reduce_max(sorted_logits, axis=1, keepdims=True) sorted_logits = sorted_logits - raw_max sums = math_ops.cumsum(math_ops.exp(sorted_logits), axis=1, reverse=True) sums = math_ops.log(sums) - sorted_logits if lambda_weight is not None and isinstance(lambda_weight, ListMLELambdaWeight): sums *= lambda_weight.individual_weights(sorted_labels) negative_log_likelihood = math_ops.reduce_sum(sums, 1) return core_losses.compute_weighted_loss( negative_log_likelihood, weights=weights, reduction=reduction)