def test_build_all_signature_defs_with_dict_alternatives(self): receiver_tensor = array_ops.placeholder(dtypes.string) receiver_tensors_alternative_1 = { "foo": array_ops.placeholder(dtypes.int64), "bar": array_ops.sparse_placeholder(dtypes.float32) } receiver_tensors_alternatives = { "other": receiver_tensors_alternative_1 } output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput(outputs={"some_output_3": output_3}), } signature_defs = export.build_all_signature_defs( receiver_tensor, export_outputs, receiver_tensors_alternatives) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}), "other:head-3": signature_def_utils.predict_signature_def( receiver_tensors_alternative_1, {"some_output_3": output_3}) # Note that the alternatives 'other:serving_default' and 'other:head-2' # are invalid, because regession and classification signatures must take # a single string input. Here we verify that these invalid signatures # are not included in the export. } self.assertDictEqual(expected_signature_defs, signature_defs)
def testExportOutputsMultiheadWithDefault(self): with ops.Graph().as_default(), self.test_session(): predictions = {'loss': constant_op.constant(1.)} output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(['2']) output_3 = constant_op.constant(['3']) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), 'head-2': export_output.ClassificationOutput(classes=output_2), 'head-3': export_output.PredictOutput(outputs={ 'some_output_3': output_3 })} estimator_spec = model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs=export_outputs) self.assertEqual(export_outputs, estimator_spec.export_outputs)
def model_fn(self, mode, features, labels, params): c = variable_scope.get_variable('c', initializer=constant_op.constant( 10, dtype=dtypes.float64), dtype=dtypes.float64) predictions = {'probabilities': math_ops.multiply(features, c)} loss = losses.absolute_difference( labels=labels, predictions=predictions['probabilities'], reduction=losses.Reduction.SUM) metrics = { 'accuracy': metrics_lib.accuracy(labels, predictions['probabilities']), 'auc': metrics_lib.auc(labels, predictions['probabilities']) } tensor_string_repr = str(features) classes = constant_op.constant(re.search('(split_inputs/split:[0-9])', tensor_string_repr).group(1), dtype=dtypes.string) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.PredictOutput(predictions), 'classification_output': export_output.ClassificationOutput(predictions['probabilities'], classes), 'classification_scores': export_output.ClassificationOutput( scores=predictions['probabilities']), 'classification_classes': export_output.ClassificationOutput(classes=classes), 'regression_output': export_output.RegressionOutput(predictions['probabilities']), } return model_fn_lib.EstimatorSpec( mode=mode, loss=math_ops.reduce_sum(loss), eval_metric_ops=metrics, predictions=predictions, train_op=loss, # This train_op isn't actually used. export_outputs=export_outputs)
def _predict_spec(tower_specs, aggregation_device): """Populate replicated EstimatorSpec for `GraphKeys.PREDICT`.""" estimator_spec = _asdict(tower_specs[0]) estimator_spec['mode'] = model_fn_lib.ModeKeys.PREDICT with ops_lib.device(aggregation_device): estimator_spec['predictions'] = _concat_tensor_dicts( *[tower_spec.predictions for tower_spec in tower_specs]) export_outputs_dict = _dict_concat( *[tower_spec.export_outputs for tower_spec in tower_specs]) export_outputs = {} for name, export_output_list in six.iteritems(export_outputs_dict): if isinstance(export_output_list[0], export_output_lib.PredictOutput): export_outputs[name] = export_output_lib.PredictOutput( outputs=_concat_tensor_dicts(*[ export_output.outputs for export_output in export_output_list ])) elif isinstance(export_output_list[0], export_output_lib.RegressionOutput): export_outputs[name] = export_output_lib.RegressionOutput( value=array_ops.concat( [export_output.value for export_output in export_output_list], axis=0)) elif isinstance(export_output_list[0], export_output_lib.ClassificationOutput): scores = None if export_output_list[0].scores is not None: scores = array_ops.concat( [export_output.scores for export_output in export_output_list], axis=0) classes = None if export_output_list[0].classes is not None: classes = array_ops.stack( [export_output.classes for export_output in export_output_list], axis=0) export_outputs[name] = export_output_lib.ClassificationOutput( scores=scores, classes=classes) estimator_spec['export_outputs'] = export_outputs return model_fn_lib.EstimatorSpec(**estimator_spec)
def _merge_predict(self, all_estimator_spec, use_tpu=False): """Merges list of `EstimatorSpec` or `TPUEstimatorSpec` for prediction. Args: all_estimator_spec: list of `EstimatorSpec` or `TPUEstimatorSpec` for the individual heads. use_tpu: If `True`, returns `TPUEstimatorSpec`. Returns: `EstimatorSpec` or `TPUEstimatorSpec` that merges all heads for PREDICT. """ predictions = {} export_outputs = { _DEFAULT_SERVING_KEY: _default_export_output(all_estimator_spec[0].export_outputs, self._heads[0].name), } merged_predict_outputs = {} for head, spec in zip(self._heads, all_estimator_spec): head_name = head.name for k, v in six.iteritems(spec.export_outputs): if k == _DEFAULT_SERVING_KEY: key = head_name else: key = '%s/%s' % (head_name, k) export_outputs[key] = v if (k == head_lib._PREDICT_SERVING_KEY and # pylint:disable=protected-access isinstance(v, export_output_lib.PredictOutput)): for kp, vp in six.iteritems(v.outputs): key = '%s/%s' % (head_name, kp) merged_predict_outputs[key] = vp for k, v in six.iteritems(spec.predictions): predictions[(head_name, k)] = v export_outputs[head_lib._PREDICT_SERVING_KEY] = ( # pylint:disable=protected-access export_output_lib.PredictOutput(merged_predict_outputs)) spec_type = ( model_fn._TPUEstimatorSpec if use_tpu else model_fn.EstimatorSpec) # pylint:disable=protected-access return spec_type(mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs=export_outputs)
def testExportOutputsMultiheadMissingDefault(self): with ops.Graph().as_default(), self.test_session(): predictions = {'loss': constant_op.constant(1.)} output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(['2']) output_3 = constant_op.constant(['3']) export_outputs = { 'head-1': export_output.RegressionOutput(value=output_1), 'head-2': export_output.ClassificationOutput(classes=output_2), 'head-3': export_output.PredictOutput(outputs={ 'some_output_3': output_3 })} with self.assertRaisesRegexp( ValueError, 'Multiple export_outputs were provided, but none of them is ' 'specified as the default. Do this by naming one of them with ' 'signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY.'): model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs=export_outputs)
def _model_fn(features, labels, mode): _ = labels x = features['x'] y = features['y'] with ops.name_scope('outputs'): predictions = { 'sum': math_ops.add(x, y, name='sum'), 'product': math_ops.multiply(x, y, name='product'), 'difference': math_ops.subtract(x, y, name='difference') } if core: export_outputs = { k: export_output.PredictOutput({k: v}) for k, v in predictions.items() } export_outputs[ signature_constants. DEFAULT_SERVING_SIGNATURE_DEF_KEY] = export_outputs['sum'] return model_fn.EstimatorSpec(mode=mode, predictions=predictions, export_outputs=export_outputs, loss=constant_op.constant(0), train_op=control_flow_ops.no_op()) else: output_alternatives = { k: (constants.ProblemType.UNSPECIFIED, { k: v }) for k, v in predictions.items() } return contrib_model_fn.ModelFnOps( mode=mode, predictions=predictions, output_alternatives=output_alternatives, loss=constant_op.constant(0), train_op=control_flow_ops.no_op())
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """Returns an `EstimatorSpec`. Please note that, + All args must be passed via name. Args: features: Input `dict` of `Tensor` or `SparseTensor` objects. mode: Estimator's `ModeKeys`. logits: logits `Tensor` with shape `[D0, D1, ... DN, logits_dimension]`. For many applications, the shape is `[batch_size, logits_dimension]`. labels: Labels `Tensor` with shape matching `logits`, namely `[D0, D1, ... DN, logits_dimension]`. When `logits_dimension=1`, shape `[D0, D1, ... DN]` is also supported. `labels` is required argument when `mode` equals `TRAIN` or `EVAL`. train_op_fn: Function that takes a scalar loss `Tensor` and returns `train_op`. Required in TRAIN mode. Returns: `EstimatorSpec`. Raises: ValueError: If `train_op_fn` is `None` in TRAIN mode. """ # Predict. with ops.name_scope(self._name, 'head'): logits = _check_logits_final_dim(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: regression_output = export_output.RegressionOutput(value=logits) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: regression_output, _REGRESS_SERVING_KEY: regression_output, _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) weighted_sum_loss, example_weight_sum, _ = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) # Eval. if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. eval_metric_ops = { metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean( # Both values and weights here are reduced, scalar Tensors. # values is the actual mean we want -- weights represents # the total weight of the batch and is needed to calculate # update_op over many batches. values=(weighted_sum_loss / example_weight_sum), weights=example_weight_sum) } return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=weighted_sum_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), weighted_sum_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), weighted_sum_loss / example_weight_sum) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=weighted_sum_loss, train_op=train_op_fn(weighted_sum_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope(self._name, 'head'): logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: regression_output = export_output.RegressionOutput( value=logits) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: regression_output, _REGRESS_SERVING_KEY: regression_output, _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, _ = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. eval_metric_ops = { metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean(unweighted_loss, weights=weights) } return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss(unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. with ops.name_scope(self._name, 'head'): with ops.name_scope(None, 'predictions', (logits, )): pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') probabilities = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES) class_ids = array_ops.reshape(math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, pred_keys.PROBABILITIES: probabilities, pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = _classification_output( scores=probabilities, n_classes=2, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, _CLASSIFY_SERVING_KEY: classifier_output, _REGRESS_SERVING_KEY: export_output.RegressionOutput(value=logistic), _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, logits=logits, logistic=logistic, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss(unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope('head'): logits = _check_logits(logits, self.logits_dimension) # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits,)): # class_ids's shape is [batch_size] class_ids = math_ops.argmax(logits, 1, name=pred_keys.CLASS_IDS) class_ids = array_ops.expand_dims(class_ids, axis=(1,)) if self._label_vocabulary: table = lookup_ops.index_to_string_table_from_tensor( vocabulary_list=self._label_vocabulary, name='class_string_lookup') classes = table.lookup(class_ids) else: classes = string_ops.as_string(class_ids, name='str_classes') probabilities = nn.softmax(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, # Expand to [batch_size, 1] pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: batch_size = array_ops.shape(probabilities)[0] export_class_list = self._label_vocabulary if not export_class_list: export_class_list = string_ops.as_string( math_ops.range(self._n_classes)) export_output_classes = array_ops.tile( input=array_ops.expand_dims(input=export_class_list, axis=0), multiples=[batch_size, 1]) classifier_output = export_output.ClassificationOutput( scores=probabilities, # `ClassificationOutput` requires string classes. classes=export_output_classes) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, _CLASSIFY_SERVING_KEY: classifier_output, _PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. unweighted_loss, label_ids = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) weights = _weights(features, self._weight_column) training_loss = losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, probabilities=probabilities, logits=logits, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), training_loss) summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, weights=weights, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None, regularization_losses=None): """Returns an `EstimatorSpec`. Args: features: Input `dict` of `Tensor` or `SparseTensor` objects. mode: Estimator's `ModeKeys`. logits: logits `Tensor` with shape `[D0, D1, ... DN, n_classes]`. For many applications, the shape is `[batch_size, n_classes]`. labels: Labels with shape matching `logits`. Can be multi-hot `Tensor` with shape `[D0, D1, ... DN, n_classes]` or `SparseTensor` with `dense_shape` `[D0, D1, ... DN, ?]`. `labels` is required argument when `mode` equals `TRAIN` or `EVAL`. train_op_fn: Function that takes a scalar loss `Tensor` and returns `train_op`. Required in TRAIN mode. regularization_losses: A list of additional scalar losses to be added to the training loss, such as regularization losses. These losses are usually expressed as a batch average, so for best results users need to set `loss_reduction=SUM_OVER_BATCH_SIZE` or `loss_reduction=SUM_OVER_NONZERO_WEIGHTS` when creating the head to avoid scaling errors. Returns: `EstimatorSpec`. Raises: ValueError: If `train_op_fn` is `None` in TRAIN mode. """ with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits_final_dim(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits, )): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = head_lib._classification_output( # pylint:disable=protected-access scores=probabilities, n_classes=self._n_classes, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access export_output.PredictOutput(predictions)) }) (training_loss, unreduced_loss, weights, processed_labels) = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) if regularization_losses: regularization_loss = math_ops.add_n(regularization_losses) regularized_training_loss = math_ops.add_n( [training_loss, regularization_loss]) else: regularization_loss = None regularized_training_loss = training_loss # Eval. if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=regularized_training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, probabilities=probabilities, weights=weights, unreduced_loss=unreduced_loss, regularization_loss=regularization_loss)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') # Only summarize mean_loss for SUM reduction to preserve backwards # compatibility. Otherwise skip it to avoid unnecessary computation. if self._loss_reduction == losses.Reduction.SUM: example_weight_sum = math_ops.reduce_sum( weights * array_ops.ones_like(unreduced_loss)) mean_loss = training_loss / example_weight_sum else: mean_loss = None with ops.name_scope(''): keys = metric_keys.MetricKeys summary.scalar( head_lib._summary_key(self._name, keys.LOSS), # pylint:disable=protected-access regularized_training_loss) if mean_loss is not None: summary.scalar( head_lib._summary_key(self._name, keys.LOSS_MEAN), # pylint:disable=protected-access mean_loss) if regularization_loss is not None: summary.scalar( head_lib._summary_key(self._name, keys.LOSS_REGULARIZATION), # pylint:disable=protected-access regularization_loss) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=regularized_training_loss, train_op=train_op_fn(regularized_training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits_final_dim(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits, )): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = head_lib._classification_output( # pylint:disable=protected-access scores=probabilities, n_classes=self._n_classes, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access export_output.PredictOutput(predictions)) }) (weighted_sum_loss, example_weight_sum, processed_labels) = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) # Eval. if mode == model_fn.ModeKeys.EVAL: weights = head_lib._get_weights_and_check_match_logits( # pylint:disable=protected-access, features=features, weight_column=self._weight_column, logits=logits) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=weighted_sum_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, probabilities=probabilities, weights=weights, weighted_sum_loss=weighted_sum_loss, example_weight_sum=example_weight_sum)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access weighted_sum_loss) summary.scalar( head_lib._summary_key( # pylint:disable=protected-access self._name, metric_keys.MetricKeys.LOSS_MEAN), weighted_sum_loss / example_weight_sum) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=weighted_sum_loss, train_op=train_op_fn(weighted_sum_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=tfgan_train.gan_train_ops): """Returns `EstimatorSpec` that a model_fn can return. See `Head` for more details. Args: features: Must be `None`. mode: Estimator's `ModeKeys`. logits: A GANModel tuple. labels: Must be `None`. train_op_fn: Function that takes a GANModel, GANLoss, generator optimizer, and discriminator optimizer, and returns a `GANTrainOps` tuple. For example, this function can come from TFGAN's `train.py` library, or can be custom. Returns: `EstimatorSpec`. Raises: ValueError: If `features` isn't `None`. ValueError: If `train_op_fn` isn't provided in train mode. """ _validate_logits_and_labels(logits, labels) if features is not None: raise ValueError( '`features` should be `None`. Instead, found: %s' % features) gan_model = logits # rename variable for clarity with ops.name_scope('GANHead'): if mode == model_fn_lib.ModeKeys.PREDICT: return model_fn_lib.EstimatorSpec( mode=model_fn_lib.ModeKeys.PREDICT, predictions=gan_model.generated_data, export_outputs={ 'predict': export_output.PredictOutput(gan_model.generated_data) }) elif mode == model_fn_lib.ModeKeys.EVAL: gan_loss = self.create_loss(features=None, mode=mode, logits=gan_model, labels=None) scalar_loss = gan_loss.generator_loss + gan_loss.discriminator_loss with ops.name_scope( None, 'metrics', [gan_loss.generator_loss, gan_loss.discriminator_loss]): eval_metric_ops = { _summary_key(self._name, 'generator_loss'): metrics_lib.mean(gan_loss.generator_loss), _summary_key(self._name, 'discriminator_loss'): metrics_lib.mean(gan_loss.discriminator_loss) } if self._get_eval_metric_ops_fn is not None: custom_eval_metric_ops = self._get_eval_metric_ops_fn( gan_model) if not isinstance(custom_eval_metric_ops, dict): raise TypeError( 'get_eval_metric_ops_fn must return a dict, ' 'received: {}'.format(custom_eval_metric_ops)) eval_metric_ops.update(custom_eval_metric_ops) return model_fn_lib.EstimatorSpec( mode=model_fn_lib.ModeKeys.EVAL, predictions=gan_model.generated_data, loss=scalar_loss, eval_metric_ops=eval_metric_ops) elif mode == model_fn_lib.ModeKeys.TRAIN: if train_op_fn is None: raise ValueError('train_op_fn can not be None.') gan_loss = self.create_loss(None, mode, gan_model, None) scalar_loss = gan_loss.generator_loss + gan_loss.discriminator_loss train_ops = train_op_fn(gan_model, gan_loss, self._generator_optimizer, self._discriminator_optimizer) training_hooks = self._get_hooks_fn(train_ops) return model_fn_lib.EstimatorSpec( loss=scalar_loss, mode=model_fn_lib.ModeKeys.TRAIN, train_op=train_ops.global_step_inc_op, training_hooks=training_hooks) else: raise ValueError('Mode not recognized: %s' % mode)
def create_estimator_spec(self, features, mode, logits, labels=None, optimizer=None, train_op_fn=None, regularization_losses=None, params=None): """Returns an `EstimatorSpec`. Args: features: Input `dict` of `Tensor` or `SparseTensor` objects. mode: Estimator's `ModeKeys`. logits: a tensor array with 2 element, one for start positon probilities , and one for the end postion probilities. the shape is `[batch_size, logits_dimension]`. labels: a tensor with demention [batch_size, 2], 2 position for true position in a doc optimizer: `Optimizer` instance to optimize the loss in TRAIN mode. Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which updates variables and increments `global_step`. train_op_fn: Function that takes a scalar loss `Tensor` and returns `train_op`. Used if `optimizer` is `None`. regularization_losses: A list of additional scalar losses to be added to the training loss, such as regularization losses. These losses are usually expressed as a batch average, so for best results users need to set `loss_reduction=SUM_OVER_BATCH_SIZE` or `loss_reduction=SUM_OVER_NONZERO_WEIGHTS` when creating the head to avoid scaling errors. Returns: `EstimatorSpec`. Raises: ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN mode, or if both are set. """ with tf.name_scope(self._name, 'head'): # Predict. pred_keys = prediction_keys.PredictionKeys with tf.name_scope(None, 'predictions', (logits, )): [start_logits, end_logits] = logits shape = tf.shape(start_logits) d1 = shape[0] d2 = shape[1] d3 = shape[2] start_logits = tf.reshape( tf.nn.softmax(tf.reshape(start_logits, [-1, d2 * d3]), -1), [d1, d2, d3, 1]) end_logits = tf.reshape( tf.nn.softmax(tf.reshape(end_logits, [-1, d2 * d3]), -1), [d1, d2, 1, d3]) #[batch_size, sent_number, seq_len,seq_len] mul = tf.multiply(start_logits, end_logits) x_tensor_band = tf.matrix_band_part(mul, 0, params.ans_limit) x_tensor_reshape = tf.reshape(x_tensor_band, [d1, d2 * d3 * d3]) #[batch_size] max = tf.reduce_max(x_tensor_reshape, -1) #[batch_size,1] row_index = tf.where(tf.not_equal(max, tf.ones_like(max) + 2.1)) indice = tf.expand_dims(tf.argmax(x_tensor_reshape, -1), -1) indice = tf.concat([row_index, indice], axis=-1) only_max = tf.sparse_to_dense( indice, tf.shape(x_tensor_reshape, out_type=tf.int64), max) only_max = tf.reshape(only_max, [d1, d2, d3, d3]) loc = tf.where(tf.cast(only_max, tf.bool)) loc = tf.slice( loc, [0, 1], [-1, -1], name="locations" ) #[batch_size,postions=3] positions = (sent_no,start_pos,end_pos) predictions = { "s_logits": start_logits, "e_logits": end_logits, pred_keys.PROBABILITIES: max, "locations": loc } if mode == model_fn.ModeKeys.PREDICT: output = export_output.PredictOutput(predictions) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: output, _PREDICT_SERVING_KEY: output }) training_loss, unreduced_loss, weights, label_ids = self.create_loss( features=None, mode=mode, logits=logits, labels=labels) if regularization_losses: regularization_loss = tf.add_n(regularization_losses) regularized_training_loss = tf.add_n( [training_loss, regularization_loss]) else: regularization_loss = None regularized_training_loss = training_loss # Eval. if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=regularized_training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, predict=tf.sign( tf.abs(tf.reshape(only_max, [d1, d2 * d3 * d3]))), location=loc, pro=max, unreduced_loss=unreduced_loss, regularization_loss=regularization_loss)) # Train. if optimizer is not None: if train_op_fn is not None: raise ValueError( 'train_op_fn and optimizer cannot both be set.') train_op = optimizer.minimize( regularized_training_loss, global_step=training_util.get_global_step()) elif train_op_fn is not None: train_op = train_op_fn(regularized_training_loss) else: raise ValueError( 'train_op_fn and optimizer cannot both be None.') with tf.name_scope(''): keys = metric_keys.MetricKeys tf.summary.scalar(_summary_key(self._name, keys.LOSS), regularized_training_loss) if regularization_loss is not None: tf.summary.scalar( _summary_key(self._name, keys.LOSS_REGULARIZATION), regularization_loss) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=regularized_training_loss, train_op=train_op)
def create_estimator_spec( self, features, logits, mode, labels=None, train_op_fn=None): """See `Head`.""" # split logits into mu, sigma and alpha components = array_ops.reshape(logits, [-1, 3, self._m]) mus = components[:, 0, :] sigmas = components[:, 1, :] alphas = components[:, 2, :] alphas = nn_ops.softmax(clip_ops.clip_by_value(alphas, 1e-2, 1.)) # Predict. with ops.name_scope('head'): #logits = head_lib._check_logits(logits, self._logits_dimension) means = math_ops.reduce_sum(alphas*mus, axis=1, keepdims=True) uncertainty = math_ops.reduce_sum( alphas*sigmas, axis=1, keepdims=True) predicted_value = array_ops.concat([means, uncertainty], 1) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: predicted_value} if mode == model_fn.ModeKeys.PREDICT: regression_output = export_output.RegressionOutput( value=predicted_value) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ head_lib._DEFAULT_SERVING_KEY: regression_output, head_lib._REGRESS_SERVING_KEY: regression_output, head_lib._PREDICT_SERVING_KEY: export_output.PredictOutput(predictions) }) # Eval. if mode == model_fn.ModeKeys.EVAL: # Estimator already adds a metric for loss. mus = math_ops.reduce_sum(alphas*mus, axis=1, keepdims=True) #mus = utils.tf_print(mus, "mus:") #labels = utils.tf_print(labels, "labels:") training_loss, unweighted_loss, _ = self.create_loss2( features=features, mode=mode, logits=mus, labels=labels) keys = metric_keys.MetricKeys eval_metric_ops = { head_lib._summary_key(self._name, keys.LOSS_MEAN) : metrics_lib.mean( unweighted_loss, weights=None) } return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=eval_metric_ops) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') training_loss, unweighted_loss, _ = self.create_loss( features=features, mode=mode, mus=mus, sigmas=sigmas, alphas=alphas, labels=labels) with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN), losses.compute_weighted_loss( unweighted_loss, reduction=losses.Reduction.MEAN)) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def create_estimator_spec(self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits_final_dim(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. pred_keys = prediction_keys.PredictionKeys with ops.name_scope(None, 'predictions', (logits, )): probabilities = math_ops.sigmoid(logits, name=pred_keys.PROBABILITIES) predictions = { pred_keys.LOGITS: logits, pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: classifier_output = head_lib._classification_output( # pylint:disable=protected-access scores=probabilities, n_classes=self._n_classes, label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: classifier_output, head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access export_output.PredictOutput(predictions)) }) (training_loss, unreduced_loss, weights, processed_labels) = self.create_loss(features=features, mode=mode, logits=logits, labels=labels) # Eval. if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, predictions=predictions, loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=processed_labels, probabilities=probabilities, weights=weights, unreduced_loss=unreduced_loss)) # Train. if train_op_fn is None: raise ValueError('train_op_fn can not be None.') # Only summarize mean_loss for SUM reduction to preserve backwards # compatibility. Otherwise skip it to avoid unnecessary computation. if self._loss_reduction == losses.Reduction.SUM: example_weight_sum = math_ops.reduce_sum( weights * array_ops.ones_like(unreduced_loss)) mean_loss = training_loss / example_weight_sum else: mean_loss = None with ops.name_scope(''): summary.scalar( head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access training_loss) if mean_loss is not None: summary.scalar( head_lib._summary_key( # pylint:disable=protected-access self._name, metric_keys.MetricKeys.LOSS_MEAN), mean_loss) return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN, predictions=predictions, loss=training_loss, train_op=train_op_fn(training_loss))
def model_fn(self, features, mode, config): """Model function for the estimator. Note that this does not take a `labels` arg. This works, but `input_fn` must return either `features` or, equivalently, `(features, None)`. Args: features: The input points. See `tf.estimator.Estimator`. mode: See `tf.estimator.Estimator`. config: See `tf.estimator.Estimator`. Returns: A `tf.estimator.EstimatorSpec` (see `tf.estimator.Estimator`) specifying this behavior: * `train_op`: Execute one mini-batch or full-batch run of Lloyd's algorithm. * `loss`: The sum of the squared distances from each input point to its closest center. * `eval_metric_ops`: Maps `SCORE` to `loss`. * `predictions`: Maps `ALL_DISTANCES` to the distance from each input point to each cluster center; maps `CLUSTER_INDEX` to the index of the closest cluster center for each input point. """ # input_points is a single Tensor. Therefore, the sharding functionality # in clustering_ops is unused, and some of the values below are lists of a # single item. input_points = _parse_features_if_necessary(features, self._feature_columns) # Let N = the number of input_points. # all_distances: A list of one matrix of shape (N, num_clusters). Each value # is the distance from an input point to a cluster center. # model_predictions: A list of one vector of shape (N). Each value is the # cluster id of an input point. # losses: Similar to cluster_idx but provides the distance to the cluster # center. # is_initialized: scalar indicating whether the initial cluster centers # have been chosen; see init_op. # init_op: an op to choose the initial cluster centers. A single worker # repeatedly executes init_op until is_initialized becomes True. # training_op: an op that runs an iteration of training, either an entire # Lloyd iteration or a mini-batch of a Lloyd iteration. Multiple workers # may execute this op, but only after is_initialized becomes True. (all_distances, model_predictions, losses, is_initialized, init_op, training_op) = clustering_ops.KMeans( inputs=input_points, num_clusters=self._num_clusters, initial_clusters=self._initial_clusters, distance_metric=self._distance_metric, use_mini_batch=self._use_mini_batch, mini_batch_steps_per_iteration=self. _mini_batch_steps_per_iteration, random_seed=self._random_seed, kmeans_plus_plus_num_retries=self._kmeans_plus_plus_num_retries ).training_graph() loss = math_ops.reduce_sum(losses) summary.scalar('loss/raw', loss) incr_step = state_ops.assign_add(training_util.get_global_step(), 1) training_op = control_flow_ops.with_dependencies( [training_op, incr_step], loss) training_hooks = [ _InitializeClustersHook(init_op, is_initialized, config.is_chief) ] if self._relative_tolerance is not None: training_hooks.append( _LossRelativeChangeHook(loss, self._relative_tolerance)) export_outputs = { KMeansClustering.ALL_DISTANCES: export_output.PredictOutput(all_distances[0]), KMeansClustering.CLUSTER_INDEX: export_output.PredictOutput(model_predictions[0]), signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.PredictOutput(model_predictions[0]) } return model_fn_lib.EstimatorSpec( mode=mode, predictions={ KMeansClustering.ALL_DISTANCES: all_distances[0], KMeansClustering.CLUSTER_INDEX: model_predictions[0], }, loss=loss, train_op=training_op, eval_metric_ops={KMeansClustering.SCORE: metrics.mean(loss)}, training_hooks=training_hooks, export_outputs=export_outputs)
def create_estimator_spec(self, features, mode, logits, labels=None, optimizer=None, train_op_fn=None, regularization_losses=None): """Returns an `EstimatorSpec`. Args: features: Input `dict` of `Tensor` or `SparseTensor` objects. mode: Estimator's `ModeKeys`. logits: a tensor array with 2 element, one for start positon probilities , and one for the end postion probilities. the shape is `[batch_size, logits_dimension]`. labels: a tensor with demention [batch_size, 2], 2 position for true position in a doc optimizer: `Optimizer` instance to optimize the loss in TRAIN mode. Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which updates variables and increments `global_step`. train_op_fn: Function that takes a scalar loss `Tensor` and returns `train_op`. Used if `optimizer` is `None`. regularization_losses: A list of additional scalar losses to be added to the training loss, such as regularization losses. These losses are usually expressed as a batch average, so for best results users need to set `loss_reduction=SUM_OVER_BATCH_SIZE` or `loss_reduction=SUM_OVER_NONZERO_WEIGHTS` when creating the head to avoid scaling errors. Returns: `EstimatorSpec`. Raises: ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN mode, or if both are set. """ with tf.name_scope(self._name, 'head'): # Predict. pred_keys = prediction_keys.PredictionKeys predictions = {} if mode == tf.estimator.ModeKeys.PREDICT: output = export_output.PredictOutput(predictions) return tf.estimator.EstimatorSpec( mode=tf.estimator.ModeKeys.PREDICT, predictions=predictions, export_outputs={ _DEFAULT_SERVING_KEY: output, _PREDICT_SERVING_KEY: output }) training_loss, unreduced_loss, weights, label_ids = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) if regularization_losses: regularization_loss = tf.add_n(regularization_losses) regularized_training_loss = tf.add_n( [training_loss, regularization_loss]) else: regularization_loss = None regularized_training_loss = training_loss # Eval. if mode == tf.estimator.ModeKeys.EVAL: return tf.estimator.EstimatorSpec( mode=tf.estimator.ModeKeys.EVAL, predictions=predictions, loss=regularized_training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, predict= None, #tf.sign(tf.abs(tf.reshape(only_max,[d1,d2*d3*d3]))), location=None, # loc, pro=max, unreduced_loss=unreduced_loss, regularization_loss=regularization_loss)) # Train. if optimizer is not None: if train_op_fn is not None: raise ValueError( 'train_op_fn and optimizer cannot both be set.') train_op = optimizer.minimize( regularized_training_loss, global_step=tf.train.get_global_step()) elif train_op_fn is not None: train_op = train_op_fn(regularized_training_loss) else: raise ValueError( 'train_op_fn and optimizer cannot both be None.') with tf.name_scope(''): keys = metric_keys.MetricKeys tf.summary.scalar(_summary_key(self._name, keys.LOSS), regularized_training_loss) if regularization_loss is not None: tf.summary.scalar( _summary_key(self._name, keys.LOSS_REGULARIZATION), regularization_loss) return tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.TRAIN, predictions=predictions, loss=regularized_training_loss, train_op=train_op)