def test_build_all_signature_defs_with_dict_alternatives(self):
receiver_tensor = array_ops.placeholder(dtypes.string)
receiver_tensors_alternative_1 = {
"foo": array_ops.placeholder(dtypes.int64),
"bar": array_ops.sparse_placeholder(dtypes.float32)
}
receiver_tensors_alternatives = {
"other": receiver_tensors_alternative_1
}
output_1 = constant_op.constant([1.])
output_2 = constant_op.constant(["2"])
output_3 = constant_op.constant(["3"])
export_outputs = {
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
export_output.RegressionOutput(value=output_1),
"head-2":
export_output.ClassificationOutput(classes=output_2),
"head-3":
export_output.PredictOutput(outputs={"some_output_3": output_3}),
}
signature_defs = export.build_all_signature_defs(
receiver_tensor, export_outputs, receiver_tensors_alternatives)
expected_signature_defs = {
"serving_default":
signature_def_utils.regression_signature_def(
receiver_tensor, output_1),
"head-2":
signature_def_utils.classification_signature_def(
receiver_tensor, output_2, None),
"head-3":
signature_def_utils.predict_signature_def(
{"input": receiver_tensor}, {"some_output_3": output_3}),
"other:head-3":
signature_def_utils.predict_signature_def(
receiver_tensors_alternative_1, {"some_output_3": output_3})
# Note that the alternatives 'other:serving_default' and 'other:head-2'
# are invalid, because regession and classification signatures must take
# a single string input. Here we verify that these invalid signatures
# are not included in the export.
}
self.assertDictEqual(expected_signature_defs, signature_defs)
def testExportOutputsMultiheadWithDefault(self):
with ops.Graph().as_default(), self.test_session():
predictions = {'loss': constant_op.constant(1.)}
output_1 = constant_op.constant([1.])
output_2 = constant_op.constant(['2'])
output_3 = constant_op.constant(['3'])
export_outputs = {
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
export_output.RegressionOutput(value=output_1),
'head-2': export_output.ClassificationOutput(classes=output_2),
'head-3': export_output.PredictOutput(outputs={
'some_output_3': output_3
})}
estimator_spec = model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs=export_outputs)
self.assertEqual(export_outputs, estimator_spec.export_outputs)
def model_fn(self, mode, features, labels, params):
c = variable_scope.get_variable('c',
initializer=constant_op.constant(
10, dtype=dtypes.float64),
dtype=dtypes.float64)
predictions = {'probabilities': math_ops.multiply(features, c)}
loss = losses.absolute_difference(
labels=labels,
predictions=predictions['probabilities'],
reduction=losses.Reduction.SUM)
metrics = {
'accuracy': metrics_lib.accuracy(labels,
predictions['probabilities']),
'auc': metrics_lib.auc(labels, predictions['probabilities'])
}
tensor_string_repr = str(features)
classes = constant_op.constant(re.search('(split_inputs/split:[0-9])',
tensor_string_repr).group(1),
dtype=dtypes.string)
export_outputs = {
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
export_output.PredictOutput(predictions),
'classification_output':
export_output.ClassificationOutput(predictions['probabilities'],
classes),
'classification_scores':
export_output.ClassificationOutput(
scores=predictions['probabilities']),
'classification_classes':
export_output.ClassificationOutput(classes=classes),
'regression_output':
export_output.RegressionOutput(predictions['probabilities']),
}
return model_fn_lib.EstimatorSpec(
mode=mode,
loss=math_ops.reduce_sum(loss),
eval_metric_ops=metrics,
predictions=predictions,
train_op=loss, # This train_op isn't actually used.
export_outputs=export_outputs)
def _predict_spec(tower_specs, aggregation_device):
"""Populate replicated EstimatorSpec for `GraphKeys.PREDICT`."""
estimator_spec = _asdict(tower_specs[0])
estimator_spec['mode'] = model_fn_lib.ModeKeys.PREDICT
with ops_lib.device(aggregation_device):
estimator_spec['predictions'] = _concat_tensor_dicts(
*[tower_spec.predictions for tower_spec in tower_specs])
export_outputs_dict = _dict_concat(
*[tower_spec.export_outputs for tower_spec in tower_specs])
export_outputs = {}
for name, export_output_list in six.iteritems(export_outputs_dict):
if isinstance(export_output_list[0], export_output_lib.PredictOutput):
export_outputs[name] = export_output_lib.PredictOutput(
outputs=_concat_tensor_dicts(*[
export_output.outputs for export_output in export_output_list
]))
elif isinstance(export_output_list[0],
export_output_lib.RegressionOutput):
export_outputs[name] = export_output_lib.RegressionOutput(
value=array_ops.concat(
[export_output.value for export_output in export_output_list],
axis=0))
elif isinstance(export_output_list[0],
export_output_lib.ClassificationOutput):
scores = None
if export_output_list[0].scores is not None:
scores = array_ops.concat(
[export_output.scores for export_output in export_output_list],
axis=0)
classes = None
if export_output_list[0].classes is not None:
classes = array_ops.stack(
[export_output.classes for export_output in export_output_list],
axis=0)
export_outputs[name] = export_output_lib.ClassificationOutput(
scores=scores, classes=classes)
estimator_spec['export_outputs'] = export_outputs
return model_fn_lib.EstimatorSpec(**estimator_spec)
def _merge_predict(self, all_estimator_spec, use_tpu=False):
"""Merges list of `EstimatorSpec` or `TPUEstimatorSpec` for prediction.
Args:
all_estimator_spec: list of `EstimatorSpec` or `TPUEstimatorSpec` for the
individual heads.
use_tpu: If `True`, returns `TPUEstimatorSpec`.
Returns:
`EstimatorSpec` or `TPUEstimatorSpec` that merges all heads for PREDICT.
"""
predictions = {}
export_outputs = {
_DEFAULT_SERVING_KEY:
_default_export_output(all_estimator_spec[0].export_outputs,
self._heads[0].name),
}
merged_predict_outputs = {}
for head, spec in zip(self._heads, all_estimator_spec):
head_name = head.name
for k, v in six.iteritems(spec.export_outputs):
if k == _DEFAULT_SERVING_KEY:
key = head_name
else:
key = '%s/%s' % (head_name, k)
export_outputs[key] = v
if (k == head_lib._PREDICT_SERVING_KEY and # pylint:disable=protected-access
isinstance(v, export_output_lib.PredictOutput)):
for kp, vp in six.iteritems(v.outputs):
key = '%s/%s' % (head_name, kp)
merged_predict_outputs[key] = vp
for k, v in six.iteritems(spec.predictions):
predictions[(head_name, k)] = v
export_outputs[head_lib._PREDICT_SERVING_KEY] = ( # pylint:disable=protected-access
export_output_lib.PredictOutput(merged_predict_outputs))
spec_type = (
model_fn._TPUEstimatorSpec if use_tpu else model_fn.EstimatorSpec) # pylint:disable=protected-access
return spec_type(mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs=export_outputs)
def testExportOutputsMultiheadMissingDefault(self):
with ops.Graph().as_default(), self.test_session():
predictions = {'loss': constant_op.constant(1.)}
output_1 = constant_op.constant([1.])
output_2 = constant_op.constant(['2'])
output_3 = constant_op.constant(['3'])
export_outputs = {
'head-1': export_output.RegressionOutput(value=output_1),
'head-2': export_output.ClassificationOutput(classes=output_2),
'head-3': export_output.PredictOutput(outputs={
'some_output_3': output_3
})}
with self.assertRaisesRegexp(
ValueError,
'Multiple export_outputs were provided, but none of them is '
'specified as the default. Do this by naming one of them with '
'signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY.'):
model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs=export_outputs)
def _model_fn(features, labels, mode):
_ = labels
x = features['x']
y = features['y']
with ops.name_scope('outputs'):
predictions = {
'sum': math_ops.add(x, y, name='sum'),
'product': math_ops.multiply(x, y, name='product'),
'difference': math_ops.subtract(x, y, name='difference')
}
if core:
export_outputs = {
k: export_output.PredictOutput({k: v})
for k, v in predictions.items()
}
export_outputs[
signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY] = export_outputs['sum']
return model_fn.EstimatorSpec(mode=mode,
predictions=predictions,
export_outputs=export_outputs,
loss=constant_op.constant(0),
train_op=control_flow_ops.no_op())
else:
output_alternatives = {
k: (constants.ProblemType.UNSPECIFIED, {
k: v
})
for k, v in predictions.items()
}
return contrib_model_fn.ModelFnOps(
mode=mode,
predictions=predictions,
output_alternatives=output_alternatives,
loss=constant_op.constant(0),
train_op=control_flow_ops.no_op())
def create_estimator_spec(
self, features, mode, logits, labels=None, train_op_fn=None):
"""Returns an `EstimatorSpec`.
Please note that,
+ All args must be passed via name.
Args:
features: Input `dict` of `Tensor` or `SparseTensor` objects.
mode: Estimator's `ModeKeys`.
logits: logits `Tensor` with shape `[D0, D1, ... DN, logits_dimension]`.
For many applications, the shape is `[batch_size, logits_dimension]`.
labels: Labels `Tensor` with shape matching `logits`, namely
`[D0, D1, ... DN, logits_dimension]`. When `logits_dimension=1`, shape
`[D0, D1, ... DN]` is also supported. `labels` is required argument when
`mode` equals `TRAIN` or `EVAL`.
train_op_fn: Function that takes a scalar loss `Tensor` and returns
`train_op`. Required in TRAIN mode.
Returns:
`EstimatorSpec`.
Raises:
ValueError: If `train_op_fn` is `None` in TRAIN mode.
"""
# Predict.
with ops.name_scope(self._name, 'head'):
logits = _check_logits_final_dim(logits, self._logits_dimension)
predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits}
if mode == model_fn.ModeKeys.PREDICT:
regression_output = export_output.RegressionOutput(value=logits)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY: regression_output,
_REGRESS_SERVING_KEY: regression_output,
_PREDICT_SERVING_KEY: export_output.PredictOutput(predictions)
})
weighted_sum_loss, example_weight_sum, _ = self.create_loss(
features=features, mode=mode, logits=logits, labels=labels)
# Eval.
if mode == model_fn.ModeKeys.EVAL:
# Estimator already adds a metric for loss.
eval_metric_ops = {
metric_keys.MetricKeys.LOSS_MEAN:
metrics_lib.mean(
# Both values and weights here are reduced, scalar Tensors.
# values is the actual mean we want -- weights represents
# the total weight of the batch and is needed to calculate
# update_op over many batches.
values=(weighted_sum_loss / example_weight_sum),
weights=example_weight_sum)
}
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=weighted_sum_loss,
eval_metric_ops=eval_metric_ops)
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
with ops.name_scope(''):
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS),
weighted_sum_loss)
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN),
weighted_sum_loss / example_weight_sum)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=weighted_sum_loss,
train_op=train_op_fn(weighted_sum_loss))
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
train_op_fn=None):
"""See `Head`."""
# Predict.
with ops.name_scope(self._name, 'head'):
logits = _check_logits(logits, self._logits_dimension)
predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits}
if mode == model_fn.ModeKeys.PREDICT:
regression_output = export_output.RegressionOutput(
value=logits)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY:
regression_output,
_REGRESS_SERVING_KEY:
regression_output,
_PREDICT_SERVING_KEY:
export_output.PredictOutput(predictions)
})
# Eval.
unweighted_loss, _ = self.create_loss(features=features,
mode=mode,
logits=logits,
labels=labels)
weights = _weights(features, self._weight_column)
training_loss = losses.compute_weighted_loss(
unweighted_loss,
weights=weights,
reduction=losses.Reduction.SUM)
if mode == model_fn.ModeKeys.EVAL:
# Estimator already adds a metric for loss.
eval_metric_ops = {
metric_keys.MetricKeys.LOSS_MEAN:
metrics_lib.mean(unweighted_loss, weights=weights)
}
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=training_loss,
eval_metric_ops=eval_metric_ops)
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
with ops.name_scope(''):
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS),
training_loss)
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN),
losses.compute_weighted_loss(unweighted_loss,
weights=weights,
reduction=losses.Reduction.MEAN))
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=training_loss,
train_op=train_op_fn(training_loss))
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
train_op_fn=None):
"""See `Head`."""
# Predict.
with ops.name_scope(self._name, 'head'):
with ops.name_scope(None, 'predictions', (logits, )):
pred_keys = prediction_keys.PredictionKeys
logits = _check_logits(logits, self.logits_dimension)
logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC)
two_class_logits = array_ops.concat(
(array_ops.zeros_like(logits), logits),
1,
name='two_class_logits')
probabilities = nn.softmax(two_class_logits,
name=pred_keys.PROBABILITIES)
class_ids = array_ops.reshape(math_ops.argmax(two_class_logits,
axis=1), (-1, 1),
name='classes')
if self._label_vocabulary:
table = lookup_ops.index_to_string_table_from_tensor(
vocabulary_list=self._label_vocabulary,
name='class_string_lookup')
classes = table.lookup(class_ids)
else:
classes = string_ops.as_string(class_ids,
name='str_classes')
predictions = {
pred_keys.LOGITS: logits,
pred_keys.LOGISTIC: logistic,
pred_keys.PROBABILITIES: probabilities,
pred_keys.CLASS_IDS: class_ids,
pred_keys.CLASSES: classes,
}
if mode == model_fn.ModeKeys.PREDICT:
classifier_output = _classification_output(
scores=probabilities,
n_classes=2,
label_vocabulary=self._label_vocabulary)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY:
classifier_output,
_CLASSIFY_SERVING_KEY:
classifier_output,
_REGRESS_SERVING_KEY:
export_output.RegressionOutput(value=logistic),
_PREDICT_SERVING_KEY:
export_output.PredictOutput(predictions)
})
# Eval.
unweighted_loss, processed_labels = self.create_loss(
features=features, mode=mode, logits=logits, labels=labels)
weights = _weights(features, self._weight_column)
training_loss = losses.compute_weighted_loss(
unweighted_loss,
weights=weights,
reduction=losses.Reduction.SUM)
if mode == model_fn.ModeKeys.EVAL:
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=training_loss,
eval_metric_ops=self._eval_metric_ops(
labels=processed_labels,
logits=logits,
logistic=logistic,
class_ids=class_ids,
unweighted_loss=unweighted_loss,
weights=weights))
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
with ops.name_scope(''):
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS),
training_loss)
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN),
losses.compute_weighted_loss(unweighted_loss,
weights=weights,
reduction=losses.Reduction.MEAN))
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=training_loss,
train_op=train_op_fn(training_loss))
def create_estimator_spec(
self, features, mode, logits, labels=None, train_op_fn=None):
"""See `Head`."""
with ops.name_scope('head'):
logits = _check_logits(logits, self.logits_dimension)
# Predict.
pred_keys = prediction_keys.PredictionKeys
with ops.name_scope(None, 'predictions', (logits,)):
# class_ids's shape is [batch_size]
class_ids = math_ops.argmax(logits, 1, name=pred_keys.CLASS_IDS)
class_ids = array_ops.expand_dims(class_ids, axis=(1,))
if self._label_vocabulary:
table = lookup_ops.index_to_string_table_from_tensor(
vocabulary_list=self._label_vocabulary,
name='class_string_lookup')
classes = table.lookup(class_ids)
else:
classes = string_ops.as_string(class_ids, name='str_classes')
probabilities = nn.softmax(logits, name=pred_keys.PROBABILITIES)
predictions = {
pred_keys.LOGITS: logits,
pred_keys.PROBABILITIES: probabilities,
# Expand to [batch_size, 1]
pred_keys.CLASS_IDS: class_ids,
pred_keys.CLASSES: classes,
}
if mode == model_fn.ModeKeys.PREDICT:
batch_size = array_ops.shape(probabilities)[0]
export_class_list = self._label_vocabulary
if not export_class_list:
export_class_list = string_ops.as_string(
math_ops.range(self._n_classes))
export_output_classes = array_ops.tile(
input=array_ops.expand_dims(input=export_class_list, axis=0),
multiples=[batch_size, 1])
classifier_output = export_output.ClassificationOutput(
scores=probabilities,
# `ClassificationOutput` requires string classes.
classes=export_output_classes)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY: classifier_output,
_CLASSIFY_SERVING_KEY: classifier_output,
_PREDICT_SERVING_KEY: export_output.PredictOutput(predictions)
})
# Eval.
unweighted_loss, label_ids = self.create_loss(
features=features, mode=mode, logits=logits, labels=labels)
weights = _weights(features, self._weight_column)
training_loss = losses.compute_weighted_loss(
unweighted_loss, weights=weights, reduction=losses.Reduction.SUM)
if mode == model_fn.ModeKeys.EVAL:
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=training_loss,
eval_metric_ops=self._eval_metric_ops(
labels=label_ids,
probabilities=probabilities,
logits=logits,
class_ids=class_ids,
unweighted_loss=unweighted_loss,
weights=weights))
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
with ops.name_scope(''):
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS),
training_loss)
summary.scalar(
_summary_key(self._name, metric_keys.MetricKeys.LOSS_MEAN),
losses.compute_weighted_loss(
unweighted_loss, weights=weights,
reduction=losses.Reduction.MEAN))
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=training_loss,
train_op=train_op_fn(training_loss))
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
train_op_fn=None,
regularization_losses=None):
"""Returns an `EstimatorSpec`.
Args:
features: Input `dict` of `Tensor` or `SparseTensor` objects.
mode: Estimator's `ModeKeys`.
logits: logits `Tensor` with shape `[D0, D1, ... DN, n_classes]`.
For many applications, the shape is `[batch_size, n_classes]`.
labels: Labels with shape matching `logits`. Can be multi-hot `Tensor`
with shape `[D0, D1, ... DN, n_classes]` or `SparseTensor` with
`dense_shape` `[D0, D1, ... DN, ?]`. `labels` is required argument when
`mode` equals `TRAIN` or `EVAL`.
train_op_fn: Function that takes a scalar loss `Tensor` and returns
`train_op`. Required in TRAIN mode.
regularization_losses: A list of additional scalar losses to be added to
the training loss, such as regularization losses. These losses are
usually expressed as a batch average, so for best results users need to
set `loss_reduction=SUM_OVER_BATCH_SIZE` or
`loss_reduction=SUM_OVER_NONZERO_WEIGHTS` when creating the head to
avoid scaling errors.
Returns:
`EstimatorSpec`.
Raises:
ValueError: If `train_op_fn` is `None` in TRAIN mode.
"""
with ops.name_scope(self._name, 'head'):
logits = head_lib._check_logits_final_dim(logits,
self.logits_dimension) # pylint:disable=protected-access
# Predict.
pred_keys = prediction_keys.PredictionKeys
with ops.name_scope(None, 'predictions', (logits, )):
probabilities = math_ops.sigmoid(logits,
name=pred_keys.PROBABILITIES)
predictions = {
pred_keys.LOGITS: logits,
pred_keys.PROBABILITIES: probabilities,
}
if mode == model_fn.ModeKeys.PREDICT:
classifier_output = head_lib._classification_output( # pylint:disable=protected-access
scores=probabilities,
n_classes=self._n_classes,
label_vocabulary=self._label_vocabulary)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY:
classifier_output,
head_lib._CLASSIFY_SERVING_KEY:
classifier_output, # pylint:disable=protected-access
head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access
export_output.PredictOutput(predictions))
})
(training_loss, unreduced_loss, weights,
processed_labels) = self.create_loss(features=features,
mode=mode,
logits=logits,
labels=labels)
if regularization_losses:
regularization_loss = math_ops.add_n(regularization_losses)
regularized_training_loss = math_ops.add_n(
[training_loss, regularization_loss])
else:
regularization_loss = None
regularized_training_loss = training_loss
# Eval.
if mode == model_fn.ModeKeys.EVAL:
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=regularized_training_loss,
eval_metric_ops=self._eval_metric_ops(
labels=processed_labels,
probabilities=probabilities,
weights=weights,
unreduced_loss=unreduced_loss,
regularization_loss=regularization_loss))
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
# Only summarize mean_loss for SUM reduction to preserve backwards
# compatibility. Otherwise skip it to avoid unnecessary computation.
if self._loss_reduction == losses.Reduction.SUM:
example_weight_sum = math_ops.reduce_sum(
weights * array_ops.ones_like(unreduced_loss))
mean_loss = training_loss / example_weight_sum
else:
mean_loss = None
with ops.name_scope(''):
keys = metric_keys.MetricKeys
summary.scalar(
head_lib._summary_key(self._name, keys.LOSS), # pylint:disable=protected-access
regularized_training_loss)
if mean_loss is not None:
summary.scalar(
head_lib._summary_key(self._name, keys.LOSS_MEAN), # pylint:disable=protected-access
mean_loss)
if regularization_loss is not None:
summary.scalar(
head_lib._summary_key(self._name,
keys.LOSS_REGULARIZATION), # pylint:disable=protected-access
regularization_loss)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=regularized_training_loss,
train_op=train_op_fn(regularized_training_loss))
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
train_op_fn=None):
"""See `Head`."""
with ops.name_scope(self._name, 'head'):
logits = head_lib._check_logits_final_dim(logits,
self.logits_dimension) # pylint:disable=protected-access
# Predict.
pred_keys = prediction_keys.PredictionKeys
with ops.name_scope(None, 'predictions', (logits, )):
probabilities = math_ops.sigmoid(logits,
name=pred_keys.PROBABILITIES)
predictions = {
pred_keys.LOGITS: logits,
pred_keys.PROBABILITIES: probabilities,
}
if mode == model_fn.ModeKeys.PREDICT:
classifier_output = head_lib._classification_output( # pylint:disable=protected-access
scores=probabilities,
n_classes=self._n_classes,
label_vocabulary=self._label_vocabulary)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY:
classifier_output,
head_lib._CLASSIFY_SERVING_KEY:
classifier_output, # pylint:disable=protected-access
head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access
export_output.PredictOutput(predictions))
})
(weighted_sum_loss, example_weight_sum,
processed_labels) = self.create_loss(features=features,
mode=mode,
logits=logits,
labels=labels)
# Eval.
if mode == model_fn.ModeKeys.EVAL:
weights = head_lib._get_weights_and_check_match_logits( # pylint:disable=protected-access,
features=features,
weight_column=self._weight_column,
logits=logits)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=weighted_sum_loss,
eval_metric_ops=self._eval_metric_ops(
labels=processed_labels,
probabilities=probabilities,
weights=weights,
weighted_sum_loss=weighted_sum_loss,
example_weight_sum=example_weight_sum))
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
with ops.name_scope(''):
summary.scalar(
head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access
weighted_sum_loss)
summary.scalar(
head_lib._summary_key( # pylint:disable=protected-access
self._name, metric_keys.MetricKeys.LOSS_MEAN),
weighted_sum_loss / example_weight_sum)
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=weighted_sum_loss,
train_op=train_op_fn(weighted_sum_loss))
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
train_op_fn=tfgan_train.gan_train_ops):
"""Returns `EstimatorSpec` that a model_fn can return.
See `Head` for more details.
Args:
features: Must be `None`.
mode: Estimator's `ModeKeys`.
logits: A GANModel tuple.
labels: Must be `None`.
train_op_fn: Function that takes a GANModel, GANLoss, generator optimizer,
and discriminator optimizer, and returns a `GANTrainOps` tuple. For
example, this function can come from TFGAN's `train.py` library, or can
be custom.
Returns:
`EstimatorSpec`.
Raises:
ValueError: If `features` isn't `None`.
ValueError: If `train_op_fn` isn't provided in train mode.
"""
_validate_logits_and_labels(logits, labels)
if features is not None:
raise ValueError(
'`features` should be `None`. Instead, found: %s' % features)
gan_model = logits # rename variable for clarity
with ops.name_scope('GANHead'):
if mode == model_fn_lib.ModeKeys.PREDICT:
return model_fn_lib.EstimatorSpec(
mode=model_fn_lib.ModeKeys.PREDICT,
predictions=gan_model.generated_data,
export_outputs={
'predict':
export_output.PredictOutput(gan_model.generated_data)
})
elif mode == model_fn_lib.ModeKeys.EVAL:
gan_loss = self.create_loss(features=None,
mode=mode,
logits=gan_model,
labels=None)
scalar_loss = gan_loss.generator_loss + gan_loss.discriminator_loss
with ops.name_scope(
None, 'metrics',
[gan_loss.generator_loss, gan_loss.discriminator_loss]):
eval_metric_ops = {
_summary_key(self._name, 'generator_loss'):
metrics_lib.mean(gan_loss.generator_loss),
_summary_key(self._name, 'discriminator_loss'):
metrics_lib.mean(gan_loss.discriminator_loss)
}
if self._get_eval_metric_ops_fn is not None:
custom_eval_metric_ops = self._get_eval_metric_ops_fn(
gan_model)
if not isinstance(custom_eval_metric_ops, dict):
raise TypeError(
'get_eval_metric_ops_fn must return a dict, '
'received: {}'.format(custom_eval_metric_ops))
eval_metric_ops.update(custom_eval_metric_ops)
return model_fn_lib.EstimatorSpec(
mode=model_fn_lib.ModeKeys.EVAL,
predictions=gan_model.generated_data,
loss=scalar_loss,
eval_metric_ops=eval_metric_ops)
elif mode == model_fn_lib.ModeKeys.TRAIN:
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
gan_loss = self.create_loss(None, mode, gan_model, None)
scalar_loss = gan_loss.generator_loss + gan_loss.discriminator_loss
train_ops = train_op_fn(gan_model, gan_loss,
self._generator_optimizer,
self._discriminator_optimizer)
training_hooks = self._get_hooks_fn(train_ops)
return model_fn_lib.EstimatorSpec(
loss=scalar_loss,
mode=model_fn_lib.ModeKeys.TRAIN,
train_op=train_ops.global_step_inc_op,
training_hooks=training_hooks)
else:
raise ValueError('Mode not recognized: %s' % mode)
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
optimizer=None,
train_op_fn=None,
regularization_losses=None,
params=None):
"""Returns an `EstimatorSpec`.
Args:
features: Input `dict` of `Tensor` or `SparseTensor` objects.
mode: Estimator's `ModeKeys`.
logits: a tensor array with 2 element, one for start positon probilities
, and one for the end postion probilities.
the shape is `[batch_size, logits_dimension]`.
labels: a tensor with demention [batch_size, 2], 2 position for true position in a doc
optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
updates variables and increments `global_step`.
train_op_fn: Function that takes a scalar loss `Tensor` and returns
`train_op`. Used if `optimizer` is `None`.
regularization_losses: A list of additional scalar losses to be added to
the training loss, such as regularization losses. These losses are
usually expressed as a batch average, so for best results users need to
set `loss_reduction=SUM_OVER_BATCH_SIZE` or
`loss_reduction=SUM_OVER_NONZERO_WEIGHTS` when creating the head to
avoid scaling errors.
Returns:
`EstimatorSpec`.
Raises:
ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
mode, or if both are set.
"""
with tf.name_scope(self._name, 'head'):
# Predict.
pred_keys = prediction_keys.PredictionKeys
with tf.name_scope(None, 'predictions', (logits, )):
[start_logits, end_logits] = logits
shape = tf.shape(start_logits)
d1 = shape[0]
d2 = shape[1]
d3 = shape[2]
start_logits = tf.reshape(
tf.nn.softmax(tf.reshape(start_logits, [-1, d2 * d3]), -1),
[d1, d2, d3, 1])
end_logits = tf.reshape(
tf.nn.softmax(tf.reshape(end_logits, [-1, d2 * d3]), -1),
[d1, d2, 1, d3])
#[batch_size, sent_number, seq_len,seq_len]
mul = tf.multiply(start_logits, end_logits)
x_tensor_band = tf.matrix_band_part(mul, 0, params.ans_limit)
x_tensor_reshape = tf.reshape(x_tensor_band,
[d1, d2 * d3 * d3])
#[batch_size]
max = tf.reduce_max(x_tensor_reshape, -1)
#[batch_size,1]
row_index = tf.where(tf.not_equal(max,
tf.ones_like(max) + 2.1))
indice = tf.expand_dims(tf.argmax(x_tensor_reshape, -1), -1)
indice = tf.concat([row_index, indice], axis=-1)
only_max = tf.sparse_to_dense(
indice, tf.shape(x_tensor_reshape, out_type=tf.int64), max)
only_max = tf.reshape(only_max, [d1, d2, d3, d3])
loc = tf.where(tf.cast(only_max, tf.bool))
loc = tf.slice(
loc, [0, 1], [-1, -1], name="locations"
) #[batch_size,postions=3] positions = (sent_no,start_pos,end_pos)
predictions = {
"s_logits": start_logits,
"e_logits": end_logits,
pred_keys.PROBABILITIES: max,
"locations": loc
}
if mode == model_fn.ModeKeys.PREDICT:
output = export_output.PredictOutput(predictions)
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY: output,
_PREDICT_SERVING_KEY: output
})
training_loss, unreduced_loss, weights, label_ids = self.create_loss(
features=None, mode=mode, logits=logits, labels=labels)
if regularization_losses:
regularization_loss = tf.add_n(regularization_losses)
regularized_training_loss = tf.add_n(
[training_loss, regularization_loss])
else:
regularization_loss = None
regularized_training_loss = training_loss
# Eval.
if mode == model_fn.ModeKeys.EVAL:
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=regularized_training_loss,
eval_metric_ops=self._eval_metric_ops(
labels=label_ids,
predict=tf.sign(
tf.abs(tf.reshape(only_max, [d1, d2 * d3 * d3]))),
location=loc,
pro=max,
unreduced_loss=unreduced_loss,
regularization_loss=regularization_loss))
# Train.
if optimizer is not None:
if train_op_fn is not None:
raise ValueError(
'train_op_fn and optimizer cannot both be set.')
train_op = optimizer.minimize(
regularized_training_loss,
global_step=training_util.get_global_step())
elif train_op_fn is not None:
train_op = train_op_fn(regularized_training_loss)
else:
raise ValueError(
'train_op_fn and optimizer cannot both be None.')
with tf.name_scope(''):
keys = metric_keys.MetricKeys
tf.summary.scalar(_summary_key(self._name, keys.LOSS),
regularized_training_loss)
if regularization_loss is not None:
tf.summary.scalar(
_summary_key(self._name, keys.LOSS_REGULARIZATION),
regularization_loss)
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=regularized_training_loss,
train_op=train_op)
def create_estimator_spec(
self, features, logits, mode, labels=None, train_op_fn=None):
"""See `Head`."""
# split logits into mu, sigma and alpha
components = array_ops.reshape(logits, [-1, 3, self._m])
mus = components[:, 0, :]
sigmas = components[:, 1, :]
alphas = components[:, 2, :]
alphas = nn_ops.softmax(clip_ops.clip_by_value(alphas, 1e-2, 1.))
# Predict.
with ops.name_scope('head'):
#logits = head_lib._check_logits(logits, self._logits_dimension)
means = math_ops.reduce_sum(alphas*mus, axis=1, keepdims=True)
uncertainty = math_ops.reduce_sum(
alphas*sigmas, axis=1, keepdims=True)
predicted_value = array_ops.concat([means, uncertainty], 1)
predictions = {prediction_keys.PredictionKeys.PREDICTIONS:
predicted_value}
if mode == model_fn.ModeKeys.PREDICT:
regression_output = export_output.RegressionOutput(
value=predicted_value)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
head_lib._DEFAULT_SERVING_KEY: regression_output,
head_lib._REGRESS_SERVING_KEY: regression_output,
head_lib._PREDICT_SERVING_KEY:
export_output.PredictOutput(predictions)
})
# Eval.
if mode == model_fn.ModeKeys.EVAL:
# Estimator already adds a metric for loss.
mus = math_ops.reduce_sum(alphas*mus, axis=1, keepdims=True)
#mus = utils.tf_print(mus, "mus:")
#labels = utils.tf_print(labels, "labels:")
training_loss, unweighted_loss, _ = self.create_loss2(
features=features, mode=mode, logits=mus, labels=labels)
keys = metric_keys.MetricKeys
eval_metric_ops = {
head_lib._summary_key(self._name,
keys.LOSS_MEAN) :
metrics_lib.mean(
unweighted_loss, weights=None)
}
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=training_loss,
eval_metric_ops=eval_metric_ops)
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
training_loss, unweighted_loss, _ = self.create_loss(
features=features, mode=mode, mus=mus,
sigmas=sigmas, alphas=alphas, labels=labels)
with ops.name_scope(''):
summary.scalar(
head_lib._summary_key(self._name,
metric_keys.MetricKeys.LOSS_MEAN),
losses.compute_weighted_loss(
unweighted_loss,
reduction=losses.Reduction.MEAN))
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=training_loss,
train_op=train_op_fn(training_loss))
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
train_op_fn=None):
"""See `Head`."""
with ops.name_scope(self._name, 'head'):
logits = head_lib._check_logits_final_dim(logits,
self.logits_dimension) # pylint:disable=protected-access
# Predict.
pred_keys = prediction_keys.PredictionKeys
with ops.name_scope(None, 'predictions', (logits, )):
probabilities = math_ops.sigmoid(logits,
name=pred_keys.PROBABILITIES)
predictions = {
pred_keys.LOGITS: logits,
pred_keys.PROBABILITIES: probabilities,
}
if mode == model_fn.ModeKeys.PREDICT:
classifier_output = head_lib._classification_output( # pylint:disable=protected-access
scores=probabilities,
n_classes=self._n_classes,
label_vocabulary=self._label_vocabulary)
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY:
classifier_output,
head_lib._CLASSIFY_SERVING_KEY:
classifier_output, # pylint:disable=protected-access
head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access
export_output.PredictOutput(predictions))
})
(training_loss, unreduced_loss, weights,
processed_labels) = self.create_loss(features=features,
mode=mode,
logits=logits,
labels=labels)
# Eval.
if mode == model_fn.ModeKeys.EVAL:
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=training_loss,
eval_metric_ops=self._eval_metric_ops(
labels=processed_labels,
probabilities=probabilities,
weights=weights,
unreduced_loss=unreduced_loss))
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
# Only summarize mean_loss for SUM reduction to preserve backwards
# compatibility. Otherwise skip it to avoid unnecessary computation.
if self._loss_reduction == losses.Reduction.SUM:
example_weight_sum = math_ops.reduce_sum(
weights * array_ops.ones_like(unreduced_loss))
mean_loss = training_loss / example_weight_sum
else:
mean_loss = None
with ops.name_scope(''):
summary.scalar(
head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), # pylint:disable=protected-access
training_loss)
if mean_loss is not None:
summary.scalar(
head_lib._summary_key( # pylint:disable=protected-access
self._name, metric_keys.MetricKeys.LOSS_MEAN),
mean_loss)
return model_fn.EstimatorSpec(mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=training_loss,
train_op=train_op_fn(training_loss))
def model_fn(self, features, mode, config):
"""Model function for the estimator.
Note that this does not take a `labels` arg. This works, but `input_fn` must
return either `features` or, equivalently, `(features, None)`.
Args:
features: The input points. See `tf.estimator.Estimator`.
mode: See `tf.estimator.Estimator`.
config: See `tf.estimator.Estimator`.
Returns:
A `tf.estimator.EstimatorSpec` (see `tf.estimator.Estimator`) specifying
this behavior:
* `train_op`: Execute one mini-batch or full-batch run of Lloyd's
algorithm.
* `loss`: The sum of the squared distances from each input point to its
closest center.
* `eval_metric_ops`: Maps `SCORE` to `loss`.
* `predictions`: Maps `ALL_DISTANCES` to the distance from each input
point to each cluster center; maps `CLUSTER_INDEX` to the index of
the closest cluster center for each input point.
"""
# input_points is a single Tensor. Therefore, the sharding functionality
# in clustering_ops is unused, and some of the values below are lists of a
# single item.
input_points = _parse_features_if_necessary(features,
self._feature_columns)
# Let N = the number of input_points.
# all_distances: A list of one matrix of shape (N, num_clusters). Each value
# is the distance from an input point to a cluster center.
# model_predictions: A list of one vector of shape (N). Each value is the
# cluster id of an input point.
# losses: Similar to cluster_idx but provides the distance to the cluster
# center.
# is_initialized: scalar indicating whether the initial cluster centers
# have been chosen; see init_op.
# init_op: an op to choose the initial cluster centers. A single worker
# repeatedly executes init_op until is_initialized becomes True.
# training_op: an op that runs an iteration of training, either an entire
# Lloyd iteration or a mini-batch of a Lloyd iteration. Multiple workers
# may execute this op, but only after is_initialized becomes True.
(all_distances, model_predictions, losses, is_initialized, init_op,
training_op) = clustering_ops.KMeans(
inputs=input_points,
num_clusters=self._num_clusters,
initial_clusters=self._initial_clusters,
distance_metric=self._distance_metric,
use_mini_batch=self._use_mini_batch,
mini_batch_steps_per_iteration=self.
_mini_batch_steps_per_iteration,
random_seed=self._random_seed,
kmeans_plus_plus_num_retries=self._kmeans_plus_plus_num_retries
).training_graph()
loss = math_ops.reduce_sum(losses)
summary.scalar('loss/raw', loss)
incr_step = state_ops.assign_add(training_util.get_global_step(), 1)
training_op = control_flow_ops.with_dependencies(
[training_op, incr_step], loss)
training_hooks = [
_InitializeClustersHook(init_op, is_initialized, config.is_chief)
]
if self._relative_tolerance is not None:
training_hooks.append(
_LossRelativeChangeHook(loss, self._relative_tolerance))
export_outputs = {
KMeansClustering.ALL_DISTANCES:
export_output.PredictOutput(all_distances[0]),
KMeansClustering.CLUSTER_INDEX:
export_output.PredictOutput(model_predictions[0]),
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
export_output.PredictOutput(model_predictions[0])
}
return model_fn_lib.EstimatorSpec(
mode=mode,
predictions={
KMeansClustering.ALL_DISTANCES: all_distances[0],
KMeansClustering.CLUSTER_INDEX: model_predictions[0],
},
loss=loss,
train_op=training_op,
eval_metric_ops={KMeansClustering.SCORE: metrics.mean(loss)},
training_hooks=training_hooks,
export_outputs=export_outputs)
def create_estimator_spec(self,
features,
mode,
logits,
labels=None,
optimizer=None,
train_op_fn=None,
regularization_losses=None):
"""Returns an `EstimatorSpec`.
Args:
features: Input `dict` of `Tensor` or `SparseTensor` objects.
mode: Estimator's `ModeKeys`.
logits: a tensor array with 2 element, one for start positon probilities
, and one for the end postion probilities.
the shape is `[batch_size, logits_dimension]`.
labels: a tensor with demention [batch_size, 2], 2 position for true position in a doc
optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
updates variables and increments `global_step`.
train_op_fn: Function that takes a scalar loss `Tensor` and returns
`train_op`. Used if `optimizer` is `None`.
regularization_losses: A list of additional scalar losses to be added to
the training loss, such as regularization losses. These losses are
usually expressed as a batch average, so for best results users need to
set `loss_reduction=SUM_OVER_BATCH_SIZE` or
`loss_reduction=SUM_OVER_NONZERO_WEIGHTS` when creating the head to
avoid scaling errors.
Returns:
`EstimatorSpec`.
Raises:
ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
mode, or if both are set.
"""
with tf.name_scope(self._name, 'head'):
# Predict.
pred_keys = prediction_keys.PredictionKeys
predictions = {}
if mode == tf.estimator.ModeKeys.PREDICT:
output = export_output.PredictOutput(predictions)
return tf.estimator.EstimatorSpec(
mode=tf.estimator.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
_DEFAULT_SERVING_KEY: output,
_PREDICT_SERVING_KEY: output
})
training_loss, unreduced_loss, weights, label_ids = self.create_loss(
features=features, mode=mode, logits=logits, labels=labels)
if regularization_losses:
regularization_loss = tf.add_n(regularization_losses)
regularized_training_loss = tf.add_n(
[training_loss, regularization_loss])
else:
regularization_loss = None
regularized_training_loss = training_loss
# Eval.
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode=tf.estimator.ModeKeys.EVAL,
predictions=predictions,
loss=regularized_training_loss,
eval_metric_ops=self._eval_metric_ops(
labels=label_ids,
predict=
None, #tf.sign(tf.abs(tf.reshape(only_max,[d1,d2*d3*d3]))),
location=None, # loc,
pro=max,
unreduced_loss=unreduced_loss,
regularization_loss=regularization_loss))
# Train.
if optimizer is not None:
if train_op_fn is not None:
raise ValueError(
'train_op_fn and optimizer cannot both be set.')
train_op = optimizer.minimize(
regularized_training_loss,
global_step=tf.train.get_global_step())
elif train_op_fn is not None:
train_op = train_op_fn(regularized_training_loss)
else:
raise ValueError(
'train_op_fn and optimizer cannot both be None.')
with tf.name_scope(''):
keys = metric_keys.MetricKeys
tf.summary.scalar(_summary_key(self._name, keys.LOSS),
regularized_training_loss)
if regularization_loss is not None:
tf.summary.scalar(
_summary_key(self._name, keys.LOSS_REGULARIZATION),
regularization_loss)
return tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.TRAIN,
predictions=predictions,
loss=regularized_training_loss,
train_op=train_op)
