def test_train_create_loss_logits_tensor_multi_dim(self):
"""Tests create_loss with multi-dimensional logits of shape [2, 2, 5]."""
head1 = head_lib.regression_head(label_dimension=2, name='head1')
head2 = head_lib.regression_head(label_dimension=3, name='head2')
multi_head = multi_head_lib.multi_head([head1, head2])
logits = np.array(
[[[-1., 1., 2., -2., 2.], [-1., 1., 2., -2., 2.]],
[[-1.5, 1.5, -2., 2., -2.], [-1.5, 1.5, -2., 2., -2.]]],
dtype=np.float32)
labels = {
'head1': np.array([[[1., 0.], [1., 0.]],
[[1.5, 1.5], [1.5, 1.5]]], dtype=np.float32),
'head2': np.array([[[0., 1., 0.], [0., 1., 0.]],
[[2., 2., 0.], [2., 2., 0.]]], dtype=np.float32),
}
# Loss for the first head:
# loss1 = ((1+1)^2 + (0-1)^2 + (1+1)^2 + (0-1)^2 +
# (1.5+1.5)^2 + (1.5-1.5)^2 + (1.5+1.5)^2 + (1.5-1.5)^2) / 8
# = 3.5
# Loss for the second head:
# loss2 = ((0-2)^2 + (1+2)^2 + (0-2)^2 + (0-2)^2 + (1+2)^2 + (0-2)^2 +
# (2+2)^2 + (2-2)^2 + (0+2)^2 + (2+2)^2 + (2-2)^2 + (0+2)^2) / 12
# = 6.167
expected_training_loss = 3.5 + 6.167
training_loss = multi_head.create_loss(
features={},
mode=model_fn.ModeKeys.TRAIN,
logits=logits,
labels=labels)[0]
tol = 1e-3
with self.cached_session():
self.assertAllClose(
expected_training_loss, training_loss.eval(), rtol=tol, atol=tol)
def test_predict_two_heads_logits_tensor_multi_dim(self):
"""Tests predict with multi-dimensional logits of shape [2, 2, 5]."""
head1 = head_lib.regression_head(label_dimension=2, name='head1')
head2 = head_lib.regression_head(label_dimension=3, name='head2')
multi_head = multi_head_lib.multi_head([head1, head2])
logits = np.array(
[[[-1., 1., 2., -2., 2.], [-1., 1., 2., -2., 2.]],
[[-1.5, 1., -3., 2., -2.], [-1.5, 1., -3., 2., -2.]]],
dtype=np.float32)
expected_logits1 = np.array(
[[[-1., 1.], [-1., 1.]],
[[-1.5, 1.], [-1.5, 1.]]],
dtype=np.float32)
expected_logits2 = np.array(
[[[2., -2., 2.], [2., -2., 2.]],
[[-3., 2., -2.], [-3., 2., -2.]]],
dtype=np.float32)
spec = multi_head.create_estimator_spec(
features={'x': np.array(((42,),), dtype=np.int32)},
mode=model_fn.ModeKeys.PREDICT,
logits=logits)
self.assertItemsEqual(
(_DEFAULT_SERVING_KEY, 'predict', 'head1', 'head1/regression',
'head1/predict', 'head2', 'head2/regression', 'head2/predict'),
spec.export_outputs.keys())
# Assert predictions and export_outputs.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
self.assertIsNone(spec.scaffold.summary_op)
predictions = sess.run(spec.predictions)
self.assertAllClose(
expected_logits1,
predictions[('head1', prediction_keys.PredictionKeys.PREDICTIONS)])
self.assertAllClose(
expected_logits2,
predictions[('head2', prediction_keys.PredictionKeys.PREDICTIONS)])
self.assertAllClose(
expected_logits1,
sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].value))
self.assertAllClose(
expected_logits1,
sess.run(spec.export_outputs['head1'].value))
self.assertAllClose(
expected_logits2,
sess.run(spec.export_outputs['head2'].value))
def _test_complete_flow(self, train_input_fn, eval_input_fn,
predict_input_fn, input_dimension, label_dimension,
batch_size):
feature_columns = [
feature_column.numeric_column('x', shape=(input_dimension, ))
]
est = linear.LinearEstimator(
head=head_lib.regression_head(label_dimension=label_dimension),
feature_columns=feature_columns,
model_dir=self._model_dir)
# TRAIN
num_steps = 10
est.train(train_input_fn, steps=num_steps)
# EVALUTE
scores = est.evaluate(eval_input_fn)
self.assertEqual(num_steps, scores[ops.GraphKeys.GLOBAL_STEP])
self.assertIn('loss', six.iterkeys(scores))
# PREDICT
predictions = np.array([
x[prediction_keys.PredictionKeys.PREDICTIONS]
for x in est.predict(predict_input_fn)
])
self.assertAllEqual((batch_size, label_dimension), predictions.shape)
# EXPORT
feature_spec = feature_column.make_parse_example_spec(feature_columns)
serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
feature_spec)
export_dir = est.export_savedmodel(tempfile.mkdtemp(),
serving_input_receiver_fn)
self.assertTrue(gfile.Exists(export_dir))
def make_regression_head(use_tpu):
if use_tpu and head_lib:
# AdaNet TPU currently requires the old head.
return head_lib.regression_head(
loss_reduction=tf_compat.v1.losses.Reduction.SUM_OVER_BATCH_SIZE)
# TF 2.0 eliminates tf.contrib.
return regression_head.RegressionHead(
loss_reduction=tf_compat.SUM_OVER_BATCH_SIZE)
def _dnn_estimator_fn(weight_column=None, label_dimension=1, *args, **kwargs): # pylint: disable=keyword-arg-before-vararg
"""Returns a DNNEstimator that uses regression_head."""
return dnn.DNNEstimator(
head=head_lib.regression_head(
weight_column=weight_column, label_dimension=label_dimension,
# Tests in core (from which this test inherits) test the sum loss.
loss_reduction=losses.Reduction.SUM),
*args, **kwargs)
def _baseline_estimator_fn(
weight_column=None, label_dimension=1, *args, **kwargs):
"""Returns a BaselineEstimator that uses regression_head."""
return baseline.BaselineEstimator(
head=head_lib.regression_head(
weight_column=weight_column, label_dimension=label_dimension,
# Tests in core (from which this test inherits) test the sum loss.
loss_reduction=losses.Reduction.SUM),
*args, **kwargs)
def test_tpu_estimator_simple_lifecycle(self, use_tpu,
subnetwork_generator, want_loss):
config = tf_compat.v1.estimator.tpu.RunConfig(master="",
tf_random_seed=42)
estimator = TPUEstimator(
# TODO: Add test with estimator Head v2.
head=head_lib.regression_head(loss_reduction=tf_compat.v1.losses.
Reduction.SUM_OVER_BATCH_SIZE),
subnetwork_generator=subnetwork_generator,
max_iteration_steps=10,
model_dir=self.test_subdirectory,
config=config,
use_tpu=use_tpu,
train_batch_size=64 if use_tpu else 0)
max_steps = 30
xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]]
xor_labels = [[1.], [0.], [1.], [0.]]
train_input_fn = tu.dummy_input_fn(xor_features, xor_labels)
# Train.
estimator.train(input_fn=train_input_fn,
steps=None,
max_steps=max_steps,
hooks=None)
# Evaluate.
eval_results = estimator.evaluate(input_fn=train_input_fn,
steps=1,
hooks=None)
# Predict.
predictions = estimator.predict(
input_fn=tu.dataset_input_fn(features=[0., 0.], labels=None))
# Export SavedModel.
def serving_input_fn():
"""Input fn for serving export, starting from serialized example."""
serialized_example = tf.compat.v1.placeholder(
dtype=tf.string, shape=(None), name="serialized_example")
return tf.estimator.export.ServingInputReceiver(
features={"x": tf.constant([[0., 0.]], name="serving_x")},
receiver_tensors=serialized_example)
estimator.export_saved_model(
export_dir_base=estimator.model_dir,
serving_input_receiver_fn=serving_input_fn)
self.assertAlmostEqual(want_loss, eval_results["loss"], places=2)
self.assertEqual(max_steps, eval_results["global_step"])
self.assertEqual(2, eval_results["iteration"])
for prediction in predictions:
self.assertIsNotNone(prediction["predictions"])
def test_train_create_loss_logits_tensor_multi_dim_wrong_shape(self):
"""Tests create_loss with a multi-dimensional logits tensor of wrong shape."""
head1 = head_lib.regression_head(label_dimension=2, name='head1')
head2 = head_lib.regression_head(label_dimension=3, name='head2')
multi_head = multi_head_lib.multi_head([head1, head2])
# logits tensor is 2x2x4 instead of 2x2x5
logits = np.array([[[-1., 1., 2., -2.], [-1., 1., 2., -2.]],
[[-1.5, 1.5, -2., 2.], [-1.5, 1.5, -2., 2.]]],
dtype=np.float32)
labels = {
'head1':
np.array([[[1., 0.], [1., 0.]], [[1.5, 1.5], [1.5, 1.5]]],
dtype=np.float32),
'head2':
np.array(
[[[0., 1., 0.], [0., 1., 0.]], [[2., 2., 0.], [2., 2., 0.]]],
dtype=np.float32),
}
with self.assertRaisesRegexp(ValueError, r'Could not split logits'):
multi_head.create_loss(features={},
mode=model_fn.ModeKeys.TRAIN,
logits=logits,
labels=labels)
def _linear_only_estimator_fn(
feature_columns,
model_dir=None,
label_dimension=1,
weight_column=None,
optimizer='Ftrl',
config=None,
partitioner=None,
sparse_combiner='sum'):
return dnn_linear_combined.DNNLinearCombinedEstimator(
head=head_lib.regression_head(
weight_column=weight_column, label_dimension=label_dimension,
# Tests in core (from which this test inherits) test the sum loss.
loss_reduction=losses.Reduction.SUM),
model_dir=model_dir,
linear_feature_columns=feature_columns,
linear_optimizer=optimizer,
input_layer_partitioner=partitioner,
config=config,
linear_sparse_combiner=sparse_combiner)
def _dnn_only_estimator_fn(
hidden_units,
feature_columns,
model_dir=None,
label_dimension=1,
weight_column=None,
optimizer='Adagrad',
activation_fn=nn.relu,
dropout=None,
input_layer_partitioner=None,
config=None):
return dnn_linear_combined.DNNLinearCombinedEstimator(
head=head_lib.regression_head(
weight_column=weight_column, label_dimension=label_dimension,
# Tests in core (from which this test inherits) test the sum loss.
loss_reduction=losses.Reduction.SUM),
model_dir=model_dir,
dnn_feature_columns=feature_columns,
dnn_optimizer=optimizer,
dnn_hidden_units=hidden_units,
dnn_activation_fn=activation_fn,
dnn_dropout=dropout,
input_layer_partitioner=input_layer_partitioner,
config=config)
def test_tpu_estimator_summaries(self, use_tpu, want_loss,
want_adanet_loss, want_eval_summary_loss,
want_predictions):
max_steps = 10
config = tf_compat.v1.estimator.tpu.RunConfig(
tf_random_seed=42,
save_summary_steps=2,
log_step_count_steps=max_steps)
assert config.log_step_count_steps
def metric_fn(predictions):
return {
"predictions":
tf_compat.v1.metrics.mean(predictions["predictions"])
}
estimator = TPUEstimator(head=head_lib.regression_head(
loss_reduction=tf_compat.v1.losses.Reduction.SUM_OVER_BATCH_SIZE),
subnetwork_generator=SimpleGenerator(
[_DNNBuilder("dnn", use_tpu=use_tpu)]),
max_iteration_steps=max_steps,
model_dir=self.test_subdirectory,
metric_fn=metric_fn,
config=config,
use_tpu=use_tpu,
train_batch_size=64 if use_tpu else 0)
xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]]
xor_labels = [[1.], [0.], [1.], [0.]]
train_input_fn = tu.dummy_input_fn(xor_features, xor_labels)
estimator.train(input_fn=train_input_fn, max_steps=max_steps)
eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1)
self.assertAlmostEqual(want_loss, eval_results["loss"], places=2)
self.assertEqual(max_steps, eval_results["global_step"])
self.assertEqual(0, eval_results["iteration"])
subnetwork_subdir = os.path.join(self.test_subdirectory,
"subnetwork/t0_dnn")
ensemble_subdir = os.path.join(
self.test_subdirectory,
"ensemble/t0_dnn_grow_complexity_regularized")
# TODO: Why is the adanet_loss written to 'loss'?
self.assertAlmostEqual(want_adanet_loss,
tu.check_eventfile_for_keyword(
"loss", self.test_subdirectory),
places=1)
self.assertEqual(
0.,
tu.check_eventfile_for_keyword("iteration/adanet/iteration",
self.test_subdirectory))
self.assertAlmostEqual(3.,
tu.check_eventfile_for_keyword(
"scalar", subnetwork_subdir),
places=3)
self.assertEqual(
(3, 3, 1),
tu.check_eventfile_for_keyword(
# When TF 2 behavior is enabled AdaNet uses V2 summaries.
"image"
if tf_compat.is_v2_behavior_enabled() else "image/image/0",
subnetwork_subdir))
self.assertAlmostEqual(5.,
tu.check_eventfile_for_keyword(
"nested/scalar", subnetwork_subdir),
places=3)
self.assertAlmostEqual(
want_adanet_loss,
tu.check_eventfile_for_keyword(
"adanet_loss/adanet/adanet_weighted_ensemble",
ensemble_subdir),
places=1)
self.assertAlmostEqual(
0.,
tu.check_eventfile_for_keyword(
"complexity_regularization/adanet/adanet_weighted_ensemble",
ensemble_subdir),
places=1)
self.assertAlmostEqual(1.,
tu.check_eventfile_for_keyword(
"mixture_weight_norms/adanet/"
"adanet_weighted_ensemble/subnetwork_0",
ensemble_subdir),
places=1)
# Eval metric summaries are always written out during eval.
subnetwork_eval_subdir = os.path.join(subnetwork_subdir, "eval")
self.assertAlmostEqual(want_eval_summary_loss,
tu.check_eventfile_for_keyword(
"loss", subnetwork_eval_subdir),
places=1)
self.assertAlmostEqual(want_eval_summary_loss,
tu.check_eventfile_for_keyword(
"average_loss", subnetwork_eval_subdir),
places=1)
self.assertAlmostEqual(want_predictions,
tu.check_eventfile_for_keyword(
"predictions", subnetwork_eval_subdir),
places=3)
eval_subdir = os.path.join(self.test_subdirectory, "eval")
ensemble_eval_subdir = os.path.join(ensemble_subdir, "eval")
for subdir in [ensemble_eval_subdir, eval_subdir]:
self.assertEqual([b"| dnn |"],
tu.check_eventfile_for_keyword(
"architecture/adanet/ensembles/0", subdir))
if subdir == eval_subdir:
self.assertAlmostEqual(want_loss,
tu.check_eventfile_for_keyword(
"loss", subdir),
places=1)
self.assertAlmostEqual(want_eval_summary_loss,
tu.check_eventfile_for_keyword(
"average_loss", subdir),
places=1)
def test_auto_ensemble_estimator_lifecycle(self, use_tpu):
head = head_lib.regression_head()
feature_columns = [tf.feature_column.numeric_column("xor", shape=[2])]
def optimizer_fn():
optimizer = tf.compat.v1.train.GradientDescentOptimizer(
learning_rate=.01)
if use_tpu:
optimizer = tf.compat.v1.tpu.CrossShardOptimizer(optimizer)
return optimizer
candidate_pool = {
"tpu_estimator_dnn":
_DNNTPUEstimator(head=head,
feature_columns=feature_columns,
optimizer=optimizer_fn,
hidden_units=[3],
use_tpu=True),
"tpu_estimator_wider_dnn":
_DNNTPUEstimator(head=head,
feature_columns=feature_columns,
optimizer=optimizer_fn,
hidden_units=[6],
use_tpu=True),
"estimator_dnn":
tf.compat.v1.estimator.DNNEstimator(
head=head,
feature_columns=feature_columns,
optimizer=optimizer_fn,
hidden_units=[3]),
"estimator_linear":
tf.compat.v1.estimator.LinearEstimator(
head=head,
feature_columns=feature_columns,
optimizer=optimizer_fn),
}
run_config = tf.compat.v1.estimator.tpu.RunConfig(master="",
tf_random_seed=42)
estimator = AutoEnsembleTPUEstimator(head=head,
candidate_pool=candidate_pool,
max_iteration_steps=10,
model_dir=self.test_subdirectory,
config=run_config,
use_tpu=use_tpu,
train_batch_size=64,
force_grow=True)
features = {"xor": [[0., 0.], [0., 1.], [1., 0.], [1., 1.]]}
labels = [[0.], [1.], [1.], [0.]]
def train_input_fn(params):
del params # Unused.
input_features = {}
for key, feature in features.items():
input_features[key] = tf.constant(feature, name=key)
input_labels = tf.constant(labels, name="labels")
return input_features, input_labels
def test_input_fn(params):
del params # Unused.
input_features = tf.compat.v1.data.Dataset.from_tensors(
tf.constant(
features["xor"])).make_one_shot_iterator().get_next()
return {"xor": input_features}, None
# Train for three iterations.
estimator.train(input_fn=train_input_fn, max_steps=30)
# Evaluate.
eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1)
self.assertAllClose(30, eval_results["global_step"])
self.assertAllClose(0.315863, eval_results["loss"], atol=.3)
# Predict.
predictions = estimator.predict(input_fn=test_input_fn)
for prediction in predictions:
self.assertIsNotNone(prediction["predictions"])
# Export SavedModel.
def serving_input_fn():
"""Input fn for serving export, starting from serialized example."""
serialized_example = tf.compat.v1.placeholder(
dtype=tf.string, shape=(None), name="serialized_example")
for key, value in features.items():
features[key] = tf.constant(value)
return tf.estimator.export.ServingInputReceiver(
features=features, receiver_tensors=serialized_example)
export_dir_base = os.path.join(self.test_subdirectory, "export")
export_saved_model_fn = getattr(estimator, "export_saved_model", None)
if not callable(export_saved_model_fn):
export_saved_model_fn = estimator.export_savedmodel
export_saved_model_fn(export_dir_base=export_dir_base,
serving_input_receiver_fn=serving_input_fn)