def test_metric_ops_not_duplicated_on_cpu(self):
with context.graph_mode():
self.setup_graph()
metric_fn = lambda: {"metric": (tf.constant(5), tf.constant(5))}
best_candidate_index = 3
mode = tf.estimator.ModeKeys.EVAL
ensemble_metrics = tu.create_ensemble_metrics(metric_fn)
subnetwork_metrics = tu.create_subnetwork_metrics(metric_fn)
iteration_metrics = tu.create_iteration_metrics(
ensemble_metrics=[ensemble_metrics],
subnetwork_metrics=[subnetwork_metrics])
ensemble_ops1 = ensemble_metrics.eval_metrics_ops()
ensemble_ops2 = ensemble_metrics.eval_metrics_ops()
subnetwork_ops1 = subnetwork_metrics.eval_metrics_ops()
subnetwork_ops2 = subnetwork_metrics.eval_metrics_ops()
iteration_ops1 = iteration_metrics.best_eval_metric_ops(
best_candidate_index, mode)
iteration_ops2 = iteration_metrics.best_eval_metric_ops(
best_candidate_index, mode)
self.assertEqual(subnetwork_ops1, subnetwork_ops2)
self.assertEqual(ensemble_ops1, ensemble_ops2)
self.assertEqual(iteration_ops1, iteration_ops2)
for ops in [ensemble_ops1, subnetwork_ops1, iteration_ops1]:
self.assertIsNotNone(ops)
def test_ensemble_metrics(self):
with context.graph_mode():
self.setup_graph()
architecture = _Architecture("test_ensemble_candidate",
"test_ensembler")
architecture.add_subnetwork(iteration_number=0,
builder_name="b_0_0")
architecture.add_subnetwork(iteration_number=0,
builder_name="b_0_1")
architecture.add_subnetwork(iteration_number=1,
builder_name="b_1_0")
architecture.add_subnetwork(iteration_number=2,
builder_name="b_2_0")
metrics = tu.create_ensemble_metrics(
self._metric_fn,
features=self._features,
labels=self._labels,
estimator_spec=self._estimator_spec,
architecture=architecture)
actual = self._run_metrics(metrics.eval_metrics_tuple())
serialized_arch_proto = actual["architecture/adanet/ensembles"]
expected_arch_string = b"| b_0_0 | b_0_1 | b_1_0 | b_2_0 |"
self.assertIn(expected_arch_string, serialized_arch_proto)
def build_ensemble_spec(self,
name,
candidate,
ensembler,
subnetwork_specs,
summary,
features,
mode,
iteration_number,
labels=None,
previous_ensemble_spec=None,
my_ensemble_index=None,
params=None,
previous_iteration_checkpoint=None):
del ensembler
del subnetwork_specs
del summary
del features
del mode
del labels
del iteration_number
del params
del my_ensemble_index
del previous_iteration_checkpoint
num_subnetworks = 0
if previous_ensemble_spec:
num_subnetworks += 1
return tu.dummy_ensemble_spec(
name=name,
num_subnetworks=num_subnetworks,
random_seed=candidate.subnetwork_builders[0].seed,
subnetwork_builders=candidate.subnetwork_builders,
dict_predictions=self._dict_predictions,
eval_metrics=tu.create_ensemble_metrics(
metric_fn=self._eval_metric_ops_fn),
export_output_key=self._export_output_key,
variables=[tf.Variable(1.)])
def test_iteration_metrics(self, use_tpu, mode):
with context.graph_mode():
self.setup_graph()
best_candidate_index = 3
ensemble_metrics = []
for i in range(10):
def metric_fn(val=i):
metric = tf.keras.metrics.Mean()
metric.update_state(tf.constant(val))
return {
"ensemble_v1_metric":
tf_compat.v1.metrics.mean(tf.constant(val)),
"ensemble_keras_metric":
metric
}
ensemble_metrics.append(tu.create_ensemble_metrics(metric_fn))
metrics = tu.create_iteration_metrics(
ensemble_metrics=ensemble_metrics)
metrics_fn = (metrics.best_eval_metrics_tuple
if use_tpu else metrics.best_eval_metric_ops)
actual = self._run_metrics(
metrics_fn(tf.constant(best_candidate_index), mode) or {})
if mode == tf.estimator.ModeKeys.EVAL:
expected = {
"ensemble_v1_metric": best_candidate_index,
"ensemble_keras_metric": best_candidate_index,
"iteration": 1
}
# We don't actually provide an architecture, so the default will be
# inside.
del actual["architecture/adanet/ensembles"]
else:
expected = {}
self.assertEqual(actual, expected)
class IterationBuilderTest(tu.AdanetTestCase):
@parameterized.named_parameters(
{
"testcase_name": "negative_max_steps",
"max_steps": -1,
}, {
"testcase_name": "zero_max_steps",
"max_steps": 0,
})
@test_util.run_in_graph_and_eager_modes
def test_init_errors(self, max_steps):
with self.assertRaises(ValueError):
_IterationBuilder(_FakeCandidateBuilder(),
_FakeSubnetworkManager(),
_FakeEnsembleBuilder(),
summary_maker=_ScopedSummary,
ensemblers=[_FakeEnsembler()],
max_steps=max_steps)
# pylint: disable=g-long-lambda
@parameterized.named_parameters(
{
"testcase_name": "single_subnetwork_fn",
"ensemble_builder": _FakeEnsembleBuilder(),
"subnetwork_builders": [_FakeBuilder("training")],
"features": lambda: [[1., -1., 0.]],
"labels": lambda: [1],
"want_loss": 1.403943,
"want_predictions": 2.129,
"want_best_candidate_index": 0,
}, {
"testcase_name":
"single_subnetwork_fn_mock_summary",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders": [_FakeBuilder("training")],
"summary_maker":
functools.partial(_TPUScopedSummary, logdir="/tmp/fakedir"),
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_best_candidate_index":
0,
}, {
"testcase_name":
"single_subnetwork_with_eval_metrics",
"ensemble_builder":
_FakeEnsembleBuilder(eval_metric_ops_fn=lambda:
{"a": (tf.constant(1), tf.constant(2))}),
"subnetwork_builders": [
_FakeBuilder("training", ),
],
"mode":
tf.estimator.ModeKeys.EVAL,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_eval_metric_ops": ["a", "iteration"],
"want_best_candidate_index":
0,
}, {
"testcase_name":
"single_subnetwork_with_non_tensor_eval_metric_op",
"ensemble_builder":
_FakeEnsembleBuilder(eval_metric_ops_fn=lambda:
{"a": (tf.constant(1), tf.no_op())}),
"subnetwork_builders": [
_FakeBuilder("training", ),
],
"mode":
tf.estimator.ModeKeys.EVAL,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_eval_metric_ops": ["a", "iteration"],
"want_best_candidate_index":
0,
}, {
"testcase_name": "single_subnetwork_done_training_fn",
"ensemble_builder": _FakeEnsembleBuilder(),
"subnetwork_builders": [_FakeBuilder("done")],
"features": lambda: [[1., -1., 0.]],
"labels": lambda: [1],
"want_loss": 1.403943,
"want_predictions": 2.129,
"want_best_candidate_index": 0,
}, {
"testcase_name": "single_dict_predictions_subnetwork_fn",
"ensemble_builder": _FakeEnsembleBuilder(dict_predictions=True),
"subnetwork_builders": [_FakeBuilder("training")],
"features": lambda: [[1., -1., 0.]],
"labels": lambda: [1],
"want_loss": 1.403943,
"want_predictions": {
"classes": 2,
"logits": 2.129
},
"want_best_candidate_index": 0,
}, {
"testcase_name": "previous_ensemble",
"ensemble_builder": _FakeEnsembleBuilder(),
"subnetwork_builders": [_FakeBuilder("training")],
"features": lambda: [[1., -1., 0.]],
"labels": lambda: [1],
"previous_ensemble_spec": lambda: tu.dummy_ensemble_spec("old"),
"want_loss": 1.403943,
"want_predictions": 2.129,
"want_best_candidate_index": 1,
}, {
"testcase_name":
"previous_ensemble_is_best",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders": [_FakeBuilder("training")],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"previous_ensemble_spec":
lambda: tu.dummy_ensemble_spec("old", random_seed=12),
"want_loss":
-.437,
"want_predictions":
.688,
"want_best_candidate_index":
0,
}, {
"testcase_name":
"previous_ensemble_spec_and_eval_metrics",
"ensemble_builder":
_FakeEnsembleBuilder(eval_metric_ops_fn=lambda:
{"a": (tf.constant(1), tf.constant(2))}),
"subnetwork_builders": [_FakeBuilder("training")],
"mode":
tf.estimator.ModeKeys.EVAL,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"previous_ensemble_spec":
lambda: tu.dummy_ensemble_spec(
"old",
eval_metrics=tu.create_ensemble_metrics(
metric_fn=lambda: {"a":
(tf.constant(1), tf.constant(2))})),
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_eval_metric_ops": ["a", "iteration"],
"want_best_candidate_index":
1,
}, {
"testcase_name":
"two_subnetwork_fns",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.40394,
"want_predictions":
2.129,
"want_best_candidate_index":
0,
}, {
"testcase_name":
"two_subnetwork_fns_other_best",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=12)
],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
-.437,
"want_predictions":
.688,
"want_best_candidate_index":
1,
}, {
"testcase_name":
"two_subnetwork_one_training_fns",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders":
[_FakeBuilder("training"),
_FakeBuilder("done", random_seed=7)],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_best_candidate_index":
0,
}, {
"testcase_name":
"two_subnetwork_done_training_fns",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders":
[_FakeBuilder("done"),
_FakeBuilder("done1", random_seed=7)],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_best_candidate_index":
0,
}, {
"testcase_name":
"two_dict_predictions_subnetwork_fns",
"ensemble_builder":
_FakeEnsembleBuilder(dict_predictions=True),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.404,
"want_predictions": {
"classes": 2,
"logits": 2.129
},
"want_best_candidate_index":
0,
}, {
"testcase_name":
"two_dict_predictions_subnetwork_fns_predict_classes",
"ensemble_builder":
_FakeEnsembleBuilder(
dict_predictions=True,
export_output_key=tu.ExportOutputKeys.CLASSIFICATION_CLASSES),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"mode":
tf.estimator.ModeKeys.PREDICT,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.404,
"want_predictions": {
"classes": 2,
"logits": 2.129
},
"want_best_candidate_index":
0,
"want_export_outputs": {
tu.ExportOutputKeys.CLASSIFICATION_CLASSES: [2.129],
"serving_default": [2.129],
},
}, {
"testcase_name":
"two_dict_predictions_subnetwork_fns_predict_scores",
"ensemble_builder":
_FakeEnsembleBuilder(
dict_predictions=True,
export_output_key=tu.ExportOutputKeys.CLASSIFICATION_SCORES),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"mode":
tf.estimator.ModeKeys.PREDICT,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.404,
"want_predictions": {
"classes": 2,
"logits": 2.129
},
"want_best_candidate_index":
0,
"want_export_outputs": {
tu.ExportOutputKeys.CLASSIFICATION_SCORES: [2.129],
"serving_default": [2.129],
},
}, {
"testcase_name":
"two_dict_predictions_subnetwork_fns_predict_regression",
"ensemble_builder":
_FakeEnsembleBuilder(
dict_predictions=True,
export_output_key=tu.ExportOutputKeys.REGRESSION),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"mode":
tf.estimator.ModeKeys.PREDICT,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_predictions": {
"classes": 2,
"logits": 2.129
},
"want_best_candidate_index":
0,
"want_export_outputs": {
tu.ExportOutputKeys.REGRESSION: 2.129,
"serving_default": 2.129,
},
}, {
"testcase_name":
"two_dict_predictions_subnetwork_fns_predict_prediction",
"ensemble_builder":
_FakeEnsembleBuilder(
dict_predictions=True,
export_output_key=tu.ExportOutputKeys.PREDICTION),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"mode":
tf.estimator.ModeKeys.PREDICT,
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_predictions": {
"classes": 2,
"logits": 2.129
},
"want_best_candidate_index":
0,
"want_export_outputs": {
tu.ExportOutputKeys.PREDICTION: {
"classes": 2,
"logits": 2.129
},
"serving_default": {
"classes": 2,
"logits": 2.129
},
},
}, {
"testcase_name":
"chief_session_run_hook",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders":
[_FakeBuilder("training", chief_hook=tu.ModifierSessionRunHook())],
"features":
lambda: [[1., -1., 0.]],
"labels":
lambda: [1],
"want_loss":
1.403943,
"want_predictions":
2.129,
"want_best_candidate_index":
0,
"want_chief_hooks":
True,
})
@test_util.run_in_graph_and_eager_modes
def test_build_iteration(self,
ensemble_builder,
subnetwork_builders,
features,
labels,
want_predictions,
want_best_candidate_index,
want_eval_metric_ops=(),
previous_ensemble_spec=lambda: None,
want_loss=None,
want_export_outputs=None,
mode=tf.estimator.ModeKeys.TRAIN,
summary_maker=_ScopedSummary,
want_chief_hooks=False):
with context.graph_mode():
tf_compat.v1.train.create_global_step()
builder = _IterationBuilder(_FakeCandidateBuilder(),
_FakeSubnetworkManager(),
ensemble_builder,
summary_maker=summary_maker,
ensemblers=[_FakeEnsembler()],
max_steps=1)
iteration = builder.build_iteration(
base_global_step=0,
iteration_number=0,
ensemble_candidates=[
EnsembleCandidate(b.name, [b], None)
for b in subnetwork_builders
],
subnetwork_builders=subnetwork_builders,
features=features(),
labels=labels(),
mode=mode,
config=tf.estimator.RunConfig(
model_dir=self.test_subdirectory),
previous_ensemble_spec=previous_ensemble_spec())
init = tf.group(tf_compat.v1.global_variables_initializer(),
tf_compat.v1.local_variables_initializer())
self.evaluate(init)
estimator_spec = iteration.estimator_spec
if want_chief_hooks:
self.assertNotEmpty(
iteration.estimator_spec.training_chief_hooks)
self.assertAllClose(want_predictions,
self.evaluate(estimator_spec.predictions),
atol=1e-3)
# A default architecture metric is always included, even if we don't
# specify one.
eval_metric_ops = estimator_spec.eval_metric_ops
if "architecture/adanet/ensembles" in eval_metric_ops:
del eval_metric_ops["architecture/adanet/ensembles"]
self.assertEqual(set(want_eval_metric_ops),
set(eval_metric_ops.keys()))
self.assertEqual(want_best_candidate_index,
self.evaluate(iteration.best_candidate_index))
if mode == tf.estimator.ModeKeys.PREDICT:
self.assertIsNotNone(estimator_spec.export_outputs)
self.assertAllClose(want_export_outputs,
self.evaluate(
_export_output_tensors(
estimator_spec.export_outputs)),
atol=1e-3)
self.assertIsNone(iteration.estimator_spec.train_op)
self.assertIsNone(iteration.estimator_spec.loss)
self.assertIsNotNone(want_export_outputs)
return
self.assertAlmostEqual(want_loss,
self.evaluate(
iteration.estimator_spec.loss),
places=3)
self.assertIsNone(iteration.estimator_spec.export_outputs)
if mode == tf.estimator.ModeKeys.TRAIN:
self.evaluate(iteration.estimator_spec.train_op)
@parameterized.named_parameters(
{
"testcase_name": "empty_subnetwork_builders",
"ensemble_builder": _FakeEnsembleBuilder(),
"subnetwork_builders": [],
"want_raises": ValueError,
}, {
"testcase_name":
"same_subnetwork_builder_names",
"ensemble_builder":
_FakeEnsembleBuilder(),
"subnetwork_builders":
[_FakeBuilder("same_name"),
_FakeBuilder("same_name")],
"want_raises":
ValueError,
}, {
"testcase_name":
"same_name_as_previous_ensemble_spec",
"ensemble_builder":
_FakeEnsembleBuilder(),
"previous_ensemble_spec_fn":
lambda: tu.dummy_ensemble_spec("same_name"),
"subnetwork_builders": [
_FakeBuilder("same_name"),
],
"want_raises":
ValueError,
}, {
"testcase_name":
"predict_invalid",
"ensemble_builder":
_FakeEnsembleBuilder(
dict_predictions=True,
export_output_key=tu.ExportOutputKeys.INVALID),
"subnetwork_builders": [
_FakeBuilder("training"),
_FakeBuilder("training2", random_seed=7)
],
"mode":
tf.estimator.ModeKeys.PREDICT,
"want_raises":
TypeError,
})
@test_util.run_in_graph_and_eager_modes
def test_build_iteration_error(self,
ensemble_builder,
subnetwork_builders,
want_raises,
previous_ensemble_spec_fn=lambda: None,
mode=tf.estimator.ModeKeys.TRAIN,
summary_maker=_ScopedSummary):
with context.graph_mode():
builder = _IterationBuilder(_FakeCandidateBuilder(),
_FakeSubnetworkManager(),
ensemble_builder,
summary_maker=summary_maker,
ensemblers=[_FakeEnsembler()],
max_steps=100)
features = [[1., -1., 0.]]
labels = [1]
with self.assertRaises(want_raises):
builder.build_iteration(
base_global_step=0,
iteration_number=0,
ensemble_candidates=[
EnsembleCandidate("test", subnetwork_builders, None)
],
subnetwork_builders=subnetwork_builders,
features=features,
labels=labels,
mode=mode,
config=tf.estimator.RunConfig(
model_dir=self.test_subdirectory),
previous_ensemble_spec=previous_ensemble_spec_fn())