def test_metric_ops_not_duplicated_on_cpu(self): with context.graph_mode(): self.setup_graph() metric_fn = lambda: {"metric": (tf.constant(5), tf.constant(5))} best_candidate_index = 3 mode = tf.estimator.ModeKeys.EVAL ensemble_metrics = tu.create_ensemble_metrics(metric_fn) subnetwork_metrics = tu.create_subnetwork_metrics(metric_fn) iteration_metrics = tu.create_iteration_metrics( ensemble_metrics=[ensemble_metrics], subnetwork_metrics=[subnetwork_metrics]) ensemble_ops1 = ensemble_metrics.eval_metrics_ops() ensemble_ops2 = ensemble_metrics.eval_metrics_ops() subnetwork_ops1 = subnetwork_metrics.eval_metrics_ops() subnetwork_ops2 = subnetwork_metrics.eval_metrics_ops() iteration_ops1 = iteration_metrics.best_eval_metric_ops( best_candidate_index, mode) iteration_ops2 = iteration_metrics.best_eval_metric_ops( best_candidate_index, mode) self.assertEqual(subnetwork_ops1, subnetwork_ops2) self.assertEqual(ensemble_ops1, ensemble_ops2) self.assertEqual(iteration_ops1, iteration_ops2) for ops in [ensemble_ops1, subnetwork_ops1, iteration_ops1]: self.assertIsNotNone(ops)
def test_ensemble_metrics(self): with context.graph_mode(): self.setup_graph() architecture = _Architecture("test_ensemble_candidate", "test_ensembler") architecture.add_subnetwork(iteration_number=0, builder_name="b_0_0") architecture.add_subnetwork(iteration_number=0, builder_name="b_0_1") architecture.add_subnetwork(iteration_number=1, builder_name="b_1_0") architecture.add_subnetwork(iteration_number=2, builder_name="b_2_0") metrics = tu.create_ensemble_metrics( self._metric_fn, features=self._features, labels=self._labels, estimator_spec=self._estimator_spec, architecture=architecture) actual = self._run_metrics(metrics.eval_metrics_tuple()) serialized_arch_proto = actual["architecture/adanet/ensembles"] expected_arch_string = b"| b_0_0 | b_0_1 | b_1_0 | b_2_0 |" self.assertIn(expected_arch_string, serialized_arch_proto)
def build_ensemble_spec(self, name, candidate, ensembler, subnetwork_specs, summary, features, mode, iteration_number, labels=None, previous_ensemble_spec=None, my_ensemble_index=None, params=None, previous_iteration_checkpoint=None): del ensembler del subnetwork_specs del summary del features del mode del labels del iteration_number del params del my_ensemble_index del previous_iteration_checkpoint num_subnetworks = 0 if previous_ensemble_spec: num_subnetworks += 1 return tu.dummy_ensemble_spec( name=name, num_subnetworks=num_subnetworks, random_seed=candidate.subnetwork_builders[0].seed, subnetwork_builders=candidate.subnetwork_builders, dict_predictions=self._dict_predictions, eval_metrics=tu.create_ensemble_metrics( metric_fn=self._eval_metric_ops_fn), export_output_key=self._export_output_key, variables=[tf.Variable(1.)])
def test_iteration_metrics(self, use_tpu, mode): with context.graph_mode(): self.setup_graph() best_candidate_index = 3 ensemble_metrics = [] for i in range(10): def metric_fn(val=i): metric = tf.keras.metrics.Mean() metric.update_state(tf.constant(val)) return { "ensemble_v1_metric": tf_compat.v1.metrics.mean(tf.constant(val)), "ensemble_keras_metric": metric } ensemble_metrics.append(tu.create_ensemble_metrics(metric_fn)) metrics = tu.create_iteration_metrics( ensemble_metrics=ensemble_metrics) metrics_fn = (metrics.best_eval_metrics_tuple if use_tpu else metrics.best_eval_metric_ops) actual = self._run_metrics( metrics_fn(tf.constant(best_candidate_index), mode) or {}) if mode == tf.estimator.ModeKeys.EVAL: expected = { "ensemble_v1_metric": best_candidate_index, "ensemble_keras_metric": best_candidate_index, "iteration": 1 } # We don't actually provide an architecture, so the default will be # inside. del actual["architecture/adanet/ensembles"] else: expected = {} self.assertEqual(actual, expected)
class IterationBuilderTest(tu.AdanetTestCase): @parameterized.named_parameters( { "testcase_name": "negative_max_steps", "max_steps": -1, }, { "testcase_name": "zero_max_steps", "max_steps": 0, }) @test_util.run_in_graph_and_eager_modes def test_init_errors(self, max_steps): with self.assertRaises(ValueError): _IterationBuilder(_FakeCandidateBuilder(), _FakeSubnetworkManager(), _FakeEnsembleBuilder(), summary_maker=_ScopedSummary, ensemblers=[_FakeEnsembler()], max_steps=max_steps) # pylint: disable=g-long-lambda @parameterized.named_parameters( { "testcase_name": "single_subnetwork_fn", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("training")], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 0, }, { "testcase_name": "single_subnetwork_fn_mock_summary", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("training")], "summary_maker": functools.partial(_TPUScopedSummary, logdir="/tmp/fakedir"), "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 0, }, { "testcase_name": "single_subnetwork_with_eval_metrics", "ensemble_builder": _FakeEnsembleBuilder(eval_metric_ops_fn=lambda: {"a": (tf.constant(1), tf.constant(2))}), "subnetwork_builders": [ _FakeBuilder("training", ), ], "mode": tf.estimator.ModeKeys.EVAL, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_eval_metric_ops": ["a", "iteration"], "want_best_candidate_index": 0, }, { "testcase_name": "single_subnetwork_with_non_tensor_eval_metric_op", "ensemble_builder": _FakeEnsembleBuilder(eval_metric_ops_fn=lambda: {"a": (tf.constant(1), tf.no_op())}), "subnetwork_builders": [ _FakeBuilder("training", ), ], "mode": tf.estimator.ModeKeys.EVAL, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_eval_metric_ops": ["a", "iteration"], "want_best_candidate_index": 0, }, { "testcase_name": "single_subnetwork_done_training_fn", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("done")], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 0, }, { "testcase_name": "single_dict_predictions_subnetwork_fn", "ensemble_builder": _FakeEnsembleBuilder(dict_predictions=True), "subnetwork_builders": [_FakeBuilder("training")], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": { "classes": 2, "logits": 2.129 }, "want_best_candidate_index": 0, }, { "testcase_name": "previous_ensemble", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("training")], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "previous_ensemble_spec": lambda: tu.dummy_ensemble_spec("old"), "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 1, }, { "testcase_name": "previous_ensemble_is_best", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("training")], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "previous_ensemble_spec": lambda: tu.dummy_ensemble_spec("old", random_seed=12), "want_loss": -.437, "want_predictions": .688, "want_best_candidate_index": 0, }, { "testcase_name": "previous_ensemble_spec_and_eval_metrics", "ensemble_builder": _FakeEnsembleBuilder(eval_metric_ops_fn=lambda: {"a": (tf.constant(1), tf.constant(2))}), "subnetwork_builders": [_FakeBuilder("training")], "mode": tf.estimator.ModeKeys.EVAL, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "previous_ensemble_spec": lambda: tu.dummy_ensemble_spec( "old", eval_metrics=tu.create_ensemble_metrics( metric_fn=lambda: {"a": (tf.constant(1), tf.constant(2))})), "want_loss": 1.403943, "want_predictions": 2.129, "want_eval_metric_ops": ["a", "iteration"], "want_best_candidate_index": 1, }, { "testcase_name": "two_subnetwork_fns", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.40394, "want_predictions": 2.129, "want_best_candidate_index": 0, }, { "testcase_name": "two_subnetwork_fns_other_best", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=12) ], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": -.437, "want_predictions": .688, "want_best_candidate_index": 1, }, { "testcase_name": "two_subnetwork_one_training_fns", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("training"), _FakeBuilder("done", random_seed=7)], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 0, }, { "testcase_name": "two_subnetwork_done_training_fns", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("done"), _FakeBuilder("done1", random_seed=7)], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 0, }, { "testcase_name": "two_dict_predictions_subnetwork_fns", "ensemble_builder": _FakeEnsembleBuilder(dict_predictions=True), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.404, "want_predictions": { "classes": 2, "logits": 2.129 }, "want_best_candidate_index": 0, }, { "testcase_name": "two_dict_predictions_subnetwork_fns_predict_classes", "ensemble_builder": _FakeEnsembleBuilder( dict_predictions=True, export_output_key=tu.ExportOutputKeys.CLASSIFICATION_CLASSES), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "mode": tf.estimator.ModeKeys.PREDICT, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.404, "want_predictions": { "classes": 2, "logits": 2.129 }, "want_best_candidate_index": 0, "want_export_outputs": { tu.ExportOutputKeys.CLASSIFICATION_CLASSES: [2.129], "serving_default": [2.129], }, }, { "testcase_name": "two_dict_predictions_subnetwork_fns_predict_scores", "ensemble_builder": _FakeEnsembleBuilder( dict_predictions=True, export_output_key=tu.ExportOutputKeys.CLASSIFICATION_SCORES), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "mode": tf.estimator.ModeKeys.PREDICT, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.404, "want_predictions": { "classes": 2, "logits": 2.129 }, "want_best_candidate_index": 0, "want_export_outputs": { tu.ExportOutputKeys.CLASSIFICATION_SCORES: [2.129], "serving_default": [2.129], }, }, { "testcase_name": "two_dict_predictions_subnetwork_fns_predict_regression", "ensemble_builder": _FakeEnsembleBuilder( dict_predictions=True, export_output_key=tu.ExportOutputKeys.REGRESSION), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "mode": tf.estimator.ModeKeys.PREDICT, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_predictions": { "classes": 2, "logits": 2.129 }, "want_best_candidate_index": 0, "want_export_outputs": { tu.ExportOutputKeys.REGRESSION: 2.129, "serving_default": 2.129, }, }, { "testcase_name": "two_dict_predictions_subnetwork_fns_predict_prediction", "ensemble_builder": _FakeEnsembleBuilder( dict_predictions=True, export_output_key=tu.ExportOutputKeys.PREDICTION), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "mode": tf.estimator.ModeKeys.PREDICT, "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_predictions": { "classes": 2, "logits": 2.129 }, "want_best_candidate_index": 0, "want_export_outputs": { tu.ExportOutputKeys.PREDICTION: { "classes": 2, "logits": 2.129 }, "serving_default": { "classes": 2, "logits": 2.129 }, }, }, { "testcase_name": "chief_session_run_hook", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("training", chief_hook=tu.ModifierSessionRunHook())], "features": lambda: [[1., -1., 0.]], "labels": lambda: [1], "want_loss": 1.403943, "want_predictions": 2.129, "want_best_candidate_index": 0, "want_chief_hooks": True, }) @test_util.run_in_graph_and_eager_modes def test_build_iteration(self, ensemble_builder, subnetwork_builders, features, labels, want_predictions, want_best_candidate_index, want_eval_metric_ops=(), previous_ensemble_spec=lambda: None, want_loss=None, want_export_outputs=None, mode=tf.estimator.ModeKeys.TRAIN, summary_maker=_ScopedSummary, want_chief_hooks=False): with context.graph_mode(): tf_compat.v1.train.create_global_step() builder = _IterationBuilder(_FakeCandidateBuilder(), _FakeSubnetworkManager(), ensemble_builder, summary_maker=summary_maker, ensemblers=[_FakeEnsembler()], max_steps=1) iteration = builder.build_iteration( base_global_step=0, iteration_number=0, ensemble_candidates=[ EnsembleCandidate(b.name, [b], None) for b in subnetwork_builders ], subnetwork_builders=subnetwork_builders, features=features(), labels=labels(), mode=mode, config=tf.estimator.RunConfig( model_dir=self.test_subdirectory), previous_ensemble_spec=previous_ensemble_spec()) init = tf.group(tf_compat.v1.global_variables_initializer(), tf_compat.v1.local_variables_initializer()) self.evaluate(init) estimator_spec = iteration.estimator_spec if want_chief_hooks: self.assertNotEmpty( iteration.estimator_spec.training_chief_hooks) self.assertAllClose(want_predictions, self.evaluate(estimator_spec.predictions), atol=1e-3) # A default architecture metric is always included, even if we don't # specify one. eval_metric_ops = estimator_spec.eval_metric_ops if "architecture/adanet/ensembles" in eval_metric_ops: del eval_metric_ops["architecture/adanet/ensembles"] self.assertEqual(set(want_eval_metric_ops), set(eval_metric_ops.keys())) self.assertEqual(want_best_candidate_index, self.evaluate(iteration.best_candidate_index)) if mode == tf.estimator.ModeKeys.PREDICT: self.assertIsNotNone(estimator_spec.export_outputs) self.assertAllClose(want_export_outputs, self.evaluate( _export_output_tensors( estimator_spec.export_outputs)), atol=1e-3) self.assertIsNone(iteration.estimator_spec.train_op) self.assertIsNone(iteration.estimator_spec.loss) self.assertIsNotNone(want_export_outputs) return self.assertAlmostEqual(want_loss, self.evaluate( iteration.estimator_spec.loss), places=3) self.assertIsNone(iteration.estimator_spec.export_outputs) if mode == tf.estimator.ModeKeys.TRAIN: self.evaluate(iteration.estimator_spec.train_op) @parameterized.named_parameters( { "testcase_name": "empty_subnetwork_builders", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [], "want_raises": ValueError, }, { "testcase_name": "same_subnetwork_builder_names", "ensemble_builder": _FakeEnsembleBuilder(), "subnetwork_builders": [_FakeBuilder("same_name"), _FakeBuilder("same_name")], "want_raises": ValueError, }, { "testcase_name": "same_name_as_previous_ensemble_spec", "ensemble_builder": _FakeEnsembleBuilder(), "previous_ensemble_spec_fn": lambda: tu.dummy_ensemble_spec("same_name"), "subnetwork_builders": [ _FakeBuilder("same_name"), ], "want_raises": ValueError, }, { "testcase_name": "predict_invalid", "ensemble_builder": _FakeEnsembleBuilder( dict_predictions=True, export_output_key=tu.ExportOutputKeys.INVALID), "subnetwork_builders": [ _FakeBuilder("training"), _FakeBuilder("training2", random_seed=7) ], "mode": tf.estimator.ModeKeys.PREDICT, "want_raises": TypeError, }) @test_util.run_in_graph_and_eager_modes def test_build_iteration_error(self, ensemble_builder, subnetwork_builders, want_raises, previous_ensemble_spec_fn=lambda: None, mode=tf.estimator.ModeKeys.TRAIN, summary_maker=_ScopedSummary): with context.graph_mode(): builder = _IterationBuilder(_FakeCandidateBuilder(), _FakeSubnetworkManager(), ensemble_builder, summary_maker=summary_maker, ensemblers=[_FakeEnsembler()], max_steps=100) features = [[1., -1., 0.]] labels = [1] with self.assertRaises(want_raises): builder.build_iteration( base_global_step=0, iteration_number=0, ensemble_candidates=[ EnsembleCandidate("test", subnetwork_builders, None) ], subnetwork_builders=subnetwork_builders, features=features, labels=labels, mode=mode, config=tf.estimator.RunConfig( model_dir=self.test_subdirectory), previous_ensemble_spec=previous_ensemble_spec_fn())