class EvaluatorTest(parameterized.TestCase, tf.test.TestCase): @parameterized.named_parameters( { "testcase_name": "choose_index_0", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_0, "want_adanet_losses": [3, 6], }, { "testcase_name": "choose_index_1", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [6, 3], }, { "testcase_name": "none_steps", "input_fn": tu.dataset_input_fn(), "steps": None, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }, { "testcase_name": "input_fn_out_of_range", "input_fn": tu.dataset_input_fn(), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }) def test_adanet_losses(self, input_fn, steps, adanet_losses, want_adanet_losses): with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, steps=steps) adanet_losses = evaluator.evaluate_adanet_losses( sess, adanet_losses(input_fn)) self.assertEqual(want_adanet_losses, adanet_losses)
def test_tpu_estimator_simple_lifecycle(self, use_tpu): config = tf.contrib.tpu.RunConfig(master="", tf_random_seed=42) estimator = TPUEstimator( head=tu.head(), subnetwork_generator=SimpleGenerator( [_DNNBuilder("dnn", use_tpu=use_tpu)]), max_iteration_steps=200, mixture_weight_initializer=tf.zeros_initializer(), use_bias=True, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) max_steps = 300 xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) # Train. estimator.train(input_fn=train_input_fn, steps=None, max_steps=max_steps, hooks=None) # Evaluate. eval_results = estimator.evaluate(input_fn=train_input_fn, steps=10, hooks=None) # Predict. # TODO: skip predictions on TF versions 1.11 and 1.12 since # some TPU hooks seem to be failing on predict. predictions = [] tf_version = LooseVersion(tf.VERSION) if (tf_version != LooseVersion("1.11.0") and tf_version != LooseVersion("1.12.0")): predictions = estimator.predict( input_fn=tu.dataset_input_fn(features=[0., 0.], labels=None)) # Export SavedModel. def serving_input_fn(): """Input fn for serving export, starting from serialized example.""" serialized_example = tf.placeholder(dtype=tf.string, shape=(None), name="serialized_example") return tf.estimator.export.ServingInputReceiver( features={"x": tf.constant([[0., 0.]], name="serving_x")}, receiver_tensors=serialized_example) export_saved_model_fn = getattr(estimator, "export_saved_model", None) if not callable(export_saved_model_fn): export_saved_model_fn = estimator.export_savedmodel export_saved_model_fn(export_dir_base=estimator.model_dir, serving_input_receiver_fn=serving_input_fn) self.assertAlmostEqual(0.32416, eval_results["loss"], places=3) self.assertEqual(max_steps, eval_results["global_step"]) for prediction in predictions: self.assertIsNotNone(prediction["predictions"])
def test_tpu_estimator_simple_lifecycle(self, use_tpu, subnetwork_generator, want_loss): config = tf_compat.v1.estimator.tpu.RunConfig(master="", tf_random_seed=42) estimator = TPUEstimator(head=tf.contrib.estimator.regression_head( loss_reduction=tf_compat.v1.losses.Reduction.SUM_OVER_BATCH_SIZE), subnetwork_generator=subnetwork_generator, max_iteration_steps=10, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) max_steps = 30 xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) # Train. estimator.train(input_fn=train_input_fn, steps=None, max_steps=max_steps, hooks=None) # Evaluate. eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1, hooks=None) # Predict. predictions = estimator.predict( input_fn=tu.dataset_input_fn(features=[0., 0.], labels=None)) # Export SavedModel. def serving_input_fn(): """Input fn for serving export, starting from serialized example.""" serialized_example = tf.placeholder(dtype=tf.string, shape=(None), name="serialized_example") return tf.estimator.export.ServingInputReceiver( features={"x": tf.constant([[0., 0.]], name="serving_x")}, receiver_tensors=serialized_example) export_saved_model_fn = getattr(estimator, "export_saved_model", None) if not callable(export_saved_model_fn): export_saved_model_fn = estimator.export_savedmodel export_saved_model_fn(export_dir_base=estimator.model_dir, serving_input_receiver_fn=serving_input_fn) self.assertAlmostEqual(want_loss, eval_results["loss"], places=2) self.assertEqual(max_steps, eval_results["global_step"]) self.assertEqual(2, eval_results["iteration"]) for prediction in predictions: self.assertIsNotNone(prediction["predictions"])
def test_tpu_estimator_simple_lifecycle(self, use_tpu, subnetwork_generator, want_loss): config = tf.compat.v1.estimator.tpu.RunConfig(master="", tf_random_seed=42) estimator = TPUEstimator( # TODO: Add test with estimator Head v2. head=make_regression_head(use_tpu), subnetwork_generator=subnetwork_generator, max_iteration_steps=10, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) max_steps = 30 xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) # Train. estimator.train(input_fn=train_input_fn, steps=None, max_steps=max_steps, hooks=None) # Evaluate. eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1, hooks=None) # Predict. predictions = estimator.predict(input_fn=tu.dataset_input_fn( features=[0., 0.], return_dataset=True)) # We need to iterate over all the predictions before moving on, otherwise # the TPU will not be shut down. for prediction in predictions: self.assertIsNotNone(prediction["predictions"]) # Export SavedModel. def serving_input_fn(): """Input fn for serving export, starting from serialized example.""" serialized_example = tf.compat.v1.placeholder( dtype=tf.string, shape=(None), name="serialized_example") return tf.estimator.export.ServingInputReceiver( features={"x": tf.constant([[0., 0.]], name="serving_x")}, receiver_tensors=serialized_example) estimator.export_saved_model( export_dir_base=estimator.model_dir, serving_input_receiver_fn=serving_input_fn) self.assertAlmostEqual(want_loss, eval_results["loss"], places=2) self.assertEqual(max_steps, eval_results["global_step"]) self.assertEqual(2, eval_results["iteration"])
def test_tpu_estimator_summaries(self): config = tf.contrib.tpu.RunConfig(tf_random_seed=42) estimator = TPUEstimator( head=tu.head(), subnetwork_generator=SimpleGenerator([_DNNBuilder("dnn")]), max_iteration_steps=200, model_dir=self.test_subdirectory, config=config) train_input_fn = tu.dummy_input_fn([[1., 0.]], [[1.]]) with fake_run_on_tpu(): estimator.train(input_fn=train_input_fn, max_steps=3) estimator.evaluate(input_fn=train_input_fn, steps=3) self.assertFalse( _summaries_exist(self.test_subdirectory + "/candidate/t0_dnn")) self.assertTrue( _summaries_exist(self.test_subdirectory + "/candidate/t0_dnn/eval"))
def test_evaluate_invalid_metric(self): input_fn = tu.dummy_input_fn([[1., 2]], [[3.]]) _, labels = input_fn() predictions = [labels * 2, labels * 3] metrics = [] for preds in predictions: metrics.append({ "mse": tf.metrics.mean_squared_error(labels, preds), "other_metric_1": (tf.constant(1), tf.constant(1)), "other_metric_2": (tf.constant(2), tf.constant(2)) }) with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, metric_name="dne", steps=3) with self.assertRaises(KeyError): metrics = evaluator.evaluate(sess, ensemble_metrics=metrics)
def test_evaluate(self): with context.graph_mode(): input_fn = tu.dummy_input_fn([[1., 2]], [[3.]]) _, labels = input_fn() predictions = [labels * 2, labels * 3] metrics = [] for preds in predictions: metrics.append({ "mse": tf_compat.v1.metrics.mean_squared_error(labels, preds), "other_metric_1": (tf.constant(1), tf.constant(1)), "other_metric_2": (tf.constant(2), tf.constant(2)) }) with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, metric_name="mse", steps=3) metrics = evaluator.evaluate(sess, ensemble_metrics=metrics) self.assertEqual([9, 36], metrics)
def test_tpu_estimator_summaries(self, use_tpu, want_loss, want_adanet_loss, want_eval_summary_loss, want_predictions): config = tf.contrib.tpu.RunConfig(tf_random_seed=42, save_summary_steps=100, log_step_count_steps=100) assert config.log_step_count_steps def metric_fn(predictions): return { "predictions": tf_compat.v1.metrics.mean(predictions["predictions"]) } max_steps = 100 estimator = TPUEstimator(head=tu.head(), subnetwork_generator=SimpleGenerator( [_DNNBuilder("dnn", use_tpu=use_tpu)]), max_iteration_steps=max_steps, model_dir=self.test_subdirectory, metric_fn=metric_fn, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) estimator.train(input_fn=train_input_fn, max_steps=max_steps) eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1) self.assertAlmostEqual(want_loss, eval_results["loss"], places=2) self.assertEqual(max_steps, eval_results["global_step"]) self.assertEqual(0, eval_results["iteration"]) subnetwork_subdir = os.path.join(self.test_subdirectory, "subnetwork/t0_dnn") ensemble_subdir = os.path.join( self.test_subdirectory, "ensemble/t0_dnn_grow_complexity_regularized") # TODO: Why is the adanet_loss written to 'loss'? self.assertAlmostEqual(want_adanet_loss, _get_summary_value("loss", self.test_subdirectory), places=1) self.assertEqual( 0., _get_summary_value("iteration/adanet/iteration", self.test_subdirectory)) self.assertAlmostEqual(3., _get_summary_value("scalar", subnetwork_subdir), places=3) self.assertEqual((3, 3, 1), _get_summary_value("image/image/0", subnetwork_subdir)) self.assertAlmostEqual(5., _get_summary_value("nested/scalar", subnetwork_subdir), places=3) self.assertAlmostEqual( want_adanet_loss, _get_summary_value("adanet_loss/adanet/adanet_weighted_ensemble", ensemble_subdir), places=1) self.assertAlmostEqual( 0., _get_summary_value( "complexity_regularization/adanet/adanet_weighted_ensemble", ensemble_subdir), places=1) self.assertAlmostEqual(1., _get_summary_value( "mixture_weight_norms/adanet/" "adanet_weighted_ensemble/subnetwork_0", ensemble_subdir), places=1) # Eval metric summaries are always written out during eval. subnetwork_eval_subdir = os.path.join(subnetwork_subdir, "eval") self.assertAlmostEqual(want_eval_summary_loss, _get_summary_value("loss", subnetwork_eval_subdir), places=1) self.assertAlmostEqual(want_eval_summary_loss, _get_summary_value("average_loss", subnetwork_eval_subdir), places=1) self.assertAlmostEqual(want_predictions, _get_summary_value("predictions", subnetwork_eval_subdir), places=3) eval_subdir = os.path.join(self.test_subdirectory, "eval") ensemble_eval_subdir = os.path.join(ensemble_subdir, "eval") for subdir in [ensemble_eval_subdir, eval_subdir]: self.assertEqual([b"| dnn |"], _get_summary_value( "architecture/adanet/ensembles/0", subdir)) if subdir == eval_subdir: self.assertAlmostEqual(want_loss, _get_summary_value("loss", subdir), places=1) self.assertAlmostEqual(want_eval_summary_loss, _get_summary_value("average_loss", subdir), places=1)
class ReportMaterializerTest(parameterized.TestCase, tf.test.TestCase): # pylint: disable=g-long-lambda @parameterized.named_parameters( { "testcase_name": "one_empty_subnetwork", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report(hparams={}, attributes={}, metrics={}), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "one_subnetwork", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "one_subnetwork_iteration_2", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "iteration_number": 2, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=2, name="foo", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "two_subnetworks", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), "foo2": subnetwork.Report( hparams={ "learning_rate": 1.e-6, "optimizer": "sgd", "num_layers": 1, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.1445), "foo": tf.constant("baz"), "parameters": tf.constant(7788), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo2"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=False, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={ "learning_rate": 1.e-6, "optimizer": "sgd", "num_layers": 1, "use_side_inputs": True, }, attributes={ "weight_norms": 3.1445, "foo": "baz", "parameters": 7788, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "two_subnetworks_zero_included", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), "foo2": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), }, "steps": 3, "included_subnetwork_names": [], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=False, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=False, ), ], }, { "testcase_name": "two_subnetworks_both_included", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), "foo2": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo1", "foo2"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics", "input_fn": tu.dummy_input_fn([[1., 1.], [1., 1.], [1., 1.]], [[1.], [2.], [3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": tf_compat.v1.metrics.mean(labels)}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 2.}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics_none_steps", "input_fn": tu.dataset_input_fn([[1., 1.], [1., 1.], [1., 1.]], [[1.], [2.], [3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": tf_compat.v1.metrics.mean(labels)}, ), }, "steps": None, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 2.}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics_non_tensor_op", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": (tf.constant(42), tf.no_op())}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 42}, included_in_final_ensemble=True, ), ], }) @test_util.run_in_graph_and_eager_modes def test_materialize_subnetwork_reports(self, input_fn, subnetwork_reports_fn, steps, iteration_number=0, included_subnetwork_names=None, want_materialized_reports=None): with context.graph_mode(): tf.constant(0.) # dummy op so that the session graph is never empty. features, labels = input_fn() subnetwork_reports = subnetwork_reports_fn(features, labels) with self.test_session() as sess: sess.run(tf_compat.v1.initializers.local_variables()) report_materializer = ReportMaterializer(input_fn=input_fn, steps=steps) materialized_reports = ( report_materializer.materialize_subnetwork_reports( sess, iteration_number, subnetwork_reports, included_subnetwork_names)) self.assertEqual( len(want_materialized_reports), len(materialized_reports)) materialized_reports_dict = { blrm.name: blrm for blrm in materialized_reports } for want_materialized_report in want_materialized_reports: materialized_report = ( materialized_reports_dict[want_materialized_report.name]) self.assertEqual(iteration_number, materialized_report.iteration_number) self.assertEqual( set(want_materialized_report.hparams.keys()), set(materialized_report.hparams.keys())) for hparam_key, want_hparam in ( want_materialized_report.hparams.items()): if isinstance(want_hparam, float): self.assertAllClose(want_hparam, materialized_report.hparams[hparam_key]) else: self.assertEqual(want_hparam, materialized_report.hparams[hparam_key]) self.assertSetEqual( set(want_materialized_report.attributes.keys()), set(materialized_report.attributes.keys())) for attribute_key, want_attribute in ( want_materialized_report.attributes.items()): if isinstance(want_attribute, float): self.assertAllClose( want_attribute, decode(materialized_report.attributes[attribute_key])) else: self.assertEqual( want_attribute, decode(materialized_report.attributes[attribute_key])) self.assertSetEqual( set(want_materialized_report.metrics.keys()), set(materialized_report.metrics.keys())) for metric_key, want_metric in ( want_materialized_report.metrics.items()): if isinstance(want_metric, float): self.assertAllClose( want_metric, decode(materialized_report.metrics[metric_key])) else: self.assertEqual(want_metric, decode(materialized_report.metrics[metric_key]))
def test_tpu_estimator_summaries(self, use_tpu): config = tf.contrib.tpu.RunConfig(tf_random_seed=42, save_summary_steps=2, log_step_count_steps=1) assert config.log_step_count_steps estimator = TPUEstimator(head=tu.head(), subnetwork_generator=SimpleGenerator( [_DNNBuilder("dnn", use_tpu=use_tpu)]), max_iteration_steps=200, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) estimator.train(input_fn=train_input_fn, max_steps=3) estimator.evaluate(input_fn=train_input_fn, steps=3) ensemble_loss = .5 self.assertAlmostEqual(ensemble_loss, _check_eventfile_for_keyword( "loss", self.test_subdirectory), places=1) self.assertIsNotNone( _check_eventfile_for_keyword("global_step/sec", self.test_subdirectory)) eval_subdir = os.path.join(self.test_subdirectory, "eval") self.assertAlmostEqual(ensemble_loss, _check_eventfile_for_keyword( "loss", eval_subdir), places=1) self.assertEqual( 0., _check_eventfile_for_keyword("iteration/adanet/iteration", self.test_subdirectory)) candidate_subdir = os.path.join(self.test_subdirectory, "candidate/t0_dnn") self.assertAlmostEqual(3., _check_eventfile_for_keyword( "scalar", candidate_subdir), places=3) self.assertEqual((3, 3, 1), _check_eventfile_for_keyword("image/image/0", candidate_subdir)) self.assertAlmostEqual(5., _check_eventfile_for_keyword( "nested/scalar", candidate_subdir), places=1) self.assertAlmostEqual( ensemble_loss, _check_eventfile_for_keyword( "adanet_loss/adanet/adanet_weighted_ensemble", candidate_subdir), places=1) self.assertAlmostEqual( 0., _check_eventfile_for_keyword( "complexity_regularization/adanet/adanet_weighted_ensemble", candidate_subdir), places=1) self.assertAlmostEqual(1., _check_eventfile_for_keyword( "mixture_weight_norms/adanet/" "adanet_weighted_ensemble/subnetwork_0", candidate_subdir), places=1)
def test_tpu_estimator_summaries(self, use_tpu): config = tf.contrib.tpu.RunConfig(tf_random_seed=42, save_summary_steps=2, log_step_count_steps=1) assert config.log_step_count_steps estimator = TPUEstimator(head=tu.head(), subnetwork_generator=SimpleGenerator( [_DNNBuilder("dnn", use_tpu=use_tpu)]), max_iteration_steps=200, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) estimator.train(input_fn=train_input_fn, max_steps=3) estimator.evaluate(input_fn=train_input_fn, steps=3) subnetwork_subdir = os.path.join(self.test_subdirectory, "subnetwork/t0_dnn") ensemble_loss = .5 ensemble_subdir = os.path.join( self.test_subdirectory, "ensemble/t0_dnn_complexity_regularized") self.assertAlmostEqual(ensemble_loss, _get_summary_value("loss", self.test_subdirectory), places=1) self.assertEqual( 0., _get_summary_value("iteration/adanet/iteration", self.test_subdirectory)) self.assertAlmostEqual(3., _get_summary_value("scalar", subnetwork_subdir), places=3) self.assertEqual((3, 3, 1), _get_summary_value("image/image/0", subnetwork_subdir)) self.assertAlmostEqual(5., _get_summary_value("nested/scalar", subnetwork_subdir), places=3) self.assertAlmostEqual( ensemble_loss, _get_summary_value("adanet_loss/adanet/adanet_weighted_ensemble", ensemble_subdir), places=1) self.assertAlmostEqual( 0., _get_summary_value( "complexity_regularization/adanet/adanet_weighted_ensemble", ensemble_subdir), places=1) self.assertAlmostEqual(1., _get_summary_value( "mixture_weight_norms/adanet/" "adanet_weighted_ensemble/subnetwork_0", ensemble_subdir), places=1) # Eval metric summaries are always written out during eval. subnetwork_eval_subdir = os.path.join(subnetwork_subdir, "eval") if use_tpu: # TODO: Why is subnetwork eval loss 0.0 when use_tpu=False? self.assertAlmostEqual(ensemble_loss, _get_summary_value("loss", subnetwork_eval_subdir), places=1) self.assertAlmostEqual(ensemble_loss, _get_summary_value("average_loss", subnetwork_eval_subdir), places=1) eval_subdir = os.path.join(self.test_subdirectory, "eval") ensemble_eval_subdir = os.path.join(ensemble_subdir, "eval") for subdir in [ensemble_eval_subdir, eval_subdir]: self.assertEqual([b"| dnn |"], _get_summary_value( "architecture/adanet/ensembles/0", subdir)) if subdir == eval_subdir: self.assertAlmostEqual(ensemble_loss, _get_summary_value("loss", subdir), places=1) self.assertAlmostEqual(ensemble_loss, _get_summary_value("average_loss", subdir), places=1)
class EvaluatorTest(parameterized.TestCase, tf.test.TestCase): @parameterized.named_parameters( { "testcase_name": "choose_index_0", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_0, "want_adanet_losses": [3, 6], }, { "testcase_name": "choose_index_1", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [6, 3], }, { "testcase_name": "none_steps", "input_fn": tu.dataset_input_fn(), "steps": None, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }, { "testcase_name": "input_fn_out_of_range", "input_fn": tu.dataset_input_fn(), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }) def test_evaluate_no_metric_fn_falls_back_to_adanet_losses( self, input_fn, steps, adanet_losses, want_adanet_losses): adanet_losses = adanet_losses(input_fn) metrics = [{ "adanet_loss": tf.metrics.mean(loss) } for loss in adanet_losses] with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, steps=steps) adanet_losses = evaluator.evaluate(sess, ensemble_metrics=metrics) self.assertEqual(want_adanet_losses, adanet_losses) @parameterized.named_parameters( { "testcase_name": "minimize_returns_nanargmin", "objective": Evaluator.Objective.MAXIMIZE, "expected_objective_fn": np.nanargmax, "metric_fn": lambda x, y: None }, { "testcase_name": "maximize_returns_nanargmax", "objective": Evaluator.Objective.MINIMIZE, "expected_objective_fn": np.nanargmin, "metric_fn": lambda x, y: None }) def test_objective(self, objective, expected_objective_fn, metric_fn=None): evaluator = Evaluator(input_fn=None, objective=objective) self.assertEqual(expected_objective_fn, evaluator.objective_fn) def test_objective_unsupported_objective(self): with self.assertRaises(ValueError): Evaluator(input_fn=None, objective="non_existent_objective") def test_evaluate(self): input_fn = tu.dummy_input_fn([[1., 2]], [[3.]]) _, labels = input_fn() predictions = [labels * 2, labels * 3] metrics = [] for preds in predictions: metrics.append({ "mse": tf.metrics.mean_squared_error(labels, preds), "other_metric_1": (tf.constant(1), tf.constant(1)), "other_metric_2": (tf.constant(2), tf.constant(2)) }) with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, metric_name="mse", steps=3) metrics = evaluator.evaluate(sess, ensemble_metrics=metrics) self.assertEqual([9, 36], metrics) def test_evaluate_invalid_metric(self): input_fn = tu.dummy_input_fn([[1., 2]], [[3.]]) _, labels = input_fn() predictions = [labels * 2, labels * 3] metrics = [] for preds in predictions: metrics.append({ "mse": tf.metrics.mean_squared_error(labels, preds), "other_metric_1": (tf.constant(1), tf.constant(1)), "other_metric_2": (tf.constant(2), tf.constant(2)) }) with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, metric_name="dne", steps=3) with self.assertRaises(KeyError): metrics = evaluator.evaluate(sess, ensemble_metrics=metrics)
def test_summaries(self): """Tests that summaries are written to candidate directory.""" run_config = tf.estimator.RunConfig(tf_random_seed=42, log_step_count_steps=2, save_summary_steps=2, model_dir=self.test_subdirectory) subnetwork_generator = SimpleGenerator([_SimpleBuilder("dnn")]) report_materializer = ReportMaterializer(input_fn=tu.dummy_input_fn( [[1., 1.]], [[0.]]), steps=1) estimator = Estimator(head=regression_head.RegressionHead( loss_reduction=tf_compat.SUM_OVER_BATCH_SIZE), subnetwork_generator=subnetwork_generator, report_materializer=report_materializer, max_iteration_steps=10, config=run_config) train_input_fn = tu.dummy_input_fn([[1., 0.]], [[1.]]) estimator.train(input_fn=train_input_fn, max_steps=3) ensemble_loss = 1.52950 self.assertAlmostEqual(ensemble_loss, tu.check_eventfile_for_keyword( "loss", self.test_subdirectory), places=3) self.assertIsNotNone( tu.check_eventfile_for_keyword("global_step/sec", self.test_subdirectory)) self.assertEqual( 0., tu.check_eventfile_for_keyword("iteration/adanet/iteration", self.test_subdirectory)) subnetwork_subdir = os.path.join(self.test_subdirectory, "subnetwork/t0_dnn") self.assertAlmostEqual(3., tu.check_eventfile_for_keyword( "scalar", subnetwork_subdir), places=3) self.assertEqual( (3, 3, 1), tu.check_eventfile_for_keyword("image", subnetwork_subdir)) self.assertAlmostEqual(5., tu.check_eventfile_for_keyword( "nested/scalar", subnetwork_subdir), places=3) ensemble_subdir = os.path.join( self.test_subdirectory, "ensemble/t0_dnn_grow_complexity_regularized") self.assertAlmostEqual( ensemble_loss, tu.check_eventfile_for_keyword( "adanet_loss/adanet/adanet_weighted_ensemble", ensemble_subdir), places=1) self.assertAlmostEqual( 0., tu.check_eventfile_for_keyword( "complexity_regularization/adanet/adanet_weighted_ensemble", ensemble_subdir), places=3) self.assertAlmostEqual(1., tu.check_eventfile_for_keyword( "mixture_weight_norms/adanet/" "adanet_weighted_ensemble/subnetwork_0", ensemble_subdir), places=3)