class EvaluatorTest(parameterized.TestCase, tf.test.TestCase): @parameterized.named_parameters( { "testcase_name": "choose_index_0", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_0, "want_adanet_losses": [3, 6], }, { "testcase_name": "choose_index_1", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [6, 3], }, { "testcase_name": "none_steps", "input_fn": tu.dataset_input_fn(), "steps": None, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }, { "testcase_name": "input_fn_out_of_range", "input_fn": tu.dataset_input_fn(), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }) def test_adanet_losses(self, input_fn, steps, adanet_losses, want_adanet_losses): with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, steps=steps) adanet_losses = evaluator.evaluate_adanet_losses( sess, adanet_losses(input_fn)) self.assertEqual(want_adanet_losses, adanet_losses)
def test_tpu_estimator_simple_lifecycle(self, use_tpu): config = tf.contrib.tpu.RunConfig(master="", tf_random_seed=42) estimator = TPUEstimator( head=tu.head(), subnetwork_generator=SimpleGenerator( [_DNNBuilder("dnn", use_tpu=use_tpu)]), max_iteration_steps=200, mixture_weight_initializer=tf.zeros_initializer(), use_bias=True, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) max_steps = 300 xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) # Train. estimator.train(input_fn=train_input_fn, steps=None, max_steps=max_steps, hooks=None) # Evaluate. eval_results = estimator.evaluate(input_fn=train_input_fn, steps=10, hooks=None) # Predict. # TODO: skip predictions on TF versions 1.11 and 1.12 since # some TPU hooks seem to be failing on predict. predictions = [] tf_version = LooseVersion(tf.VERSION) if (tf_version != LooseVersion("1.11.0") and tf_version != LooseVersion("1.12.0")): predictions = estimator.predict( input_fn=tu.dataset_input_fn(features=[0., 0.], labels=None)) # Export SavedModel. def serving_input_fn(): """Input fn for serving export, starting from serialized example.""" serialized_example = tf.placeholder(dtype=tf.string, shape=(None), name="serialized_example") return tf.estimator.export.ServingInputReceiver( features={"x": tf.constant([[0., 0.]], name="serving_x")}, receiver_tensors=serialized_example) export_saved_model_fn = getattr(estimator, "export_saved_model", None) if not callable(export_saved_model_fn): export_saved_model_fn = estimator.export_savedmodel export_saved_model_fn(export_dir_base=estimator.model_dir, serving_input_receiver_fn=serving_input_fn) self.assertAlmostEqual(0.32416, eval_results["loss"], places=3) self.assertEqual(max_steps, eval_results["global_step"]) for prediction in predictions: self.assertIsNotNone(prediction["predictions"])
def test_tpu_estimator_simple_lifecycle(self, use_tpu, subnetwork_generator, want_loss): config = tf_compat.v1.estimator.tpu.RunConfig(master="", tf_random_seed=42) estimator = TPUEstimator(head=tf.contrib.estimator.regression_head( loss_reduction=tf_compat.v1.losses.Reduction.SUM_OVER_BATCH_SIZE), subnetwork_generator=subnetwork_generator, max_iteration_steps=10, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) max_steps = 30 xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) # Train. estimator.train(input_fn=train_input_fn, steps=None, max_steps=max_steps, hooks=None) # Evaluate. eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1, hooks=None) # Predict. predictions = estimator.predict( input_fn=tu.dataset_input_fn(features=[0., 0.], labels=None)) # Export SavedModel. def serving_input_fn(): """Input fn for serving export, starting from serialized example.""" serialized_example = tf.placeholder(dtype=tf.string, shape=(None), name="serialized_example") return tf.estimator.export.ServingInputReceiver( features={"x": tf.constant([[0., 0.]], name="serving_x")}, receiver_tensors=serialized_example) export_saved_model_fn = getattr(estimator, "export_saved_model", None) if not callable(export_saved_model_fn): export_saved_model_fn = estimator.export_savedmodel export_saved_model_fn(export_dir_base=estimator.model_dir, serving_input_receiver_fn=serving_input_fn) self.assertAlmostEqual(want_loss, eval_results["loss"], places=2) self.assertEqual(max_steps, eval_results["global_step"]) self.assertEqual(2, eval_results["iteration"]) for prediction in predictions: self.assertIsNotNone(prediction["predictions"])
def test_tpu_estimator_simple_lifecycle(self, use_tpu, subnetwork_generator, want_loss): config = tf.compat.v1.estimator.tpu.RunConfig(master="", tf_random_seed=42) estimator = TPUEstimator( # TODO: Add test with estimator Head v2. head=make_regression_head(use_tpu), subnetwork_generator=subnetwork_generator, max_iteration_steps=10, model_dir=self.test_subdirectory, config=config, use_tpu=use_tpu, train_batch_size=64 if use_tpu else 0) max_steps = 30 xor_features = [[1., 0.], [0., 0], [0., 1.], [1., 1.]] xor_labels = [[1.], [0.], [1.], [0.]] train_input_fn = tu.dummy_input_fn(xor_features, xor_labels) # Train. estimator.train(input_fn=train_input_fn, steps=None, max_steps=max_steps, hooks=None) # Evaluate. eval_results = estimator.evaluate(input_fn=train_input_fn, steps=1, hooks=None) # Predict. predictions = estimator.predict(input_fn=tu.dataset_input_fn( features=[0., 0.], return_dataset=True)) # We need to iterate over all the predictions before moving on, otherwise # the TPU will not be shut down. for prediction in predictions: self.assertIsNotNone(prediction["predictions"]) # Export SavedModel. def serving_input_fn(): """Input fn for serving export, starting from serialized example.""" serialized_example = tf.compat.v1.placeholder( dtype=tf.string, shape=(None), name="serialized_example") return tf.estimator.export.ServingInputReceiver( features={"x": tf.constant([[0., 0.]], name="serving_x")}, receiver_tensors=serialized_example) estimator.export_saved_model( export_dir_base=estimator.model_dir, serving_input_receiver_fn=serving_input_fn) self.assertAlmostEqual(want_loss, eval_results["loss"], places=2) self.assertEqual(max_steps, eval_results["global_step"]) self.assertEqual(2, eval_results["iteration"])
class ReportMaterializerTest(parameterized.TestCase, tf.test.TestCase): # pylint: disable=g-long-lambda @parameterized.named_parameters( { "testcase_name": "one_empty_subnetwork", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report(hparams={}, attributes={}, metrics={}), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "one_subnetwork", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "one_subnetwork_iteration_2", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "iteration_number": 2, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=2, name="foo", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "two_subnetworks", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), "foo2": subnetwork.Report( hparams={ "learning_rate": 1.e-6, "optimizer": "sgd", "num_layers": 1, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.1445), "foo": tf.constant("baz"), "parameters": tf.constant(7788), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo2"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=False, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={ "learning_rate": 1.e-6, "optimizer": "sgd", "num_layers": 1, "use_side_inputs": True, }, attributes={ "weight_norms": 3.1445, "foo": "baz", "parameters": 7788, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "two_subnetworks_zero_included", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), "foo2": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), }, "steps": 3, "included_subnetwork_names": [], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=False, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=False, ), ], }, { "testcase_name": "two_subnetworks_both_included", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), "foo2": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo1", "foo2"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics", "input_fn": tu.dummy_input_fn([[1., 1.], [1., 1.], [1., 1.]], [[1.], [2.], [3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": tf_compat.v1.metrics.mean(labels)}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 2.}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics_none_steps", "input_fn": tu.dataset_input_fn([[1., 1.], [1., 1.], [1., 1.]], [[1.], [2.], [3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": tf_compat.v1.metrics.mean(labels)}, ), }, "steps": None, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 2.}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics_non_tensor_op", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": (tf.constant(42), tf.no_op())}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 42}, included_in_final_ensemble=True, ), ], }) @test_util.run_in_graph_and_eager_modes def test_materialize_subnetwork_reports(self, input_fn, subnetwork_reports_fn, steps, iteration_number=0, included_subnetwork_names=None, want_materialized_reports=None): with context.graph_mode(): tf.constant(0.) # dummy op so that the session graph is never empty. features, labels = input_fn() subnetwork_reports = subnetwork_reports_fn(features, labels) with self.test_session() as sess: sess.run(tf_compat.v1.initializers.local_variables()) report_materializer = ReportMaterializer(input_fn=input_fn, steps=steps) materialized_reports = ( report_materializer.materialize_subnetwork_reports( sess, iteration_number, subnetwork_reports, included_subnetwork_names)) self.assertEqual( len(want_materialized_reports), len(materialized_reports)) materialized_reports_dict = { blrm.name: blrm for blrm in materialized_reports } for want_materialized_report in want_materialized_reports: materialized_report = ( materialized_reports_dict[want_materialized_report.name]) self.assertEqual(iteration_number, materialized_report.iteration_number) self.assertEqual( set(want_materialized_report.hparams.keys()), set(materialized_report.hparams.keys())) for hparam_key, want_hparam in ( want_materialized_report.hparams.items()): if isinstance(want_hparam, float): self.assertAllClose(want_hparam, materialized_report.hparams[hparam_key]) else: self.assertEqual(want_hparam, materialized_report.hparams[hparam_key]) self.assertSetEqual( set(want_materialized_report.attributes.keys()), set(materialized_report.attributes.keys())) for attribute_key, want_attribute in ( want_materialized_report.attributes.items()): if isinstance(want_attribute, float): self.assertAllClose( want_attribute, decode(materialized_report.attributes[attribute_key])) else: self.assertEqual( want_attribute, decode(materialized_report.attributes[attribute_key])) self.assertSetEqual( set(want_materialized_report.metrics.keys()), set(materialized_report.metrics.keys())) for metric_key, want_metric in ( want_materialized_report.metrics.items()): if isinstance(want_metric, float): self.assertAllClose( want_metric, decode(materialized_report.metrics[metric_key])) else: self.assertEqual(want_metric, decode(materialized_report.metrics[metric_key]))
class EvaluatorTest(parameterized.TestCase, tf.test.TestCase): @parameterized.named_parameters( { "testcase_name": "choose_index_0", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_0, "want_adanet_losses": [3, 6], }, { "testcase_name": "choose_index_1", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [6, 3], }, { "testcase_name": "none_steps", "input_fn": tu.dataset_input_fn(), "steps": None, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }, { "testcase_name": "input_fn_out_of_range", "input_fn": tu.dataset_input_fn(), "steps": 3, "adanet_losses": _fake_adanet_losses_1, "want_adanet_losses": [18, 9], }) def test_evaluate_no_metric_fn_falls_back_to_adanet_losses( self, input_fn, steps, adanet_losses, want_adanet_losses): adanet_losses = adanet_losses(input_fn) metrics = [{ "adanet_loss": tf.metrics.mean(loss) } for loss in adanet_losses] with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, steps=steps) adanet_losses = evaluator.evaluate(sess, ensemble_metrics=metrics) self.assertEqual(want_adanet_losses, adanet_losses) @parameterized.named_parameters( { "testcase_name": "minimize_returns_nanargmin", "objective": Evaluator.Objective.MAXIMIZE, "expected_objective_fn": np.nanargmax, "metric_fn": lambda x, y: None }, { "testcase_name": "maximize_returns_nanargmax", "objective": Evaluator.Objective.MINIMIZE, "expected_objective_fn": np.nanargmin, "metric_fn": lambda x, y: None }) def test_objective(self, objective, expected_objective_fn, metric_fn=None): evaluator = Evaluator(input_fn=None, objective=objective) self.assertEqual(expected_objective_fn, evaluator.objective_fn) def test_objective_unsupported_objective(self): with self.assertRaises(ValueError): Evaluator(input_fn=None, objective="non_existent_objective") def test_evaluate(self): input_fn = tu.dummy_input_fn([[1., 2]], [[3.]]) _, labels = input_fn() predictions = [labels * 2, labels * 3] metrics = [] for preds in predictions: metrics.append({ "mse": tf.metrics.mean_squared_error(labels, preds), "other_metric_1": (tf.constant(1), tf.constant(1)), "other_metric_2": (tf.constant(2), tf.constant(2)) }) with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, metric_name="mse", steps=3) metrics = evaluator.evaluate(sess, ensemble_metrics=metrics) self.assertEqual([9, 36], metrics) def test_evaluate_invalid_metric(self): input_fn = tu.dummy_input_fn([[1., 2]], [[3.]]) _, labels = input_fn() predictions = [labels * 2, labels * 3] metrics = [] for preds in predictions: metrics.append({ "mse": tf.metrics.mean_squared_error(labels, preds), "other_metric_1": (tf.constant(1), tf.constant(1)), "other_metric_2": (tf.constant(2), tf.constant(2)) }) with self.test_session() as sess: evaluator = Evaluator(input_fn=input_fn, metric_name="dne", steps=3) with self.assertRaises(KeyError): metrics = evaluator.evaluate(sess, ensemble_metrics=metrics)