def build_iteration(self, iteration_number, subnetwork_builders, features, mode, labels=None, previous_ensemble_summary=None, previous_ensemble_spec=None): """Builds and returns AdaNet iteration t. This method uses the generated the candidate subnetworks given the ensemble at iteration t-1 and creates graph operations to train them. The returned `_Iteration` tracks the training of all candidates to know when the iteration is over, and tracks the best candidate's predictions and loss, as defined by lowest complexity-regularized loss on the train set. Args: iteration_number: The iteration number. subnetwork_builders: A list of `Builders` for adding ` Subnetworks` to the graph. Each subnetwork is then wrapped in a `_Candidate` to train. features: Dictionary of `Tensor` objects keyed by feature name. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. labels: `Tensor` of labels. Can be `None`. previous_ensemble_summary: The `_ScopedSummary` for the previous ensemble. previous_ensemble_spec: Optional `_EnsembleSpec` for iteration t-1. Returns: An _Iteration instance. Raises: ValueError: If subnetwork_builders is empty. ValueError: If two `Builder` instances share the same name. """ if not subnetwork_builders: raise ValueError("Each iteration must have at least one Builder.") training = mode == tf.estimator.ModeKeys.TRAIN skip_summaries = mode == tf.estimator.ModeKeys.PREDICT with tf.variable_scope("iteration_{}".format(iteration_number)): # Iteration step to use instead of global step. iteration_step = tf.get_variable( "step", shape=[], initializer=tf.zeros_initializer(), trainable=False, dtype=tf.int64) # Convert to tensor so that users cannot mutate it. iteration_step_tensor = tf.convert_to_tensor(iteration_step) seen_builder_names = {} candidates = [] summaries = [] subnetwork_reports = {} # TODO: Consolidate building subnetwork into # candidate_builder. if previous_ensemble_spec: # Include previous best subnetwork as a candidate so that its # predictions are returned until a new candidate outperforms. seen_builder_names = {previous_ensemble_spec.name: True} previous_best_candidate = self._candidate_builder.build_candidate( ensemble_spec=previous_ensemble_spec, training=training, iteration_step=iteration_step_tensor, summary=previous_ensemble_summary, is_previous_best=True) candidates.append(previous_best_candidate) summaries.append(previous_ensemble_summary) # Generate subnetwork reports. if mode != tf.estimator.ModeKeys.PREDICT: subnetwork_report = subnetwork.Report( hparams={}, attributes={}, metrics=(previous_ensemble_spec.eval_metric_ops.copy() if previous_ensemble_spec.eval_metric_ops is not None else {}), ) subnetwork_report.metrics["adanet_loss"] = tf.metrics.mean( previous_ensemble_spec.adanet_loss) subnetwork_reports["previous_ensemble"] = subnetwork_report for subnetwork_builder in subnetwork_builders: if subnetwork_builder.name in seen_builder_names: raise ValueError( "Two ensembles have the same name '{}'".format( subnetwork_builder.name)) seen_builder_names[subnetwork_builder.name] = True summary = _ScopedSummary(subnetwork_builder.name, skip_summary=skip_summaries) summaries.append(summary) ensemble_spec = self._ensemble_builder.append_new_subnetwork( ensemble_spec=previous_ensemble_spec, subnetwork_builder=subnetwork_builder, summary=summary, features=features, mode=mode, iteration_step=iteration_step_tensor, labels=labels) candidate = self._candidate_builder.build_candidate( ensemble_spec=ensemble_spec, training=training, iteration_step=iteration_step_tensor, summary=summary) candidates.append(candidate) # Generate subnetwork reports. if mode != tf.estimator.ModeKeys.PREDICT: subnetwork_report = subnetwork_builder.build_subnetwork_report( ) if not subnetwork_report: subnetwork_report = subnetwork.Report(hparams={}, attributes={}, metrics={}) if ensemble_spec.eval_metric_ops is not None: for metric_name, metric in ensemble_spec.eval_metric_ops.items( ): subnetwork_report.metrics[metric_name] = metric subnetwork_report.metrics["adanet_loss"] = tf.metrics.mean( ensemble_spec.adanet_loss) subnetwork_reports[ subnetwork_builder.name] = subnetwork_report best_candidate_index = 0 best_predictions = candidates[0].ensemble_spec.predictions best_loss = candidates[0].ensemble_spec.loss best_eval_metric_ops = candidates[0].ensemble_spec.eval_metric_ops best_export_outputs = candidates[0].ensemble_spec.export_outputs if len(candidates) >= 1: # Dynamically select the outputs of best candidate. best_candidate_index = self._best_candidate_index(candidates) best_predictions = self._best_predictions( candidates, best_candidate_index) best_loss = self._best_loss(candidates, best_candidate_index, mode) best_eval_metric_ops = self._best_eval_metric_ops( candidates, best_candidate_index) best_export_outputs = self._best_export_outputs( candidates, best_candidate_index, mode, best_predictions) estimator_spec = tf.estimator.EstimatorSpec( mode=mode, predictions=best_predictions, loss=best_loss, train_op=self._create_train_op(candidates, mode, iteration_step), eval_metric_ops=best_eval_metric_ops, export_outputs=best_export_outputs) return _Iteration(number=iteration_number, candidates=candidates, estimator_spec=estimator_spec, best_candidate_index=best_candidate_index, summaries=summaries, is_over=self._is_over(candidates), subnetwork_reports=subnetwork_reports, step=iteration_step_tensor)
def build_iteration(self, iteration_number, subnetwork_builders, features, mode, labels=None, previous_ensemble_summary=None, previous_ensemble_spec=None, rebuilding=False): """Builds and returns AdaNet iteration t. This method uses the generated the candidate subnetworks given the ensemble at iteration t-1 and creates graph operations to train them. The returned `_Iteration` tracks the training of all candidates to know when the iteration is over, and tracks the best candidate's predictions and loss, as defined by lowest complexity-regularized loss on the train set. Args: iteration_number: Integer iteration number. subnetwork_builders: A list of `Builders` for adding ` Subnetworks` to the graph. Each subnetwork is then wrapped in a `_Candidate` to train. features: Dictionary of `Tensor` objects keyed by feature name. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. labels: `Tensor` of labels. Can be `None`. previous_ensemble_summary: The `_ScopedSummary` for the previous ensemble. previous_ensemble_spec: Optional `_EnsembleSpec` for iteration t-1. rebuilding: Boolean whether the iteration is being rebuilt only to restore the previous best subnetworks and ensembles. Returns: An _Iteration instance. Raises: ValueError: If subnetwork_builders is empty. ValueError: If two `Builder` instances share the same name. """ tf.logging.info("%s iteration %s", "Rebuilding" if rebuilding else "Building", iteration_number) if not subnetwork_builders: raise ValueError("Each iteration must have at least one Builder.") # TODO: Consider moving ensemble mode logic to ensemble.py. ensemble_mode = mode if rebuilding: # Create the frozen ensemble in EVAL mode by default. This way their # outputs aren't affected by dropout etc. ensemble_mode = tf.estimator.ModeKeys.EVAL if mode == tf.estimator.ModeKeys.PREDICT: ensemble_mode = mode # Only replicate in training mode when the user requests it. if self._replicate_ensemble_in_training and ( mode == tf.estimator.ModeKeys.TRAIN): ensemble_mode = mode training = mode == tf.estimator.ModeKeys.TRAIN skip_summaries = mode == tf.estimator.ModeKeys.PREDICT with tf.variable_scope("iteration_{}".format(iteration_number)): # Iteration step to use instead of global step. iteration_step = tf.get_variable( "step", shape=[], initializer=tf.zeros_initializer(), trainable=False, dtype=tf.int64) # Convert to tensor so that users cannot mutate it. iteration_step_tensor = tf.convert_to_tensor(iteration_step) seen_builder_names = {} candidates = [] summaries = [] subnetwork_reports = {} # TODO: Consolidate building subnetwork into # candidate_builder. if previous_ensemble_spec: # Include previous best subnetwork as a candidate so that its # predictions are returned until a new candidate outperforms. seen_builder_names = {previous_ensemble_spec.name: True} previous_best_candidate = self._candidate_builder.build_candidate( ensemble_spec=previous_ensemble_spec, training=training, iteration_step=iteration_step_tensor, summary=previous_ensemble_summary, is_previous_best=True) candidates.append(previous_best_candidate) summaries.append(previous_ensemble_summary) # Generate subnetwork reports. if mode == tf.estimator.ModeKeys.EVAL: subnetwork_report = subnetwork.Report( hparams={}, attributes={}, metrics=(previous_ensemble_spec.eval_metric_ops.copy() if previous_ensemble_spec.eval_metric_ops is not None else {}), ) subnetwork_report.metrics["adanet_loss"] = tf.metrics.mean( previous_ensemble_spec.adanet_loss) subnetwork_reports["previous_ensemble"] = subnetwork_report for subnetwork_builder in subnetwork_builders: if subnetwork_builder.name in seen_builder_names: raise ValueError( "Two ensembles have the same name '{}'".format( subnetwork_builder.name)) seen_builder_names[subnetwork_builder.name] = True ensemble_name = "t{}_{}".format(iteration_number, subnetwork_builder.name) summary = _ScopedSummary(ensemble_name, skip_summary=skip_summaries or rebuilding) summaries.append(summary) ensemble_spec = self._ensemble_builder.append_new_subnetwork( ensemble_name=ensemble_name, ensemble_spec=previous_ensemble_spec, iteration_number=iteration_number, subnetwork_builder=subnetwork_builder, summary=summary, features=features, mode=ensemble_mode, iteration_step=iteration_step_tensor, labels=labels) candidate = self._candidate_builder.build_candidate( ensemble_spec=ensemble_spec, training=training, iteration_step=iteration_step_tensor, summary=summary) candidates.append(candidate) # Generate subnetwork reports. if mode != tf.estimator.ModeKeys.PREDICT: subnetwork_report = subnetwork_builder.build_subnetwork_report( ) if not subnetwork_report: subnetwork_report = subnetwork.Report(hparams={}, attributes={}, metrics={}) if ensemble_spec.eval_metric_ops is not None: for metric_name in sorted( ensemble_spec.eval_metric_ops): metric = ensemble_spec.eval_metric_ops[metric_name] subnetwork_report.metrics[metric_name] = metric subnetwork_report.metrics["adanet_loss"] = tf.metrics.mean( ensemble_spec.adanet_loss) subnetwork_reports[ subnetwork_builder.name] = subnetwork_report # Dynamically select the outputs of best candidate. best_candidate_index = self._best_candidate_index(candidates) best_predictions = self._best_predictions(candidates, best_candidate_index) best_loss = self._best_loss(candidates, best_candidate_index, mode) best_eval_metric_ops = self._best_eval_metric_ops( candidates, best_candidate_index, mode) best_export_outputs = self._best_export_outputs( candidates, best_candidate_index, mode, best_predictions) # Hooks on TPU cannot depend on any graph `Tensors`. Instead the value of # `is_over` is stored in a `Variable` that can later be retrieved from # inside a training hook. is_over_var_fn = tf.make_template("is_over_var_fn", is_over_var) estimator_spec = tf.estimator.EstimatorSpec( mode=mode, predictions=best_predictions, loss=best_loss, train_op=self._create_train_op(candidates, mode, iteration_step, is_over_var_fn), eval_metric_ops=best_eval_metric_ops, export_outputs=best_export_outputs) return _Iteration(number=iteration_number, candidates=candidates, estimator_spec=estimator_spec, best_candidate_index=best_candidate_index, summaries=summaries, is_over_fn=is_over_var_fn, subnetwork_reports=subnetwork_reports, step=iteration_step_tensor)
def build_iteration(self, iteration_number, ensemble_candidates, subnetwork_builders, features, mode, labels=None, previous_ensemble_summary=None, previous_ensemble_spec=None, rebuilding=False): """Builds and returns AdaNet iteration t. This method uses the generated the candidate subnetworks given the ensemble at iteration t-1 and creates graph operations to train them. The returned `_Iteration` tracks the training of all candidates to know when the iteration is over, and tracks the best candidate's predictions and loss, as defined by lowest complexity-regularized loss on the train set. Args: iteration_number: Integer iteration number. ensemble_candidates: Iterable of `adanet.ensemble.Candidate` instances. subnetwork_builders: A list of `Builders` for adding ` Subnetworks` to the graph. Each subnetwork is then wrapped in a `_Candidate` to train. features: Dictionary of `Tensor` objects keyed by feature name. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. labels: `Tensor` of labels. Can be `None`. previous_ensemble_summary: The `adanet.Summary` for the previous ensemble. previous_ensemble_spec: Optional `_EnsembleSpec` for iteration t-1. rebuilding: Boolean whether the iteration is being rebuilt only to restore the previous best subnetworks and ensembles. Returns: An _Iteration instance. Raises: ValueError: If subnetwork_builders is empty. ValueError: If two subnetworks share the same name. ValueError: If two ensembles share the same name. """ tf.logging.info("%s iteration %s", "Rebuilding" if rebuilding else "Building", iteration_number) if not subnetwork_builders: raise ValueError("Each iteration must have at least one Builder.") # TODO: Consider moving builder mode logic to ensemble_builder.py. builder_mode = mode if rebuilding: # Build the subnetworks and ensembles in EVAL mode by default. This way # their outputs aren't affected by dropout etc. builder_mode = tf.estimator.ModeKeys.EVAL if mode == tf.estimator.ModeKeys.PREDICT: builder_mode = mode # Only replicate in training mode when the user requests it. if self._replicate_ensemble_in_training and ( mode == tf.estimator.ModeKeys.TRAIN): builder_mode = mode features, labels = self._check_numerics(features, labels) training = mode == tf.estimator.ModeKeys.TRAIN skip_summaries = mode == tf.estimator.ModeKeys.PREDICT with tf.variable_scope("iteration_{}".format(iteration_number)): # Iteration step to use instead of global step. iteration_step = tf.get_variable( "step", shape=[], initializer=tf.zeros_initializer(), trainable=False, dtype=tf.int64) # Convert to tensor so that users cannot mutate it. iteration_step_tensor = tf.convert_to_tensor(iteration_step) seen_builder_names = {} candidates = [] summaries = [] subnetwork_reports = {} previous_ensemble = None if previous_ensemble_spec: previous_ensemble = previous_ensemble_spec.ensemble # Include previous best subnetwork as a candidate so that its # predictions are returned until a new candidate outperforms. seen_builder_names = {previous_ensemble_spec.name: True} previous_best_candidate = self._candidate_builder.build_candidate( ensemble_spec=previous_ensemble_spec, training=training, iteration_step=iteration_step_tensor, summary=previous_ensemble_summary, is_previous_best=True) candidates.append(previous_best_candidate) summaries.append(previous_ensemble_summary) # Generate subnetwork reports. if mode == tf.estimator.ModeKeys.EVAL: metrics = call_eval_metrics( previous_ensemble_spec.eval_metrics) subnetwork_report = subnetwork.Report( hparams={}, attributes={}, metrics=metrics, ) subnetwork_report.metrics["adanet_loss"] = tf.metrics.mean( previous_ensemble_spec.adanet_loss) subnetwork_reports["previous_ensemble"] = subnetwork_report for subnetwork_builder in subnetwork_builders: if subnetwork_builder.name in seen_builder_names: raise ValueError( "Two subnetworks have the same name '{}'".format( subnetwork_builder.name)) seen_builder_names[subnetwork_builder.name] = True subnetwork_specs = [] num_subnetworks = len(subnetwork_builders) for i, subnetwork_builder in enumerate(subnetwork_builders): if not self._placement_strategy.should_build_subnetwork( num_subnetworks, i) and not rebuilding: continue subnetwork_name = "t{}_{}".format(iteration_number, subnetwork_builder.name) subnetwork_summary = self._summary_maker( namespace="subnetwork", scope=subnetwork_name, skip_summary=skip_summaries or rebuilding) summaries.append(subnetwork_summary) tf.logging.info("%s subnetwork '%s'", "Rebuilding" if rebuilding else "Building", subnetwork_builder.name) subnetwork_spec = self._subnetwork_manager.build_subnetwork_spec( name=subnetwork_name, subnetwork_builder=subnetwork_builder, iteration_step=iteration_step_tensor, summary=subnetwork_summary, features=features, mode=builder_mode, labels=labels, previous_ensemble=previous_ensemble) subnetwork_specs.append(subnetwork_spec) if not self._placement_strategy.should_build_ensemble( num_subnetworks) and not rebuilding: # Workers that don't build ensembles need a dummy candidate in order # to train the subnetwork. # Because only ensembles can be considered candidates, we need to # convert the subnetwork into a dummy ensemble and subsequently a # dummy candidate. However, this dummy candidate is never considered a # true candidate during candidate evaluation and selection. # TODO: Eliminate need for candidates. dummy_candidate = self._candidate_builder.build_candidate( # pylint: disable=protected-access ensemble_spec=ensemble_builder_lib._EnsembleSpec( name=subnetwork_name, ensemble=None, architecture=None, subnetwork_builders=subnetwork_builders, predictions=subnetwork_spec.predictions, loss=subnetwork_spec.loss, adanet_loss=0.), # pylint: enable=protected-access training=training, iteration_step=iteration_step_tensor, summary=subnetwork_summary, track_moving_average=False) candidates.append(dummy_candidate) # Generate subnetwork reports. if mode != tf.estimator.ModeKeys.PREDICT: subnetwork_report = subnetwork_builder.build_subnetwork_report( ) if not subnetwork_report: subnetwork_report = subnetwork.Report(hparams={}, attributes={}, metrics={}) metrics = call_eval_metrics(subnetwork_spec.eval_metrics) for metric_name in sorted(metrics): metric = metrics[metric_name] subnetwork_report.metrics[metric_name] = metric subnetwork_reports[ subnetwork_builder.name] = subnetwork_report # Create (ensembler_candidate*ensembler) ensembles. seen_ensemble_names = {} for ensembler in self._ensemblers: for ensemble_candidate in ensemble_candidates: if not self._placement_strategy.should_build_ensemble( num_subnetworks) and not rebuilding: continue ensemble_name = "t{}_{}_{}".format(iteration_number, ensemble_candidate.name, ensembler.name) if ensemble_name in seen_ensemble_names: raise ValueError( "Two ensembles have the same name '{}'".format( ensemble_name)) seen_ensemble_names[ensemble_name] = True summary = self._summary_maker(namespace="ensemble", scope=ensemble_name, skip_summary=skip_summaries or rebuilding) summaries.append(summary) ensemble_spec = self._ensemble_builder.build_ensemble_spec( name=ensemble_name, candidate=ensemble_candidate, ensembler=ensembler, subnetwork_specs=subnetwork_specs, summary=summary, features=features, mode=builder_mode, iteration_step=iteration_step_tensor, iteration_number=iteration_number, labels=labels, previous_ensemble_spec=previous_ensemble_spec) candidate = self._candidate_builder.build_candidate( ensemble_spec=ensemble_spec, training=training, iteration_step=iteration_step_tensor, summary=summary) candidates.append(candidate) # TODO: Move adanet_loss from subnetwork report to a new # ensemble report, since the adanet_loss is associated with an # ensemble, and only when using a ComplexityRegularizedEnsemblers. # Keep adanet_loss in subnetwork report for backwards compatibility. if len(ensemble_candidates) != len(subnetwork_builders): continue if len(ensemble_candidate.subnetwork_builders) > 1: continue if mode == tf.estimator.ModeKeys.PREDICT: continue builder_name = ensemble_candidate.subnetwork_builders[ 0].name subnetwork_reports[builder_name].metrics[ "adanet_loss"] = tf.metrics.mean( ensemble_spec.adanet_loss) # Dynamically select the outputs of best candidate. best_candidate_index = self._best_candidate_index(candidates) best_predictions = self._best_predictions(candidates, best_candidate_index) best_loss = self._best_loss(candidates, best_candidate_index, mode) best_export_outputs = self._best_export_outputs( candidates, best_candidate_index, mode, best_predictions) # Hooks on TPU cannot depend on any graph `Tensors`. Instead the value of # `is_over` is stored in a `Variable` that can later be retrieved from # inside a training hook. is_over_var_template = tf.make_template("is_over_var_template", _is_over_var) training_chief_hooks, training_hooks = (), () for subnetwork_spec in subnetwork_specs: if not self._placement_strategy.should_train_subnetworks( num_subnetworks) and not rebuilding: continue if not subnetwork_spec.train_op: continue training_chief_hooks += subnetwork_spec.train_op.chief_hooks or ( ) training_hooks += subnetwork_spec.train_op.hooks or () for candidate in candidates: spec = candidate.ensemble_spec if not spec.train_op: continue training_chief_hooks += spec.train_op.chief_hooks or () training_hooks += spec.train_op.hooks or () summary = self._summary_maker(namespace=None, scope=None, skip_summary=skip_summaries or rebuilding) summaries.append(summary) with summary.current_scope(): summary.scalar("iteration/adanet/iteration", iteration_number) summary.scalar("iteration_step/adanet/iteration_step", iteration_step_tensor) if best_loss is not None: summary.scalar("loss", best_loss) train_op = self._create_train_op(subnetwork_specs, candidates, mode, iteration_step, is_over_var_template, num_subnetworks) iteration_metrics = _IterationMetrics(candidates, subnetwork_specs) if self._use_tpu: estimator_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=mode, predictions=best_predictions, loss=best_loss, train_op=train_op, eval_metrics=iteration_metrics.best_eval_metrics_tuple( best_candidate_index, mode), export_outputs=best_export_outputs, training_hooks=training_hooks) else: estimator_spec = tf.estimator.EstimatorSpec( mode=mode, predictions=best_predictions, loss=best_loss, train_op=train_op, eval_metric_ops=iteration_metrics.best_eval_metric_ops( best_candidate_index, mode), export_outputs=best_export_outputs, training_chief_hooks=training_chief_hooks, training_hooks=training_hooks) return _Iteration(number=iteration_number, candidates=candidates, subnetwork_specs=subnetwork_specs, estimator_spec=estimator_spec, best_candidate_index=best_candidate_index, summaries=summaries, is_over_fn=is_over_var_template, subnetwork_reports=subnetwork_reports, step=iteration_step_tensor)
class ReportMaterializerTest(parameterized.TestCase, tf.test.TestCase): # pylint: disable=g-long-lambda @parameterized.named_parameters( { "testcase_name": "one_empty_subnetwork", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report(hparams={}, attributes={}, metrics={} ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "one_subnetwork", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "one_subnetwork_iteration_2", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "iteration_number": 2, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=2, name="foo", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "two_subnetworks", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.14), "foo": tf.constant("bar"), "parameters": tf.constant(7777), "boo": tf.constant(True), }, metrics={}, ), "foo2": subnetwork.Report( hparams={ "learning_rate": 1.e-6, "optimizer": "sgd", "num_layers": 1, "use_side_inputs": True, }, attributes={ "weight_norms": tf.constant(3.1445), "foo": tf.constant("baz"), "parameters": tf.constant(7788), "boo": tf.constant(True), }, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo2"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={ "learning_rate": 1.e-5, "optimizer": "sgd", "num_layers": 0, "use_side_inputs": True, }, attributes={ "weight_norms": 3.14, "foo": "bar", "parameters": 7777, "boo": True, }, metrics={}, included_in_final_ensemble=False, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={ "learning_rate": 1.e-6, "optimizer": "sgd", "num_layers": 1, "use_side_inputs": True, }, attributes={ "weight_norms": 3.1445, "foo": "baz", "parameters": 7788, "boo": True, }, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "two_subnetworks_zero_included", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), "foo2": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), }, "steps": 3, "included_subnetwork_names": [], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=False, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=False, ), ], }, { "testcase_name": "two_subnetworks_both_included", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo1": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), "foo2": subnetwork.Report( hparams={}, attributes={}, metrics={}, ), }, "steps": 3, "included_subnetwork_names": ["foo1", "foo2"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo1", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), subnetwork.MaterializedReport( iteration_number=0, name="foo2", hparams={}, attributes={}, metrics={}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics", "input_fn": tu.dummy_input_fn([[1., 1.], [1., 1.], [1., 1.]], [[1.], [2.], [3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": tf.metrics.mean(labels)}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 2.}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics_none_steps", "input_fn": tu.dataset_input_fn([[1., 1.], [1., 1.], [1., 1.]], [[1.], [2.], [3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": tf.metrics.mean(labels)}, ), }, "steps": None, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 2.}, included_in_final_ensemble=True, ), ], }, { "testcase_name": "materialize_metrics_non_tensor_op", "input_fn": tu.dummy_input_fn([[1., 2]], [[3.]]), "subnetwork_reports_fn": lambda features, labels: { "foo": subnetwork.Report( hparams={}, attributes={}, metrics={"moo": (tf.constant(42), tf.no_op())}, ), }, "steps": 3, "included_subnetwork_names": ["foo"], "want_materialized_reports": [ subnetwork.MaterializedReport( iteration_number=0, name="foo", hparams={}, attributes={}, metrics={"moo": 42}, included_in_final_ensemble=True, ), ], }) def test_materialize_subnetwork_reports(self, input_fn, subnetwork_reports_fn, steps, iteration_number=0, included_subnetwork_names=None, want_materialized_reports=None): tf.constant(0.) # dummy op so that the session graph is never empty. features, labels = input_fn() subnetwork_reports = subnetwork_reports_fn(features, labels) with self.test_session() as sess: sess.run(tf.initializers.local_variables()) report_materializer = ReportMaterializer(input_fn=input_fn, steps=steps) materialized_reports = ( report_materializer.materialize_subnetwork_reports( sess, iteration_number, subnetwork_reports, included_subnetwork_names)) self.assertEqual(len(want_materialized_reports), len(materialized_reports)) materialized_reports_dict = { blrm.name: blrm for blrm in materialized_reports } for want_materialized_report in want_materialized_reports: materialized_report = ( materialized_reports_dict[want_materialized_report.name]) self.assertEqual(iteration_number, materialized_report.iteration_number) self.assertEqual(set(want_materialized_report.hparams.keys()), set(materialized_report.hparams.keys())) for hparam_key, want_hparam in ( want_materialized_report.hparams.items()): if isinstance(want_hparam, float): self.assertAllClose( want_hparam, materialized_report.hparams[hparam_key]) else: self.assertEqual( want_hparam, materialized_report.hparams[hparam_key]) self.assertSetEqual( set(want_materialized_report.attributes.keys()), set(materialized_report.attributes.keys())) for attribute_key, want_attribute in ( want_materialized_report.attributes.items()): if isinstance(want_attribute, float): self.assertAllClose( want_attribute, decode( materialized_report.attributes[attribute_key])) else: self.assertEqual( want_attribute, decode( materialized_report.attributes[attribute_key])) self.assertSetEqual( set(want_materialized_report.metrics.keys()), set(materialized_report.metrics.keys())) for metric_key, want_metric in ( want_materialized_report.metrics.items()): if isinstance(want_metric, float): self.assertAllClose( want_metric, decode(materialized_report.metrics[metric_key])) else: self.assertEqual( want_metric, decode(materialized_report.metrics[metric_key]))