def test_build_ensemble_subnetwork_has_scalar_logits(self): logits = tf.ones(shape=(100, )) ensemble_spec = self._build_easy_ensemble([ subnetwork.Subnetwork(last_layer=logits, logits=logits, complexity=0.) ]) self.assertEqual([1], ensemble_spec.bias.shape.as_list())
def _build_subnetwork(self, multi_head=False): last_layer = tf.Variable( tf_compat.random_normal(shape=(2, 3)), trainable=False).read_value() def new_logits(): return tf_compat.v1.layers.dense( last_layer, units=1, kernel_initializer=tf_compat.v1.glorot_uniform_initializer()) if multi_head: logits = {k: new_logits() for k in multi_head} last_layer = {k: last_layer for k in multi_head} else: logits = new_logits() return subnetwork.Subnetwork(last_layer=logits, logits=logits, complexity=2)
def dummy_ensemble_spec(name, random_seed=42, num_subnetworks=1, bias=0., loss=None, adanet_loss=None, eval_metrics=None, variables=None, dict_predictions=False, export_output_key=None, subnetwork_builders=None, train_op=None): """Creates a dummy `_EnsembleSpec` instance. Args: name: _EnsembleSpec's name. random_seed: A scalar random seed. num_subnetworks: The number of fake subnetworks in this ensemble. bias: Bias value. loss: Float loss to return. When None, it's picked from a random distribution. adanet_loss: Float AdaNet loss to return. When None, it's picked from a random distribution. eval_metrics: Optional eval metrics tuple of (metric_fn, tensor args). variables: List of `tf.Variable` instances associated with the ensemble. dict_predictions: Boolean whether to return predictions as a dictionary of `Tensor` or just a single float `Tensor`. export_output_key: An `ExportOutputKeys` for faking export outputs. subnetwork_builders: List of `adanet.subnetwork.Builder` objects. train_op: A train op. Returns: A dummy `_EnsembleSpec` instance. """ if loss is None: loss = dummy_tensor([], random_seed) if adanet_loss is None: adanet_loss = dummy_tensor([], random_seed * 2) else: adanet_loss = tf.convert_to_tensor(value=adanet_loss) logits = dummy_tensor([], random_seed * 3) if dict_predictions: predictions = { "logits": logits, "classes": tf.cast(tf.abs(logits), dtype=tf.int64) } else: predictions = logits weighted_subnetworks = [ ensemble_lib.WeightedSubnetwork( name=name, iteration_number=1, logits=dummy_tensor([2, 1], random_seed * 4), weight=dummy_tensor([2, 1], random_seed * 4), subnetwork=subnetwork_lib.Subnetwork( last_layer=dummy_tensor([1, 2], random_seed * 4), logits=dummy_tensor([2, 1], random_seed * 4), complexity=1., persisted_tensors={})) ] export_outputs = _dummy_export_outputs(export_output_key, logits, predictions) bias = tf.constant(bias) return _EnsembleSpec(name=name, ensemble=ensemble_lib.ComplexityRegularized( weighted_subnetworks=weighted_subnetworks * num_subnetworks, bias=bias, logits=logits, ), architecture=_Architecture("dummy_ensemble_candidate", "dummy_ensembler"), subnetwork_builders=subnetwork_builders, predictions=predictions, step=tf.Variable(0), variables=variables, loss=loss, adanet_loss=adanet_loss, train_op=train_op, eval_metrics=eval_metrics, export_outputs=export_outputs)
def build_subnetwork(self, features, labels, logits_dimension, training, iteration_step, summary, previous_ensemble, config=None): # We don't need an EVAL mode since AdaNet takes care of evaluation for us. mode = tf.estimator.ModeKeys.PREDICT if training: mode = tf.estimator.ModeKeys.TRAIN # Call in template to ensure that variables are created once and reused. call_model_fn_template = tf.compat.v1.make_template( "model_fn", self._call_model_fn) subestimator_features, subestimator_labels = features, labels local_init_ops = [] subestimator = self._subestimator(config) if training and subestimator.train_input_fn: # TODO: Consider tensorflow_estimator/python/estimator/util.py. inputs = subestimator.train_input_fn() if isinstance(inputs, (tf_compat.DatasetV1, tf_compat.DatasetV2)): subestimator_features, subestimator_labels = ( tf_compat.make_one_shot_iterator(inputs).get_next()) else: subestimator_features, subestimator_labels = inputs # Construct subnetwork graph first because of dependencies on scope. _, _, bagging_train_op_spec, sub_local_init_op = call_model_fn_template( subestimator, subestimator_features, subestimator_labels, mode, summary) # Graph for ensemble learning gets model_fn_1 for scope. logits, last_layer, _, ensemble_local_init_op = call_model_fn_template( subestimator, features, labels, mode, summary) if sub_local_init_op: local_init_ops.append(sub_local_init_op) if ensemble_local_init_op: local_init_ops.append(ensemble_local_init_op) # Run train op in a hook so that exceptions can be intercepted by the # AdaNet framework instead of the Estimator's monitored training session. hooks = bagging_train_op_spec.hooks + (_SecondaryTrainOpRunnerHook( bagging_train_op_spec.train_op), ) train_op_spec = subnetwork_lib.TrainOpSpec( train_op=tf.no_op(), chief_hooks=bagging_train_op_spec.chief_hooks, hooks=hooks) else: logits, last_layer, train_op_spec, local_init_op = call_model_fn_template( subestimator, features, labels, mode, summary) if local_init_op: local_init_ops.append(local_init_op) # TODO: Replace with variance complexity measure. complexity = tf.constant(0.) return subnetwork_lib.Subnetwork(logits=logits, last_layer=last_layer, shared={"train_op": train_op_spec}, complexity=complexity, local_init_ops=local_init_ops)