def test_build_ensemble_subnetwork_has_scalar_logits(self):
logits = tf.ones(shape=(100, ))
ensemble_spec = self._build_easy_ensemble([
subnetwork.Subnetwork(last_layer=logits,
logits=logits,
complexity=0.)
])
self.assertEqual([1], ensemble_spec.bias.shape.as_list())
def _build_subnetwork(self, multi_head=False):
last_layer = tf.Variable(
tf_compat.random_normal(shape=(2, 3)), trainable=False).read_value()
def new_logits():
return tf_compat.v1.layers.dense(
last_layer,
units=1,
kernel_initializer=tf_compat.v1.glorot_uniform_initializer())
if multi_head:
logits = {k: new_logits() for k in multi_head}
last_layer = {k: last_layer for k in multi_head}
else:
logits = new_logits()
return subnetwork.Subnetwork(last_layer=logits, logits=logits, complexity=2)
def dummy_ensemble_spec(name,
random_seed=42,
num_subnetworks=1,
bias=0.,
loss=None,
adanet_loss=None,
eval_metrics=None,
variables=None,
dict_predictions=False,
export_output_key=None,
subnetwork_builders=None,
train_op=None):
"""Creates a dummy `_EnsembleSpec` instance.
Args:
name: _EnsembleSpec's name.
random_seed: A scalar random seed.
num_subnetworks: The number of fake subnetworks in this ensemble.
bias: Bias value.
loss: Float loss to return. When None, it's picked from a random
distribution.
adanet_loss: Float AdaNet loss to return. When None, it's picked from a
random distribution.
eval_metrics: Optional eval metrics tuple of (metric_fn, tensor args).
variables: List of `tf.Variable` instances associated with the ensemble.
dict_predictions: Boolean whether to return predictions as a dictionary of
`Tensor` or just a single float `Tensor`.
export_output_key: An `ExportOutputKeys` for faking export outputs.
subnetwork_builders: List of `adanet.subnetwork.Builder` objects.
train_op: A train op.
Returns:
A dummy `_EnsembleSpec` instance.
"""
if loss is None:
loss = dummy_tensor([], random_seed)
if adanet_loss is None:
adanet_loss = dummy_tensor([], random_seed * 2)
else:
adanet_loss = tf.convert_to_tensor(value=adanet_loss)
logits = dummy_tensor([], random_seed * 3)
if dict_predictions:
predictions = {
"logits": logits,
"classes": tf.cast(tf.abs(logits), dtype=tf.int64)
}
else:
predictions = logits
weighted_subnetworks = [
ensemble_lib.WeightedSubnetwork(
name=name,
iteration_number=1,
logits=dummy_tensor([2, 1], random_seed * 4),
weight=dummy_tensor([2, 1], random_seed * 4),
subnetwork=subnetwork_lib.Subnetwork(
last_layer=dummy_tensor([1, 2], random_seed * 4),
logits=dummy_tensor([2, 1], random_seed * 4),
complexity=1.,
persisted_tensors={}))
]
export_outputs = _dummy_export_outputs(export_output_key, logits,
predictions)
bias = tf.constant(bias)
return _EnsembleSpec(name=name,
ensemble=ensemble_lib.ComplexityRegularized(
weighted_subnetworks=weighted_subnetworks *
num_subnetworks,
bias=bias,
logits=logits,
),
architecture=_Architecture("dummy_ensemble_candidate",
"dummy_ensembler"),
subnetwork_builders=subnetwork_builders,
predictions=predictions,
step=tf.Variable(0),
variables=variables,
loss=loss,
adanet_loss=adanet_loss,
train_op=train_op,
eval_metrics=eval_metrics,
export_outputs=export_outputs)
def build_subnetwork(self,
features,
labels,
logits_dimension,
training,
iteration_step,
summary,
previous_ensemble,
config=None):
# We don't need an EVAL mode since AdaNet takes care of evaluation for us.
mode = tf.estimator.ModeKeys.PREDICT
if training:
mode = tf.estimator.ModeKeys.TRAIN
# Call in template to ensure that variables are created once and reused.
call_model_fn_template = tf.compat.v1.make_template(
"model_fn", self._call_model_fn)
subestimator_features, subestimator_labels = features, labels
local_init_ops = []
subestimator = self._subestimator(config)
if training and subestimator.train_input_fn:
# TODO: Consider tensorflow_estimator/python/estimator/util.py.
inputs = subestimator.train_input_fn()
if isinstance(inputs, (tf_compat.DatasetV1, tf_compat.DatasetV2)):
subestimator_features, subestimator_labels = (
tf_compat.make_one_shot_iterator(inputs).get_next())
else:
subestimator_features, subestimator_labels = inputs
# Construct subnetwork graph first because of dependencies on scope.
_, _, bagging_train_op_spec, sub_local_init_op = call_model_fn_template(
subestimator, subestimator_features, subestimator_labels, mode,
summary)
# Graph for ensemble learning gets model_fn_1 for scope.
logits, last_layer, _, ensemble_local_init_op = call_model_fn_template(
subestimator, features, labels, mode, summary)
if sub_local_init_op:
local_init_ops.append(sub_local_init_op)
if ensemble_local_init_op:
local_init_ops.append(ensemble_local_init_op)
# Run train op in a hook so that exceptions can be intercepted by the
# AdaNet framework instead of the Estimator's monitored training session.
hooks = bagging_train_op_spec.hooks + (_SecondaryTrainOpRunnerHook(
bagging_train_op_spec.train_op), )
train_op_spec = subnetwork_lib.TrainOpSpec(
train_op=tf.no_op(),
chief_hooks=bagging_train_op_spec.chief_hooks,
hooks=hooks)
else:
logits, last_layer, train_op_spec, local_init_op = call_model_fn_template(
subestimator, features, labels, mode, summary)
if local_init_op:
local_init_ops.append(local_init_op)
# TODO: Replace with variance complexity measure.
complexity = tf.constant(0.)
return subnetwork_lib.Subnetwork(logits=logits,
last_layer=last_layer,
shared={"train_op": train_op_spec},
complexity=complexity,
local_init_ops=local_init_ops)
