def testCrossEntropyCustomLoop(self, loss):
model_layers = [
layers.Conv2D(7, 5, input_shape=(32, 32, 3)),
layers.Activation('relu'),
layers.Conv2D(10, (3, 3), kernel_regularizer='l2'),
layers.GlobalMaxPool2D()
]
model = tf.keras.Sequential(model_layers)
model.compile('sgd', loss)
loss_fn = utils.get_loss_fn(model=model, loss=loss)
x = np.random.random((11, 32, 32, 3)).astype(np.float32)
y = np.random.random((11, 10)).astype(np.float32)
tf_x = tf.constant(x)
tf_y = tf.constant(y)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
model_loss = sess.run(model.total_loss,
feed_dict={
'conv2d_input:0': x,
'global_max_pooling2d_target:0': y
})
fn_loss = sess.run(loss_fn((tf_x, tf_y)))
fn_loss_w_pred = sess.run(
loss_fn((tf_x, tf_y), prediction=model(tf_x)))
self.assertAlmostEqual(model_loss, fn_loss, fn_loss_w_pred)
def testCrossEntropy(self, loss, label_shape, is_logits,
use_regularization):
conv_kwargs = {
'kernel_regularizer': 'l2'
} if use_regularization else {}
model_layers = [
layers.Conv2D(7, 5, input_shape=(32, 32, 3), **conv_kwargs),
layers.Activation('relu'),
layers.Conv2D(10, (3, 3), activation='relu', **conv_kwargs),
layers.GlobalMaxPool2D()
]
if is_logits:
model_layers.append(layers.Activation('softmax'))
model = tf.keras.Sequential(model_layers)
model.compile('sgd', loss)
loss_fn = utils.get_loss_fn(model=model, loss=loss)
x = tf.constant(np.random.random((11, 32, 32, 3)).astype(np.float32))
y = tf.constant(np.random.random(label_shape).astype(np.float32))
model_loss = model.evaluate(x, y, steps=1)
fn_loss = tf.keras.backend.get_value(loss_fn((x, y)))
fn_loss_w_pred = tf.keras.backend.get_value(
loss_fn((x, y), prediction=model(x)))
self.assertAlmostEqual(model_loss, fn_loss, fn_loss_w_pred)
model.train_on_batch(np.random.random((11, 32, 32, 3)),
np.random.random(label_shape))
x = tf.constant(np.random.random((11, 32, 32, 3)).astype(np.float32))
y = tf.constant(np.random.random(label_shape).astype(np.float32))
model_loss = model.test_on_batch(x, y)
fn_loss = tf.keras.backend.get_value(loss_fn((x, y)))
fn_loss_w_pred = tf.keras.backend.get_value(
loss_fn((x, y), prediction=model(x)))
self.assertAlmostEqual(model_loss, fn_loss, fn_loss_w_pred)
def _create_optimizer(self):
"""Initializes the hyperparameters and sets the self._optimizer property."""
if self._optimizer:
return
if not self._layer_collection:
self.register_layers(self._model, self._loss)
if self._config['adapt_damping']:
if 'train_batch' not in self._kfac_kwargs:
raise ValueError(
'Must provide a train_batch tuple to use adaptive '
'damping. Use register_train_batch or pass it in '
'during optimizer construction.')
if 'loss_fn' not in self._kfac_kwargs:
self._kfac_kwargs['loss_fn'] = utils.get_loss_fn(
self._model,
self._loss,
loss_weights=self._config['loss_weights'])
with tf.name_scope(self._name):
with tf.init_scope():
# "iterations" property will create iterations if necessary.
_ = self.iterations
self._create_hypers()
self._kfac_kwargs.update(self._hyper)
try:
# We use the TF 1 variable_scope instead of the TF 2 recommended
# name_scope because we need to recover the variables created in this
# scope, which is not possible with name_scope.
with tf.variable_scope(self._tf_var_scope):
self._optimizer = _KFAC_OPT_CLASS(
layer_collection=self._layer_collection,
**self._kfac_kwargs)
except ValueError as e:
msg = str(e)
if re.search('Variable .* already exists', msg):
raise ValueError(
'You may have instantiated a KFAC Optimizer with the same name as '
'an existing one. Try resetting the default graph, instantiating '
'the optimizer with a different name, or changing the optimizer\'s '
'name.\nHere is the original ValueError:\n ' + msg)
elif re.search(
'Found the following errors with variable registration'
'.*gamma.*registered with wrong number of uses.*', msg):
# We don't regex the name batch_normalization because the user could
# have renamed the layer. We don't regex beta because they could have
# used BatchNorm without the shift.
raise ValueError(
'There may have been an issue registering BatchNormalization. Try '
'using tf.keras.backend.set_learning_phase before model '
'construction. An alternative solution is to use the unfused '
'batchnorm implementation (pass the argument fused=False to '
'BatchNormalization).\nHere is the original ValueError:\n '
+ msg)
else:
raise e
def testMSE(self, loss):
model = _mlp()
model.compile('sgd', loss)
loss_fn = utils.get_loss_fn(model=model, loss=loss)
x = tf.constant(np.random.random((23, 1)).astype(np.float32))
y = tf.constant(np.random.random((23, 1)).astype(np.float32))
model_loss = model.test_on_batch(x, y)
fn_loss = tf.keras.backend.get_value(loss_fn((x, y)))
fn_loss_w_pred = tf.keras.backend.get_value(
loss_fn((x, y), prediction=model(x)))
self.assertAlmostEqual(model_loss, fn_loss, fn_loss_w_pred)
def testMultiLoss(self, multi_loss, loss_weights):
inps, outs = _two_loss_model()
model = tf.keras.Model(inputs=inps, outputs=outs)
model.compile('sgd', multi_loss, loss_weights=loss_weights)
loss_fn = utils.get_loss_fn(model=model,
loss=multi_loss,
loss_weights=loss_weights)
x = tf.constant(np.random.random((11, 28, 28, 1)).astype(np.float32))
y_1 = tf.constant(np.random.random((11, 1)).astype(np.float32))
y_2 = tf.constant(np.random.random((11, 9)).astype(np.float32))
# test_on_batch returns the total loss and the two individual losses.
# We just want the total, so we use model_loss[0].
model_loss = model.test_on_batch(x, [y_1, y_2])[0]
fn_loss = tf.keras.backend.get_value(loss_fn((x, [y_1, y_2])))
fn_loss_w_pred = tf.keras.backend.get_value(
loss_fn((x, [y_1, y_2]), prediction=model(x)))
self.assertAlmostEqual(model_loss, fn_loss, fn_loss_w_pred)