def test_lr_plateau(self):
inputs = np.random.standard_normal((1024, 5))
outputs = (inputs.dot(np.random.standard_normal(
(5, 1))).squeeze(axis=-1) > 0).astype('int32')
model = models.Sequential()
model.add(
layers.Dense(
units=2,
input_shape=(5, ),
use_bias=False,
activation='softmax',
name='Output',
))
model.compile(
optimizer=LRMultiplier(AdamV2(), {'Output': 100.0}),
loss='sparse_categorical_crossentropy',
)
model.fit(
inputs,
outputs,
validation_split=0.1,
epochs=1000,
callbacks=[
callbacks.ReduceLROnPlateau(patience=2, verbose=True),
callbacks.EarlyStopping(patience=5),
],
)
predicted = model.predict(inputs).argmax(axis=-1)
self.assertLess(np.sum(np.abs(outputs - predicted)), 20)
def test_nested(self):
inputs = np.random.standard_normal((1024, 5))
outputs = (inputs.dot(np.random.standard_normal(
(5, 1))).squeeze(axis=-1) > 0).astype('int32')
model = models.Sequential()
model.add(
layers.Dense(
units=5,
input_shape=(5, ),
activation='tanh',
bias_constraint=constraints.max_norm(100.0),
name='Dense',
))
model.add(layers.Dense(
units=2,
activation='softmax',
name='Output',
))
if TF_KERAS:
import tensorflow as tf
base_opt = tf.keras.optimizers.Adam(amsgrad=True, decay=1e-4)
else:
base_opt = keras.optimizers.Adam(amsgrad=True, decay=1e-4)
model.compile(
optimizer=LRMultiplier(LRMultiplier(base_opt, {'Dense': 1.2}),
{'Output': 2.0}),
loss='sparse_categorical_crossentropy',
)
model.fit(
inputs,
outputs,
validation_split=0.1,
epochs=1000,
callbacks=[
callbacks.ReduceLROnPlateau(patience=2, verbose=True),
callbacks.EarlyStopping(patience=5),
],
)
model_path = os.path.join(
tempfile.gettempdir(),
'test_lr_multiplier_%f.h5' % np.random.random())
model.save(model_path)
model = models.load_model(model_path,
custom_objects=self._get_custom_objects())
predicted = model.predict(inputs).argmax(axis=-1)
self.assertLess(np.sum(np.abs(outputs - predicted)), 20)
def test_nested(self):
inputs = np.random.standard_normal((1024, 5))
outputs = (inputs.dot(np.random.standard_normal((5, 1))).squeeze(axis=-1) > 0).astype('int32')
model = models.Sequential()
model.add(layers.Dense(
units=5,
input_shape=(5,),
activation='tanh',
name='Dense',
))
model.add(layers.Dense(
units=2,
activation='softmax',
name='Output',
))
model.compile(
optimizer=LRMultiplier(LRMultiplier('adam', {'Dense': 1.2}), {'Output': 2.0}),
loss='sparse_categorical_crossentropy',
)
model.fit(
inputs,
outputs,
validation_split=0.1,
epochs=1000,
callbacks=[
callbacks.ReduceLROnPlateau(patience=2, verbose=True),
callbacks.EarlyStopping(patience=5),
],
)
if not EAGER_MODE:
model_path = os.path.join(tempfile.gettempdir(), 'test_lr_multiplier_%f.h5' % np.random.random())
model.save(model_path)
model = models.load_model(model_path, custom_objects={'LRMultiplier': LRMultiplier})
predicted = model.predict(inputs).argmax(axis=-1)
self.assertLess(np.sum(np.abs(outputs - predicted)), 20)