def test_callbacks(self):
self._apply_tests_bank([
dict(
layer_type=Sequential,
layer_params={
"layers": [
Input((5, 5, 1)),
SpectralConv2D(2, (3, 3)),
ScaledAveragePooling2D((2, 2)),
SpectralDense(4),
]
},
batch_size=250,
steps_per_epoch=125,
epochs=5,
input_shape=(5, 5, 1),
k_lip_data=1.0,
k_lip_model=1.0,
callbacks=[CondenseCallback(on_batch=True, on_epoch=False)],
),
dict(
layer_type=Sequential,
layer_params={
"layers": [
Input((5, 5, 1)),
SpectralConv2D(2, (3, 3), name="conv1"),
ScaledAveragePooling2D((2, 2)),
SpectralDense(4, name="dense1"),
]
},
batch_size=250,
steps_per_epoch=125,
epochs=5,
input_shape=(5, 5, 1),
k_lip_data=1.0,
k_lip_model=1.0,
callbacks=[
# CondenseCallback(on_batch=False, on_epoch=True),
MonitorCallback(
monitored_layers=["conv1", "dense1"],
logdir=os.path.join("logs", "lip_layers",
"Sequential"),
target="kernel",
what="all",
on_epoch=False,
on_batch=True,
),
MonitorCallback(
monitored_layers=["conv1", "dense1"],
logdir=os.path.join("logs", "lip_layers",
"Sequential"),
target="wbar",
what="all",
on_epoch=False,
on_batch=True,
),
],
),
])
def test_multilayer(self):
self._apply_tests_bank([
dict(
layer_type=Sequential,
layer_params={
"layers": [
Input((5, 5, 1)),
SpectralConv2D(2, (3, 3), use_bias=False),
SpectralDense(4),
]
},
batch_size=250,
steps_per_epoch=125,
epochs=5,
input_shape=(5, 5, 1),
k_lip_data=1.0,
k_lip_model=1.0,
callbacks=[],
),
dict(
layer_type=Sequential,
layer_params={
"layers": [
Input((5, 5, 1)),
SpectralConv2D(2, (3, 3)),
SpectralDense(4),
]
},
batch_size=250,
steps_per_epoch=125,
epochs=5,
input_shape=(5, 5, 1),
k_lip_data=5.0,
k_lip_model=1.0,
callbacks=[],
),
dict(
layer_type=Sequential,
layer_params={
"layers": [
Input((5, 5, 1)),
SpectralConv2D(2, (3, 3)),
SpectralDense(4),
]
},
batch_size=250,
steps_per_epoch=125,
epochs=5,
input_shape=(5, 5, 1),
k_lip_data=1.0,
k_lip_model=5.0,
callbacks=[],
),
])
def test_as_supertype_model(self):
input_shape = (5, 5, 3)
inp = Input(input_shape)
x = SpectralConv2D(2, (3, 3), k_coef_lip=2.0)(inp)
x = FrobeniusConv2D(2, (3, 3), k_coef_lip=2.0)(x)
x = Flatten()(x)
x = Dense(4)(x)
x = SpectralDense(4, k_coef_lip=2.0)(x)
x = FrobeniusDense(2, k_coef_lip=2.0)(x)
model = Model(inputs=inp, outputs=x)
self._test_model(model, input_shape)
def test_as_supertype_model(self):
input_shape = (8, 8, 3)
inp = Input(input_shape)
x = SpectralConv2D(2, (3, 3), k_coef_lip=2.0)(inp)
x = ScaledL2NormPooling2D((2, 2), k_coef_lip=2.0)(x)
x = FrobeniusConv2D(2, (3, 3), k_coef_lip=2.0)(x)
x = Flatten()(x)
x = Dense(4)(x)
x = SpectralDense(4, k_coef_lip=2.0)(x)
out = FrobeniusDense(2, k_coef_lip=2.0)(x)
model = Model(inputs=inp, outputs=out)
self._test_model(model, input_shape)
def get_cnn_baseline(input_shape, k_coef_lip, scale, niter_spectral,
niter_bjorck, bjorck_forward):
model = deel.lip.model.Sequential([
Input(shape=input_shape),
SpectralConv2D(filters=4 * scale,
kernel_size=(3, 3),
data_format='channels_last'),
GroupSort(n=2),
SpectralConv2D(filters=4 * scale,
kernel_size=(3, 3),
data_format='channels_last'),
GroupSort(n=2),
ScaledL2NormPooling2D(pool_size=(2, 2), data_format='channels_last'),
SpectralConv2D(filters=4 * scale,
kernel_size=(3, 3),
data_format='channels_last'),
GroupSort(n=2),
SpectralConv2D(filters=4 * scale,
kernel_size=(3, 3),
data_format='channels_last'),
GroupSort(n=2),
ScaledL2NormPooling2D(pool_size=(2, 2), data_format='channels_last'),
Flatten(),
SpectralDense(8 * scale,
niter_spectral=niter_spectral,
niter_bjorck=niter_bjorck,
bjorck_forward=bjorck_forward),
FullSort(),
SpectralDense(8 * scale,
niter_spectral=niter_spectral,
niter_bjorck=niter_bjorck,
bjorck_forward=bjorck_forward),
FullSort(),
SpectralDense(1, activation="linear", bjorck_forward=bjorck_forward),
],
k_coef_lip=k_coef_lip,
name='cnn_baseline')
return model
def __init__(self, num_heads, height, scale, niter_bjorck, niter_spectral,
bjorck_forward, **kwargs):
super().__init__(**kwargs)
self._kwargs = kwargs
self.num_heads = num_heads
self.height = height
self.scale = scale
self.niter_bjorck = niter_bjorck
self.niter_spectral = niter_spectral
self.bjorck_forward = bjorck_forward
heads = [[] for _ in range(1 + 2 * self.height)]
for _ in range(self.num_heads):
for j in range(self.height):
heads[2 * j].append(
SpectralDense(self.scale,
niter_bjorck=self.niter_bjorck,
niter_spectral=self.niter_spectral,
bjorck_forward=self.bjorck_forward))
heads[2 * j + 1].append(GroupSort2())
heads[2 * self.height].append(FrobeniusDense(1))
self.heads = heads
def test_as_supertype_sequential(self):
input_shape = (5, 5, 3)
model = Sequential(
[
# the bug occurs only when using the "as_supertype_export" function
# with:
# - lipschitz coef != 1.0
# - Frobenius layer ( not the Spectral ones )
# - Sequential model ( not Model )
# - tensorflow 2.1 ( not 2.0 )
Input(input_shape),
SpectralConv2D(2, (3, 3)),
FrobeniusConv2D(2, (3, 3)),
Flatten(),
Dense(4),
SpectralDense(4),
FrobeniusDense(2),
],
k_coef_lip=5.0,
)
self._test_model(model, input_shape)
def get_lipschitz_overfitter(ModelType, input_shape, output_shape, k_coef_lip,
scale, niter_bjorck, niter_spectral, groupsort,
conv, bjorck_forward, scaler, multihead, deep,
very_deep, final_pooling):
Act = (lambda: GroupSort2()) if groupsort else (lambda: FullSort())
layers = [tf.keras.layers.Input(shape=input_shape)]
if conv:
spectral_conv = True
conv_maker = lambda *args, **kw: (SpectralConv2D(*args, **kw)
if spectral_conv else OrthoConv2D(
*args, **kw))
pooling = 'scaled'
if pooling == 'scaled':
pooler = lambda: ScaledL2NormPooling2D(pool_size=(2, 2))
elif pooling == 'invertible':
pooler = lambda: InvertibleDownSampling(pool_size=(2, 2))
layers += [
conv_maker(scale * 1, (3, 3)),
GroupSort2(),
conv_maker(scale * 1, (3, 3)),
GroupSort2(),
pooler(),
conv_maker(scale * 2, (3, 3)),
GroupSort2(),
conv_maker(scale * 2, (3, 3)),
GroupSort2(),
pooler(),
conv_maker(scale * 4, (3, 3)),
GroupSort2(),
conv_maker(scale * 4, (3, 3)),
GroupSort2(),
]
if very_deep:
layers += [
pooler(),
conv_maker(scale * 4, (2, 2)),
GroupSort2(),
conv_maker(scale * 4, (2, 2)),
GroupSort2(),
]
if pooling == 'invertible':
layers.append(conv_maker(scale * 1, (1, 1)))
if final_pooling == 'flatten':
layers += [pooler(), tf.keras.layers.Flatten()]
elif final_pooling == 'global':
layers.append(GlobalL2NormPooling2D())
dense_scale = min(512, 4 * scale)
layers += [
SpectralDense(dense_scale,
niter_bjorck=niter_bjorck,
niter_spectral=niter_spectral,
bjorck_forward=bjorck_forward),
Act()
]
if deep:
layers += [
SpectralDense(dense_scale,
niter_bjorck=niter_bjorck,
niter_spectral=niter_spectral,
bjorck_forward=bjorck_forward),
Act()
]
if multihead is not None:
heads_width = int(scale / (output_shape**0.5)) # heuristic
layers.append(
MultiLipschitzHead(output_shape,
height=multihead,
scale=heads_width,
niter_bjorck=niter_bjorck,
niter_spectral=niter_spectral,
bjorck_forward=bjorck_forward))
else:
layers += [
SpectralDense(dense_scale,
niter_bjorck=niter_bjorck,
niter_spectral=niter_spectral,
bjorck_forward=bjorck_forward),
Act(),
UnitaryRowsDense(output_shape),
]
if scaler:
layers.append(Scaler())
model = ModelType(layers, k_coef_lip=k_coef_lip, name='overfitter')
return model