def test_prelu(self):
batch_size = 32
num_classes = 10
(x_train, y_train), (x_test, y_test) = load_cifar10()
model = PReLUDepthNet(input_shape=x_train.shape[1:],
classes=num_classes,
width=self.width,
depth=self.depth)
opt = keras.optimizers.adam(lr=self.lr)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=self.metrics)
print(model.summary())
log_dir = 'summaries/prelu_depth{}-{}-{}-{}'.format(
self.depth, self.seed, self.lr, datetime.datetime.now())
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=self.epochs,
validation_data=(x_test, y_test),
shuffle=False,
callbacks=[
TensorBoard(log_dir=log_dir),
])
model.save('checkpoints/depth{}-prelu.h5'.format(self.depth))
def test_selu(self):
batch_size = 32
num_classes = 10
(x_train, y_train), (x_test, y_test) = load_cifar10()
model = network_for_cifar10_factory('selu', x_train.shape[1:],
num_classes)
opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=self.metrics)
log_dir = 'summaries/selu-extended-{}-{}'.format(
self.seed, datetime.datetime.now())
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=self.epochs,
validation_data=(x_test, y_test),
shuffle=False,
callbacks=[
TensorBoard(log_dir=log_dir),
])
def test_lkrelu(self):
batch_size = 32
num_classes = 10
(x_train, y_train), (x_test, y_test) = load_cifar10()
inputs = Input(shape=x_train.shape[1:])
x = Conv2D(self.width, (3, 3))(inputs)
x = LKReLU()(x)
x = Flatten()(x)
x = Dense(num_classes, activation='softmax', name='fc1000')(x)
model = Model(inputs=inputs, outputs=x)
print(model.summary())
opt = keras.optimizers.sgd()
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=self.metrics)
log_dir = 'summaries/width-lkrelu-{}-cifar10-{}-{}'.format(
self.width, self.seed, datetime.datetime.now())
model.fit(
x_train,
y_train,
batch_size=batch_size,
epochs=self.epochs,
validation_data=(x_test, y_test),
shuffle=False,
callbacks=[
TensorBoard(log_dir=log_dir),
# ModelCheckpoint(
# 'checkpoints/width-lkrelu-cifar10-{epoch:02d}-{val_loss:.2f}.hdf5')
])
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
def test_extended(self):
resultdir = 'summaries/'
table_outputdir = '../../docs/article/tables/'
table_data = {}
activations = [
'relu', 'lkrelu', 'relu-sqrt2', 'prelu', 'lkprelu', 'prelu-sqrt2',
'selu', 'lkselu', 'selu-sqrt2', 'swish', 'lkswish', 'swish-sqrt2'
]
colnames = {
'relu': 'ReLU',
'lkrelu': 'LK-ReLU',
'relu-sqrt2': 'ReLU sqrt2',
'prelu': 'PReLU',
'lkprelu': 'LK-PReLU',
'prelu-sqrt2': 'PReLU sqrt2',
'selu': 'SELU',
'lkselu': 'LK-SELU',
'selu-sqrt2': 'SELU sqrt2',
'swish': 'Swish',
'lkswish': 'LK-Swish',
'swish-sqrt2': 'Swish sqrt2'
}
num_classes = 10
(x_train, y_train), (x_test, y_test) = load_cifar10()
networks = {
'relu':
network_for_cifar10_factory('relu', x_train.shape[1:],
num_classes),
'prelu':
network_for_cifar10_factory('prelu', x_train.shape[1:],
num_classes),
'selu':
network_for_cifar10_factory('selu', x_train.shape[1:],
num_classes),
'swish':
network_for_cifar10_factory('swish', x_train.shape[1:],
num_classes),
'lkrelu':
network_for_cifar10_factory('lkrelu', x_train.shape[1:],
num_classes),
'lkprelu':
network_for_cifar10_factory('lkprelu', x_train.shape[1:],
num_classes),
'lkselu':
network_for_cifar10_factory('lkselu', x_train.shape[1:],
num_classes),
'lkswish':
network_for_cifar10_factory('lkswish', x_train.shape[1:],
num_classes),
'relu-sqrt2':
network_for_cifar10_factory('relu',
x_train.shape[1:],
num_classes,
width=np.sqrt(2)),
'prelu-sqrt2':
network_for_cifar10_factory('prelu',
x_train.shape[1:],
num_classes,
width=np.sqrt(2)),
'selu-sqrt2':
network_for_cifar10_factory('selu',
x_train.shape[1:],
num_classes,
width=np.sqrt(2)),
'swish-sqrt2':
network_for_cifar10_factory('swish',
x_train.shape[1:],
num_classes,
width=np.sqrt(2)),
}
parameters = pd.DataFrame(
[
get_parameter_counts(networks[activation])['total']
for activation in activations
],
index=[colnames[activation] for activation in activations])
tags = ['val_acc']
for tag in tags:
table_tag = []
for activation in activations:
template = os.path.join(
resultdir, '{}-extended*/events*'.format(activation, tag))
filenames = glob.glob(template)
if not filenames:
raise RuntimeError(
'Files not found for {}'.format(template))
maxes = []
for filename in filenames:
maxes.append(get_best(filename, tag))
median = np.median(maxes)
table_tag.append(median)
table_tag = pd.DataFrame(
[table_tag],
index=['Best'],
columns=[colnames[activation] for activation in activations])
table_data[tag] = table_tag
accuracy_table = table_data['val_acc'].T
singles = accuracy_table.loc[['ReLU', 'PReLU', 'SELU', 'Swish']].values
lks = accuracy_table.loc[[
'LK-ReLU', 'LK-PReLU', 'LK-SELU', 'LK-Swish'
]].values
sqrt2 = accuracy_table.loc[[
'ReLU sqrt2', 'PReLU sqrt2', 'SELU sqrt2', 'Swish sqrt2'
]].values
table = np.concatenate([singles, sqrt2, lks], axis=1)
table = pd.DataFrame(table,
index=['ReLU', 'PReLU', 'SELU', 'Swish'],
columns=['Single', '$sqrt{2}$', 'LK-inked'])
table.to_latex(os.path.join(table_outputdir, 'extended.tex'))
param_singles = parameters.loc[['ReLU', 'PReLU', 'SELU',
'Swish']].values
param_lks = parameters.loc[[
'LK-ReLU', 'LK-PReLU', 'LK-SELU', 'LK-Swish'
]].values
param_sqrt2 = parameters.loc[[
'ReLU sqrt2', 'PReLU sqrt2', 'SELU sqrt2', 'Swish sqrt2'
]].values
param_table = np.concatenate([param_singles, param_sqrt2, param_lks],
axis=1)
param_table = pd.DataFrame(param_table,
index=['ReLU', 'PReLU', 'SELU', 'Swish'],
columns=['Single', '$\sqrt{2}$', 'Linked'])
param_table.to_latex(
os.path.join(table_outputdir, 'param_extended.tex'))
