def test_prelu(self): batch_size = 32 num_classes = 10 (x_train, y_train), (x_test, y_test) = load_cifar10() model = PReLUDepthNet(input_shape=x_train.shape[1:], classes=num_classes, width=self.width, depth=self.depth) opt = keras.optimizers.adam(lr=self.lr) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=self.metrics) print(model.summary()) log_dir = 'summaries/prelu_depth{}-{}-{}-{}'.format( self.depth, self.seed, self.lr, datetime.datetime.now()) model.fit(x_train, y_train, batch_size=batch_size, epochs=self.epochs, validation_data=(x_test, y_test), shuffle=False, callbacks=[ TensorBoard(log_dir=log_dir), ]) model.save('checkpoints/depth{}-prelu.h5'.format(self.depth))
def test_selu(self): batch_size = 32 num_classes = 10 (x_train, y_train), (x_test, y_test) = load_cifar10() model = network_for_cifar10_factory('selu', x_train.shape[1:], num_classes) opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=self.metrics) log_dir = 'summaries/selu-extended-{}-{}'.format( self.seed, datetime.datetime.now()) model.fit(x_train, y_train, batch_size=batch_size, epochs=self.epochs, validation_data=(x_test, y_test), shuffle=False, callbacks=[ TensorBoard(log_dir=log_dir), ])
def test_lkrelu(self): batch_size = 32 num_classes = 10 (x_train, y_train), (x_test, y_test) = load_cifar10() inputs = Input(shape=x_train.shape[1:]) x = Conv2D(self.width, (3, 3))(inputs) x = LKReLU()(x) x = Flatten()(x) x = Dense(num_classes, activation='softmax', name='fc1000')(x) model = Model(inputs=inputs, outputs=x) print(model.summary()) opt = keras.optimizers.sgd() model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=self.metrics) log_dir = 'summaries/width-lkrelu-{}-cifar10-{}-{}'.format( self.width, self.seed, datetime.datetime.now()) model.fit( x_train, y_train, batch_size=batch_size, epochs=self.epochs, validation_data=(x_test, y_test), shuffle=False, callbacks=[ TensorBoard(log_dir=log_dir), # ModelCheckpoint( # 'checkpoints/width-lkrelu-cifar10-{epoch:02d}-{val_loss:.2f}.hdf5') ]) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1])
def test_extended(self): resultdir = 'summaries/' table_outputdir = '../../docs/article/tables/' table_data = {} activations = [ 'relu', 'lkrelu', 'relu-sqrt2', 'prelu', 'lkprelu', 'prelu-sqrt2', 'selu', 'lkselu', 'selu-sqrt2', 'swish', 'lkswish', 'swish-sqrt2' ] colnames = { 'relu': 'ReLU', 'lkrelu': 'LK-ReLU', 'relu-sqrt2': 'ReLU sqrt2', 'prelu': 'PReLU', 'lkprelu': 'LK-PReLU', 'prelu-sqrt2': 'PReLU sqrt2', 'selu': 'SELU', 'lkselu': 'LK-SELU', 'selu-sqrt2': 'SELU sqrt2', 'swish': 'Swish', 'lkswish': 'LK-Swish', 'swish-sqrt2': 'Swish sqrt2' } num_classes = 10 (x_train, y_train), (x_test, y_test) = load_cifar10() networks = { 'relu': network_for_cifar10_factory('relu', x_train.shape[1:], num_classes), 'prelu': network_for_cifar10_factory('prelu', x_train.shape[1:], num_classes), 'selu': network_for_cifar10_factory('selu', x_train.shape[1:], num_classes), 'swish': network_for_cifar10_factory('swish', x_train.shape[1:], num_classes), 'lkrelu': network_for_cifar10_factory('lkrelu', x_train.shape[1:], num_classes), 'lkprelu': network_for_cifar10_factory('lkprelu', x_train.shape[1:], num_classes), 'lkselu': network_for_cifar10_factory('lkselu', x_train.shape[1:], num_classes), 'lkswish': network_for_cifar10_factory('lkswish', x_train.shape[1:], num_classes), 'relu-sqrt2': network_for_cifar10_factory('relu', x_train.shape[1:], num_classes, width=np.sqrt(2)), 'prelu-sqrt2': network_for_cifar10_factory('prelu', x_train.shape[1:], num_classes, width=np.sqrt(2)), 'selu-sqrt2': network_for_cifar10_factory('selu', x_train.shape[1:], num_classes, width=np.sqrt(2)), 'swish-sqrt2': network_for_cifar10_factory('swish', x_train.shape[1:], num_classes, width=np.sqrt(2)), } parameters = pd.DataFrame( [ get_parameter_counts(networks[activation])['total'] for activation in activations ], index=[colnames[activation] for activation in activations]) tags = ['val_acc'] for tag in tags: table_tag = [] for activation in activations: template = os.path.join( resultdir, '{}-extended*/events*'.format(activation, tag)) filenames = glob.glob(template) if not filenames: raise RuntimeError( 'Files not found for {}'.format(template)) maxes = [] for filename in filenames: maxes.append(get_best(filename, tag)) median = np.median(maxes) table_tag.append(median) table_tag = pd.DataFrame( [table_tag], index=['Best'], columns=[colnames[activation] for activation in activations]) table_data[tag] = table_tag accuracy_table = table_data['val_acc'].T singles = accuracy_table.loc[['ReLU', 'PReLU', 'SELU', 'Swish']].values lks = accuracy_table.loc[[ 'LK-ReLU', 'LK-PReLU', 'LK-SELU', 'LK-Swish' ]].values sqrt2 = accuracy_table.loc[[ 'ReLU sqrt2', 'PReLU sqrt2', 'SELU sqrt2', 'Swish sqrt2' ]].values table = np.concatenate([singles, sqrt2, lks], axis=1) table = pd.DataFrame(table, index=['ReLU', 'PReLU', 'SELU', 'Swish'], columns=['Single', '$sqrt{2}$', 'LK-inked']) table.to_latex(os.path.join(table_outputdir, 'extended.tex')) param_singles = parameters.loc[['ReLU', 'PReLU', 'SELU', 'Swish']].values param_lks = parameters.loc[[ 'LK-ReLU', 'LK-PReLU', 'LK-SELU', 'LK-Swish' ]].values param_sqrt2 = parameters.loc[[ 'ReLU sqrt2', 'PReLU sqrt2', 'SELU sqrt2', 'Swish sqrt2' ]].values param_table = np.concatenate([param_singles, param_sqrt2, param_lks], axis=1) param_table = pd.DataFrame(param_table, index=['ReLU', 'PReLU', 'SELU', 'Swish'], columns=['Single', '$\sqrt{2}$', 'Linked']) param_table.to_latex( os.path.join(table_outputdir, 'param_extended.tex'))