def plot_filters(nnet, xp_path, title_suffix, file_prefix="", pretrain=False):
"""
Plot all filters of the first convolutional layer
"""
assert hasattr(nnet, "conv1_layer")
W = nnet.conv1_layer.W.get_value()
title = "First layer filters " + title_suffix
plot_mosaic(W, title=title, canvas="black", export_pdf=(xp_path + "/" + file_prefix + "filters"))
if not pretrain and (nnet.best_weight_dict is not None):
W_best = nnet.best_weight_dict["conv1_w"]
title = ("First layer filters at epoch " + str(nnet.auc_best_epoch) + " " + title_suffix)
plot_mosaic(W_best, title=title, canvas="black", export_pdf=(xp_path + "/" + file_prefix + "filters_best_ep"))
def plot_random_reconstructions(nnet, xp_path, title_suffix, file_prefix, n_img=32):
"""
plot the reconstructions of n_img randomly drawn images
"""
# only plot reconstructions for image data
if nnet.data._X_train.ndim != 4:
return
from utils.visualization.mosaic_plot import plot_mosaic
random_idx = np.random.choice(nnet.data.n_train, n_img, replace=False)
_, _, _, _, _, _, _, reconstruction, _, _ = nnet.forward(nnet.data._X_train[random_idx, ...],
nnet.data._y_train[random_idx])
title = str(n_img) + " random autoencoder reconstructions " + title_suffix
print(reconstruction.shape)
plot_mosaic(reconstruction, title=title, export_pdf=(xp_path + "/" + file_prefix + "ae_reconstructions"))
def train_autoencoder(nnet):
if Cfg.ae_diagnostics:
nnet.initialize_ae_diagnostics(nnet.ae_n_epochs)
print("Starting training autoencoder with %s" % nnet.sgd_solver)
for epoch in range(nnet.ae_n_epochs):
start_time = time.time()
if Cfg.ae_lr_drop and (epoch == Cfg.ae_lr_drop_in_epoch):
# Drop the learning rate in epoch specified in Cfg.ae_lr_drop_after_epoch by factor Cfg.ae_lr_drop_factor
# Thus, a simple separation of learning into a "region search" and "finetuning" stage.
lr_new = Cfg.floatX(
(1.0 / Cfg.ae_lr_drop_factor) * Cfg.learning_rate.get_value())
print("")
print(
"Learning rate drop in epoch {} from {:.6f} to {:.6f}".format(
epoch, Cfg.floatX(Cfg.learning_rate.get_value()), lr_new))
print("")
Cfg.learning_rate.set_value(lr_new)
# In each epoch, we do a full pass over the training data:
l2 = 0
batches = 0
train_err = 0
train_scores = np.empty(nnet.data.n_train)
for batch in nnet.data.get_epoch_train():
inputs, _, batch_idx = batch
start_idx = batch_idx * Cfg.batch_size
stop_idx = min(nnet.data.n_train, start_idx + Cfg.batch_size)
err, l2, b_scores = nnet.ae_backprop(inputs)
train_err += err * inputs.shape[0]
train_scores[start_idx:stop_idx] = b_scores.flatten()
batches += 1
train_err /= nnet.data.n_train
# save train diagnostics and test performance on val and test data if specified
if Cfg.ae_diagnostics:
nnet.save_ae_diagnostics('train', epoch, train_err, train_scores,
l2)
# Performance on validation and test set
if nnet.data.n_val > 0:
val_err = ae_performance(nnet, which_set='val', epoch=epoch)
test_err = ae_performance(nnet, which_set='test', epoch=epoch)
# print results for epoch
print("{:32} {:.5f}".format("Train error:", train_err))
if Cfg.ae_diagnostics:
if nnet.data.n_val > 0:
print("{:32} {:.5f}".format("Val error:", val_err))
print("{:32} {:.5f}".format("Test error:", test_err))
print("Epoch {} of {} took {:.3f}s".format(epoch + 1, nnet.ae_n_epochs,
time.time() - start_time))
print("")
# Get final performance in last epoch if no running diagnostics are taken
if not Cfg.ae_diagnostics:
nnet.initialize_ae_diagnostics(1)
# perform forward passes on train, val, and test set
print("Get final performance...")
_ = ae_performance(nnet, which_set='train', epoch=0)
if nnet.data.n_val > 0:
_ = ae_performance(nnet, which_set='val', epoch=0)
_ = ae_performance(nnet, which_set='test', epoch=0)
print("Evaluation completed.")
# save weights
if Cfg.pretrain:
nnet.dump_weights("{}/ae_pretrained_weights.p".format(Cfg.xp_path),
pretrain=True)
else:
nnet.dump_weights("{}/weights_final.p".format(Cfg.xp_path))
# if image data plot some random reconstructions
if nnet.data._X_train.ndim == 4:
from utils.visualization.mosaic_plot import plot_mosaic
n_img = 32
random_idx = np.random.choice(nnet.data.n_train, n_img, replace=False)
_, _, _, reps = nnet.ae_forward(nnet.data._X_train[random_idx, ...])
title = str(n_img) + " random autoencoder reconstructions"
plot_mosaic(reps,
title=title,
export_pdf=(Cfg.xp_path + "/ae_reconstructions"))
# plot diagnostics if specified
if Cfg.ae_diagnostics & Cfg.pretrain:
from utils.visualization.diagnostics_plot import plot_ae_diagnostics
from utils.visualization.filters_plot import plot_filters
# common suffix for plot titles
str_lr = "lr = " + str(nnet.ae_learning_rate)
C = int(Cfg.C.get_value())
if not Cfg.weight_decay:
C = None
str_C = "C = " + str(C)
title_suffix = "(" + nnet.ae_solver + ", " + str_C + ", " + str_lr + ")"
# plot diagnostics
plot_ae_diagnostics(nnet, Cfg.xp_path, title_suffix)
# plot filters
plot_filters(nnet,
Cfg.xp_path,
title_suffix,
file_prefix="ae_",
pretrain=True)
def build_autoencoder(self, nnet):
# implementation of the network architecture
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, 32, 5, n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
# Code Layer
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
nnet.addReshapeLayer(shape=([0], (Cfg.cifar10_rep_dim / 16), 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(scale_factor=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(scale_factor=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(scale_factor=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=3,
filter_size=(5, 5),
pad='same',
b=None)
nnet.addSigmoidLayer()
def build_architecture(self, nnet):
# Build the encoder architecture
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=32,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 3
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss or Cfg.msvdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
def build_autoencoder(self, nnet):
# implementation of different network architectures
assert Cfg.mnist_architecture in (1, 2)
if Cfg.mnist_architecture == 1:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
8,
5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=8,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=4,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
# Code Layer
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
nnet.addReshapeLayer(shape=([0], (Cfg.mnist_rep_dim / 16), 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(
scale_factor=(2, 2)
) # TODO: is this Upscale necessary? Shouldn't there be as many Upscales as MaxPools?
addConvModule(nnet,
num_filters=4,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# to have the same output dimensions, pad must be 1 here
addConvModule(nnet,
num_filters=8,
filter_size=(5, 5),
pad=1,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mnist_bias:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same',
b=None)
nnet.addSigmoidLayer()
elif Cfg.mnist_architecture == 2:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=256,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=128,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320,
b=None)
# Code Layer
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320,
b=None)
nnet.addReshapeLayer(shape=([0], 20, 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
addConvModule(nnet,
num_filters=128,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# to have the same output dimensions, pad must be 1 here
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
pad=1,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mnist_bias:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same',
b=None)
nnet.addSigmoidLayer()
else:
raise ValueError("No valid choice of architecture")
def build_architecture(self, nnet):
# implementation of different network architectures
assert Cfg.mnist_architecture in (1, 2)
# increase number of parameters if dropout is used
if Cfg.dropout_architecture:
units_multiplier = 2
else:
units_multiplier = 1
if Cfg.mnist_architecture == 1:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=8,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=8 * units_multiplier,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2)
addConvModule(nnet,
num_filters=4 * units_multiplier,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier,
b=None)
elif Cfg.mnist_architecture == 2:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=256,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=256 * units_multiplier,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2)
addConvModule(nnet,
num_filters=256 * units_multiplier,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
addConvModule(nnet,
num_filters=128 * units_multiplier,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=320 * units_multiplier)
else:
nnet.addDenseLayer(num_units=320 * units_multiplier, b=None)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier,
b=None)
else:
raise ValueError("No valid choice of architecture")
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
def build_architecture(self, nnet):
# implementation of different network architectures
assert Cfg.cifar10_architecture in (1, 2, 3)
if Cfg.cifar10_architecture == 1:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=16,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 1
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same')
else:
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
if Cfg.cifar10_bias:
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
if Cfg.cifar10_architecture == 2:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=16,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 2
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
if Cfg.cifar10_architecture == 3:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=32,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 3
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
def plot_outliers_and_most_normal(model, n, xp_path):
"""
Plot outliers and most normal examples in ascending order of train, val, and test set.
That is, the top left image is the most normal, the bottom right image the most anomalous example.
"""
# reload images with original scale
model.data.load_data(original_scale=True)
for which_set in ['train', 'val', 'test']:
if which_set == 'train':
X = model.data._X_train
n_samples = model.data.n_train
if which_set == 'val':
X = model.data._X_val
n_samples = model.data.n_val
if which_set == 'test':
X = model.data._X_test
n_samples = model.data.n_test
if X.size > 0: # only if set is specified
idx_sort = model.diag[which_set]['scores'][:, -1].argsort()
normals = X[idx_sort, ...][:n, ...]
outliers = X[idx_sort, ...][-n:, ...]
str_samples = "(" + str(n) + " of " + str(int(n_samples)) + ")"
title_norm = which_set.title(
) + " set examples with ascending scores " + str_samples
title_out = which_set.title(
) + " set outliers with ascending scores " + str_samples
plot_mosaic(normals,
title=title_norm,
export_pdf=(xp_path + "/normals_" + which_set))
plot_mosaic(outliers,
title=title_out,
export_pdf=(xp_path + "/outliers_" + which_set))
# plot with scores at best epoch
if model.best_weight_dict is not None:
epoch = model.auc_best_epoch
str_epoch = "at epoch " + str(epoch) + " "
idx_sort = model.diag[which_set]['scores'][:, epoch].argsort()
normals = X[idx_sort, ...][:n, ...]
outliers = X[idx_sort, ...][-n:, ...]
str_samples = "(" + str(n) + " of " + str(int(n_samples)) + ")"
title_norm = (which_set.title() +
" set examples with ascending scores " +
str_epoch + str_samples)
title_out = (which_set.title() +
" set outliers with ascending scores " +
str_epoch + str_samples)
plot_mosaic(normals,
title=title_norm,
export_pdf=(xp_path + "/normals_" + which_set +
"_best_ep"))
plot_mosaic(outliers,
title=title_out,
export_pdf=(xp_path + "/outliers_" + which_set +
"_best_ep"))
else:
pass
def build_autoencoder(self, nnet):
# implementation of different network architectures
if Cfg.architecture not in (1,2,3,4):
# architecture spec A_B_C_D_E_F_G_H
tmp = Cfg.architecture.split("_")
use_pool = int(tmp[0]) # 1 or 0
n_conv = int(tmp[1])
n_dense = int(tmp[2])
c_out = int(tmp[3])
zsize = int(tmp[4])
ksize= int(tmp[5])
stride = int(tmp[6])
num_filters = c_out
if use_pool:
print("Using pooling and upscaling")
pad = 'same'
crop = 'same'
else:
print("Using strided convolutions for dim. reduction/upscaling")
deconvinpad = 0
convinpad = (ksize-stride+1)//2
crop = convinpad
outpad = 0
# Build architecture
nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width))
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, n_filters=c_out, filter_size=ksize, n_sample=Cfg.n_dict_learn)
plot_mosaic(W1_init, title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# Add all but last conv. layer
for i in range(n_conv-1):
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = convinpad,
)
num_filters *= 2
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
if n_dense > 0:
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = convinpad,
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Dense layer
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.bias:
nnet.addDenseLayer(num_units=zsize, use_batch_norm=False)
else:
nnet.addDenseLayer(num_units=zsize, use_batch_norm=Cfg.use_batch_norm,
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.debug_architecture_layers: print("Added dense layer")
else:
h = self.image_height / (2**(n_conv-1))
addConvModule(nnet,
num_filters=zsize,
filter_size=(h,h),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=False,
dropout=False,
p_dropout=0.2,
use_maxpool = False,
stride = (1,1),
pad = (0,0),
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Now image (channels, height, width) = (zsize,1,1),
nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer
if Cfg.debug_architecture_layers: print("Feature layer here")
n_deconv_layers = 0
if n_dense > 0:
h1 = self.image_height // (2**n_conv) # height = width of image going into first conv layer
num_filters = c_out * (2**(n_conv-1))
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.bias:
nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm)
else:
nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm,
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.debug_architecture_layers: print("Added dense layer")
nnet.addReshapeLayer(shape=([0], num_filters, h1, h1))
if Cfg.debug_architecture_layers: print("Reshaping to (None, %d, %d, %d)"%(num_filters, h1, h1))
num_filters = num_filters // 2
if use_pool:
nnet.addUpscale(scale_factor=(2,2)) # since maxpool is after each conv. each upscale is before corresponding deconv
output_size = None
else:
output_size = h1*2
if n_conv > 1:
addConvTransposeModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = stride,
crop = crop,
outpad = outpad,
upscale = use_pool,
inpad = deconvinpad,
output_size = output_size
)
n_deconv_layers += 1
#print("Added deconv_layer %d" % n_deconv_layers)
if Cfg.debug_architecture_layers: print("Added deconv_layer %d: (None, %d, %d, %d)" % (n_deconv_layers, num_filters, output_size,output_size))
num_filters //=2
if not use_pool:
output_size *= 2
elif n_conv > 1:
h2 = self.image_height // (2**(n_conv-1)) # height of image going in to second conv layer
num_filters = c_out * (2**(n_conv-2))
addConvTransposeModule(nnet,
num_filters=num_filters,
filter_size=(h2,h2),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = (1,1),
crop = 0,
outpad = outpad,
upscale = use_pool,
inpad = deconvinpad
)
n_deconv_layers += 1
if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers)
output_size = h2*2
num_filters //= 2
else:
if use_pool:
output_size = None
else:
output_size = self.image_height # only conv layer will be reconstruction layer
# Add remaining deconv layers
for i in range(n_conv-2):
addConvTransposeModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = stride,
crop = crop,
outpad = outpad,
upscale = use_pool,
inpad = deconvinpad,
output_size = output_size
)
n_deconv_layers += 1
if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers)
if not use_pool:
output_size *= 2
num_filters //=2
# add reconstruction layer
addConvTransposeModule(nnet,
num_filters=self.channels,
filter_size=(ksize,ksize),
W_init=W1_init,
#pad = "valid",
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=False,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = stride,
crop = crop,
outpad = outpad,
upscale = False,
inpad = deconvinpad,
output_size = output_size,
reconstruction_layer=True
)
if Cfg.debug_architecture_layers: print("Added reconstruction layer")
def build_architecture(self, nnet):
# implementation of different network architectures
if Cfg.architecture not in (1,2,3):
# architecture spec A_B_C_D_E_F_G_H
tmp = Cfg.architecture.split("_")
use_pool = int(tmp[0]) # 1 or 0
n_conv = int(tmp[1])
n_dense = int(tmp[2])
c1 = int(tmp[3])
zsize = int(tmp[4])
ksize= int(tmp[5])
stride = int(tmp[6])
num_filters = c1
# If using maxpool, we should have pad = same
if use_pool:
pad = 'same'
else:
pad = (ksize-stride+1)//2
pad = (pad,pad)
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, n_filters=c1, filter_size=ksize, n_sample=Cfg.n_dict_learn)
plot_mosaic(W1_init, title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# Build architecture
nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width))
# Add all but last conv. layer
for i in range(n_conv-1):
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = pad,
)
num_filters *= 2
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
if n_dense > 0:
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = pad,
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Dense layer
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.bias:
nnet.addDenseLayer(num_units=zsize)
else:
nnet.addDenseLayer(num_units=zsize,
b=None)
if Cfg.debug_architecture_layers: print("Added dense layer")
else:
h = self.image_height / (2**(n_conv-1))
addConvModule(nnet,
num_filters=zsize,
filter_size=(h,h),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=False,
dropout=False,
p_dropout=0.2,
use_maxpool = False,
stride = (1,1),
pad = (0,0),
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Add ouput/feature layer
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet):
# implementation of the autoencoder architectures
fs = (5, 5)
fn = [64, 32]
mps = (2, 2)
hn = fn[-1] * 5
##############################
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary_new(nnet.data._X_train,
n_filters=fn[0],
filter_shape=fs,
n_sample=500)
W1_init = np.reshape(W1_init, (fn[0], self._X_train.shape[1]) + fs)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
nnet.addInputLayer(shape=(None, ) + self._X_train.shape[1:])
addConvModule(nnet,
num_filters=fn[0],
filter_size=fs,
W_init=W1_init,
bias=Cfg.mobiFall_bias,
pool_size=mps,
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=fn[1],
filter_size=fs,
bias=Cfg.mobiFall_bias,
pool_size=mps,
use_batch_norm=Cfg.use_batch_norm)
h_units = np.prod(nnet.all_layers[-1].output_shape[1:])
d_r = nnet.all_layers[-1].output_shape[1:]
if Cfg.mobiFall_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=hn,
b=None)
# Code Layer
if Cfg.mobiFall_bias:
nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
if Cfg.mobiFall_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=hn,
b=None)
if Cfg.mobiFall_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=h_units)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=h_units,
b=None)
nnet.addReshapeLayer(shape=([0], h_units / np.prod(d_r[1:]), d_r[1],
d_r[2]))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
addConvModule(nnet,
num_filters=fn[1],
filter_size=fs,
bias=Cfg.mobiFall_bias,
pool_size=mps,
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
addConvModule(nnet,
num_filters=fn[0],
filter_size=fs,
bias=Cfg.mobiFall_bias,
pool_size=mps,
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mobiFall_bias:
nnet.addConvLayer(num_filters=1, filter_size=fs, pad='same')
else:
nnet.addConvLayer(num_filters=3,
filter_size=fs,
pad='same',
b=None)
def build_architecture(self, nnet):
# implementation of the encoder network architectures
fs = (5, 5)
fn = [64, 32]
mps = (2, 2)
hn = fn[-1] * 5
# increase number of parameters if dropout is used
if Cfg.dropout_architecture:
units_multiplier = 2
else:
units_multiplier = 1
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary_new(nnet.data._X_train,
n_filters=fn[0],
filter_shape=fs,
n_sample=500)
W1_init = np.reshape(W1_init, (fn[0], self._X_train.shape[1]) + fs)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, ) + self._X_train.shape[1:])
addConvModule(nnet,
num_filters=fn[0] * units_multiplier,
filter_size=fs,
W_init=W1_init,
bias=Cfg.mobiFall_bias,
pool_size=mps,
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2)
addConvModule(nnet,
num_filters=fn[1] * units_multiplier,
filter_size=fs,
bias=Cfg.mobiFall_bias,
pool_size=mps,
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mobiFall_bias:
nnet.addDenseLayer(num_units=hn * units_multiplier)
else:
nnet.addDenseLayer(num_units=hn * units_multiplier, b=None)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mobiFall_bias:
nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim *
units_multiplier)
else:
nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim *
units_multiplier,
b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss or Cfg.msvdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
def build_autoencoder(self, nnet):
# implementation of the autoencoder architectures
nf = [256,256,128] #[32,32,16]
hn = 320 #32
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, n_filters=nf[0], filter_size=5, n_sample=500)
plot_mosaic(W1_init, title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=nf[0],
filter_size=(5,5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=nf[1],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=nf[2],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm)
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None)
# Code Layer
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None)
nnet.addReshapeLayer(shape=([0], hn/16, 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
addConvModule(nnet,
num_filters=nf[2],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
addConvModule(nnet,
num_filters=nf[1],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# to have the same output dimensions, pad must be 1 here
addConvModule(nnet,
num_filters=nf[0],
filter_size=(5,5),
pad=1,
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mnist_bias:
nnet.addConvLayer(num_filters=1,
filter_size=(5,5),
pad='same')
else:
nnet.addConvLayer(num_filters=1,
filter_size=(5,5),
pad='same',
b=None)
nnet.addSigmoidLayer()
