def build_autoencoder(self, nnet):
# implementation of the network architecture
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, 32, 5, n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
# Code Layer
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
nnet.addReshapeLayer(shape=([0], (Cfg.cifar10_rep_dim / 16), 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(scale_factor=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(scale_factor=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(scale_factor=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=3,
filter_size=(5, 5),
pad='same',
b=None)
nnet.addSigmoidLayer()
def build_autoencoder(self, nnet):
# implementation of different network architectures
assert Cfg.mnist_architecture in (1, 2)
if Cfg.mnist_architecture == 1:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
8,
5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=8,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=4,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
# Code Layer
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
nnet.addReshapeLayer(shape=([0], (Cfg.mnist_rep_dim / 16), 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addUpscale(
scale_factor=(2, 2)
) # TODO: is this Upscale necessary? Shouldn't there be as many Upscales as MaxPools?
addConvModule(nnet,
num_filters=4,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# to have the same output dimensions, pad must be 1 here
addConvModule(nnet,
num_filters=8,
filter_size=(5, 5),
pad=1,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mnist_bias:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same',
b=None)
nnet.addSigmoidLayer()
elif Cfg.mnist_architecture == 2:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=256,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=128,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320,
b=None)
# Code Layer
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm,
num_units=320,
b=None)
nnet.addReshapeLayer(shape=([0], 20, 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
addConvModule(nnet,
num_filters=128,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# to have the same output dimensions, pad must be 1 here
addConvModule(nnet,
num_filters=256,
filter_size=(5, 5),
pad=1,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mnist_bias:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(num_filters=1,
filter_size=(5, 5),
pad='same',
b=None)
nnet.addSigmoidLayer()
else:
raise ValueError("No valid choice of architecture")
def build_architecture(self, nnet):
# Build the encoder architecture
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=32,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 3
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss or Cfg.msvdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
def build_architecture(self, nnet):
# implementation of different network architectures
assert Cfg.mnist_architecture in (1, 2)
# increase number of parameters if dropout is used
if Cfg.dropout_architecture:
units_multiplier = 2
else:
units_multiplier = 1
if Cfg.mnist_architecture == 1:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=8,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=8 * units_multiplier,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2)
addConvModule(nnet,
num_filters=4 * units_multiplier,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier,
b=None)
elif Cfg.mnist_architecture == 2:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=256,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=256 * units_multiplier,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2)
addConvModule(nnet,
num_filters=256 * units_multiplier,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
addConvModule(nnet,
num_filters=128 * units_multiplier,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=320 * units_multiplier)
else:
nnet.addDenseLayer(num_units=320 * units_multiplier, b=None)
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim *
units_multiplier,
b=None)
else:
raise ValueError("No valid choice of architecture")
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
def build_architecture(self, nnet):
# implementation of different network architectures
assert Cfg.cifar10_architecture in (1, 2, 3)
if Cfg.cifar10_architecture == 1:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=16,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 1
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same')
else:
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.cifar10_bias:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same')
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
if Cfg.cifar10_bias:
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
if Cfg.cifar10_architecture == 2:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=16,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 2
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=16,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
if Cfg.cifar10_architecture == 3:
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train,
n_filters=32,
filter_size=5,
n_sample=500)
plot_mosaic(W1_init,
title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture 3
nnet.addInputLayer(shape=(None, 3, 32, 32))
if Cfg.weight_dict_init & (not nnet.pretrained):
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
W=W1_init,
b=None)
else:
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=32,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=64,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm,
num_filters=128,
filter_size=(5, 5),
pad='same',
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
nnet.addMaxPool(pool_size=(2, 2))
nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None)
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " +
self.dataset_name)
def build_autoencoder(self, nnet):
# implementation of different network architectures
if Cfg.architecture not in (1,2,3,4):
# architecture spec A_B_C_D_E_F_G_H
tmp = Cfg.architecture.split("_")
use_pool = int(tmp[0]) # 1 or 0
n_conv = int(tmp[1])
n_dense = int(tmp[2])
c_out = int(tmp[3])
zsize = int(tmp[4])
ksize= int(tmp[5])
stride = int(tmp[6])
num_filters = c_out
if use_pool:
print("Using pooling and upscaling")
pad = 'same'
crop = 'same'
else:
print("Using strided convolutions for dim. reduction/upscaling")
deconvinpad = 0
convinpad = (ksize-stride+1)//2
crop = convinpad
outpad = 0
# Build architecture
nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width))
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, n_filters=c_out, filter_size=ksize, n_sample=Cfg.n_dict_learn)
plot_mosaic(W1_init, title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# Add all but last conv. layer
for i in range(n_conv-1):
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = convinpad,
)
num_filters *= 2
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
if n_dense > 0:
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = convinpad,
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Dense layer
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.bias:
nnet.addDenseLayer(num_units=zsize, use_batch_norm=False)
else:
nnet.addDenseLayer(num_units=zsize, use_batch_norm=Cfg.use_batch_norm,
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.debug_architecture_layers: print("Added dense layer")
else:
h = self.image_height / (2**(n_conv-1))
addConvModule(nnet,
num_filters=zsize,
filter_size=(h,h),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=False,
dropout=False,
p_dropout=0.2,
use_maxpool = False,
stride = (1,1),
pad = (0,0),
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Now image (channels, height, width) = (zsize,1,1),
nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer
if Cfg.debug_architecture_layers: print("Feature layer here")
n_deconv_layers = 0
if n_dense > 0:
h1 = self.image_height // (2**n_conv) # height = width of image going into first conv layer
num_filters = c_out * (2**(n_conv-1))
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.bias:
nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm)
else:
nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm,
b=None)
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
if Cfg.debug_architecture_layers: print("Added dense layer")
nnet.addReshapeLayer(shape=([0], num_filters, h1, h1))
if Cfg.debug_architecture_layers: print("Reshaping to (None, %d, %d, %d)"%(num_filters, h1, h1))
num_filters = num_filters // 2
if use_pool:
nnet.addUpscale(scale_factor=(2,2)) # since maxpool is after each conv. each upscale is before corresponding deconv
output_size = None
else:
output_size = h1*2
if n_conv > 1:
addConvTransposeModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = stride,
crop = crop,
outpad = outpad,
upscale = use_pool,
inpad = deconvinpad,
output_size = output_size
)
n_deconv_layers += 1
#print("Added deconv_layer %d" % n_deconv_layers)
if Cfg.debug_architecture_layers: print("Added deconv_layer %d: (None, %d, %d, %d)" % (n_deconv_layers, num_filters, output_size,output_size))
num_filters //=2
if not use_pool:
output_size *= 2
elif n_conv > 1:
h2 = self.image_height // (2**(n_conv-1)) # height of image going in to second conv layer
num_filters = c_out * (2**(n_conv-2))
addConvTransposeModule(nnet,
num_filters=num_filters,
filter_size=(h2,h2),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = (1,1),
crop = 0,
outpad = outpad,
upscale = use_pool,
inpad = deconvinpad
)
n_deconv_layers += 1
if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers)
output_size = h2*2
num_filters //= 2
else:
if use_pool:
output_size = None
else:
output_size = self.image_height # only conv layer will be reconstruction layer
# Add remaining deconv layers
for i in range(n_conv-2):
addConvTransposeModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = stride,
crop = crop,
outpad = outpad,
upscale = use_pool,
inpad = deconvinpad,
output_size = output_size
)
n_deconv_layers += 1
if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers)
if not use_pool:
output_size *= 2
num_filters //=2
# add reconstruction layer
addConvTransposeModule(nnet,
num_filters=self.channels,
filter_size=(ksize,ksize),
W_init=W1_init,
#pad = "valid",
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=False,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = False,
stride = stride,
crop = crop,
outpad = outpad,
upscale = False,
inpad = deconvinpad,
output_size = output_size,
reconstruction_layer=True
)
if Cfg.debug_architecture_layers: print("Added reconstruction layer")
def build_architecture(self, nnet):
# implementation of different network architectures
if Cfg.architecture not in (1,2,3):
# architecture spec A_B_C_D_E_F_G_H
tmp = Cfg.architecture.split("_")
use_pool = int(tmp[0]) # 1 or 0
n_conv = int(tmp[1])
n_dense = int(tmp[2])
c1 = int(tmp[3])
zsize = int(tmp[4])
ksize= int(tmp[5])
stride = int(tmp[6])
num_filters = c1
# If using maxpool, we should have pad = same
if use_pool:
pad = 'same'
else:
pad = (ksize-stride+1)//2
pad = (pad,pad)
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, n_filters=c1, filter_size=ksize, n_sample=Cfg.n_dict_learn)
plot_mosaic(W1_init, title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# Build architecture
nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width))
# Add all but last conv. layer
for i in range(n_conv-1):
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = pad,
)
num_filters *= 2
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
if n_dense > 0:
addConvModule(nnet,
num_filters=num_filters,
filter_size=(ksize,ksize),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
dropout=Cfg.dropout,
p_dropout=0.2,
use_maxpool = use_pool,
stride = stride,
pad = pad,
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Dense layer
if Cfg.dropout:
nnet.addDropoutLayer()
if Cfg.bias:
nnet.addDenseLayer(num_units=zsize)
else:
nnet.addDenseLayer(num_units=zsize,
b=None)
if Cfg.debug_architecture_layers: print("Added dense layer")
else:
h = self.image_height / (2**(n_conv-1))
addConvModule(nnet,
num_filters=zsize,
filter_size=(h,h),
W_init=W1_init,
bias=Cfg.bias,
pool_size=(2,2),
use_batch_norm=False,
dropout=False,
p_dropout=0.2,
use_maxpool = False,
stride = (1,1),
pad = (0,0),
)
if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers)
# Add ouput/feature layer
if Cfg.softmax_loss:
nnet.addDenseLayer(num_units=1)
nnet.addSigmoidLayer()
elif Cfg.svdd_loss:
nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer
else:
raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet):
# implementation of the autoencoder architectures
nf = [256,256,128] #[32,32,16]
hn = 320 #32
if Cfg.weight_dict_init & (not nnet.pretrained):
# initialize first layer filters by atoms of a dictionary
W1_init = learn_dictionary(nnet.data._X_train, n_filters=nf[0], filter_size=5, n_sample=500)
plot_mosaic(W1_init, title="First layer filters initialization",
canvas="black",
export_pdf=(Cfg.xp_path + "/filters_init"))
else:
W1_init = None
# build architecture
nnet.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(nnet,
num_filters=nf[0],
filter_size=(5,5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=nf[1],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(nnet,
num_filters=nf[2],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm)
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None)
# Code Layer
if Cfg.mnist_bias:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer
if Cfg.mnist_bias:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn)
else:
nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None)
nnet.addReshapeLayer(shape=([0], hn/16, 4, 4))
if Cfg.leaky_relu:
nnet.addLeakyReLU()
else:
nnet.addReLU()
addConvModule(nnet,
num_filters=nf[2],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
addConvModule(nnet,
num_filters=nf[1],
filter_size=(5,5),
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# to have the same output dimensions, pad must be 1 here
addConvModule(nnet,
num_filters=nf[0],
filter_size=(5,5),
pad=1,
bias=Cfg.mnist_bias,
pool_size=(2,2),
use_batch_norm=Cfg.use_batch_norm,
upscale=True)
# reconstruction
if Cfg.mnist_bias:
nnet.addConvLayer(num_filters=1,
filter_size=(5,5),
pad='same')
else:
nnet.addConvLayer(num_filters=1,
filter_size=(5,5),
pad='same',
b=None)
nnet.addSigmoidLayer()
