def __init__(self, dataset, use_weights=None, pretrain=False, profile=False): """ initialize instance """ self.initialize_variables(dataset) # whether to enable profiling in Theano functions self.profile = profile W1_init = None self.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(self, num_filters=8, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(self, num_filters=4, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) # Code Layer if Cfg.mnist_bias: self.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: self.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) self.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer self.addDenseLayer(num_units=10) self.addSoftmaxLayer() input_var = T.tensor4('inputs') target_var = T.ivector('targets') final_layer = self.all_layers[-1] prediction = lasagne.layers.get_output(final_layer, deterministic=False) loss = lasagne.objectives.categorical_crossentropy( prediction, target_var) loss = loss.mean() params = lasagne.layers.get_all_params(final_layer, trainable=True) updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=0.0001, momentum=0.9) train_fn = theano.function([input_var, target_var], loss, updates=updates) val_fn = theano.function([input_var, target_var], loss)
def build_autoencoder(self, nnet): # implementation of different network architectures assert Cfg.mnist_architecture in (1, 2) if Cfg.mnist_architecture == 1: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, 8, 5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=8, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=4, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) # Code Layer if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer nnet.addReshapeLayer(shape=([0], (Cfg.mnist_rep_dim / 16), 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addUpscale( scale_factor=(2, 2) ) # TODO: is this Upscale necessary? Shouldn't there be as many Upscales as MaxPools? addConvModule(nnet, num_filters=4, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # to have the same output dimensions, pad must be 1 here addConvModule(nnet, num_filters=8, filter_size=(5, 5), pad=1, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mnist_bias: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same', b=None) nnet.addSigmoidLayer() elif Cfg.mnist_architecture == 2: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=256, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=256, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=256, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=128, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320, b=None) # Code Layer if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320, b=None) nnet.addReshapeLayer(shape=([0], 20, 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() addConvModule(nnet, num_filters=128, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) addConvModule(nnet, num_filters=256, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # to have the same output dimensions, pad must be 1 here addConvModule(nnet, num_filters=256, filter_size=(5, 5), pad=1, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mnist_bias: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same', b=None) nnet.addSigmoidLayer() else: raise ValueError("No valid choice of architecture")
def build_architecture(self, nnet): # implementation of different network architectures assert Cfg.mnist_architecture in (1, 2) # increase number of parameters if dropout is used if Cfg.dropout_architecture: units_multiplier = 2 else: units_multiplier = 1 if Cfg.mnist_architecture == 1: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=8, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=8 * units_multiplier, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2) addConvModule(nnet, num_filters=4 * units_multiplier, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier, b=None) elif Cfg.mnist_architecture == 2: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=256, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=256 * units_multiplier, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2) addConvModule(nnet, num_filters=256 * units_multiplier, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) addConvModule(nnet, num_filters=128 * units_multiplier, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mnist_bias: nnet.addDenseLayer(num_units=320 * units_multiplier) else: nnet.addDenseLayer(num_units=320 * units_multiplier, b=None) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier, b=None) else: raise ValueError("No valid choice of architecture") if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet): # implementation of different network architectures if Cfg.architecture not in (1,2,3,4): # architecture spec A_B_C_D_E_F_G_H tmp = Cfg.architecture.split("_") use_pool = int(tmp[0]) # 1 or 0 n_conv = int(tmp[1]) n_dense = int(tmp[2]) c_out = int(tmp[3]) zsize = int(tmp[4]) ksize= int(tmp[5]) stride = int(tmp[6]) num_filters = c_out if use_pool: print("Using pooling and upscaling") pad = 'same' crop = 'same' else: print("Using strided convolutions for dim. reduction/upscaling") deconvinpad = 0 convinpad = (ksize-stride+1)//2 crop = convinpad outpad = 0 # Build architecture nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width)) if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=c_out, filter_size=ksize, n_sample=Cfg.n_dict_learn) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # Add all but last conv. layer for i in range(n_conv-1): addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = convinpad, ) num_filters *= 2 if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) if n_dense > 0: addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = convinpad, ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Dense layer if Cfg.dropout: nnet.addDropoutLayer() if Cfg.bias: nnet.addDenseLayer(num_units=zsize, use_batch_norm=False) else: nnet.addDenseLayer(num_units=zsize, use_batch_norm=Cfg.use_batch_norm, b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.debug_architecture_layers: print("Added dense layer") else: h = self.image_height / (2**(n_conv-1)) addConvModule(nnet, num_filters=zsize, filter_size=(h,h), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=False, dropout=False, p_dropout=0.2, use_maxpool = False, stride = (1,1), pad = (0,0), ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Now image (channels, height, width) = (zsize,1,1), nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer if Cfg.debug_architecture_layers: print("Feature layer here") n_deconv_layers = 0 if n_dense > 0: h1 = self.image_height // (2**n_conv) # height = width of image going into first conv layer num_filters = c_out * (2**(n_conv-1)) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.bias: nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm) else: nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm, b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.debug_architecture_layers: print("Added dense layer") nnet.addReshapeLayer(shape=([0], num_filters, h1, h1)) if Cfg.debug_architecture_layers: print("Reshaping to (None, %d, %d, %d)"%(num_filters, h1, h1)) num_filters = num_filters // 2 if use_pool: nnet.addUpscale(scale_factor=(2,2)) # since maxpool is after each conv. each upscale is before corresponding deconv output_size = None else: output_size = h1*2 if n_conv > 1: addConvTransposeModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = stride, crop = crop, outpad = outpad, upscale = use_pool, inpad = deconvinpad, output_size = output_size ) n_deconv_layers += 1 #print("Added deconv_layer %d" % n_deconv_layers) if Cfg.debug_architecture_layers: print("Added deconv_layer %d: (None, %d, %d, %d)" % (n_deconv_layers, num_filters, output_size,output_size)) num_filters //=2 if not use_pool: output_size *= 2 elif n_conv > 1: h2 = self.image_height // (2**(n_conv-1)) # height of image going in to second conv layer num_filters = c_out * (2**(n_conv-2)) addConvTransposeModule(nnet, num_filters=num_filters, filter_size=(h2,h2), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = (1,1), crop = 0, outpad = outpad, upscale = use_pool, inpad = deconvinpad ) n_deconv_layers += 1 if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers) output_size = h2*2 num_filters //= 2 else: if use_pool: output_size = None else: output_size = self.image_height # only conv layer will be reconstruction layer # Add remaining deconv layers for i in range(n_conv-2): addConvTransposeModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = stride, crop = crop, outpad = outpad, upscale = use_pool, inpad = deconvinpad, output_size = output_size ) n_deconv_layers += 1 if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers) if not use_pool: output_size *= 2 num_filters //=2 # add reconstruction layer addConvTransposeModule(nnet, num_filters=self.channels, filter_size=(ksize,ksize), W_init=W1_init, #pad = "valid", bias=Cfg.bias, pool_size=(2,2), use_batch_norm=False, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = stride, crop = crop, outpad = outpad, upscale = False, inpad = deconvinpad, output_size = output_size, reconstruction_layer=True ) if Cfg.debug_architecture_layers: print("Added reconstruction layer")
def build_architecture(self, nnet): # implementation of different network architectures if Cfg.architecture not in (1,2,3): # architecture spec A_B_C_D_E_F_G_H tmp = Cfg.architecture.split("_") use_pool = int(tmp[0]) # 1 or 0 n_conv = int(tmp[1]) n_dense = int(tmp[2]) c1 = int(tmp[3]) zsize = int(tmp[4]) ksize= int(tmp[5]) stride = int(tmp[6]) num_filters = c1 # If using maxpool, we should have pad = same if use_pool: pad = 'same' else: pad = (ksize-stride+1)//2 pad = (pad,pad) if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=c1, filter_size=ksize, n_sample=Cfg.n_dict_learn) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # Build architecture nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width)) # Add all but last conv. layer for i in range(n_conv-1): addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = pad, ) num_filters *= 2 if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) if n_dense > 0: addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = pad, ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Dense layer if Cfg.dropout: nnet.addDropoutLayer() if Cfg.bias: nnet.addDenseLayer(num_units=zsize) else: nnet.addDenseLayer(num_units=zsize, b=None) if Cfg.debug_architecture_layers: print("Added dense layer") else: h = self.image_height / (2**(n_conv-1)) addConvModule(nnet, num_filters=zsize, filter_size=(h,h), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=False, dropout=False, p_dropout=0.2, use_maxpool = False, stride = (1,1), pad = (0,0), ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Add ouput/feature layer if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet): # implementation of the autoencoder architectures fs = (5, 5) fn = [64, 32] mps = (2, 2) hn = fn[-1] * 5 ############################## if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary_new(nnet.data._X_train, n_filters=fn[0], filter_shape=fs, n_sample=500) W1_init = np.reshape(W1_init, (fn[0], self._X_train.shape[1]) + fs) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None nnet.addInputLayer(shape=(None, ) + self._X_train.shape[1:]) addConvModule(nnet, num_filters=fn[0], filter_size=fs, W_init=W1_init, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=fn[1], filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm) h_units = np.prod(nnet.all_layers[-1].output_shape[1:]) d_r = nnet.all_layers[-1].output_shape[1:] if Cfg.mobiFall_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) # Code Layer if Cfg.mobiFall_bias: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer if Cfg.mobiFall_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) if Cfg.mobiFall_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=h_units) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=h_units, b=None) nnet.addReshapeLayer(shape=([0], h_units / np.prod(d_r[1:]), d_r[1], d_r[2])) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() addConvModule(nnet, num_filters=fn[1], filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, upscale=True) addConvModule(nnet, num_filters=fn[0], filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mobiFall_bias: nnet.addConvLayer(num_filters=1, filter_size=fs, pad='same') else: nnet.addConvLayer(num_filters=3, filter_size=fs, pad='same', b=None)
def build_architecture(self, nnet): # implementation of the encoder network architectures fs = (5, 5) fn = [64, 32] mps = (2, 2) hn = fn[-1] * 5 # increase number of parameters if dropout is used if Cfg.dropout_architecture: units_multiplier = 2 else: units_multiplier = 1 if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary_new(nnet.data._X_train, n_filters=fn[0], filter_shape=fs, n_sample=500) W1_init = np.reshape(W1_init, (fn[0], self._X_train.shape[1]) + fs) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, ) + self._X_train.shape[1:]) addConvModule(nnet, num_filters=fn[0] * units_multiplier, filter_size=fs, W_init=W1_init, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2) addConvModule(nnet, num_filters=fn[1] * units_multiplier, filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mobiFall_bias: nnet.addDenseLayer(num_units=hn * units_multiplier) else: nnet.addDenseLayer(num_units=hn * units_multiplier, b=None) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mobiFall_bias: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim * units_multiplier) else: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim * units_multiplier, b=None) if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss or Cfg.msvdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet): # implementation of the autoencoder architectures nf = [256,256,128] #[32,32,16] hn = 320 #32 if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=nf[0], filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=nf[0], filter_size=(5,5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=nf[1], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=nf[2], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm) if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) # Code Layer if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) nnet.addReshapeLayer(shape=([0], hn/16, 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() addConvModule(nnet, num_filters=nf[2], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, upscale=True) addConvModule(nnet, num_filters=nf[1], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # to have the same output dimensions, pad must be 1 here addConvModule(nnet, num_filters=nf[0], filter_size=(5,5), pad=1, bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mnist_bias: nnet.addConvLayer(num_filters=1, filter_size=(5,5), pad='same') else: nnet.addConvLayer(num_filters=1, filter_size=(5,5), pad='same', b=None) nnet.addSigmoidLayer()