def plot_filters(nnet, xp_path, title_suffix, file_prefix="", pretrain=False): """ Plot all filters of the first convolutional layer """ assert hasattr(nnet, "conv1_layer") W = nnet.conv1_layer.W.get_value() title = "First layer filters " + title_suffix plot_mosaic(W, title=title, canvas="black", export_pdf=(xp_path + "/" + file_prefix + "filters")) if not pretrain and (nnet.best_weight_dict is not None): W_best = nnet.best_weight_dict["conv1_w"] title = ("First layer filters at epoch " + str(nnet.auc_best_epoch) + " " + title_suffix) plot_mosaic(W_best, title=title, canvas="black", export_pdf=(xp_path + "/" + file_prefix + "filters_best_ep"))
def plot_random_reconstructions(nnet, xp_path, title_suffix, file_prefix, n_img=32): """ plot the reconstructions of n_img randomly drawn images """ # only plot reconstructions for image data if nnet.data._X_train.ndim != 4: return from utils.visualization.mosaic_plot import plot_mosaic random_idx = np.random.choice(nnet.data.n_train, n_img, replace=False) _, _, _, _, _, _, _, reconstruction, _, _ = nnet.forward(nnet.data._X_train[random_idx, ...], nnet.data._y_train[random_idx]) title = str(n_img) + " random autoencoder reconstructions " + title_suffix print(reconstruction.shape) plot_mosaic(reconstruction, title=title, export_pdf=(xp_path + "/" + file_prefix + "ae_reconstructions"))
def train_autoencoder(nnet): if Cfg.ae_diagnostics: nnet.initialize_ae_diagnostics(nnet.ae_n_epochs) print("Starting training autoencoder with %s" % nnet.sgd_solver) for epoch in range(nnet.ae_n_epochs): start_time = time.time() if Cfg.ae_lr_drop and (epoch == Cfg.ae_lr_drop_in_epoch): # Drop the learning rate in epoch specified in Cfg.ae_lr_drop_after_epoch by factor Cfg.ae_lr_drop_factor # Thus, a simple separation of learning into a "region search" and "finetuning" stage. lr_new = Cfg.floatX( (1.0 / Cfg.ae_lr_drop_factor) * Cfg.learning_rate.get_value()) print("") print( "Learning rate drop in epoch {} from {:.6f} to {:.6f}".format( epoch, Cfg.floatX(Cfg.learning_rate.get_value()), lr_new)) print("") Cfg.learning_rate.set_value(lr_new) # In each epoch, we do a full pass over the training data: l2 = 0 batches = 0 train_err = 0 train_scores = np.empty(nnet.data.n_train) for batch in nnet.data.get_epoch_train(): inputs, _, batch_idx = batch start_idx = batch_idx * Cfg.batch_size stop_idx = min(nnet.data.n_train, start_idx + Cfg.batch_size) err, l2, b_scores = nnet.ae_backprop(inputs) train_err += err * inputs.shape[0] train_scores[start_idx:stop_idx] = b_scores.flatten() batches += 1 train_err /= nnet.data.n_train # save train diagnostics and test performance on val and test data if specified if Cfg.ae_diagnostics: nnet.save_ae_diagnostics('train', epoch, train_err, train_scores, l2) # Performance on validation and test set if nnet.data.n_val > 0: val_err = ae_performance(nnet, which_set='val', epoch=epoch) test_err = ae_performance(nnet, which_set='test', epoch=epoch) # print results for epoch print("{:32} {:.5f}".format("Train error:", train_err)) if Cfg.ae_diagnostics: if nnet.data.n_val > 0: print("{:32} {:.5f}".format("Val error:", val_err)) print("{:32} {:.5f}".format("Test error:", test_err)) print("Epoch {} of {} took {:.3f}s".format(epoch + 1, nnet.ae_n_epochs, time.time() - start_time)) print("") # Get final performance in last epoch if no running diagnostics are taken if not Cfg.ae_diagnostics: nnet.initialize_ae_diagnostics(1) # perform forward passes on train, val, and test set print("Get final performance...") _ = ae_performance(nnet, which_set='train', epoch=0) if nnet.data.n_val > 0: _ = ae_performance(nnet, which_set='val', epoch=0) _ = ae_performance(nnet, which_set='test', epoch=0) print("Evaluation completed.") # save weights if Cfg.pretrain: nnet.dump_weights("{}/ae_pretrained_weights.p".format(Cfg.xp_path), pretrain=True) else: nnet.dump_weights("{}/weights_final.p".format(Cfg.xp_path)) # if image data plot some random reconstructions if nnet.data._X_train.ndim == 4: from utils.visualization.mosaic_plot import plot_mosaic n_img = 32 random_idx = np.random.choice(nnet.data.n_train, n_img, replace=False) _, _, _, reps = nnet.ae_forward(nnet.data._X_train[random_idx, ...]) title = str(n_img) + " random autoencoder reconstructions" plot_mosaic(reps, title=title, export_pdf=(Cfg.xp_path + "/ae_reconstructions")) # plot diagnostics if specified if Cfg.ae_diagnostics & Cfg.pretrain: from utils.visualization.diagnostics_plot import plot_ae_diagnostics from utils.visualization.filters_plot import plot_filters # common suffix for plot titles str_lr = "lr = " + str(nnet.ae_learning_rate) C = int(Cfg.C.get_value()) if not Cfg.weight_decay: C = None str_C = "C = " + str(C) title_suffix = "(" + nnet.ae_solver + ", " + str_C + ", " + str_lr + ")" # plot diagnostics plot_ae_diagnostics(nnet, Cfg.xp_path, title_suffix) # plot filters plot_filters(nnet, Cfg.xp_path, title_suffix, file_prefix="ae_", pretrain=True)
def build_autoencoder(self, nnet): # implementation of the network architecture if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, 32, 5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) nnet.addInputLayer(shape=(None, 3, 32, 32)) if Cfg.weight_dict_init & (not nnet.pretrained): nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', W=W1_init, b=None) else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=128, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) # Code Layer nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer nnet.addReshapeLayer(shape=([0], (Cfg.cifar10_rep_dim / 16), 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=128, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addUpscale(scale_factor=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addUpscale(scale_factor=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addUpscale(scale_factor=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=3, filter_size=(5, 5), pad='same', b=None) nnet.addSigmoidLayer()
def build_architecture(self, nnet): # Build the encoder architecture if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=32, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture 3 nnet.addInputLayer(shape=(None, 3, 32, 32)) if Cfg.weight_dict_init & (not nnet.pretrained): nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', W=W1_init, b=None) else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=128, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None) if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss or Cfg.msvdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer
def build_autoencoder(self, nnet): # implementation of different network architectures assert Cfg.mnist_architecture in (1, 2) if Cfg.mnist_architecture == 1: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, 8, 5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=8, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=4, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) # Code Layer if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer nnet.addReshapeLayer(shape=([0], (Cfg.mnist_rep_dim / 16), 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addUpscale( scale_factor=(2, 2) ) # TODO: is this Upscale necessary? Shouldn't there be as many Upscales as MaxPools? addConvModule(nnet, num_filters=4, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # to have the same output dimensions, pad must be 1 here addConvModule(nnet, num_filters=8, filter_size=(5, 5), pad=1, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mnist_bias: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same', b=None) nnet.addSigmoidLayer() elif Cfg.mnist_architecture == 2: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=256, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=256, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=256, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=128, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm) if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320, b=None) # Code Layer if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=320, b=None) nnet.addReshapeLayer(shape=([0], 20, 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() addConvModule(nnet, num_filters=128, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) addConvModule(nnet, num_filters=256, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # to have the same output dimensions, pad must be 1 here addConvModule(nnet, num_filters=256, filter_size=(5, 5), pad=1, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mnist_bias: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(num_filters=1, filter_size=(5, 5), pad='same', b=None) nnet.addSigmoidLayer() else: raise ValueError("No valid choice of architecture")
def build_architecture(self, nnet): # implementation of different network architectures assert Cfg.mnist_architecture in (1, 2) # increase number of parameters if dropout is used if Cfg.dropout_architecture: units_multiplier = 2 else: units_multiplier = 1 if Cfg.mnist_architecture == 1: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=8, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=8 * units_multiplier, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2) addConvModule(nnet, num_filters=4 * units_multiplier, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier, b=None) elif Cfg.mnist_architecture == 2: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=256, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=256 * units_multiplier, filter_size=(5, 5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2) addConvModule(nnet, num_filters=256 * units_multiplier, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) addConvModule(nnet, num_filters=128 * units_multiplier, filter_size=(5, 5), bias=Cfg.mnist_bias, pool_size=(2, 2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mnist_bias: nnet.addDenseLayer(num_units=320 * units_multiplier) else: nnet.addDenseLayer(num_units=320 * units_multiplier, b=None) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim * units_multiplier, b=None) else: raise ValueError("No valid choice of architecture") if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_architecture(self, nnet): # implementation of different network architectures assert Cfg.cifar10_architecture in (1, 2, 3) if Cfg.cifar10_architecture == 1: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=16, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture 1 nnet.addInputLayer(shape=(None, 3, 32, 32)) if Cfg.cifar10_bias: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same') else: if Cfg.weight_dict_init & (not nnet.pretrained): nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same', W=W1_init, b=None) else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.cifar10_bias: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) if Cfg.cifar10_bias: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.cifar10_bias: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) if Cfg.cifar10_bias: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.cifar10_bias: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same') else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) if Cfg.cifar10_bias: nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None) if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name) if Cfg.cifar10_architecture == 2: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=16, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture 2 nnet.addInputLayer(shape=(None, 3, 32, 32)) if Cfg.weight_dict_init & (not nnet.pretrained): nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same', W=W1_init, b=None) else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=16, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None) if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name) if Cfg.cifar10_architecture == 3: if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=32, filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture 3 nnet.addInputLayer(shape=(None, 3, 32, 32)) if Cfg.weight_dict_init & (not nnet.pretrained): nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', W=W1_init, b=None) else: nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=32, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=64, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addConvLayer(use_batch_norm=Cfg.use_batch_norm, num_filters=128, filter_size=(5, 5), pad='same', b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() nnet.addMaxPool(pool_size=(2, 2)) nnet.addDenseLayer(num_units=Cfg.cifar10_rep_dim, b=None) if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def plot_outliers_and_most_normal(model, n, xp_path): """ Plot outliers and most normal examples in ascending order of train, val, and test set. That is, the top left image is the most normal, the bottom right image the most anomalous example. """ # reload images with original scale model.data.load_data(original_scale=True) for which_set in ['train', 'val', 'test']: if which_set == 'train': X = model.data._X_train n_samples = model.data.n_train if which_set == 'val': X = model.data._X_val n_samples = model.data.n_val if which_set == 'test': X = model.data._X_test n_samples = model.data.n_test if X.size > 0: # only if set is specified idx_sort = model.diag[which_set]['scores'][:, -1].argsort() normals = X[idx_sort, ...][:n, ...] outliers = X[idx_sort, ...][-n:, ...] str_samples = "(" + str(n) + " of " + str(int(n_samples)) + ")" title_norm = which_set.title( ) + " set examples with ascending scores " + str_samples title_out = which_set.title( ) + " set outliers with ascending scores " + str_samples plot_mosaic(normals, title=title_norm, export_pdf=(xp_path + "/normals_" + which_set)) plot_mosaic(outliers, title=title_out, export_pdf=(xp_path + "/outliers_" + which_set)) # plot with scores at best epoch if model.best_weight_dict is not None: epoch = model.auc_best_epoch str_epoch = "at epoch " + str(epoch) + " " idx_sort = model.diag[which_set]['scores'][:, epoch].argsort() normals = X[idx_sort, ...][:n, ...] outliers = X[idx_sort, ...][-n:, ...] str_samples = "(" + str(n) + " of " + str(int(n_samples)) + ")" title_norm = (which_set.title() + " set examples with ascending scores " + str_epoch + str_samples) title_out = (which_set.title() + " set outliers with ascending scores " + str_epoch + str_samples) plot_mosaic(normals, title=title_norm, export_pdf=(xp_path + "/normals_" + which_set + "_best_ep")) plot_mosaic(outliers, title=title_out, export_pdf=(xp_path + "/outliers_" + which_set + "_best_ep")) else: pass
def build_autoencoder(self, nnet): # implementation of different network architectures if Cfg.architecture not in (1,2,3,4): # architecture spec A_B_C_D_E_F_G_H tmp = Cfg.architecture.split("_") use_pool = int(tmp[0]) # 1 or 0 n_conv = int(tmp[1]) n_dense = int(tmp[2]) c_out = int(tmp[3]) zsize = int(tmp[4]) ksize= int(tmp[5]) stride = int(tmp[6]) num_filters = c_out if use_pool: print("Using pooling and upscaling") pad = 'same' crop = 'same' else: print("Using strided convolutions for dim. reduction/upscaling") deconvinpad = 0 convinpad = (ksize-stride+1)//2 crop = convinpad outpad = 0 # Build architecture nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width)) if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=c_out, filter_size=ksize, n_sample=Cfg.n_dict_learn) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # Add all but last conv. layer for i in range(n_conv-1): addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = convinpad, ) num_filters *= 2 if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) if n_dense > 0: addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = convinpad, ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Dense layer if Cfg.dropout: nnet.addDropoutLayer() if Cfg.bias: nnet.addDenseLayer(num_units=zsize, use_batch_norm=False) else: nnet.addDenseLayer(num_units=zsize, use_batch_norm=Cfg.use_batch_norm, b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.debug_architecture_layers: print("Added dense layer") else: h = self.image_height / (2**(n_conv-1)) addConvModule(nnet, num_filters=zsize, filter_size=(h,h), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=False, dropout=False, p_dropout=0.2, use_maxpool = False, stride = (1,1), pad = (0,0), ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Now image (channels, height, width) = (zsize,1,1), nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer if Cfg.debug_architecture_layers: print("Feature layer here") n_deconv_layers = 0 if n_dense > 0: h1 = self.image_height // (2**n_conv) # height = width of image going into first conv layer num_filters = c_out * (2**(n_conv-1)) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.bias: nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm) else: nnet.addDenseLayer(num_units=h1**2 * num_filters, use_batch_norm=Cfg.use_batch_norm, b=None) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() if Cfg.debug_architecture_layers: print("Added dense layer") nnet.addReshapeLayer(shape=([0], num_filters, h1, h1)) if Cfg.debug_architecture_layers: print("Reshaping to (None, %d, %d, %d)"%(num_filters, h1, h1)) num_filters = num_filters // 2 if use_pool: nnet.addUpscale(scale_factor=(2,2)) # since maxpool is after each conv. each upscale is before corresponding deconv output_size = None else: output_size = h1*2 if n_conv > 1: addConvTransposeModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = stride, crop = crop, outpad = outpad, upscale = use_pool, inpad = deconvinpad, output_size = output_size ) n_deconv_layers += 1 #print("Added deconv_layer %d" % n_deconv_layers) if Cfg.debug_architecture_layers: print("Added deconv_layer %d: (None, %d, %d, %d)" % (n_deconv_layers, num_filters, output_size,output_size)) num_filters //=2 if not use_pool: output_size *= 2 elif n_conv > 1: h2 = self.image_height // (2**(n_conv-1)) # height of image going in to second conv layer num_filters = c_out * (2**(n_conv-2)) addConvTransposeModule(nnet, num_filters=num_filters, filter_size=(h2,h2), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = (1,1), crop = 0, outpad = outpad, upscale = use_pool, inpad = deconvinpad ) n_deconv_layers += 1 if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers) output_size = h2*2 num_filters //= 2 else: if use_pool: output_size = None else: output_size = self.image_height # only conv layer will be reconstruction layer # Add remaining deconv layers for i in range(n_conv-2): addConvTransposeModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = stride, crop = crop, outpad = outpad, upscale = use_pool, inpad = deconvinpad, output_size = output_size ) n_deconv_layers += 1 if Cfg.debug_architecture_layers: print("Added deconv_layer %d" % n_deconv_layers) if not use_pool: output_size *= 2 num_filters //=2 # add reconstruction layer addConvTransposeModule(nnet, num_filters=self.channels, filter_size=(ksize,ksize), W_init=W1_init, #pad = "valid", bias=Cfg.bias, pool_size=(2,2), use_batch_norm=False, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = False, stride = stride, crop = crop, outpad = outpad, upscale = False, inpad = deconvinpad, output_size = output_size, reconstruction_layer=True ) if Cfg.debug_architecture_layers: print("Added reconstruction layer")
def build_architecture(self, nnet): # implementation of different network architectures if Cfg.architecture not in (1,2,3): # architecture spec A_B_C_D_E_F_G_H tmp = Cfg.architecture.split("_") use_pool = int(tmp[0]) # 1 or 0 n_conv = int(tmp[1]) n_dense = int(tmp[2]) c1 = int(tmp[3]) zsize = int(tmp[4]) ksize= int(tmp[5]) stride = int(tmp[6]) num_filters = c1 # If using maxpool, we should have pad = same if use_pool: pad = 'same' else: pad = (ksize-stride+1)//2 pad = (pad,pad) if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=c1, filter_size=ksize, n_sample=Cfg.n_dict_learn) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # Build architecture nnet.addInputLayer(shape=(None, self.channels, self.image_height, self.image_width)) # Add all but last conv. layer for i in range(n_conv-1): addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = pad, ) num_filters *= 2 if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) if n_dense > 0: addConvModule(nnet, num_filters=num_filters, filter_size=(ksize,ksize), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2, use_maxpool = use_pool, stride = stride, pad = pad, ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Dense layer if Cfg.dropout: nnet.addDropoutLayer() if Cfg.bias: nnet.addDenseLayer(num_units=zsize) else: nnet.addDenseLayer(num_units=zsize, b=None) if Cfg.debug_architecture_layers: print("Added dense layer") else: h = self.image_height / (2**(n_conv-1)) addConvModule(nnet, num_filters=zsize, filter_size=(h,h), W_init=W1_init, bias=Cfg.bias, pool_size=(2,2), use_batch_norm=False, dropout=False, p_dropout=0.2, use_maxpool = False, stride = (1,1), pad = (0,0), ) if Cfg.debug_architecture_layers: print("Added conv_layer %d" % nnet.n_conv_layers) # Add ouput/feature layer if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss: nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet): # implementation of the autoencoder architectures fs = (5, 5) fn = [64, 32] mps = (2, 2) hn = fn[-1] * 5 ############################## if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary_new(nnet.data._X_train, n_filters=fn[0], filter_shape=fs, n_sample=500) W1_init = np.reshape(W1_init, (fn[0], self._X_train.shape[1]) + fs) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None nnet.addInputLayer(shape=(None, ) + self._X_train.shape[1:]) addConvModule(nnet, num_filters=fn[0], filter_size=fs, W_init=W1_init, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=fn[1], filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm) h_units = np.prod(nnet.all_layers[-1].output_shape[1:]) d_r = nnet.all_layers[-1].output_shape[1:] if Cfg.mobiFall_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) # Code Layer if Cfg.mobiFall_bias: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim, b=None) nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer if Cfg.mobiFall_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) if Cfg.mobiFall_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=h_units) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=h_units, b=None) nnet.addReshapeLayer(shape=([0], h_units / np.prod(d_r[1:]), d_r[1], d_r[2])) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() addConvModule(nnet, num_filters=fn[1], filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, upscale=True) addConvModule(nnet, num_filters=fn[0], filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mobiFall_bias: nnet.addConvLayer(num_filters=1, filter_size=fs, pad='same') else: nnet.addConvLayer(num_filters=3, filter_size=fs, pad='same', b=None)
def build_architecture(self, nnet): # implementation of the encoder network architectures fs = (5, 5) fn = [64, 32] mps = (2, 2) hn = fn[-1] * 5 # increase number of parameters if dropout is used if Cfg.dropout_architecture: units_multiplier = 2 else: units_multiplier = 1 if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary_new(nnet.data._X_train, n_filters=fn[0], filter_shape=fs, n_sample=500) W1_init = np.reshape(W1_init, (fn[0], self._X_train.shape[1]) + fs) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, ) + self._X_train.shape[1:]) addConvModule(nnet, num_filters=fn[0] * units_multiplier, filter_size=fs, W_init=W1_init, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout, p_dropout=0.2) addConvModule(nnet, num_filters=fn[1] * units_multiplier, filter_size=fs, bias=Cfg.mobiFall_bias, pool_size=mps, use_batch_norm=Cfg.use_batch_norm, dropout=Cfg.dropout) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mobiFall_bias: nnet.addDenseLayer(num_units=hn * units_multiplier) else: nnet.addDenseLayer(num_units=hn * units_multiplier, b=None) if Cfg.dropout: nnet.addDropoutLayer() if Cfg.mobiFall_bias: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim * units_multiplier) else: nnet.addDenseLayer(num_units=Cfg.mobiFall_rep_dim * units_multiplier, b=None) if Cfg.softmax_loss: nnet.addDenseLayer(num_units=1) nnet.addSigmoidLayer() elif Cfg.svdd_loss or Cfg.msvdd_loss: nnet.setFeatureLayer( ) # set the currently highest layer to be the SVDD feature layer else: raise ValueError("No valid choice of loss for dataset " + self.dataset_name)
def build_autoencoder(self, nnet): # implementation of the autoencoder architectures nf = [256,256,128] #[32,32,16] hn = 320 #32 if Cfg.weight_dict_init & (not nnet.pretrained): # initialize first layer filters by atoms of a dictionary W1_init = learn_dictionary(nnet.data._X_train, n_filters=nf[0], filter_size=5, n_sample=500) plot_mosaic(W1_init, title="First layer filters initialization", canvas="black", export_pdf=(Cfg.xp_path + "/filters_init")) else: W1_init = None # build architecture nnet.addInputLayer(shape=(None, 1, 28, 28)) addConvModule(nnet, num_filters=nf[0], filter_size=(5,5), W_init=W1_init, bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=nf[1], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm) addConvModule(nnet, num_filters=nf[2], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm) if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) # Code Layer if Cfg.mnist_bias: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim) else: nnet.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None) nnet.setFeatureLayer() # set the currently highest layer to be the SVDD feature layer if Cfg.mnist_bias: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn) else: nnet.addDenseLayer(use_batch_norm=Cfg.use_batch_norm, num_units=hn, b=None) nnet.addReshapeLayer(shape=([0], hn/16, 4, 4)) if Cfg.leaky_relu: nnet.addLeakyReLU() else: nnet.addReLU() addConvModule(nnet, num_filters=nf[2], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, upscale=True) addConvModule(nnet, num_filters=nf[1], filter_size=(5,5), bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # to have the same output dimensions, pad must be 1 here addConvModule(nnet, num_filters=nf[0], filter_size=(5,5), pad=1, bias=Cfg.mnist_bias, pool_size=(2,2), use_batch_norm=Cfg.use_batch_norm, upscale=True) # reconstruction if Cfg.mnist_bias: nnet.addConvLayer(num_filters=1, filter_size=(5,5), pad='same') else: nnet.addConvLayer(num_filters=1, filter_size=(5,5), pad='same', b=None) nnet.addSigmoidLayer()