Beispiel #1
0
class LeNet:
    def __init__(self,
                 dataset,
                 use_weights=None,
                 pretrain=False,
                 profile=False):
        """
        initialize instance
        """
        self.initialize_variables(dataset)
        # whether to enable profiling in Theano functions
        self.profile = profile

        W1_init = None

        self.addInputLayer(shape=(None, 1, 28, 28))

        addConvModule(self,
                      num_filters=8,
                      filter_size=(5, 5),
                      W_init=W1_init,
                      bias=Cfg.mnist_bias,
                      pool_size=(2, 2),
                      use_batch_norm=Cfg.use_batch_norm)

        addConvModule(self,
                      num_filters=4,
                      filter_size=(5, 5),
                      bias=Cfg.mnist_bias,
                      pool_size=(2, 2),
                      use_batch_norm=Cfg.use_batch_norm)

        # Code Layer
        if Cfg.mnist_bias:
            self.addDenseLayer(num_units=Cfg.mnist_rep_dim)
        else:
            self.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
        self.setFeatureLayer(
        )  # set the currently highest layer to be the SVDD feature layer
        self.addDenseLayer(num_units=10)
        self.addSoftmaxLayer()
        input_var = T.tensor4('inputs')
        target_var = T.ivector('targets')
        final_layer = self.all_layers[-1]
        prediction = lasagne.layers.get_output(final_layer,
                                               deterministic=False)
        loss = lasagne.objectives.categorical_crossentropy(
            prediction, target_var)
        loss = loss.mean()
        params = lasagne.layers.get_all_params(final_layer, trainable=True)
        updates = lasagne.updates.nesterov_momentum(loss,
                                                    params,
                                                    learning_rate=0.0001,
                                                    momentum=0.9)
        train_fn = theano.function([input_var, target_var],
                                   loss,
                                   updates=updates)
        val_fn = theano.function([input_var, target_var], loss)

    def train(self):
        # train on epoch
        i_batch = 0
        for batch in self.data.get_epoch_train():
            # train
            inputs, targets, _ = batch
            print "#################"
            exit()
            self.train_fn(inputs, targets)

    # def adversarial(self, images, labels):
    #     # instantiate model
    #     keras.backend.set_learning_phase(0)
    #     preprocessing = (np.array([104]), 1)
    #
    #     fmodel = foolbox.models.TheanoModel(self.input_layer, self.logits, (0, 1), 2, 1, preprocessing)
    #
    #     # apply attack on source image
    #     # ::-1 reverses the color channels, because Keras ResNet50 expects BGR instead of RGB
    #     print np.shape(images)
    #     attack = foolbox.attacks.FGSM(fmodel)
    #     print 'starting attack'
    #     for i in range(len(images)):
    #         image = images[i][0]
    #         label = labels[i]
    #         adversarial = attack(np.reshape(image, (28, 28, 1)), label)
    #         print label
    #         print np.shape(images)
    #         # plt.plot(image)
    #         print np.shape(image)
    #         print image
    #         plt.show()
    #         # adversarial = attack(image[:, :, ::-1], label)
    #
    #         # if the attack fails, adversarial will be None and a warning will be printed
    #         print adversarial
    #         time.sleep(2)
    #         print 'ending attack'
    #
    #         plt.figure()
    #
    #         plt.subplot(1, 3, 1)
    #         plt.title('Original')
    #         plt.imshow(image)  # division by 255 to convert [0, 255] to [0, 1]
    #         plt.axis('off')
    #
    #         plt.subplot(1, 3, 2)
    #         plt.title('Adversarial')
    #         plt.imshow(adversarial)  # ::-1 to convert BGR to RGB
    #         plt.axis('off')
    #
    #         plt.subplot(1, 3, 3)
    #         plt.title('Difference')
    #         difference = adversarial - image
    #         plt.imshow(difference / abs(difference).max() * 0.2 + 0.5)
    #         plt.axis('off')
    #
    #         plt.show()

    def initialize_variables(self, dataset):

        self.all_layers, self.trainable_layers = (), ()

        self.n_conv_layers = 0
        self.n_dense_layers = 0
        self.n_relu_layers = 0
        self.n_leaky_relu_layers = 0
        self.n_bn_layers = 0
        self.n_norm_layers = 0
        self.n_abs_layers = 0
        self.n_maxpool_layers = 0
        self.n_upscale_layers = 0
        self.n_dropout_layers = 0
        self.n_dimshuffle_layers = 0
        self.n_reshape_layers = 0
        self.R_init = 0

        self.learning_rate_init = Cfg.learning_rate.get_value()

        self.it = 0
        self.clock = 0

        self.pretrained = False  # set to True after pretraining such that dictionary initialization mustn't be repeated

        self.diag = {}  # init an empty dictionary to hold diagnostics
        self.log = Log(dataset_name=dataset)
        self.ad_log = AD_Log()

        self.dense_layers, self.conv_layers, = [], []

    def compile_updates(self):
        """ create network from architecture given in modules (determined by dataset)
        create Theano compiled functions
        """

        opt.sgd.updates.create_update(self)

    def compile_autoencoder(self):
        """
        create network from autoencoder architecture (determined by dataset)
        and compile Theano update functions.
        """

        print("Compiling autoencoder...")
        opt.sgd.updates.create_autoencoder(self)
        print("Autoencoder compiled.")

    def load_data(self, data_loader=None, pretrain=False):

        self.data = data_loader()

        if pretrain:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)
        elif Cfg.reconstruction_loss:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)
        elif Cfg.svdd_loss and Cfg.reconstruction_penalty:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)
        else:
            self.data.build_architecture(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)

    def flush_data(self):

        self.data._X_train = None
        self.data._y_train = None
        self.data._X_val = None
        self.data._y_val = None
        self.data._X_test = None
        self.data._y_test = None

        print("Data flushed from network.")

    def next_layers(self, layer):

        flag = False
        for current_layer in self.all_layers:
            if flag:
                yield current_layer
            if current_layer is layer:
                flag = True

    def previous_layers(self, layer):

        flag = False
        for current_layer in reversed(self.all_layers):
            if flag:
                yield current_layer
            if current_layer is layer:
                flag = True

    def start_clock(self):

        self.clock = time.time()

    def stop_clock(self):

        self.clocked = time.time() - self.clock
        print("Total elapsed time: %g" % self.clocked)

    def pretrain(self, solver, lr, n_epochs):
        """
        pre-train weights with an autoencoder
        """

        self.ae_solver = solver.lower()
        self.ae_learning_rate = lr
        self.ae_n_epochs = n_epochs

        # set learning rate
        lr_tmp = Cfg.learning_rate.get_value()
        Cfg.learning_rate.set_value(Cfg.floatX(lr))

        self.compile_autoencoder()

        from opt.sgd.train import train_autoencoder
        train_autoencoder(self)

        # remove layer attributes, re-initialize network and reset learning rate
        for layer in self.all_layers:
            delattr(self, layer.name + "_layer")
        self.initialize_variables(self.data.dataset_name)
        Cfg.learning_rate.set_value(Cfg.floatX(lr_tmp))
        self.pretrained = True  # set to True that dictionary initialization mustn't be repeated

        # load network architecture
        if Cfg.svdd_loss and Cfg.reconstruction_penalty:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)
        else:
            self.data.build_architecture(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)

        # load weights learned by autoencoder
        self.load_weights(Cfg.xp_path + "/ae_pretrained_weights.p")

    def train1(self, solver, n_epochs=10, save_at=0, save_to=''):

        self.solver = solver.lower()
        self.ae_solver = solver.lower()
        self.n_epochs = n_epochs
        self.save_at = save_at
        self.save_to = save_to

        self.log['solver'] = self.solver
        self.log['save_at'] = self.save_at

        self.compile_updates()

        from opt.sgd.train import train_network

        self.start_clock()

        self.stop_clock()

        # self.log.save_to_file()

    def evaluate(self, solver):

        # this could be simplified to only compiling the forwardpropagation...
        self.solver = solver.lower()  # needed for compiling backprop
        self.compile_updates()

        print("Evaluating network with current weights...")

        self.initialize_diagnostics(1)
        self.copy_parameters()

        # perform forward passes on training, val, and test set
        _, _ = performance(self, which_set='train', epoch=0, print_=True)
        _, _ = performance(self, which_set='val', epoch=0, print_=True)
        _, _ = performance(self, which_set='test', epoch=0, print_=True)

        print("Evaluation on train, val, and test set completed.")

    def log_results(self, filename=None):
        """
        log the results relevant for anomaly detection
        """

        self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
        self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
        self.ad_log['train_time'] = self.train_time

        self.ad_log['val_auc'] = self.diag['val']['auc'][-1]
        self.ad_log['val_accuracy'] = self.diag['val']['acc'][-1]

        self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
        self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
        self.ad_log['test_time'] = self.test_time

        self.ad_log.save_to_file(filename=filename)

    def addInputLayer(self, **kwargs):

        self.input_layer = InputLayer(name="input", **kwargs)
        self.input_layer.inp_ndim = len(kwargs["shape"])

    def addConvLayer(self, use_batch_norm=False, **kwargs):
        """
        Add convolutional layer.
        If batch norm flag is True, the convolutional layer
        will be followed by a batch-normalization layer
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_conv_layers += 1
        name = "conv%i" % self.n_conv_layers

        new_layer = ConvLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        if use_batch_norm:
            self.n_bn_layers += 1
            name = "bn%i" % self.n_bn_layers
            self.all_layers += (BatchNorm(new_layer, name=name), )

    def addDenseLayer(self, use_batch_norm=False, **kwargs):
        """
        Add dense layer.
        If batch norm flag is True, the dense layer
        will be followed by a batch-normalization layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dense_layers += 1
        name = "dense%i" % self.n_dense_layers

        new_layer = DenseLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        if use_batch_norm:
            self.n_bn_layers += 1
            name = "bn%i" % self.n_bn_layers
            self.all_layers += (BatchNorm(new_layer, name=name), )

    def addSigmoidLayer(self, **kwargs):
        """
        Add sigmoid classification layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]
        new_layer = Sigmoid(input_layer, **kwargs)

        self.all_layers += (new_layer, )

        self.n_layers = len(self.all_layers)

    def addSoftmaxLayer(self, **kwargs):
        """
        Add softmax multi-class classification layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]
        new_layer = Softmax(input_layer, **kwargs)

        self.all_layers += (new_layer, )

        self.n_layers = len(self.all_layers)

    def addNormLayer(self, **kwargs):
        """
        Add layer which normalizes its input to length 1. 
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_norm_layers += 1
        name = "norm%i" % self.n_norm_layers

        new_layer = Norm(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addAbsLayer(self, **kwargs):
        """
        Add layer which returns the absolute value of its input.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_abs_layers += 1
        name = "abs%i" % self.n_abs_layers

        new_layer = Abs(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addReLU(self, **kwargs):
        """
        Add ReLU activation layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_relu_layers += 1
        name = "relu%i" % self.n_relu_layers

        new_layer = ReLU(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addLeakyReLU(self, **kwargs):
        """
        Add leaky ReLU activation layer. (with leakiness=0.01)
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_leaky_relu_layers += 1
        name = "leaky_relu%i" % self.n_leaky_relu_layers

        new_layer = LeakyReLU(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addMaxPool(self, **kwargs):
        """
        Add MaxPooling activation layer.
        """

        input_layer = self.input_layer if not self.all_layers\
            else self.all_layers[-1]

        self.n_maxpool_layers += 1
        name = "maxpool%i" % self.n_maxpool_layers

        new_layer = MaxPool(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addUpscale(self, **kwargs):
        """
        Add Upscaling activation layer.
        """

        input_layer = self.input_layer if not self.all_layers\
            else self.all_layers[-1]

        self.n_upscale_layers += 1
        name = "upscale%i" % self.n_upscale_layers

        new_layer = Upscale(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addDropoutLayer(self, **kwargs):
        """
        Add Dropout layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dropout_layers += 1
        name = "dropout%i" % self.n_dropout_layers

        new_layer = DropoutLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addDimshuffleLayer(self, **kwargs):
        """
        Add Dimshuffle layer to reorder dimensions
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dimshuffle_layers += 1
        name = "dimshuffle%i" % self.n_dimshuffle_layers

        new_layer = Dimshuffle(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addReshapeLayer(self, **kwargs):
        """
        Add reshape layer to reshape dimensions
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_reshape_layers += 1
        name = "reshape%i" % self.n_reshape_layers

        new_layer = Reshape(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def setFeatureLayer(self):
        """
        sets the currently highest layer of the current network to be the code layer (compression layer)
        """
        setattr(self, "feature_layer", self.all_layers[-1])

    def dump_weights(self, filename=None, pretrain=False):

        dump_weights(self, filename, pretrain=pretrain)

    def load_weights(self, filename=None):

        assert filename and os.path.exists(filename)

        load_weights(self, filename)

    def update_R(self):
        """
        method to update R while leaving the network parameters and center c fixed in a block coordinate optimization
        """

        print("Updating radius R...")

        # Get updates
        R = update_R(self.diag['train']['rep'],
                     self.cvar.get_value(),
                     solver=Cfg.R_update_solver,
                     scalar_method=Cfg.R_update_scalar_method,
                     lp_obj=Cfg.R_update_lp_obj)

        # Update R
        self.Rvar.set_value(Cfg.floatX(R))

        print("Radius R updated.")

    def update_R_c(self):
        """
        method to update R and c while leaving the network parameters fixed in a block coordinate optimization
        """

        print("Updating radius R and center c...")

        # Get updates
        R, c = update_R_c(self.diag['train']['rep'],
                          np.sum(self.diag['train']['rep']**2, axis=1),
                          solver=Cfg.QP_solver)

        # Update values
        self.Rvar.set_value(Cfg.floatX(R))
        self.cvar.set_value(Cfg.floatX(c))

        print("Radius R and center c updated.")

    def initialize_diagnostics(self, n_epochs):
        """
        initialize diagnostics for the neural network
        """

        # data-dependent diagnostics for train, validation, and test set
        self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
                                          n_samples=self.data.n_train)
        self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
                                        n_samples=self.data.n_val)
        self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
                                         n_samples=self.data.n_test)

        # network parameter diagnostics
        self.diag['network'] = NNetParamDiag(self, n_epochs=n_epochs)

        # Best results (highest AUC on test set)
        self.auc_best = 0
        self.auc_best_epoch = 0  # determined by highest AUC on test set
        self.best_weight_dict = None

    def save_objective_and_accuracy(self, epoch, which_set, objective,
                                    accuracy):
        """
        save objective and accuracy of epoch
        """

        self.diag[which_set]['objective'][epoch] = objective
        self.diag[which_set]['acc'][epoch] = accuracy

    def save_initial_parameters(self):
        """
        save a copy of the initial network parameters for diagnostics.
        """

        self.W_init = []
        self.b_init = []

        for layer in self.trainable_layers:
            if layer.isdense | layer.isconv:
                self.W_init.append(None)
                self.b_init.append(None)

        i = 0
        for layer in self.trainable_layers:
            if layer.isdense | layer.isconv:
                self.W_init[i] = layer.W.get_value()
                if layer.b is not None:
                    self.b_init[i] = layer.b.get_value()
                i += 1

    def copy_parameters(self):
        """
        save a copy of the current network parameters in order to monitor the difference between epochs.
        """
        i = 0
        for layer in self.trainable_layers:
            if layer.isdense | layer.isconv:
                self.diag['network']['W_copy'][i] = layer.W.get_value()
                if layer.b is not None:
                    self.diag['network']['b_copy'][i] = layer.b.get_value()
                i += 1

    def copy_initial_parameters_to_cache(self):
        """
        Save a copy of the initial parameters in cache
        """
        self.diag['network']['W_copy'] = list(self.W_init)
        self.diag['network']['b_copy'] = list(self.b_init)

    def save_network_diagnostics(self, epoch, l2, R):
        """
        save diagnostics of the network
        """

        self.diag['network']['l2_penalty'][epoch] = l2
        self.log['l2_penalty'].append(float(l2))

        i = 0
        j = 0
        for layer in self.trainable_layers:
            if layer.isdense:
                self.diag['network']['W_norms'][i][:, epoch] = np.sum(
                    layer.W.get_value()**2, axis=0)
                if layer.b is not None:
                    self.diag['network']['b_norms'][
                        i][:, epoch] = layer.b.get_value()**2
                i += 1

            if layer.isdense | layer.isconv:
                dW = np.sqrt(
                    np.sum((layer.W.get_value() -
                            self.diag['network']['W_copy'][j])**2))
                self.diag['network']['dW_norms'][j][epoch] = dW
                if layer.b is not None:
                    db = np.sqrt(
                        np.sum((layer.b.get_value() -
                                self.diag['network']['b_copy'][j])**2))
                    self.diag['network']['db_norms'][j][epoch] = db
                j += 1

        # diagnostics only relevant for the SVDD loss
        if Cfg.svdd_loss:
            self.diag['network']['R'][epoch] = R
            self.diag['network']['c_norm'][epoch] = np.sqrt(
                np.sum(self.cvar.get_value()**2))

    def track_best_results(self, epoch):
        """
        Save network parameters where AUC on the test set was highest.
        """

        if self.diag['test']['auc'][epoch] > self.auc_best:
            self.auc_best = self.diag['test']['auc'][epoch]
            self.auc_best_epoch = epoch

            self.best_weight_dict = dict()

            for layer in self.trainable_layers:
                self.best_weight_dict[layer.name + "_w"] = layer.W.get_value()
                if layer.b is not None:
                    self.best_weight_dict[layer.name +
                                          "_b"] = layer.b.get_value()

            if Cfg.svdd_loss:
                self.best_weight_dict["R"] = self.Rvar.get_value()

    def dump_best_weights(self, filename):
        """
        pickle the network parameters, where AUC on the test set was highest.
        """

        with open(filename, 'wb') as f:
            pickle.dump(self.best_weight_dict, f)

        print("Parameters of best epoch saved in %s" % filename)

    def save_diagnostics(self, which_set, epoch, scores, rep_norm, rep,
                         emp_loss, reconstruction_penalty):
        """
        save diagnostics for which_set of epoch
        """

        if self.data.n_classes == 2:

            if which_set == 'train':
                y = self.data._y_train
            if which_set == 'val':
                y = self.data._y_val
            if which_set == 'test':
                y = self.data._y_test

            self.diag[which_set]['scores'][:, epoch] = scores

            if sum(y) > 0:
                AUC = roc_auc_score(y, scores)
                self.diag[which_set]['auc'][epoch] = AUC
                self.log[which_set + '_auc'].append(float(AUC))
                print("{:32} {:.2f}%".format(which_set.title() + ' AUC:',
                                             100. * AUC))

            scores_normal = scores[y == 0]
            scores_outlier = scores[y == 1]
            normal_summary = get_five_number_summary(scores_normal)
            outlier_summary = get_five_number_summary(scores_outlier)
            self.log[which_set +
                     '_normal_scores_summary'].append(normal_summary)
            self.log[which_set +
                     '_outlier_scores_summary'].append(outlier_summary)

            self.diag[which_set]['rep'] = rep
            self.diag[which_set]['rep_norm'][:, epoch] = rep_norm

            rep_norm_normal = rep_norm[y == 0]
            rep_norm_outlier = rep_norm[y == 1]
            normal_summary = get_five_number_summary(rep_norm_normal)
            outlier_summary = get_five_number_summary(rep_norm_outlier)
            self.log[which_set +
                     '_normal_rep_norm_summary'].append(normal_summary)
            self.log[which_set +
                     '_outlier_rep_norm_summary'].append(outlier_summary)

            if Cfg.svdd_loss:
                rep_mean = np.mean(rep, axis=0)
                self.diag[which_set]['output_mean_norm'][epoch] = np.sqrt(
                    np.sum(rep_mean**2))
                self.diag[which_set]['c_mean_diff'][epoch] = np.sqrt(
                    np.sum((rep_mean - self.cvar.get_value())**2))

        self.diag[which_set]['reconstruction_penalty'][
            epoch] = reconstruction_penalty
        self.diag[which_set]['emp_loss'][epoch] = float(emp_loss)

        self.log[which_set + '_emp_loss'].append(float(emp_loss))

    def initialize_ae_diagnostics(self, n_epochs):
        """
        initialize diagnostic variables for autoencoder network.
        """

        self.train_time = 0
        self.test_time = 0
        self.best_weight_dict = None

        # data-dependent diagnostics for train, validation, and test set
        self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
                                          n_samples=self.data.n_train)
        self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
                                        n_samples=self.data.n_val)
        self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
                                         n_samples=self.data.n_test)

        # network parameters
        self.diag['network'] = {}
        self.diag['network']['l2_penalty'] = np.zeros(n_epochs,
                                                      dtype=Cfg.floatX)

    def save_ae_diagnostics(self, which_set, epoch, error, scores, l2):
        """
        save autoencoder diagnostics for which_set
        """

        self.diag[which_set]['objective'][epoch] = error + l2
        self.diag[which_set]['emp_loss'][epoch] = error
        self.diag[which_set]['scores'][:, epoch] = scores

        self.diag['network']['l2_penalty'][epoch] = l2

        if self.data.n_classes == 2:

            if which_set == 'train':
                y = self.data._y_train
            if which_set == 'val':
                y = self.data._y_val
            if which_set == 'test':
                y = self.data._y_test

            if sum(y) > 0:
                AUC = roc_auc_score(y, scores)
                self.diag[which_set]['auc'][epoch] = AUC
Beispiel #2
0
class NeuralNet:
    def __init__(self,
                 dataset,
                 use_weights=None,
                 pretrain=False,
                 profile=False):
        """
        initialize instance
        """

        # whether to enable profiling in Theano functions
        self.profile = profile

        self.initialize_variables(dataset)

        # load dataset
        load_dataset(self, dataset.lower(), pretrain)

        if use_weights and not pretrain:
            self.load_weights(use_weights)

    def initialize_variables(self, dataset):

        self.all_layers, self.trainable_layers = (), ()

        self.n_conv_layers = 0
        self.n_dense_layers = 0
        self.n_relu_layers = 0
        self.n_leaky_relu_layers = 0
        self.n_bn_layers = 0
        self.n_norm_layers = 0
        self.n_abs_layers = 0
        self.n_maxpool_layers = 0
        self.n_upscale_layers = 0
        self.n_dropout_layers = 0
        self.n_dimshuffle_layers = 0
        self.n_reshape_layers = 0
        self.R_init = 0

        self.learning_rate_init = Cfg.learning_rate.get_value()

        self.it = 0
        self.clock = 0

        self.pretrained = False  # set to True after pretraining such that dictionary initialization mustn't be repeated

        self.diag = {}  # init an empty dictionary to hold diagnostics
        self.log = Log(dataset_name=dataset)
        self.ad_log = AD_Log()

        self.dense_layers, self.conv_layers, = [], []

    def compile_updates(self):
        """ create network from architecture given in modules (determined by dataset)
        create Theano compiled functions
        """

        opt.sgd.updates.create_update(self)

    def compile_autoencoder(self):
        """
        create network from autoencoder architecture (determined by dataset)
        and compile Theano update functions.
        """

        print("Compiling autoencoder...")
        opt.sgd.updates.create_autoencoder(self)
        print("Autoencoder compiled.")

    def load_data(self, data_loader=None, pretrain=False):

        self.data = data_loader()

        if pretrain:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)
        elif Cfg.reconstruction_loss:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)
        elif Cfg.svdd_loss and Cfg.reconstruction_penalty:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)
        else:
            self.data.build_architecture(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)

    def flush_data(self):

        self.data._X_train = None
        self.data._y_train = None
        self.data._X_val = None
        self.data._y_val = None
        self.data._X_test = None
        self.data._y_test = None

        print("Data flushed from network.")

    def next_layers(self, layer):

        flag = False
        for current_layer in self.all_layers:
            if flag:
                yield current_layer
            if current_layer is layer:
                flag = True

    def previous_layers(self, layer):

        flag = False
        for current_layer in reversed(self.all_layers):
            if flag:
                yield current_layer
            if current_layer is layer:
                flag = True

    def start_clock(self):

        self.clock = time.time()

    def stop_clock(self):

        self.clocked = time.time() - self.clock
        print("Total elapsed time: %g" % self.clocked)

    def pretrain(self, solver, lr, n_epochs):
        """
        pre-train weights with an autoencoder
        """

        self.ae_solver = solver.lower()
        self.ae_learning_rate = lr
        self.ae_n_epochs = n_epochs

        # set learning rate
        lr_tmp = Cfg.learning_rate.get_value()
        Cfg.learning_rate.set_value(Cfg.floatX(lr))

        self.compile_autoencoder()

        from opt.sgd.train import train_autoencoder
        train_autoencoder(self)

        # remove layer attributes, re-initialize network and reset learning rate
        for layer in self.all_layers:
            delattr(self, layer.name + "_layer")
        self.initialize_variables(self.data.dataset_name)
        Cfg.learning_rate.set_value(Cfg.floatX(lr_tmp))
        self.pretrained = True  # set to True that dictionary initialization mustn't be repeated

        # load network architecture
        if Cfg.svdd_loss and Cfg.reconstruction_penalty:
            self.data.build_autoencoder(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)
        else:
            self.data.build_architecture(self)

            for layer in self.all_layers:
                setattr(self, layer.name + "_layer", layer)

            self.log.store_architecture(self)

        # load weights learned by autoencoder
        self.load_weights(Cfg.xp_path + "/ae_pretrained_weights.p")

    def train(self, solver, n_epochs=10, save_at=0, save_to=''):

        self.solver = solver.lower()
        self.ae_solver = solver.lower()
        self.n_epochs = n_epochs
        self.save_at = save_at
        self.save_to = save_to

        self.log['solver'] = self.solver
        self.log['save_at'] = self.save_at

        self.compile_updates()

        from opt.sgd.train import train_network

        self.start_clock()
        train_network(self)
        self.stop_clock()

        # self.log.save_to_file()

    def evaluate(self, solver):

        # this could be simplified to only compiling the forwardpropagation...
        self.solver = solver.lower()  # needed for compiling backprop
        self.compile_updates()

        print("Evaluating network with current weights...")

        self.initialize_diagnostics(1)
        self.copy_parameters()

        # perform forward passes on training, val, and test set
        _, _ = performance(self, which_set='train', epoch=0, print_=True)
        _, _ = performance(self, which_set='val', epoch=0, print_=True)
        _, _ = performance(self, which_set='test', epoch=0, print_=True)

        print("Evaluation on train, val, and test set completed.")

    def log_results(self, filename=None):
        """
        log the results relevant for anomaly detection
        """

        self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
        self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
        self.ad_log['train_time'] = self.train_time

        self.ad_log['val_auc'] = self.diag['val']['auc'][-1]
        self.ad_log['val_accuracy'] = self.diag['val']['acc'][-1]

        self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
        self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
        self.ad_log['test_time'] = self.test_time

        self.ad_log.save_to_file(filename=filename)

    def addInputLayer(self, **kwargs):

        self.input_layer = InputLayer(name="input", **kwargs)
        self.input_layer.inp_ndim = len(kwargs["shape"])

    def addConvLayer(self, use_batch_norm=False, **kwargs):
        """
        Add convolutional layer.
        If batch norm flag is True, the convolutional layer
        will be followed by a batch-normalization layer
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_conv_layers += 1
        name = "conv%i" % self.n_conv_layers

        new_layer = ConvLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        if use_batch_norm:
            self.n_bn_layers += 1
            name = "bn%i" % self.n_bn_layers
            self.all_layers += (BatchNorm(new_layer, name=name), )

    def addDenseLayer(self, use_batch_norm=False, **kwargs):
        """
        Add dense layer.
        If batch norm flag is True, the dense layer
        will be followed by a batch-normalization layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dense_layers += 1
        name = "dense%i" % self.n_dense_layers

        new_layer = DenseLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        if use_batch_norm:
            self.n_bn_layers += 1
            name = "bn%i" % self.n_bn_layers
            self.all_layers += (BatchNorm(new_layer, name=name), )

    def addSigmoidLayer(self, **kwargs):
        """
        Add sigmoid classification layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]
        new_layer = Sigmoid(input_layer, **kwargs)

        self.all_layers += (new_layer, )

        self.n_layers = len(self.all_layers)

    def addSoftmaxLayer(self, **kwargs):
        """
        Add softmax multi-class classification layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]
        new_layer = Softmax(input_layer, **kwargs)

        self.all_layers += (new_layer, )

        self.n_layers = len(self.all_layers)

    def addNormLayer(self, **kwargs):
        """
        Add layer which normalizes its input to length 1. 
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_norm_layers += 1
        name = "norm%i" % self.n_norm_layers

        new_layer = Norm(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addAbsLayer(self, **kwargs):
        """
        Add layer which returns the absolute value of its input.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_abs_layers += 1
        name = "abs%i" % self.n_abs_layers

        new_layer = Abs(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addReLU(self, **kwargs):
        """
        Add ReLU activation layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_relu_layers += 1
        name = "relu%i" % self.n_relu_layers

        new_layer = ReLU(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addLeakyReLU(self, **kwargs):
        """
        Add leaky ReLU activation layer. (with leakiness=0.01)
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_leaky_relu_layers += 1
        name = "leaky_relu%i" % self.n_leaky_relu_layers

        new_layer = LeakyReLU(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addMaxPool(self, **kwargs):
        """
        Add MaxPooling activation layer.
        """

        input_layer = self.input_layer if not self.all_layers\
            else self.all_layers[-1]

        self.n_maxpool_layers += 1
        name = "maxpool%i" % self.n_maxpool_layers

        new_layer = MaxPool(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addUpscale(self, **kwargs):
        """
        Add Upscaling activation layer.
        """

        input_layer = self.input_layer if not self.all_layers\
            else self.all_layers[-1]

        self.n_upscale_layers += 1
        name = "upscale%i" % self.n_upscale_layers

        new_layer = Upscale(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addDropoutLayer(self, **kwargs):
        """
        Add Dropout layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dropout_layers += 1
        name = "dropout%i" % self.n_dropout_layers

        new_layer = DropoutLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addDimshuffleLayer(self, **kwargs):
        """
        Add Dimshuffle layer to reorder dimensions
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dimshuffle_layers += 1
        name = "dimshuffle%i" % self.n_dimshuffle_layers

        new_layer = Dimshuffle(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addReshapeLayer(self, **kwargs):
        """
        Add reshape layer to reshape dimensions
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_reshape_layers += 1
        name = "reshape%i" % self.n_reshape_layers

        new_layer = Reshape(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def setFeatureLayer(self):
        """
        sets the currently highest layer of the current network to be the code layer (compression layer)
        """
        setattr(self, "feature_layer", self.all_layers[-1])

    def dump_weights(self, filename=None, pretrain=False):

        dump_weights(self, filename, pretrain=pretrain)

    def load_weights(self, filename=None):

        assert filename and os.path.exists(filename)

        load_weights(self, filename)

    def update_R(self):
        """
        method to update R while leaving the network parameters and center c fixed in a block coordinate optimization
        """

        print("Updating radius R...")

        # Get updates
        R = update_R(self.diag['train']['rep'],
                     self.cvar.get_value(),
                     solver=Cfg.R_update_solver,
                     scalar_method=Cfg.R_update_scalar_method,
                     lp_obj=Cfg.R_update_lp_obj)

        # Update R
        self.Rvar.set_value(Cfg.floatX(R))

        print("Radius R updated.")

    def update_R_c(self):
        """
        method to update R and c while leaving the network parameters fixed in a block coordinate optimization
        """

        print("Updating radius R and center c...")

        # Get updates
        R, c = update_R_c(self.diag['train']['rep'],
                          np.sum(self.diag['train']['rep']**2, axis=1),
                          solver=Cfg.QP_solver)

        # Update values
        self.Rvar.set_value(Cfg.floatX(R))
        self.cvar.set_value(Cfg.floatX(c))

        print("Radius R and center c updated.")

    def initialize_diagnostics(self, n_epochs):
        """
        initialize diagnostics for the neural network
        """

        # data-dependent diagnostics for train, validation, and test set
        self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
                                          n_samples=self.data.n_train)
        self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
                                        n_samples=self.data.n_val)
        self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
                                         n_samples=self.data.n_test)

        # network parameter diagnostics
        self.diag['network'] = NNetParamDiag(self, n_epochs=n_epochs)

        # Best results (highest AUC on test set)
        self.auc_best = 0
        self.auc_best_epoch = 0  # determined by highest AUC on test set
        self.best_weight_dict = None

    def save_objective_and_accuracy(self, epoch, which_set, objective,
                                    accuracy):
        """
        save objective and accuracy of epoch
        """

        self.diag[which_set]['objective'][epoch] = objective
        self.diag[which_set]['acc'][epoch] = accuracy

    def save_initial_parameters(self):
        """
        save a copy of the initial network parameters for diagnostics.
        """

        self.W_init = []
        self.b_init = []

        for layer in self.trainable_layers:
            if layer.isdense | layer.isconv:
                self.W_init.append(None)
                self.b_init.append(None)

        i = 0
        for layer in self.trainable_layers:
            if layer.isdense | layer.isconv:
                self.W_init[i] = layer.W.get_value()
                if layer.b is not None:
                    self.b_init[i] = layer.b.get_value()
                i += 1

    def copy_parameters(self):
        """
        save a copy of the current network parameters in order to monitor the difference between epochs.
        """
        i = 0
        for layer in self.trainable_layers:
            if layer.isdense | layer.isconv:
                self.diag['network']['W_copy'][i] = layer.W.get_value()
                if layer.b is not None:
                    self.diag['network']['b_copy'][i] = layer.b.get_value()
                i += 1

    def copy_initial_parameters_to_cache(self):
        """
        Save a copy of the initial parameters in cache
        """
        self.diag['network']['W_copy'] = list(self.W_init)
        self.diag['network']['b_copy'] = list(self.b_init)

    def save_network_diagnostics(self, epoch, l2, R):
        """
        save diagnostics of the network
        """

        self.diag['network']['l2_penalty'][epoch] = l2
        self.log['l2_penalty'].append(float(l2))

        i = 0
        j = 0
        for layer in self.trainable_layers:
            if layer.isdense:
                self.diag['network']['W_norms'][i][:, epoch] = np.sum(
                    layer.W.get_value()**2, axis=0)
                if layer.b is not None:
                    self.diag['network']['b_norms'][
                        i][:, epoch] = layer.b.get_value()**2
                i += 1

            if layer.isdense | layer.isconv:
                dW = np.sqrt(
                    np.sum((layer.W.get_value() -
                            self.diag['network']['W_copy'][j])**2))
                self.diag['network']['dW_norms'][j][epoch] = dW
                if layer.b is not None:
                    db = np.sqrt(
                        np.sum((layer.b.get_value() -
                                self.diag['network']['b_copy'][j])**2))
                    self.diag['network']['db_norms'][j][epoch] = db
                j += 1

        # diagnostics only relevant for the SVDD loss
        if Cfg.svdd_loss:
            self.diag['network']['R'][epoch] = R
            self.diag['network']['c_norm'][epoch] = np.sqrt(
                np.sum(self.cvar.get_value()**2))

    def track_best_results(self, epoch):
        """
        Save network parameters where AUC on the test set was highest.
        """

        if self.diag['test']['auc'][epoch] > self.auc_best:
            self.auc_best = self.diag['test']['auc'][epoch]
            self.auc_best_epoch = epoch

            self.best_weight_dict = dict()

            for layer in self.trainable_layers:
                self.best_weight_dict[layer.name + "_w"] = layer.W.get_value()
                if layer.b is not None:
                    self.best_weight_dict[layer.name +
                                          "_b"] = layer.b.get_value()

            if Cfg.svdd_loss:
                self.best_weight_dict["R"] = self.Rvar.get_value()

    def dump_best_weights(self, filename):
        """
        pickle the network parameters, where AUC on the test set was highest.
        """

        with open(filename, 'wb') as f:
            pickle.dump(self.best_weight_dict, f)

        print("Parameters of best epoch saved in %s" % filename)

    def save_diagnostics(self, which_set, epoch, scores, rep_norm, rep,
                         emp_loss, reconstruction_penalty):
        """
        save diagnostics for which_set of epoch
        """

        if self.data.n_classes == 2:

            if which_set == 'train':
                y = self.data._y_train
            if which_set == 'val':
                y = self.data._y_val
            if which_set == 'test':
                y = self.data._y_test

            self.diag[which_set]['scores'][:, epoch] = scores

            if sum(y) > 0:
                AUC = roc_auc_score(y, scores)
                self.diag[which_set]['auc'][epoch] = AUC
                self.log[which_set + '_auc'].append(float(AUC))
                print("{:32} {:.2f}%".format(which_set.title() + ' AUC:',
                                             100. * AUC))

            scores_normal = scores[y == 0]
            scores_outlier = scores[y == 1]
            normal_summary = get_five_number_summary(scores_normal)
            outlier_summary = get_five_number_summary(scores_outlier)
            self.log[which_set +
                     '_normal_scores_summary'].append(normal_summary)
            self.log[which_set +
                     '_outlier_scores_summary'].append(outlier_summary)

            self.diag[which_set]['rep'] = rep
            self.diag[which_set]['rep_norm'][:, epoch] = rep_norm

            rep_norm_normal = rep_norm[y == 0]
            rep_norm_outlier = rep_norm[y == 1]
            normal_summary = get_five_number_summary(rep_norm_normal)
            outlier_summary = get_five_number_summary(rep_norm_outlier)
            self.log[which_set +
                     '_normal_rep_norm_summary'].append(normal_summary)
            self.log[which_set +
                     '_outlier_rep_norm_summary'].append(outlier_summary)

            if Cfg.svdd_loss:
                rep_mean = np.mean(rep, axis=0)
                self.diag[which_set]['output_mean_norm'][epoch] = np.sqrt(
                    np.sum(rep_mean**2))
                self.diag[which_set]['c_mean_diff'][epoch] = np.sqrt(
                    np.sum((rep_mean - self.cvar.get_value())**2))

        self.diag[which_set]['reconstruction_penalty'][
            epoch] = reconstruction_penalty
        self.diag[which_set]['emp_loss'][epoch] = float(emp_loss)

        self.log[which_set + '_emp_loss'].append(float(emp_loss))

    def initialize_ae_diagnostics(self, n_epochs):
        """
        initialize diagnostic variables for autoencoder network.
        """

        self.train_time = 0
        self.test_time = 0
        self.best_weight_dict = None

        # data-dependent diagnostics for train, validation, and test set
        self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
                                          n_samples=self.data.n_train)
        self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
                                        n_samples=self.data.n_val)
        self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
                                         n_samples=self.data.n_test)

        # network parameters
        self.diag['network'] = {}
        self.diag['network']['l2_penalty'] = np.zeros(n_epochs,
                                                      dtype=Cfg.floatX)

    def save_ae_diagnostics(self, which_set, epoch, error, scores, l2):
        """
        save autoencoder diagnostics for which_set
        """

        self.diag[which_set]['objective'][epoch] = error + l2
        self.diag[which_set]['emp_loss'][epoch] = error
        self.diag[which_set]['scores'][:, epoch] = scores

        self.diag['network']['l2_penalty'][epoch] = l2

        if self.data.n_classes == 2:

            if which_set == 'train':
                y = self.data._y_train
            if which_set == 'val':
                y = self.data._y_val
            if which_set == 'test':
                y = self.data._y_test

            if sum(y) > 0:
                AUC = roc_auc_score(y, scores)
                self.diag[which_set]['auc'][epoch] = AUC
Beispiel #3
0
class NeuralNet:
    def __init__(self, dataset, use_weights=None, profile=False):
        """ initialize instance
        """

        # whether to enable profiling in Theano functions
        self.profile = profile

        # patch lasagne creation of parameters
        # (otherwise broadcasting issue with latest versions)
        patch_lasagne()

        self.initialize_variables(dataset)

        # load dataset
        load_dataset(self, dataset.lower())

        if use_weights:
            # special case for VGG pre-trained network
            if use_weights.endswith('vgg16.pkl'):
                self.data.load_weights(self)
            else:
                self.load_weights(use_weights)

    def initialize_variables(self, dataset):

        self.all_layers, self.trainable_layers = (), ()

        self.n_conv_layers = 0
        self.n_dense_layers = 0
        self.n_relu_layers = 0
        self.n_bn_layers = 0
        self.n_maxpool_layers = 0
        self.n_dropout_layers = 0

        self.it = 0
        self.clock = 0

        self.log = Log(dataset_name=dataset)

        if Cfg.draw_on_board:
            self.xp_on_board(dataset)

        self.dense_layers, self.conv_layers, = [], []

    def xp_on_board(self, dataset):
        """
        Connect to tensorboard server and create pycrayon experiment
        """

        hostname = socket.gethostname()
        hostname = "local" if hostname == Cfg.hostname \
            else hostname.split(".").pop(0)
        client = pycrayon.CrayonClient(hostname=Cfg.hostname)
        count = 0
        while True:
            experiment_name_train = "{}_{}_{}_train"\
                .format(dataset, hostname, count)
            experiment_name_val = "{}_{}_{}_val"\
                .format(dataset, hostname, count)
            try:
                self.board_monitor_train = \
                    client.create_experiment(experiment_name_train)
                self.board_monitor_val = \
                    client.create_experiment(experiment_name_val)
                print("Connected to pycrayon server: experiment '{}_{}_{}'".
                      format(dataset, hostname, count))
                break
            except:
                count += 1
            # safeguard
            if count > 1000:
                raise ValueError("Failed to connect to server board")

    def compile_updates(self):
        """ create network from architecture given in modules (determined by dataset)
        create Theano compiled functions
        """

        opt.sgd.updates.create_update(self)

        # avoid unnecessary compilation of not using lwsvm
        if self.solver not in ('svm', 'bpfw'):
            return

        if self.solver == 'bpfw':
            self.update_bpfw = opt.bpfw.updates.compile_update_bpfw(self)
            return

        if Cfg.store_on_gpu:
            from opt.svm.full_gpu import compile_update_conv,\
                compile_update_dense, compile_update_svm
        else:
            from opt.svm.part_gpu import compile_update_conv,\
                compile_update_dense, compile_update_svm

        for layer in self.trainable_layers:

            print("Compiling updates for {}...".format(layer.name))

            layer.hinge_avg = layers.fun.compile_hinge_avg(self, layer)

            if layer.isconv:
                layer.svm_update = compile_update_conv(self, layer)

            if layer.isdense:
                layer.svm_update = compile_update_dense(self, layer)

            if layer.issvm:
                layer.svm_update = compile_update_svm(self, layer)

            flush_last_line()

        print("Updates compiled.")

    def load_data(self, data_loader=None):

        self.data = data_loader()
        self.data.build_architecture(self)

        for layer in self.all_layers:
            setattr(self, layer.name + "_layer", layer)

        self.log.store_architecture(self)

    def next_layers(self, layer):

        flag = False
        for current_layer in self.all_layers:
            if flag:
                yield current_layer
            if current_layer is layer:
                flag = True

    def previous_layers(self, layer):

        flag = False
        for current_layer in reversed(self.all_layers):
            if flag:
                yield current_layer
            if current_layer is layer:
                flag = True

    def start_clock(self):

        self.clock = time.time()

    def stop_clock(self):

        self.clocked = time.time() - self.clock
        print("Total elapsed time: %g" % self.clocked)

    def train(self, solver, n_epochs=10, save_at=0, save_to=''):

        self.solver = solver.lower()
        self.n_epochs = n_epochs
        self.save_at = save_at
        self.save_to = save_to

        self.log['solver'] = self.solver
        self.log['save_at'] = self.save_at

        self.compile_updates()

        if solver == "svm":
            assert Cfg.compile_lwsvm
            from opt.svm.train import train_network
        elif solver == "bpfw":
            from opt.bpfw.train import train_network
        else:
            from opt.sgd.train import train_network

        self.start_clock()
        train_network(self)
        self.stop_clock()

        self.log.save_to_file()

    def addInputLayer(self, **kwargs):

        self.input_layer = InputLayer(name="input", **kwargs)
        self.input_layer.inp_ndim = len(kwargs["shape"])

    def addConvLayer(self, use_batch_norm=False, **kwargs):
        """
        Add convolutional layer.
        If batch norm flag is True, the convolutional layer
        will be followed by a batch-normalization layer
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_conv_layers += 1
        name = "conv%i" % self.n_conv_layers

        new_layer = ConvLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        if use_batch_norm:
            self.n_bn_layers += 1
            name = "bn%i" % self.n_bn_layers
            self.all_layers += (BatchNorm(new_layer, name=name), )

    def addDenseLayer(self, use_batch_norm=False, **kwargs):
        """
        Add dense layer.
        If batch norm flag is True, the dense layer
        will be followed by a batch-normalization layer
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dense_layers += 1
        name = "dense%i" % self.n_dense_layers

        new_layer = DenseLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        if use_batch_norm:
            self.n_bn_layers += 1
            name = "bn%i" % self.n_bn_layers
            self.all_layers += (BatchNorm(new_layer, name=name), )

    def addSVMLayer(self, **kwargs):
        """
        Add classification layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]
        new_layer = SVMLayer(input_layer,
                             num_units=self.data.n_classes,
                             **kwargs)

        self.all_layers += (new_layer, )
        self.trainable_layers += (new_layer, )

        self.n_layers = len(self.all_layers)

    def addReLU(self, **kwargs):
        """
        Add ReLU activation layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_relu_layers += 1
        name = "relu%i" % self.n_relu_layers

        new_layer = ReLU(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addMaxPool(self, **kwargs):
        """
        Add MaxPooling activation layer.
        """

        input_layer = self.input_layer if not self.all_layers\
            else self.all_layers[-1]

        self.n_maxpool_layers += 1
        name = "maxpool%i" % self.n_maxpool_layers

        new_layer = MaxPool(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def addDropoutLayer(self, **kwargs):
        """
        Add Dropout layer.
        """

        input_layer = self.input_layer if not self.all_layers \
            else self.all_layers[-1]

        self.n_dropout_layers += 1
        name = "dropout%i" % self.n_dropout_layers

        new_layer = DropoutLayer(input_layer, name=name, **kwargs)

        self.all_layers += (new_layer, )

    def dump_weights(self, filename=None):

        dump_weights(self, filename)

    def load_weights(self, filename=None):

        assert filename and os.path.exists(filename)

        load_weights(self, filename)