Exemple #1
0
    def __init__(self,
                 model,
                 dataset,
                 train_cost,
                 valid_cost,
                 learning_method,
                 stop_criteria,
                 log=None,
                 verbose=True):
        self.model = model
        self.dataset = dataset
        self.train_cost = train_cost
        self.valid_cost = valid_cost
        self.learning_method = learning_method
        self.stop_criteria = stop_criteria
        self.log = log
        self.verbose = verbose

        if self.log is None:
            # use default Log setting
            self.log = Log(logger=internal_logger)

        elif self.log.save_to_database:
            self.log.print_records()
            self.log.info('\n')
Exemple #2
0
def train():

    # build dataset
    batch_size = 64
    data = Mnist(batch_size=batch_size, train_valid_test_ratio=[5, 1, 1])

    # build model
    model = Sequential(input_var=T.matrix(), output_var=T.matrix())
    model.add(Linear(prev_dim=28 * 28, this_dim=200))
    model.add(RELU())
    model.add(Linear(prev_dim=200, this_dim=100))
    model.add(RELU())
    model.add(Dropout(0.5))
    model.add(Linear(prev_dim=100, this_dim=10))
    model.add(Softmax())

    # build learning method
    decay_batch = int(data.train.X.shape[0] * 2 / batch_size)
    learning_method = SGD(learning_rate=0.1,
                          momentum=0.9,
                          lr_decay_factor=0.9,
                          decay_batch=decay_batch)

    # Build Logger
    log = Log(
        experiment_name='MLP',
        description='This is a tutorial',
        save_outputs=True,  # log all the outputs from the screen
        save_model=True,  # save the best model
        save_epoch_error=True,  # log error at every epoch
        save_to_database={
            'name': 'Example.sqlite3',
            'records': {
                'Batch_Size': batch_size,
                'Learning_Rate': learning_method.learning_rate,
                'Momentum': learning_method.momentum
            }
        })  # end log

    # put everything into the train object
    train_object = TrainObject(model=model,
                               log=log,
                               dataset=data,
                               train_cost=mse,
                               valid_cost=error,
                               learning_method=learning_method,
                               stop_criteria={
                                   'max_epoch': 100,
                                   'epoch_look_back': 5,
                                   'percent_decrease': 0.01
                               })
    # finally run the code
    train_object.setup()
    train_object.run()

    ypred = model.fprop(data.get_test().X)
    ypred = np.argmax(ypred, axis=1)
    y = np.argmax(data.get_test().y, axis=1)
    accuracy = np.equal(ypred, y).astype('f4').sum() / len(y)
    print('test accuracy:', accuracy)
Exemple #3
0
    def __init__(self,
                 train_valid_test_ratio=[8, 1, 1],
                 batch_size=100,
                 num_batches=None,
                 iter_class='SequentialSubsetIterator',
                 rng=None,
                 log=None):
        '''
        DESCRIPTION: Abstract class
        PARAMS:
            split_mode(sequential | random): sequentially or randomly split the dataset
        '''

        assert len(train_valid_test_ratio) == 3, 'the size of list is not 3'
        self.ratio = train_valid_test_ratio
        self.iter_class = iter_class
        self.batch_size = batch_size
        self.num_batches = num_batches
        self.rng = rng

        self.log = log

        if self.log is None:
            # use default Log setting, using the internal logger
            self.log = Log(logger=internal_logger)
Exemple #4
0
    def __init__(self, model, dataset, train_cost, valid_cost, learning_method, stop_criteria, log=None):
        self.model = model
        self.dataset = dataset
        self.train_cost = train_cost
        self.valid_cost = valid_cost
        self.learning_method = learning_method
        self.stop_criteria = stop_criteria
        self.log = log

        if self.log is None:
            # use default Log setting
            self.log = Log(logger=internal_logger)

        elif self.log.save_to_database:
            self.log.print_records()
            self.log.info('\n')
Exemple #5
0
    def __init__(self,
                 train_valid_test_ratio=[8, 1, 1],
                 log=None,
                 batch_size=100,
                 num_batches=None,
                 iter_class='SequentialSubsetIterator',
                 rng=None):

        assert len(train_valid_test_ratio) == 3, 'the size of list is not 3'
        self.ratio = train_valid_test_ratio
        self.iter_class = iter_class
        self.batch_size = batch_size
        self.num_batches = num_batches
        self.rng = rng

        self.log = log

        if self.log is None:
            # use default Log setting, using the internal logger
            self.log = Log(logger=internal_logger)
Exemple #6
0
 def __init__(self, log):
     self.log = log
     if self.log is None:
         # use default Log setting, using the internal logger
         self.log = Log(logger=internal_logger)
Exemple #7
0
class TrainObject():

    def __init__(self, model, dataset, train_cost, valid_cost, learning_method, stop_criteria, log=None):
        self.model = model
        self.dataset = dataset
        self.train_cost = train_cost
        self.valid_cost = valid_cost
        self.learning_method = learning_method
        self.stop_criteria = stop_criteria
        self.log = log

        if self.log is None:
            # use default Log setting
            self.log = Log(logger=internal_logger)

        elif self.log.save_to_database:
            self.log.print_records()
            self.log.info('\n')


    def setup(self):

        self.log.info( '..begin setting up train object')

        #===================[ build params and deltas list ]==================#

        params = []
        deltas = []

        for layer in self.model.layers:
            for param in layer.params:
                # checked that the param to be updated is shared variable
                if is_shared_var(param):
                    param.name += '_' + layer.__class__.__name__
                    params += [param]
                    deltas += [shared_zeros(shape=param.shape.eval())]

        #=====================[ training params updates ]=====================#

        self.log.info("..update params: " + str(params))
        train_y_pred, train_layers_stats = self.model.train_fprop(self.model.input_var)
        train_cost = self.train_cost(self.model.output_var, train_y_pred).astype(floatX)

        train_updates = []
        gparams = T.grad(train_cost, params)
        for delta, param, gparam in zip(deltas, params, gparams):
            train_updates += self.learning_method.update(delta, gparam)
            train_updates += [(param, param+delta)]

        #----[ append updates of stats from each layer to train updates ]-----#

        self.train_stats_names, train_stats_vars = split_list(train_layers_stats)
        train_stats_vars = [var.astype(floatX) for var in train_stats_vars]
        self.train_stats_shared = generate_shared_list(train_stats_vars)
        train_stats_updates = merge_lists(self.train_stats_shared, train_stats_vars)
        train_updates += train_stats_updates

        #-------------------------[ train functions ]-------------------------#

        self.log.info('..begin compiling functions')
        self.training = theano.function(inputs=[self.model.input_var, self.model.output_var],
                                        outputs=train_cost,
                                        updates=train_updates,
                                        on_unused_input='warn',
                                        allow_input_downcast=True)

        self.log.info('..training function compiled')

        #=============================[ testing ]=============================#

        test_y_pred, test_layers_stats = self.model.test_fprop(self.model.input_var)

        #-----[ append updates of stats from each layer to test updates ]-----#

        self.test_stats_names, test_stats_vars = split_list(test_layers_stats)
        test_stats_vars = [var.astype(floatX) for var in test_stats_vars]
        self.test_stats_shared = generate_shared_list(test_stats_vars)
        test_stats_updates = merge_lists(self.test_stats_shared, test_stats_vars)

        #-------------------------[ test functions ]--------------------------#

        test_stopping_error = self.valid_cost(self.model.output_var, test_y_pred).astype(floatX)
        test_cost = self.train_cost(self.model.output_var, test_y_pred).astype(floatX)

        self.testing = theano.function(inputs=[self.model.input_var, self.model.output_var],
                                       outputs=(test_stopping_error, test_cost),
                                       updates=test_stats_updates,
                                       on_unused_input='warn',
                                       allow_input_downcast=True)

        self.log.info('..testing function compiled')


    def run(self):

        best_valid_error = float(sys.maxint)
        valid_error = float(sys.maxint)

        train_cost = float(sys.maxint)
        valid_cost = float(sys.maxint)

        train_stats_values = []
        valid_stats_values = []

        epoch = 0
        error_dcr = 0
        self.best_epoch_last_update = 0
        self.best_valid_last_update = float(sys.maxint)

        train_stats_names = ['train_' + name for name in self.train_stats_names]
        valid_stats_names = ['valid_' + name for name in self.test_stats_names]

        job_start = time.time()

        while (self.continue_learning(epoch, error_dcr, best_valid_error)):

            if epoch > 0:
                self.log.info("best_epoch_last_update: %d"%self.best_epoch_last_update)
                self.log.info("valid_error_decrease: %f"%error_dcr)
                self.log.info("best_valid_last_update: %f"%self.best_valid_last_update)
                self.log.info("========[ End of Epoch ]========\n\n")

            epoch += 1

            start_time = time.time()

            num_train_examples = 0
            total_train_cost = 0.
            train_stats_values = np.zeros(len(train_stats_names), dtype=floatX)

            num_valid_examples = 0
            total_valid_cost = 0.
            total_valid_stopping_cost = 0.
            valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX)

            blk = 0

            for block in self.dataset:
                block_time = time.time()
                blk += 1

                train_set = block.get_train()
                valid_set = block.get_valid()

                #====================[ Training Progress ]====================#
                if train_set.dataset_size > 0:

                    self.log.info('..training '+ self.dataset.__class__.__name__
                                + ' block %s/%s'%(blk, self.dataset.nblocks))

                    progbar = Progbar(target=train_set.dataset_size)
                    blk_sz = 0
                    for idx in train_set:
                        cost = self.training(train_set.X[idx], train_set.y[idx])
                        total_train_cost += cost * len(idx)
                        num_train_examples += len(idx)
                        train_stats_values += len(idx) * get_shared_values(self.train_stats_shared)
                        blk_sz += len(idx)
                        progbar.update(blk_sz)
                    print

                    #-------[ Update train best cost and error values ]-------#
                    train_cost = total_train_cost / num_train_examples
                    train_stats_values /= num_train_examples

                #===================[ Validating Progress ]===================#
                if valid_set.dataset_size > 0:

                    self.log.info('..validating ' + self.dataset.__class__.__name__
                                + ' block %s/%s'%(blk, self.dataset.nblocks))

                    progbar = Progbar(target=valid_set.dataset_size)
                    blk_sz = 0
                    for idx in valid_set:
                        stopping_cost, cost = self.testing(valid_set.X[idx], valid_set.y[idx])
                        total_valid_cost += cost * len(idx)
                        total_valid_stopping_cost += stopping_cost * len(idx)
                        num_valid_examples += len(idx)
                        valid_stats_values += len(idx) * get_shared_values(self.test_stats_shared)
                        blk_sz += len(idx)
                        progbar.update(blk_sz)
                    print

                    #-------[ Update valid best cost and error values ]-------#
                    valid_error = total_valid_stopping_cost / num_valid_examples
                    valid_cost = total_valid_cost / num_valid_examples
                    valid_stats_values /= num_valid_examples

                    if valid_error < best_valid_error:
                        best_valid_error = valid_error
                        self.log.info('..best validation error so far')
                        if self.log.save_model:
                            self.log._save_model(self.model)
                            self.log.info('..model saved')

                    if valid_error < self.best_valid_last_update:
                        error_dcr = self.best_valid_last_update - valid_error
                    else:
                        error_dcr = 0

                self.log.info('block time: %0.2fs'%(time.time()-block_time))
                self.log.info(get_mem_usage())

            #==============[ save to database, save epoch error]==============#
            if self.log.save_to_database:
                self.log._save_to_database(epoch, train_cost, valid_cost, best_valid_error)
                self.log.info('..sent to database: %s:%s' % (self.log.save_to_database['name'],
                                                             self.log.experiment_name))

            if self.log.save_epoch_error:
                self.log._save_epoch_error(epoch, valid_error)
                self.log.info('..epoch error saved')

            end_time = time.time()

            #=====================[ log outputs to file ]=====================#
            merged_train = merge_lists(train_stats_names, train_stats_values)
            merged_valid = merge_lists(valid_stats_names, valid_stats_values)

            outputs = [('epoch', epoch),
                        ('runtime(s)', int(end_time-start_time)),
                        ('train_' + self.train_cost.func_name, train_cost),
                        ('valid_' + self.train_cost.func_name, valid_cost),
                        ('valid_' + self.valid_cost.func_name, valid_error),
                        ('best_valid_' + self.valid_cost.func_name, best_valid_error)]

            outputs += merged_train + merged_valid
            self.log._log_outputs(outputs)


        job_end = time.time()
        self.log.info('Job Completed on %s'%time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end)))
        ttl_time = int(job_end - job_start)
        dt = datetime.timedelta(seconds=ttl_time)
        self.log.info('Total Time Taken: %s'%str(dt))
        self.log.info("========[ End of Job ]========\n\n")


    def continue_learning(self, epoch, error_dcr, best_valid_error):

        if epoch > self.stop_criteria['max_epoch']:
            return False

        elif self.stop_criteria['percent_decrease'] is None or \
            self.stop_criteria['epoch_look_back'] is None:
            return True

        elif np.abs(float(error_dcr) / self.best_valid_last_update) \
            >= self.stop_criteria['percent_decrease']:
            self.best_valid_last_update = best_valid_error
            self.best_epoch_last_update = epoch
            return True

        elif epoch - self.best_epoch_last_update > \
            self.stop_criteria['epoch_look_back']:
            return False

        else:
            return True
Exemple #8
0
def train():
    batch_size = 256
    short_memory = 0.9
    learning_rate = 0.005
    data = Cifar10(batch_size=batch_size, train_valid_test_ratio=[4, 1, 1])
    _, c, h, w = data.train.X.shape

    model = Sequential(input_var=T.tensor4(), output_var=T.matrix())
    model.add(
        Convolution2D(input_channels=c,
                      filters=8,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      border_mode='full'))
    h, w = full(h, w, kernel=3, stride=1)
    model.add(
        BatchNormalization(dim=8, layer_type='conv',
                           short_memory=short_memory))
    model.add(RELU())
    model.add(
        Convolution2D(input_channels=8,
                      filters=16,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      border_mode='valid'))
    h, w = valid(h, w, kernel=3, stride=1)
    model.add(
        BatchNormalization(dim=16,
                           layer_type='conv',
                           short_memory=short_memory))
    model.add(RELU())
    model.add(Pooling2D(poolsize=(4, 4), stride=(4, 4), mode='max'))
    h, w = valid(h, w, kernel=4, stride=4)
    model.add(Flatten())
    model.add(Linear(16 * h * w, 512))
    model.add(
        BatchNormalization(dim=512, layer_type='fc',
                           short_memory=short_memory))
    model.add(RELU())

    model.add(Linear(512, 10))
    model.add(Softmax())

    # learning_method = RMSprop(learning_rate=learning_rate)
    learning_method = Adam(learning_rate=learning_rate)
    # learning_method = SGD(learning_rate=0.001)

    # Build Logger
    log = Log(
        experiment_name='cifar10_cnn_tutorial',
        description='This is a tutorial',
        save_outputs=True,  # log all the outputs from the screen
        save_model=True,  # save the best model
        save_epoch_error=True,  # log error at every epoch
        save_to_database={
            'name': 'hyperparam.sqlite3',
            'records': {
                'Batch_Size': batch_size,
                'Learning_Rate': learning_method.learning_rate
            }
        })  # end log

    # put everything into the train object
    train_object = TrainObject(model=model,
                               log=log,
                               dataset=data,
                               train_cost=entropy,
                               valid_cost=error,
                               learning_method=learning_method,
                               stop_criteria={
                                   'max_epoch': 100,
                                   'epoch_look_back': 10,
                                   'percent_decrease': 0.01
                               })
    # finally run the code
    train_object.setup()
    train_object.run()

    # test the model on test set
    ypred = model.fprop(data.get_test().X)
    ypred = np.argmax(ypred, axis=1)
    y = np.argmax(data.get_test().y, axis=1)
    accuracy = np.equal(ypred, y).astype('f4').sum() / len(y)
    print 'test accuracy:', accuracy
Exemple #9
0
def train():
    max_features=20000
    maxseqlen = 100 # cut texts after this number of words (among top max_features most common words)
    batch_size = 16
    word_vec_len = 256
    iter_class = 'SequentialRecurrentIterator'
    seq_len = 10

    data = IMDB(pad_zero=True, maxlen=100, nb_words=max_features, batch_size=batch_size,
                train_valid_test_ratio=[8,2,0], iter_class=iter_class, seq_len=seq_len)

    print('Build model...')
    model = Sequential(input_var=T.matrix(), output_var=T.matrix())
    model.add(Embedding(max_features, word_vec_len))

    # MLP layers
    model.add(Transform((word_vec_len,))) # transform from 3d dimensional input to 2d input for mlp
    model.add(Linear(word_vec_len, 100))
    model.add(RELU())
    model.add(BatchNormalization(dim=100, layer_type='fc'))
    model.add(Linear(100,100))
    model.add(RELU())
    model.add(BatchNormalization(dim=100, layer_type='fc'))
    model.add(Linear(100, word_vec_len))
    model.add(RELU())
    model.add(Transform((maxseqlen, word_vec_len))) # transform back from 2d to 3d for recurrent input

    # Stacked up BiLSTM layers
    model.add(BiLSTM(word_vec_len, 50, output_mode='concat', return_sequences=True))
    model.add(BiLSTM(100, 24, output_mode='sum', return_sequences=True))
    model.add(LSTM(24, 24, return_sequences=True))

    # MLP layers
    model.add(Reshape((24 * maxseqlen,)))
    model.add(BatchNormalization(dim=24 * maxseqlen, layer_type='fc'))
    model.add(Linear(24 * maxseqlen, 50))
    model.add(RELU())
    model.add(Dropout(0.2))
    model.add(Linear(50, 1))
    model.add(Sigmoid())

    # build learning method
    decay_batch = int(data.train.X.shape[0] * 5 / batch_size)
    learning_method = SGD(learning_rate=0.1, momentum=0.9,
                          lr_decay_factor=1.0, decay_batch=decay_batch)

    # Build Logger
    log = Log(experiment_name = 'MLP',
              description = 'This is a tutorial',
              save_outputs = True, # log all the outputs from the screen
              save_model = True, # save the best model
              save_epoch_error = True, # log error at every epoch
              save_to_database = {'name': 'Example.sqlite3',
                                  'records': {'Batch_Size': batch_size,
                                              'Learning_Rate': learning_method.learning_rate,
                                              'Momentum': learning_method.momentum}}
             ) # end log

    # put everything into the train object
    train_object = TrainObject(model = model,
                               log = log,
                               dataset = data,
                               train_cost = mse,
                               valid_cost = error,
                               learning_method = learning_method,
                               stop_criteria = {'max_epoch' : 100,
                                                'epoch_look_back' : 5,
                                                'percent_decrease' : 0.01}
                               )
    # finally run the code
    train_object.setup()
    train_object.run()
Exemple #10
0
def train(args):
    # build dataset

    xpath = os.environ['MOZI_DATA_PATH'] + '/X_{}_augment_{}.npy'.format(
        '_'.join([str(d) for d in _IMG_INPUT_DIM_]), str(img_augment))
    ypath = os.environ['MOZI_DATA_PATH'] + '/y_{}_augment_{}.npy'.format(
        '_'.join([str(d) for d in _IMG_INPUT_DIM_]), str(img_augment))
    if not os.path.exists(xpath) or not os.path.exists(ypath):
        X, y = make_Xy(args, img_augment)
        with open(xpath, 'wb') as fout:
            np.save(fout, X)
            print '..saved to', xpath

        with open(ypath, 'wb') as fout:
            np.save(fout, y)
            print '..saved to', ypath
    else:
        with open(xpath) as xin, open(ypath) as yin:
            X = np.load(xin)
            y = np.load(yin)
            print '..data loaded'

    if img_preprocess:
        X = img_preprocess.apply(X)
    # import pdb; pdb.set_trace()
    idxs = np.arange(len(X))
    np.random.shuffle(idxs)
    data = MultiInputsData(X=X[idxs][:10000],
                           y=y[idxs][:10000],
                           train_valid_test_ratio=train_valid_test_ratio,
                           batch_size=batch_size)

    if load_model:
        print '..loading model', load_model
        model_path = os.environ[
            'MOZI_SAVE_PATH'] + '/' + load_model + '/model.pkl'
        with open(model_path) as fin:
            model = cPickle.load(fin)
    else:
        # c, h, w = _IMG_INPUT_DIM_
        # build the master model
        model = Sequential(input_var=T.tensor4(),
                           output_var=T.tensor4(),
                           verbose=verbose)

        ks = 11
        model.add(
            Convolution2D(input_channels=3,
                          filters=16,
                          kernel_size=(ks, ks),
                          stride=(1, 1),
                          border_mode='full'))
        model.add(Crop(border=(ks / 2, ks / 2)))
        model.add(
            BatchNormalization(dim=16,
                               layer_type='conv',
                               short_memory=short_memory))
        model.add(RELU())
        model.add(
            Pooling2D(poolsize=(3, 3),
                      stride=(1, 1),
                      padding=(1, 1),
                      mode='max'))
        # model.add(RELU())
        # h, w = full(h, w, 5, 1)

        ks = 9
        model.add(
            Convolution2D(input_channels=16,
                          filters=32,
                          kernel_size=(ks, ks),
                          stride=(1, 1),
                          border_mode='full'))
        model.add(Crop(border=(ks / 2, ks / 2)))
        model.add(
            BatchNormalization(dim=32,
                               layer_type='conv',
                               short_memory=short_memory))
        model.add(RELU())
        model.add(
            Pooling2D(poolsize=(3, 3),
                      stride=(1, 1),
                      padding=(1, 1),
                      mode='max'))

        ks = 5
        model.add(
            Convolution2D(input_channels=32,
                          filters=1,
                          kernel_size=(ks, ks),
                          stride=(1, 1),
                          border_mode='full'))
        # model.add(BatchNormalization(dim=1, layer_type='conv', short_memory=short_memory))

        model.add(Crop(border=(ks / 2, ks / 2)))
        model.add(Sigmoid())

    # build learning method
    # learning_method = SGD(learning_rate=lr, momentum=momentum,
    #                       lr_decay_factor=lr_decay_factor, decay_batch=decay_batch)
    learning_method = Adam(learning_rate=lr)
    # learning_method = RMSprop(learning_rate=lr)

    # Build Logger
    log = Log(
        experiment_name=experiment_name,
        description=desc,
        save_outputs=True,  # log all the outputs from the screen
        save_model=save_model,  # save the best model
        save_epoch_error=True,  # log error at every epoch
        save_to_database={
            'name': 'skin_segment.sqlite3',
            'records': {
                'learning_rate': lr,
                'valid_cost_func': valid_cost,
                'train_cost_func': train_cost
            }
        })  # end log

    os.system('cp {} {}'.format(__file__, log.exp_dir))
    dname = os.path.dirname(os.path.realpath(__file__))

    # put everything into the train object
    train_object = TrainObject(model=model,
                               log=log,
                               dataset=data,
                               train_cost=train_cost,
                               valid_cost=valid_cost,
                               learning_method=learning_method,
                               stop_criteria={
                                   'max_epoch': 100,
                                   'epoch_look_back': 5,
                                   'percent_decrease': 0.01
                               })
    # finally run the code
    train_object.setup()
    train_object.run()
Exemple #11
0
class TrainObject():
    def __init__(self,
                 model,
                 dataset,
                 train_cost,
                 valid_cost,
                 learning_method,
                 stop_criteria,
                 log=None):
        self.model = model
        self.dataset = dataset
        self.train_cost = train_cost
        self.valid_cost = valid_cost
        self.learning_method = learning_method
        self.stop_criteria = stop_criteria
        self.log = log

        if self.log is None:
            # use default Log setting
            self.log = Log(logger=internal_logger)

        elif self.log.save_to_database:
            self.log.print_records()
            self.log.info('\n')

    def setup(self):

        self.log.info('..begin setting up train object')

        #===================[ build params and deltas list ]==================#

        params = []
        deltas = []

        for layer in self.model.layers:
            for param in layer.params:
                # checked that the param to be updated is shared variable
                if is_shared_var(param):
                    param.name += '_' + layer.__class__.__name__
                    params += [param]
                    deltas += [shared_zeros(shape=param.shape.eval())]

        #=====================[ training params updates ]=====================#

        self.log.info("..update params: " + str(params))
        train_y_pred, train_layers_stats = self.model.train_fprop(
            self.model.input_var)
        train_cost = self.train_cost(self.model.output_var,
                                     train_y_pred).astype(floatX)

        train_updates = []
        gparams = T.grad(train_cost, params)
        for delta, param, gparam in zip(deltas, params, gparams):
            train_updates += self.learning_method.update(delta, gparam)
            train_updates += [(param, param + delta)]

        #----[ append updates of stats from each layer to train updates ]-----#

        self.train_stats_names, train_stats_vars = split_list(
            train_layers_stats)
        train_stats_vars = [var.astype(floatX) for var in train_stats_vars]
        self.train_stats_shared = generate_shared_list(train_stats_vars)
        train_stats_updates = merge_lists(self.train_stats_shared,
                                          train_stats_vars)
        train_updates += train_stats_updates

        #-------------------------[ train functions ]-------------------------#

        self.log.info('..begin compiling functions')
        self.training = theano.function(
            inputs=[self.model.input_var, self.model.output_var],
            outputs=train_cost,
            updates=train_updates,
            on_unused_input='warn',
            allow_input_downcast=True)

        self.log.info('..training function compiled')

        #======================[ testing params updates ]=====================#

        test_y_pred, test_layers_stats = self.model.test_fprop(
            self.model.input_var)

        #-----[ append updates of stats from each layer to test updates ]-----#

        self.test_stats_names, test_stats_vars = split_list(test_layers_stats)
        test_stats_vars = [var.astype(floatX) for var in test_stats_vars]
        self.test_stats_shared = generate_shared_list(test_stats_vars)
        test_stats_updates = merge_lists(self.test_stats_shared,
                                         test_stats_vars)

        #-------------------------[ test functions ]--------------------------#

        test_stopping_error = self.valid_cost(self.model.output_var,
                                              test_y_pred).astype(floatX)
        test_cost = self.train_cost(self.model.output_var,
                                    test_y_pred).astype(floatX)

        self.testing = theano.function(
            inputs=[self.model.input_var, self.model.output_var],
            outputs=(test_stopping_error, test_cost),
            updates=test_stats_updates,
            on_unused_input='warn',
            allow_input_downcast=True)

        self.log.info('..testing function compiled')

    def run(self):

        best_valid_error = float(sys.maxint)
        valid_error = float(sys.maxint)

        train_cost = float(sys.maxint)
        valid_cost = float(sys.maxint)

        train_stats_values = []
        valid_stats_values = []

        epoch = 0
        error_dcr = 0
        self.best_epoch_last_update = 0
        self.best_valid_last_update = float(sys.maxint)

        train_stats_names = [
            'train_' + name for name in self.train_stats_names
        ]
        valid_stats_names = ['valid_' + name for name in self.test_stats_names]

        job_start = time.time()

        while (self.continue_learning(epoch, error_dcr, best_valid_error)):

            if epoch > 0:
                self.log.info("best_epoch_last_update: %d" %
                              self.best_epoch_last_update)
                self.log.info("valid_error_decrease: %f" % error_dcr)
                self.log.info("best_valid_last_update: %f" %
                              self.best_valid_last_update)
                self.log.info("========[ End of Epoch ]========\n\n")

            epoch += 1

            start_time = time.time()

            num_train_examples = 0
            total_train_cost = 0.
            train_stats_values = np.zeros(len(train_stats_names), dtype=floatX)

            num_valid_examples = 0
            total_valid_cost = 0.
            total_valid_stopping_cost = 0.
            valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX)

            blk = 0

            for block in self.dataset:
                block_time = time.time()
                blk += 1

                train_set = block.get_train()
                valid_set = block.get_valid()

                #====================[ Training Progress ]====================#
                if train_set.dataset_size > 0:

                    self.log.info('..training ' +
                                  self.dataset.__class__.__name__ +
                                  ' block %s/%s' % (blk, self.dataset.nblocks))

                    progbar = Progbar(target=train_set.dataset_size)
                    for idx in train_set:
                        cost = self.training(train_set.X[idx],
                                             train_set.y[idx])
                        total_train_cost += cost * len(idx)
                        num_train_examples += len(idx)
                        train_stats_values += len(idx) * get_shared_values(
                            self.train_stats_shared)
                        progbar.update(num_train_examples)
                    print

                    #-------[ Update train best cost and error values ]-------#
                    train_cost = total_train_cost / num_train_examples
                    train_stats_values /= num_train_examples

                #===================[ Validating Progress ]===================#
                if valid_set.dataset_size > 0:

                    self.log.info('..validating ' +
                                  self.dataset.__class__.__name__ +
                                  ' block %s/%s' % (blk, self.dataset.nblocks))

                    progbar = Progbar(target=valid_set.dataset_size)
                    for idx in valid_set:
                        stopping_cost, cost = self.testing(
                            valid_set.X[idx], valid_set.y[idx])
                        total_valid_cost += cost * len(idx)
                        total_valid_stopping_cost += stopping_cost * len(idx)
                        num_valid_examples += len(idx)
                        valid_stats_values += len(idx) * get_shared_values(
                            self.test_stats_shared)
                        progbar.update(num_valid_examples)
                    print

                    #-------[ Update valid best cost and error values ]-------#
                    valid_error = total_valid_stopping_cost / num_valid_examples
                    valid_cost = total_valid_cost / num_valid_examples
                    valid_stats_values /= num_valid_examples

                    if valid_error < best_valid_error:
                        best_valid_error = valid_error
                        self.log.info('..best validation error so far')
                        if self.log.save_model:
                            self.log._save_model(self.model)
                            self.log.info('..model saved')

                    if valid_error < self.best_valid_last_update:
                        error_dcr = self.best_valid_last_update - valid_error
                    else:
                        error_dcr = 0

                self.log.info('block time: %0.2fs' %
                              (time.time() - block_time))
                self.log.info(get_mem_usage())

            #==============[ save to database, save epoch error]==============#
            if self.log.save_to_database:
                self.log._save_to_database(epoch, train_cost, valid_cost,
                                           best_valid_error)
                self.log.info('..sent to database: %s:%s' %
                              (self.log.save_to_database['name'],
                               self.log.experiment_name))

            if self.log.save_epoch_error:
                self.log._save_epoch_error(epoch, valid_error)
                self.log.info('..epoch error saved')

            end_time = time.time()

            #=====================[ log outputs to file ]=====================#
            merged_train = merge_lists(train_stats_names, train_stats_values)
            merged_valid = merge_lists(valid_stats_names, valid_stats_values)

            outputs = [('epoch', epoch),
                       ('runtime(s)', int(end_time - start_time)),
                       ('train_' + self.train_cost.func_name, train_cost),
                       ('valid_' + self.train_cost.func_name, valid_cost),
                       ('valid_' + self.valid_cost.func_name, valid_error),
                       ('best_valid_' + self.valid_cost.func_name,
                        best_valid_error)]

            outputs += merged_train + merged_valid
            self.log._log_outputs(outputs)

        job_end = time.time()
        self.log.info(
            'Job Completed on %s' %
            time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end)))
        ttl_time = int(job_end - job_start)
        dt = datetime.timedelta(seconds=ttl_time)
        self.log.info('Total Time Taken: %s' % str(dt))
        self.log.info("========[ End of Job ]========\n\n")

    def continue_learning(self, epoch, error_dcr, best_valid_error):

        if epoch > self.stop_criteria['max_epoch']:
            return False

        elif self.stop_criteria['percent_decrease'] is None or \
            self.stop_criteria['epoch_look_back'] is None:
            return True

        elif np.abs(float(error_dcr) / self.best_valid_last_update) \
            >= self.stop_criteria['percent_decrease']:
            self.best_valid_last_update = best_valid_error
            self.best_epoch_last_update = epoch
            return True

        elif epoch - self.best_epoch_last_update > \
            self.stop_criteria['epoch_look_back']:
            return False

        else:
            return True
Exemple #12
0
def train():

    data = Cifar10(batch_size=32, train_valid_test_ratio=[4, 1, 1])

    model = Sequential(input_var=T.tensor4(), output_var=T.matrix())
    model.add(
        Convolution2D(input_channels=3,
                      filters=8,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      border_mode='full'))
    model.add(RELU())
    model.add(
        Convolution2D(input_channels=8,
                      filters=16,
                      kernel_size=(3, 3),
                      stride=(1, 1)))
    model.add(RELU())
    model.add(Pooling2D(poolsize=(4, 4), stride=(4, 4), mode='max'))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Linear(16 * 8 * 8, 512))
    model.add(RELU())
    model.add(Dropout(0.5))

    model.add(Linear(512, 10))
    model.add(Softmax())

    # build learning method
    learning_method = SGD(learning_rate=0.01,
                          momentum=0.9,
                          lr_decay_factor=0.9,
                          decay_batch=5000)

    # Build Logger
    log = Log(
        experiment_name='cifar10_cnn',
        description='This is a tutorial',
        save_outputs=True,  # log all the outputs from the screen
        save_model=True,  # save the best model
        save_epoch_error=True,  # log error at every epoch
        save_to_database={
            'name': 'hyperparam.sqlite3',
            'records': {
                'Batch_Size': data.batch_size,
                'Learning_Rate': learning_method.learning_rate,
                'Momentum': learning_method.momentum
            }
        })  # end log

    # put everything into the train object
    train_object = TrainObject(model=model,
                               log=log,
                               dataset=data,
                               train_cost=entropy,
                               valid_cost=error,
                               learning_method=learning_method,
                               stop_criteria={
                                   'max_epoch': 30,
                                   'epoch_look_back': 5,
                                   'percent_decrease': 0.01
                               })
    # finally run the code
    train_object.setup()
    train_object.run()

    # test the model on test set
    ypred = model.fprop(data.get_test().X)
    ypred = np.argmax(ypred, axis=1)
    y = np.argmax(data.get_test().y, axis=1)
    accuracy = np.equal(ypred, y).astype('f4').sum() / len(y)
    print 'test accuracy:', accuracy