예제 #1
0
def test_DBN(dataset, hyper):
    datasets = dataset.sharedTrain, dataset.sharedValid, dataset.sharedTest

    train_set_x, train_set_y = dataset.sharedTrain
    valid_set_x, valid_set_y = dataset.sharedValid
    test_set_x, test_set_y = dataset.sharedTest

    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / hyper.batchSize

    numpy_rng = numpy.random.RandomState(123)
    print '... building the model'

    dbn = DBN(numpy_rng=numpy_rng, n_ins=dataset.n_in,
              hidden_layers_sizes=hyper.nHidden,
              n_outs=dataset.n_out)

    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=hyper.batchSize,
                                                k=hyper.k)

    # Start timeout timer
    wait_timeout(test_DBN, 30)

    print '... pre-training the model'
    start_time = time.time()

    for i in xrange(dbn.n_layers):
        for epoch in xrange(hyper.pretrainingEpochs):
            print "Pretraining epoch ", epoch, ", time ", (time.time() - start_time) / 60.0
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=hyper.pretrainingLearningRate))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.time()
    print('The pretraining code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))

    print '... getting the finetuning functions'
    train_fn, validate_model, test_model = dbn.build_finetune_functions(
                datasets=datasets, batch_size=hyper.batchSize,
                learning_rate=hyper.learningRate)

    print '... finetunning the model'
    # early-stopping parameters
    patience = 4 * n_train_batches  # look as this many examples regardless
    validation_frequency = min(n_train_batches, patience / 2)

    best_params = None
    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = time.time()

    done_looping = False
    epoch = 0

    while (epoch < hyper.numberEpochs) and (not done_looping):
        epoch = epoch + 1
	print "Finetuning epoch ", epoch, ", time ", (time.time() - start_time) / 60.0 
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' % \
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))

                if this_validation_loss < best_validation_loss:

                    if (this_validation_loss < best_validation_loss *
                        hyper.improvementThreshold):
                        patience = max(patience, iter * hyper.patienceIncrease)

                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    test_losses = test_model()
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    end_time = time.time()
    print(('Optimization complete with best validation score of %f %%,'
           'with test performance %f %%') %
                 (best_validation_loss * 100., test_score * 100.))
    print('The fine tuning code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time)
                                              / 60.))
def run_dbn_model(data_set,
                  finetune_lr=0.1, pretraining_epochs=10,
                  pretrain_lr=0.01, k=1, training_epochs=1000,
                  batch_size=10):


    # get partitioned data sets
    train_data, test_data, validation_data = data_set.get_data()

    # get train x,y, and id
    train_data_x, train_data_y, train_data_id = train_data

    train_ten_x, train_ten_y = theano.shared(np.asarray(train_data_x, dtype=theano.config.floatX)), \
                               theano.shared(np.asarray(train_data_y, dtype='int32'))
                               #theano.shared(np.asarray(pd.get_dummies(train_data_y).as_matrix(), dtype='int32'))

    # get test x,y, and id
    test_data_x,  test_data_y, test_data_id  = test_data

    test_ten_x, test_ten_y = theano.shared(np.asarray(test_data_x, dtype=theano.config.floatX)), \
                             theano.shared(np.asarray(test_data_y, dtype='int32'))
                             #theano.shared(np.asarray(pd.get_dummies(test_data_y).as_matrix(), dtype='int32'))

    # get validation x,y and id
    validation_data_x, validation_data_y, validation_data_id = validation_data

    validation_ten_x, validation_ten_y = theano.shared(np.asarray(validation_data_x, dtype=theano.config.floatX)), \
                                         theano.shared(np.asarray(validation_data_y, dtype='int32'))
                                         #theano.shared(np.asarray(pd.get_dummies(validation_data_y).as_matrix(), dtype='int32'))

    ten_data_set = [(train_ten_x,train_ten_y), (validation_ten_x, validation_ten_y), (test_ten_x, test_ten_y)]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_ten_x.get_value(borrow=True).shape[0] / batch_size
    print ("n_train_batches: " + str(n_train_batches))

    cols = train_data_x.shape[1]
    assert(train_data_x.shape[1] == test_data_x.shape[1]  and test_data_x.shape[1] == validation_data_x.shape[1])
    print("cols: " + str(cols))

    print("train_x" + str(train_ten_x.get_value(borrow=True).shape))
    print("train_y" + str(train_ten_y.get_value(borrow=True).shape))
    print("valid_x" + str(validation_ten_x.get_value(borrow=True).shape))
    print("valid_y" + str(validation_ten_y.get_value(borrow=True).shape))
    print("test_x" + str(test_ten_x.get_value(borrow=True).shape))
    print("test_y" + str(test_ten_y.get_value(borrow=True).shape))

    # numpy random generator
    numpy_rng = np.random.RandomState(123)
    print '... building the model'
    # construct the Deep Belief Network
    dbn = DBN(numpy_rng=numpy_rng, n_ins= cols,
              hidden_layers_sizes=[cols*10, cols*10, cols*10],
              n_outs=5)

    # start-snippet-2
    #########################
    # PRETRAINING THE MODEL #
    #########################
    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_ten_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = timeit.default_timer()
    ## Pre-train layer-wise
    for i in range(dbn.n_layers):
        # go through pretraining epochs
        for epoch in range(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in range(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print np.mean(c)

    end_time = timeit.default_timer()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model = dbn.build_finetune_functions(
        datasets=ten_data_set,
        batch_size=batch_size,
        learning_rate=finetune_lr
    )

    print '... finetuning the model'
    # early-stopping parameters
    patience = 4 * n_train_batches  # look as this many examples regardless
    patience_increase = 2.    # wait this much longer when a new best is
                              # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatches before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = np.inf
    test_score = 0.
    start_time = timeit.default_timer()

    done_looping = False
    epoch = 0

    while (epoch < training_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):

            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = np.mean(validation_losses)
                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%'
                    % (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.
                    )
                )

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if (
                        this_validation_loss < best_validation_loss *
                        improvement_threshold
                    ):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = np.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

           # if patience <= iter:
           #     done_looping = True
           #     break

    end_time = timeit.default_timer()
    print(
        (
            'Optimization complete with best validation score of %f %%, '
            'obtained at iteration %i, '
            'with test performance %f %%'
        ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)
    )
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time)
                                              / 60.))
예제 #3
0
파일: test_dbn.py 프로젝트: xuyinan/CDBN
def test_DBN(finetune_lr=0.05, pretraining_epochs=20, pretrain_lr=0.01,
            k=1, training_epochs=1000, batch_size=10,DEBUG=0):

    # datasets = LF.load_cifar()
    # IMAGE_SIZE = 32
    # if DEBUG:
    #     N1 = 400
    #     N2 = 600
    # else:
    #     N1 = 40000
    #     N2 = 50000
    # x, y = datasets[0]
    # train_set_x = x[:N1]
    # train_set_y = y[:N1]
    # valid_set_x = x[N1:N2]
    # valid_set_y = y[N1:N2]
    # test_set_x, test_set_y = datasets[1]
    # datasets = [(train_set_x, train_set_y),
    #             (valid_set_x, valid_set_y),
    #             (test_set_x, test_set_y)]

    datasets = load_data(DEBUG=DEBUG)
    IMAGE_SIZE = 28

    if DEBUG:
        N1 = 400
        N2 = 600
    else:
        N1 = 40000
        N2 = 50000
    x, y = datasets[0]
    train_set_x = x[:N1]
    train_set_y = y[:N1]
    valid_set_x = x[N1:N2]
    valid_set_y = y[N1:N2]
    test_set_x, test_set_y = datasets[1]
    datasets = [(train_set_x, train_set_y),
                (valid_set_x, valid_set_y),
                (test_set_x, test_set_y)]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.shape[0].eval()/batch_size
    n_valid_batches = valid_set_x.shape[0].eval()/batch_size
    n_test_batches = test_set_x.shape[0].eval()/batch_size

    print '... building the model'
    # construct the Deep Belief Network
    dbn = DBN(n_ins=IMAGE_SIZE**2, hidden_layers_sizes=[500, 500],
              n_outs=10)

    #########################
    # PRETRAINING THE MODEL #
    #########################
    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = timeit.default_timer()
    ## Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost %.4f' % (i+1, epoch+1, numpy.mean(c))

    end_time = timeit.default_timer()
    print 'The pretraining for DBN ran for %.2fm' % ((end_time-start_time)/60.))
    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_fn, test_fn = dbn.build_finetune_functions(
        datasets=datasets,
        batch_size=batch_size,
        learning_rate=finetune_lr
예제 #4
0
파일: train_dbm.py 프로젝트: pdikang/kaggle
def train_dbn(
    datasets, finetune_lr=0.1, pretraining_epochs=100, pretrain_lr=0.01, k=1, training_epochs=1000, batch_size=10
):

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = np.random.RandomState(123)
    logging.info("... building the model")
    # construct the Deep Belief Network
    dbn = DBN(
        numpy_rng=numpy_rng,
        n_ins=train_set_x.get_value(borrow=True).shape[1],
        hidden_layers_sizes=[200, 200, 200],
        n_outs=2,
    )

    #########################
    # PRETRAINING THE MODEL #
    #########################
    logging.info("... getting the pretraining functions")
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size, k=k)

    logging.info("... pre-training the model")
    start_time = time.clock()
    ## Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
            logging.info("Pre-training layer %i, epoch %d, cost " % (i, epoch))
            logging.info(np.mean(c))

    end_time = time.clock()
    logging.warn(
        "The pretraining code for file "
        + os.path.split(__file__)[1]
        + " ran for %.2fm" % ((end_time - start_time) / 60.0)
    )

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    logging.info("... getting the finetuning functions")
    train_fn, validate_model, test_model, validate_auc, test_auc = dbn.build_finetune_functions(
        datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr
    )

    logging.info("... finetunning the model")
    # early-stopping parameters
    patience = 10 * n_train_batches  # look as this many examples regardless
    patience_increase = 2.0  # wait this much longer when a new best is
    # found
    improvement_threshold = 0.995  # a relative improvement of this much is
    # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
    # go through this many
    # minibatche before checking the network
    # on the validation set; in this case we
    # check every epoch

    best_params = None
    best_validation_auc = 0
    test_score = 0.0
    start_time = time.clock()

    done_looping = False
    epoch = 0

    while (epoch < training_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = np.mean(validation_losses)
                validation_auc_score = validate_auc()
                logging.info(
                    "epoch %i, minibatch %i/%i, validation error %f %%, validation auc %f %%"
                    % (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.0,
                        validation_auc_score * 100.0,
                    )
                )

                # if we got the best validation score until now
                if validation_auc_score > best_validation_auc:

                    # improve patience if loss improvement is good enough
                    if validation_auc_score > best_validation_auc / improvement_threshold:
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_auc = validation_auc_score
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = np.mean(test_losses)
                    test_auc_score = test_auc()
                    logging.info(
                        ("     epoch %i, minibatch %i/%i, test error of " "best model %f %%, auc of best model %f %%")
                        % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.0, test_auc_score * 100.0)
                    )

            if patience <= iter:
                done_looping = True
                break

    end_time = time.clock()
    logging.info(
        ("Optimization complete with best validation auc score of %f %%," "with test auc %f %% (zero-one %f %%)")
        % (best_validation_auc * 100.0, test_auc_score * 100.0, test_score * 100.0)
    )
    logging.warn(
        "The fine tuning code for file "
        + os.path.split(__file__)[1]
        + " ran for %.2fm" % ((end_time - start_time) / 60.0)
    )

    return dbn
예제 #5
0
def test_DBN(finetune_lr=0.1,
             pretraining_epochs=100,
             pretrain_lr=0.01,
             k=1,
             training_epochs=1000,
             dataset='../mnist.pkl.gz',
             batch_size=10):
    """
    Demonstrates how to train and test a Deep Belief Network.

    This is demonstrated on MNIST.

    :type finetune_lr: float
    :param finetune_lr: learning rate used in the finetune stage
    :type pretraining_epochs: int
    :param pretraining_epochs: number of epoch to do pretraining
    :type pretrain_lr: float
    :param pretrain_lr: learning rate to be used during pre-training
    :type k: int
    :param k: number of Gibbs steps in CD/PCD
    :type training_epochs: int
    :param training_epochs: maximal number of iterations ot run the optimizer
    :type dataset: string
    :param dataset: path the the pickled dataset
    :type batch_size: int
    :param batch_size: the size of a minibatch
    """

    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)
    print('... building the model')
    # construct the Deep Belief Network
    dbn = DBN(numpy_rng=numpy_rng,
              n_ins=28 * 28,
              hidden_layers_sizes=[1000, 1000, 1000],
              n_outs=10)

    # start-snippet-2
    #########################
    # PRETRAINING THE MODEL #
    #########################
    print('... getting the pretraining functions')
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print('... pre-training the model')
    start_time = timeit.default_timer()
    # Pre-train layer-wise
    for i in range(dbn.n_layers):
        # go through pretraining epochs
        for epoch in range(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in range(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
            print('Pre-training layer %i, epoch %d, cost ' % (i, epoch),
                  end=' ')
            print(numpy.mean(c, dtype='float64'))

    end_time = timeit.default_timer()
    # end-snippet-2
    print('The pretraining code for file ' + os.path.split(__file__)[1] +
          ' ran for %.2fm' % ((end_time - start_time) / 60.),
          file=sys.stderr)
    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print('... getting the finetuning functions')
    train_fn, validate_model, test_model = dbn.build_finetune_functions(
        datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr)

    print('... finetuning the model')
    # early-stopping parameters

    # look as this many examples regardless
    patience = 4 * n_train_batches

    # wait this much longer when a new best is found
    patience_increase = 2.

    # a relative improvement of this much is considered significant
    improvement_threshold = 0.995

    # go through this many minibatches before checking the network on
    # the validation set; in this case we check every epoch
    validation_frequency = min(n_train_batches, patience / 2)

    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = timeit.default_timer()

    done_looping = False
    epoch = 0

    while (epoch < training_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):

            train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses,
                                                  dtype='float64')
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    # improve patience if loss improvement is good enough
                    if (this_validation_loss <
                            best_validation_loss * improvement_threshold):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = numpy.mean(test_losses, dtype='float64')
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    end_time = timeit.default_timer()
    print(('Optimization complete with best validation score of %f %%, '
           'obtained at iteration %i, '
           'with test performance %f %%') %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print('The fine tuning code for file ' + os.path.split(__file__)[1] +
          ' ran for %.2fm' % ((end_time - start_time) / 60.),
          file=sys.stderr)
예제 #6
0
def run_dbn_model(data_set,
                  finetune_lr=0.1,
                  pretraining_epochs=10,
                  pretrain_lr=0.01,
                  k=1,
                  training_epochs=1000,
                  batch_size=10):

    # get partitioned data sets
    train_data, test_data, validation_data = data_set.get_data()

    # get train x,y, and id
    train_data_x, train_data_y, train_data_id = train_data

    train_ten_x, train_ten_y = theano.shared(np.asarray(train_data_x, dtype=theano.config.floatX)), \
                               theano.shared(np.asarray(train_data_y, dtype='int32'))
    #theano.shared(np.asarray(pd.get_dummies(train_data_y).as_matrix(), dtype='int32'))

    # get test x,y, and id
    test_data_x, test_data_y, test_data_id = test_data

    test_ten_x, test_ten_y = theano.shared(np.asarray(test_data_x, dtype=theano.config.floatX)), \
                             theano.shared(np.asarray(test_data_y, dtype='int32'))
    #theano.shared(np.asarray(pd.get_dummies(test_data_y).as_matrix(), dtype='int32'))

    # get validation x,y and id
    validation_data_x, validation_data_y, validation_data_id = validation_data

    validation_ten_x, validation_ten_y = theano.shared(np.asarray(validation_data_x, dtype=theano.config.floatX)), \
                                         theano.shared(np.asarray(validation_data_y, dtype='int32'))
    #theano.shared(np.asarray(pd.get_dummies(validation_data_y).as_matrix(), dtype='int32'))

    ten_data_set = [(train_ten_x, train_ten_y),
                    (validation_ten_x, validation_ten_y),
                    (test_ten_x, test_ten_y)]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_ten_x.get_value(borrow=True).shape[0] / batch_size
    print("n_train_batches: " + str(n_train_batches))

    cols = train_data_x.shape[1]
    assert (train_data_x.shape[1] == test_data_x.shape[1]
            and test_data_x.shape[1] == validation_data_x.shape[1])
    print("cols: " + str(cols))

    print("train_x" + str(train_ten_x.get_value(borrow=True).shape))
    print("train_y" + str(train_ten_y.get_value(borrow=True).shape))
    print("valid_x" + str(validation_ten_x.get_value(borrow=True).shape))
    print("valid_y" + str(validation_ten_y.get_value(borrow=True).shape))
    print("test_x" + str(test_ten_x.get_value(borrow=True).shape))
    print("test_y" + str(test_ten_y.get_value(borrow=True).shape))

    # numpy random generator
    numpy_rng = np.random.RandomState(123)
    print '... building the model'
    # construct the Deep Belief Network
    dbn = DBN(numpy_rng=numpy_rng,
              n_ins=cols,
              hidden_layers_sizes=[cols * 10, cols * 10, cols * 10],
              n_outs=5)

    # start-snippet-2
    #########################
    # PRETRAINING THE MODEL #
    #########################
    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_ten_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = timeit.default_timer()
    ## Pre-train layer-wise
    for i in range(dbn.n_layers):
        # go through pretraining epochs
        for epoch in range(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in range(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print np.mean(c)

    end_time = timeit.default_timer()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model = dbn.build_finetune_functions(
        datasets=ten_data_set,
        batch_size=batch_size,
        learning_rate=finetune_lr)

    print '... finetuning the model'
    # early-stopping parameters
    patience = 4 * n_train_batches  # look as this many examples regardless
    patience_increase = 2.  # wait this much longer when a new best is
    # found
    improvement_threshold = 0.995  # a relative improvement of this much is
    # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
    # go through this many
    # minibatches before checking the network
    # on the validation set; in this case we
    # check every epoch

    best_validation_loss = np.inf
    test_score = 0.
    start_time = timeit.default_timer()

    done_looping = False
    epoch = 0

    while (epoch < training_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):

            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = np.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if (this_validation_loss <
                            best_validation_loss * improvement_threshold):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = np.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

        # if patience <= iter:
        #     done_looping = True
        #     break

    end_time = timeit.default_timer()
    print(('Optimization complete with best validation score of %f %%, '
           'obtained at iteration %i, '
           'with test performance %f %%') %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))