def test_DBN(finetune_lr=0.1,
             pretraining_epochs=100,
             pretrain_lr=0.1,
             k=1,
             training_epochs=5000,
             batch_size=5):

    datasets = load10sec_ECGII_data()

    pretrain_set_x = datasets[0]
    trdatasets = [datasets[1], datasets[2]]

    train_set_x, train_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)

    print '... building the model'

    dbn = DBN(numpy_rng=numpy_rng,
              n_ins=2500,
              hidden_layers_sizes=[1000, 1000, 1000],
              n_outs=2)

    #########################
    # PRETRAINING THE MODEL #
    #########################

    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=pretrain_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    # Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, train_cost_model, test_score_model, test_cost_model, test_confmatrix = dbn.build_finetune_functions(
        datasets=trdatasets,
        batch_size=batch_size,
    )

    print '... finetuning the model'

    tr_cost = []
    te_cost = []

    test_score = 0.
    start_time = time.clock()

    epoch = 0
    while (epoch < training_epochs):
        epoch = epoch + 1
        learning_rate = 0.01 / (1 + 0.001 * epoch)
        # go through the training set
        for minibatch_index in xrange(n_train_batches):
            minibatch_avg_cost = train_fn(minibatch_index, learning_rate)
        print(('In epoch %d,') % (epoch))
        epoch_trcost = numpy.mean(train_cost_model())
        epoch_tecost = numpy.mean(test_cost_model())
        print 'Training cost = ', epoch_trcost
        tr_cost.append(epoch_trcost)
        print 'Testing cost = ', epoch_tecost
        te_cost.append(epoch_tecost)

    test_losses = test_score_model()
    test_score = numpy.mean(test_losses)

    test_confmatrices = test_confmatrix()
    test_confelement = numpy.sum(test_confmatrices, axis=0)
    true_pos = test_confelement[0]
    true_neg = test_confelement[1]
    false_pos = test_confelement[2]
    false_neg = test_confelement[3]
    f_score = (true_pos + true_neg) / float(true_pos + true_neg + false_pos +
                                            5 * false_neg)

    print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
    print(('TP %i, TN %i, FP %i, FN %i') %
          (true_pos, true_neg, false_pos, false_neg))

    end_time = time.clock()
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))

    x_axis = numpy.arange(training_epochs)
    plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost),
             '.')
    plt.show()
Exemple #2
0
def test_DBN(finetune_lr=0.001, pretraining_epochs=100,
             pretrain_lr=0.001, k=1, training_epochs=100,
             batch_size=25):
    """
    Demonstrates how to train and main a Deep Belief Network.

    This is demonstrated on MNIST.

    :type finetune_lr: float
    :param finetune_lr: learning rate used in the finetune stage
    :type pretraining_epochs: int
    :param pretraining_epochs: number of epoch to do pretraining
    :type pretrain_lr: float
    :param pretrain_lr: learning rate to be used during pre-training
    :type k: int
    :param k: number of Gibbs steps in CD/PCD
    :type training_epochs: int
    :param training_epochs: maximal number of iterations ot run the optimizer
    :type dataset: string
    :param dataset: path the the pickled dataset
    :type batch_size: int
    :param batch_size: the size of a minibatch
    """

    # datasets = loadFeaturedData()
    datasets = load10secData()

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)
    print '... building the model'

    # construct the Deep Belief Network
    # Test for debugging
    # dbn = DBN(numpy_rng=numpy_rng, n_ins=10,
    #           hidden_layers_sizes=[100, 100, 100],
    #           n_outs=2)

    dbn = DBN(numpy_rng=numpy_rng, n_ins=7500,
              hidden_layers_sizes=[5000, 5000, 1000],
              n_outs=2)

    # start-snippet-2
    #########################
    # PRETRAINING THE MODEL #
    #########################
    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    ## Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))
    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model, test_confmatrix = dbn.build_finetune_functions(
        datasets=datasets,
        batch_size=batch_size,
        learning_rate=finetune_lr
    )

    print '... finetuning the model'
    # early-stopping parameters
    patience = 4 * n_train_batches  # look as this many examples regardless
    patience_increase = 2.    # wait this much longer when a new best is
                              # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatches before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = time.clock()

    done_looping = False
    epoch = 0

    while (epoch < training_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses)
                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%'
                    % (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.
                    )
                )
                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if (
                        this_validation_loss < best_validation_loss *
                        improvement_threshold
                    ):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # main it on the main set
                    test_losses = test_model()
                    test_score = numpy.mean(test_losses)
                    test_confmatrices = test_confmatrix()
                    test_confelement = numpy.sum(test_confmatrices, axis=0)
                    true_pos = test_confelement[0]
                    true_neg = test_confelement[1]
                    false_pos = test_confelement[2]
                    false_neg = test_confelement[3]

                    print(('epoch %i, minibatch %i/%i, main error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))
                    print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))

            if patience <= iter:
                done_looping = True
                break

    end_time = time.clock()
    print(
        (
            'Optimization complete with best validation score of %f %%, '
            'obtained at iteration %i, '
            'with main performance %f %% '
        ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)
    )
    print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time)
                                              / 60.))
Exemple #3
0
def test_DBN(finetune_lr=0.01,
             pretraining_epochs=100,
             pretrain_lr=0.01,
             k=1,
             training_epochs=100,
             batch_size=5):
    """
    Demonstrates how to train and main a Deep Belief Network.

    This is demonstrated on MNIST.

    :type finetune_lr: float
    :param finetune_lr: learning rate used in the finetune stage
    :type pretraining_epochs: int
    :param pretraining_epochs: number of epoch to do pretraining
    :type pretrain_lr: float
    :param pretrain_lr: learning rate to be used during pre-training
    :type k: int
    :param k: number of Gibbs steps in CD/PCD
    :type training_epochs: int
    :param training_epochs: maximal number of iterations ot run the optimizer
    :type dataset: string
    :param dataset: path the the pickled dataset
    :type batch_size: int
    :param batch_size: the size of a minibatch
    """

    datasets = loadFeaturedData()
    # datasets = load10secData()

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)
    print '... building the model'

    # construct the Deep Belief Network
    # Test for debugging
    # dbn = DBN(numpy_rng=numpy_rng, n_ins=10,
    #           hidden_layers_sizes=[100, 100, 100],
    #           n_outs=2)

    dbn = DBN(numpy_rng=numpy_rng,
              n_ins=10,
              hidden_layers_sizes=[50, 50],
              n_outs=2)

    # start-snippet-2
    #########################
    # PRETRAINING THE MODEL #
    #########################
    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    ## Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))
    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model, test_confmatrix = dbn.build_finetune_functions(
        datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr)

    print '... finetuning the model'
    # early-stopping parameters
    patience = 4 * n_train_batches  # look as this many examples regardless
    patience_increase = 2.  # wait this much longer when a new best is
    # found
    improvement_threshold = 0.995  # a relative improvement of this much is
    # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
    # go through this many
    # minibatches before checking the network
    # on the validation set; in this case we
    # check every epoch

    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = time.clock()

    done_looping = False
    epoch = 0

    while (epoch < training_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:

                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))
                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if (this_validation_loss <
                            best_validation_loss * improvement_threshold):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # main it on the main set
                    test_losses = test_model()
                    test_score = numpy.mean(test_losses)
                    test_confmatrices = test_confmatrix()
                    test_confelement = numpy.sum(test_confmatrices, axis=0)
                    true_pos = test_confelement[0]
                    true_neg = test_confelement[1]
                    false_pos = test_confelement[2]
                    false_neg = test_confelement[3]

                    print(('epoch %i, minibatch %i/%i, main error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))
                    print(('TP %i, TN %i, FP %i, FN %i') %
                          (true_pos, true_neg, false_pos, false_neg))

            if patience <= iter:
                done_looping = True
                break

    end_time = time.clock()
    print(('Optimization complete with best validation score of %f %%, '
           'obtained at iteration %i, '
           'with main performance %f %% ') %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print(('TP %i, TN %i, FP %i, FN %i') %
          (true_pos, true_neg, false_pos, false_neg))
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))
def test_DBN(finetune_lr=0.01,
             pretraining_epochs=100,
             pretrain_lr=0.1,
             k=1,
             training_epochs=100,
             batch_size=5):

    datasets = loadFeaturedData()

    train_set_x, train_set_y = datasets[0]
    test_set_x, test_set_y = datasets[1]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)

    print '... building the model'

    dbn = DBN(numpy_rng=numpy_rng,
              n_ins=10,
              hidden_layers_sizes=[100],
              n_outs=2)

    #########################
    # PRETRAINING THE MODEL #
    #########################

    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    # Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, test_model, test_confmatrix = dbn.build_finetune_functions(
        datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr)

    index = T.lscalar('index')  # index to a [mini]batch

    # Custom function
    stopcheck_model = theano.function(
        inputs=[index],
        outputs=dbn.finetune_cost,
        givens={
            dbn.x: test_set_x[index * batch_size:(index + 1) * batch_size],
            dbn.y: test_set_y[index * batch_size:(index + 1) * batch_size]
        })

    train_check = theano.function(
        inputs=[index],
        outputs=dbn.errors,
        givens={
            dbn.x: train_set_x[index * batch_size:(index + 1) * batch_size],
            dbn.y: train_set_y[index * batch_size:(index + 1) * batch_size]
        })

    print '... finetuning the model'

    test_score = 0.
    start_time = time.clock()

    tr_cost = []
    te_cost = []

    epoch = 0
    while (epoch < training_epochs):
        epoch = epoch + 1
        # go through the training set
        d = []
        for minibatch_index in xrange(n_train_batches):
            minibatch_avg_cost = train_fn(minibatch_index)
            d.append(minibatch_avg_cost)
        print(('In epoch %d, ') % (epoch))
        print 'Training cost = ', numpy.mean(d)
        tr_cost.append(numpy.mean(d))

        tc = []
        for test_batch_index in xrange(n_test_batches):
            testing_cost = stopcheck_model(test_batch_index)
            tc.append(testing_cost)
        print 'Testing cost = ', numpy.mean(tc)
        te_cost.append(numpy.mean(tc))

    train_losses = [train_check(i) for i in xrange(n_train_batches)]
    print 'Training error = ', numpy.mean(train_losses) * 100, ' %'

    test_losses = test_model()
    test_score = numpy.mean(test_losses)

    test_confmatrices = test_confmatrix()
    test_confelement = numpy.sum(test_confmatrices, axis=0)
    true_pos = test_confelement[0]
    true_neg = test_confelement[1]
    false_pos = test_confelement[2]
    false_neg = test_confelement[3]
    f_score = (true_pos + true_neg) / float(true_pos + true_neg + false_pos +
                                            5 * false_neg)

    print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
    print(('TP %i, TN %i, FP %i, FN %i') %
          (true_pos, true_neg, false_pos, false_neg))

    end_time = time.clock()
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] + ' ran for %.2fm' %
                          ((end_time - start_time) / 60.))

    x_axis = numpy.arange(training_epochs)
    plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost),
             '.')
    plt.show()
def test_DBN(finetune_lr=0.01, pretraining_epochs=100,
             pretrain_lr=0.1, k=1, training_epochs=100,
             batch_size=5):

    datasets = loadFeaturedData()

    train_set_x, train_set_y = datasets[0]
    test_set_x, test_set_y = datasets[1]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)

    print '... building the model'

    dbn = DBN(numpy_rng=numpy_rng, n_ins=10,
              hidden_layers_sizes=[100],
              n_outs=2)

    #########################
    # PRETRAINING THE MODEL #
    #########################

    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    # Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, test_model, test_confmatrix = dbn.build_finetune_functions(
        datasets=datasets,
        batch_size=batch_size,
        learning_rate=finetune_lr
    )

    index = T.lscalar('index')  # index to a [mini]batch

    # Custom function
    stopcheck_model = theano.function(
        inputs=[index],
        outputs=dbn.finetune_cost,
        givens={
            dbn.x: test_set_x[index * batch_size:(index + 1) * batch_size],
            dbn.y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    train_check = theano.function(
        inputs=[index],
        outputs=dbn.errors,
        givens={
            dbn.x: train_set_x[index * batch_size: (index + 1) * batch_size],
            dbn.y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )

    print '... finetuning the model'

    test_score = 0.
    start_time = time.clock()

    tr_cost = []
    te_cost = []

    epoch = 0
    while (epoch < training_epochs):
        epoch = epoch + 1
        # go through the training set
        d = []
        for minibatch_index in xrange(n_train_batches):
            minibatch_avg_cost = train_fn(minibatch_index)
            d.append(minibatch_avg_cost)
        print(('In epoch %d, ') % (epoch))
        print 'Training cost = ', numpy.mean(d)
        tr_cost.append(numpy.mean(d))

        tc = []
        for test_batch_index in xrange(n_test_batches):
            testing_cost = stopcheck_model(test_batch_index)
            tc.append(testing_cost)
        print 'Testing cost = ', numpy.mean(tc)
        te_cost.append(numpy.mean(tc))


    train_losses = [train_check(i) for i in xrange(n_train_batches)]
    print 'Training error = ', numpy.mean(train_losses)*100, ' %'

    test_losses = test_model()
    test_score = numpy.mean(test_losses)

    test_confmatrices = test_confmatrix()
    test_confelement = numpy.sum(test_confmatrices, axis=0)
    true_pos = test_confelement[0]
    true_neg = test_confelement[1]
    false_pos = test_confelement[2]
    false_neg = test_confelement[3]
    f_score = (true_pos + true_neg)/float(true_pos + true_neg + false_pos + 5*false_neg)

    print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
    print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))

    end_time = time.clock()
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time)
                                              / 60.))

    x_axis = numpy.arange(training_epochs)
    plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost), '.')
    plt.show()
def test_DBN(finetune_lr=0.1, pretraining_epochs=100,
             pretrain_lr=0.1, k=1, training_epochs=5000,
             batch_size=5):

    datasets = load10sec_ECGII_data()

    pretrain_set_x = datasets[0]
    trdatasets = [datasets[1], datasets[2]]

    train_set_x, train_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)

    print '... building the model'

    dbn = DBN(numpy_rng=numpy_rng, n_ins=2500,
              hidden_layers_sizes=[1000, 1000, 1000],
              n_outs=2)

    #########################
    # PRETRAINING THE MODEL #
    #########################

    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=pretrain_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    # Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))

    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, train_cost_model, test_score_model, test_cost_model, test_confmatrix = dbn.build_finetune_functions(
        datasets=trdatasets,
        batch_size=batch_size,
    )

    print '... finetuning the model'

    tr_cost = []
    te_cost = []

    test_score = 0.
    start_time = time.clock()

    epoch = 0
    while (epoch < training_epochs):
        epoch = epoch + 1
        learning_rate = 0.01/(1+0.001*epoch)
        # go through the training set
        for minibatch_index in xrange(n_train_batches):
            minibatch_avg_cost = train_fn(minibatch_index, learning_rate)
        print(('In epoch %d,') % (epoch))
        epoch_trcost = numpy.mean(train_cost_model())
        epoch_tecost = numpy.mean(test_cost_model())
        print 'Training cost = ', epoch_trcost
        tr_cost.append(epoch_trcost)
        print 'Testing cost = ', epoch_tecost
        te_cost.append(epoch_tecost)

    test_losses = test_score_model()
    test_score = numpy.mean(test_losses)

    test_confmatrices = test_confmatrix()
    test_confelement = numpy.sum(test_confmatrices, axis=0)
    true_pos = test_confelement[0]
    true_neg = test_confelement[1]
    false_pos = test_confelement[2]
    false_neg = test_confelement[3]
    f_score = (true_pos + true_neg)/float(true_pos + true_neg + false_pos + 5*false_neg)

    print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
    print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))

    end_time = time.clock()
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time)
                                              / 60.))

    x_axis = numpy.arange(training_epochs)
    plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost), '.')
    plt.show()
Exemple #7
0
def test_DBN(finetune_lr=0.1, decay=False, training_epochs=100,
             pretraining_epochs=10, pretrain_lr=0.01, k=1,
             batch_size=10):
    """
    Demonstrates how to train and test a Deep Belief Network.

    This is demonstrated on CIFAR-10.

    :type finetune_lr: float
    :param finetune_lr: learning rate used in the finetune stage
    :type decay: boolean
    :param decay: whether use weight decay or not in the finetune stage
    :type pretraining_epochs: int
    :param pretraining_epochs: number of epoch to do pretraining
    :type pretrain_lr: float
    :param pretrain_lr: learning rate to be used during pre-training
    :type k: int
    :param k: number of Gibbs steps in CD/PCD
    :type training_epochs: int
    :param training_epochs: maximal number of iterations ot run the optimizer
    :type batch_size: int
    :param batch_size: the size of a minibatch
    """

    datasets = load_data()

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size

    # numpy random generator
    numpy_rng = numpy.random.RandomState(123)
    print '... building the model'
    # construct the Deep Belief Network
    dbn = DBN(numpy_rng=numpy_rng, n_ins=32 * 32 * 3,
              hidden_layers_sizes=[1000, 1000, 1000],
              n_outs=10)

    # start-snippet-2
    #########################
    # PRETRAINING THE MODEL #
    #########################
    print '... getting the pretraining functions'
    pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
                                                batch_size=batch_size,
                                                k=k)

    print '... pre-training the model'
    start_time = time.clock()
    ## Pre-train layer-wise
    for i in xrange(dbn.n_layers):
        # go through pretraining epochs
        for epoch in xrange(pretraining_epochs):
            # go through the training set
            c = []
            for batch_index in xrange(n_train_batches):
                c.append(pretraining_fns[i](index=batch_index,
                                            lr=pretrain_lr))
            print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
            print numpy.mean(c)

    end_time = time.clock()
    # end-snippet-2
    print >> sys.stderr, ('The pretraining code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))
    ########################
    # FINETUNING THE MODEL #
    ########################

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model = dbn.build_finetune_functions(
        datasets=datasets,
        batch_size=batch_size
    )

    print '... finetuning the model'
    # early-stopping parameters
    patience = 4 * n_train_batches  # look as this many examples regardless
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatches before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    test_score = 0.
    epoch = 0
    start_time = time.clock()

    while (epoch < training_epochs):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):
            tlearning_rate = finetune_lr
            if decay:
                tlearning_rate = finetune_lr/(1.0 + epoch / 10.0)
               
            minibatch_avg_cost = train_fn(minibatch_index, tlearning_rate)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses)
                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%, learning rate %f'
                    % (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.,
                        tlearning_rate
                    )
                )

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))


    end_time = time.clock()
    print(
        (
            'Optimization complete with best validation score of %f %%, '
            'obtained at iteration %i, '
            'with test performance %f %%'
        ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)
    )
    print >> sys.stderr, ('The fine tuning code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time)
                                              / 60.))