Esempio n. 1
0
def test_cnn_predict():
    X = np.random.standard_normal((10, 2 * 100 * 50))
    X, = theano_floatx(X)

    m = Cnn(
        100 * 50,
        [10, 15],
        [20, 12],
        1,
        ['sigmoid', 'sigmoid'],
        ['rectifier', 'rectifier'],
        'sigmoid',
        'cat_ce',
        100,
        50,
        2,
        optimizer=('rmsprop', {
            'step_rate': 1e-4,
            'decay': 0.9
        }),
        batch_size=2,
        max_iter=10,
        pool_shapes=[(2, 2), (2, 2)],
        filter_shapes=[(4, 4), (3, 3)],
    )
    m.predict(X)
Esempio n. 2
0
def test_cnn_iter_fit():
    X = np.random.standard_normal((10, 2 * 100 * 50))
    Z = np.random.random((10, 1)) > 0.5
    X, Z = theano_floatx(X, Z)

    m = Cnn(
        100 * 50,
        [10, 15],
        [20, 12],
        1,
        ['sigmoid', 'sigmoid'],
        ['rectifier', 'rectifier'],
        'sigmoid',
        'cat_ce',
        100,
        50,
        2,
        optimizer=('rmsprop', {
            'step_rate': 1e-4,
            'decay': 0.9
        }),
        batch_size=2,
        max_iter=10,
        pool_shapes=[(2, 2), (2, 2)],
        filter_shapes=[(4, 4), (3, 3)],
    )
    for i, info in enumerate(m.iter_fit(X, Z)):
        if i >= 10:
            break
Esempio n. 3
0
def new_trainer(pars, data):

    # 3700 for binning
    input_size = 3700
    # 13 as there are 12 fields
    output_size = 13
    n_channels = 2
    bin_cm = 10
    max_x_cm = 440
    min_x_cm = 70
    max_y_cm = 250
    x_range = max_x_cm/bin_cm - min_x_cm/bin_cm
    y_range = max_y_cm*2/bin_cm
    im_width = y_range
    im_height = x_range
    batch_size = pars['batch_size']
    m = Cnn(input_size, pars['n_hidden_conv'], pars['n_hidden_full'], output_size,
            pars['hidden_conv_transfers'], pars['hidden_full_transfers'], 'softmax',
            loss='cat_ce',image_height=im_height,image_width=im_width,n_image_channel=n_channels,pool_size=pars['pool_size'],filter_shapes=pars['filter_shapes'],
            batch_size = batch_size, optimizer=pars['optimizer'])
    climin.initialize.randomize_normal(m.parameters.data, 0, pars['par_std'])

    weight_decay = ((m.parameters.hidden_conv_to_hidden_full**2).sum()
                    + (m.parameters.hidden_full_to_hidden_full_0**2).sum()
                    + (m.parameters.hidden_to_out**2).sum())
    weight_decay /= m.exprs['inpt'].shape[0]
    m.exprs['true_loss'] = m.exprs['loss']
    c_wd = pars['L2']
    m.exprs['loss'] = m.exprs['loss'] + c_wd * weight_decay

    # length of dataset should be 270000 (for no time-integration)
    n_report = 270000/batch_size
    max_iter = n_report * 100

    interrupt = climin.stops.OnSignal()
    print dir(climin.stops)
    stop = climin.stops.Any([
        climin.stops.AfterNIterations(max_iter),
        climin.stops.OnSignal(signal.SIGTERM),
        #climin.stops.NotBetterThanAfter(1e-1,500,key='train_loss'),
    ])

    pause = climin.stops.ModuloNIterations(n_report)
    reporter = KeyPrinter(['n_iter', 'train_loss', 'val_loss'])

    t = Trainer(
        m,
        stop=stop, pause=pause, report=reporter,
        interrupt=interrupt)

    make_data_dict(t,data)

    return t
Esempio n. 4
0
def test_cnn_predict():
    X = np.random.standard_normal((10, 2 * 100 * 50))
    X, = theano_floatx(X)

    m = Cnn(100 * 50, [10, 15], [20, 12], 1,
            ['sigmoid', 'sigmoid'], ['rectifier', 'rectifier'],
            'sigmoid',
            'cat_ce', 100, 50, 2,
            optimizer=('rmsprop', {'step_rate': 1e-4, 'decay': 0.9}),
            batch_size=2,
            max_iter=10,
            pool_shapes=[(2, 2), (2, 2)],
            filter_shapes=[(4, 4), (3, 3)],
            )
    m.predict(X)
Esempio n. 5
0
def test_cnn_iter_fit():
    X = np.random.standard_normal((10, 2 * 100 * 50))
    Z = np.random.random((10, 1)) > 0.5
    X, Z = theano_floatx(X, Z)

    m = Cnn(100 * 50, [10, 15], [20, 12], 1,
            ['sigmoid', 'sigmoid'], ['rectifier', 'rectifier'],
            'sigmoid',
            'cat_ce', 100, 50, 2,
            optimizer=('rmsprop', {'step_rate': 1e-4, 'decay': 0.9}),
            batch_size=2,
            max_iter=10,
            pool_shapes=[(2, 2), (2, 2)],
            filter_shapes=[(4, 4), (3, 3)],
            )
    for i, info in enumerate(m.iter_fit(X, Z)):
        if i >= 10:
            break
Esempio n. 6
0
def load_model(filename, n_outputs):
    m = Cnn(3072, [64, 64, 64], [256], n_outputs,
            ['rectifier', 'rectifier', 'rectifier'], ['rectifier'],
            out_transfer='softmax', loss='nce', image_height=32,
            image_width=32, n_image_channel=3, optimizer=None,
            batch_size=128, max_iter=12,
            pool_shapes=[[3, 3], [3, 3], [3, 3]],
            filter_shapes=[[5, 5], [5, 5], [5, 5]],
            pool_strides=[[2, 2], [2, 2], [2, 2]], padding=[2,2,2],
            lrnorm=[True, True, False],
            init_weights_stdev=[0.01, 0.1, 0.1, 0.1, 0.1])
    m.parameters.data[:] = np.load(filename)
    return m
Esempio n. 7
0
def convolutional_nets_on_CIFAR10():

    #### load data ####
    train_file = 'pylearn2_gcn_whitened/train.pkl'
    test_file = 'pylearn2_gcn_whitened/test.pkl'
    # Load data.

    f = open(train_file,'rb')
    train_set = cPickle.load(f)
    f = open(test_file)
    test_set = cPickle.load(f)

    X, Z = train_set.get_data()
    VX, VZ = test_set.get_data()

    Z = one_hot(Z, 10)
    VZ = one_hot(VZ, 10)

    X = X[:128*390]#390]
    Z = Z[:128*390]#390]
    VX = VX[:128*78]#*78]
    VZ = VZ[:128*78]#*78]
    X = np.array(X, dtype=np.float32)
    Z = np.array(Z, dtype=np.float32)
    VZ = np.array(VZ, dtype=np.float32)
    VX = np.array(VX, dtype=np.float32)
    #### initialize model ####

    max_passes = 500
    batch_size = 128
    max_iter = max_passes * X.shape[0] / batch_size
    n_report = X.shape[0] / (5*batch_size)

    stop = climin.stops.any_([
        climin.stops.after_n_iterations(max_iter),
        ])

    pause = climin.stops.modulo_n_iterations(n_report)
    #optimizer = 'rmsprop', {'steprate': 0.1, 'momentum': 0.8, 'decay': 0.9, 'step_adapt': 0.001}
    optimizer = 'gd', {'steprate': 0.01, 'momentum': 0.9}
    #optimizer = dropout_optimizer_conf(steprate_0=1, n_repeats=1)
    #m = Cnn(3072, [96, 192, 192], [500], 10, ['tanh', 'tanh', 'tanh'], ['tanh'], out_transfer='softmax',
                #loss='nce', image_height=32, image_width=32, n_image_channel=3, optimizer=optimizer,
                #batch_size=batch_size, max_iter=max_iter, pool_shapes=[[4, 4], [4, 4], [2, 2]],
                #filter_shapes=[[8, 8], [8, 8], [5, 5]], pool_strides=[[2, 2], [2, 2], [2, 2]],
                #padding=[4,3,3])
    m = Cnn(3072, [32, 64, 128], [50], 10, ['rectifier', 'rectifier', 'rectifier'], ['rectifier'], out_transfer='softmax',
                loss='nce', image_height=32, image_width=32, n_image_channel=3, optimizer=optimizer,
                batch_size=batch_size, max_iter=max_iter, pool_shapes=[[3, 3], [3, 3], [3, 3]],
                filter_shapes=[[5, 5], [5, 5], [5, 5]], pool_strides=[[2, 2], [2, 2], [2, 2]],
                padding=[2,2,2], lrnorm=[True, True, False], init_weights_stdev=[0.01, 0.1, 0.1, 0.1, 0.1])

    #m = Cnn(3072, [32, 32, 64], [64, 10], 10, ['rectifier', 'rectifier', 'rectifier'],  ['rectifier', 'rectifier'], out_transfer='softmax',
    #            loss='nce', image_height=32, image_width=32, n_image_channel=3, optimizer=optimizer,
    #            batch_size=batch_size, max_iter=max_iter, pool_shapes=[[2, 2], [2, 2], [1, 1]],
    #            filter_shapes=[[5, 5], [5, 5], [5, 5]], pool_strides=[[2, 2], [2, 2], [1, 1]])


    #m.parameters.data[...] = np.random.normal(0, 0.1, m.parameters.data.shape)
    #inits = m.sample_conv_weights()
    #for name, val in inits:
    #    m.parameters[name] = val
    weight_decay = 0.04*((m.parameters.in_to_hidden**2).sum()) + 0.04*((m.parameters.hidden_conv_to_hidden_conv_0**2).sum()) + 0.04*((m.parameters.hidden_conv_to_hidden_conv_1**2).sum()) + 2*(m.parameters.hidden_conv_to_hidden_full**2).sum()
    weight_decay /= m.exprs['inpt'].shape[0]
    m.exprs['true_loss'] = m.exprs['loss']
    m.exprs['loss'] = m.exprs['loss'] + weight_decay

    n_wrong = 1 - T.eq(T.argmax(m.exprs['output'], axis=1), T.argmax(m.exprs['target'], axis=1)).mean()
    f_n_wrong = m.function(['inpt', 'target'], n_wrong)
    
    losses = []
    v_losses = []
    print 'max iter', max_iter

    #### train model ####

    start = time.time()
    # Set up a nice printout.
    keys = '#', 'val loss', 'seconds', 'train emp', 'val emp'
    max_len = max(len(i) for i in keys)
    header = '\t'.join(i for i in keys)
    print header
    print '-' * len(header)

    f_loss = m.function(['inpt', 'target'], ['loss'])
    for i, info in enumerate(m.powerfit((X, Z), (VX, VZ), stop, pause, eval_train_loss=False)):
        if info['n_iter'] % n_report != 0:
            continue
        passed = time.time() - start
        v_losses.append(info['val_loss'])

        #img = tile_raster_images(fe.parameters['in_to_hidden'].T, image_dims, feature_dims, (1, 1))
        #save_and_display(img, 'filters-%i.png' % i
        f_wrong_val = m.apply_minibatches_function(f_n_wrong, VX, VZ)*VX.shape[0]
	f_wrong_train = m.apply_minibatches_function(f_n_wrong, X[:len(VX)], Z[:len(VZ)])*len(VX)
        info.update({
            'time': passed,
            'val_emp': f_wrong_val,
	    'train_emp': f_wrong_train
        })
        row = '%(n_iter)i\t%(val_loss)g\t%(time)g\t%(train_emp)g\t%(val_emp)g' % info
        print row
Esempio n. 8
0
    def run(self):
        self.prepare_data()
        X, Z, VX, VZ = self.data
	del(self.data)
        max_iter = self.max_passes * X.shape[0] / self.batch_size
        n_report = X.shape[0] / self.batch_size
        stop = climin.stops.any_([
            climin.stops.after_n_iterations(max_iter),
            ])

        pause = climin.stops.modulo_n_iterations(n_report)
        optimizer = 'gd', {'steprate': 0.01, 'momentum': 0.9}
        print "Zshape", Z.shape
	print "Xshape", X.shape
	print "VXshape", VX.shape
	print "VZshape", VZ.shape
	m = Cnn(3072, [64, 64, 64], [256], Z.shape[1],
                ['rectifier', 'rectifier', 'rectifier'], ['rectifier'],
                out_transfer='softmax', loss='nce', image_height=32,
                image_width=32, n_image_channel=3, optimizer=optimizer,
                batch_size=self.batch_size, max_iter=max_iter,
                pool_shapes=[[3, 3], [3, 3], [3, 3]],
                filter_shapes=[[5, 5], [5, 5], [5, 5]],
                pool_strides=[[2, 2], [2, 2], [2, 2]], padding=[2,2,2],
                lrnorm=[True, True, False],
                init_weights_stdev=[0.01, 0.1, 0.1, 0.1, 0.1])

        weight_decay = 0.4*((m.parameters.in_to_hidden**2).sum()) + \
                       0.4*((m.parameters.hidden_conv_to_hidden_conv_0**2)
                            .sum()) + \
                       0.4*((m.parameters.hidden_conv_to_hidden_conv_1**2)
                            .sum()) \
                       + 2*(m.parameters.hidden_conv_to_hidden_full**2).sum()
        weight_decay /= m.exprs['inpt'].shape[0]
        m.exprs['true_loss'] = m.exprs['loss']
        m.exprs['loss'] += weight_decay

        n_wrong = 1 - T.eq(T.argmax(m.exprs['output'], axis=1),
                           T.argmax(m.exprs['target'], axis=1)).mean()
        f_n_wrong = m.function(['inpt', 'target'], n_wrong)

        v_losses = []
        print 'max iter', max_iter

        start = time.time()
        keys = '#', 'val loss', 'seconds', 'train emp', 'val emp'
        header = '\t'.join(i for i in keys)
        print header
        print '-' * len(header)
        info = None
        for i, info in enumerate(m.powerfit((X, Z), (VX, VZ), stop, pause,
                                            eval_train_loss=False)):
            if info['n_iter'] % n_report != 0:
                continue
            passed = time.time() - start
            v_losses.append(info['val_loss'])
            f_wrong_val = m.apply_minibatches_function(f_n_wrong, VX, VZ)
            f_wrong_val = f_wrong_val*VX.shape[0]
            f_wrong_train = m.apply_minibatches_function(f_n_wrong, X[:len(VX)],
                                                         Z[:len(VZ)])*len(VX)
            info.update({
                'time': passed,
                'val_emp': f_wrong_val,
                'train_emp': f_wrong_train
            })
            row = '%(n_iter)i\t%(val_loss)g\t%(time)g\t%(train_emp)g\t%(' \
                  'val_emp)g' % info
            print row
            if (i % 5) == 0:
                np.save(self.model_name, info['best_pars'])
        np.save(self.model_name, info['best_pars'])
        m.parameters.data[:] = info['best_pars']
        return m
Esempio n. 9
0
    def run(self):
        self.prepare_data()
        X, Z, VX, VZ = self.data
        del (self.data)
        max_iter = self.max_passes * X.shape[0] / self.batch_size
        n_report = X.shape[0] / self.batch_size
        stop = climin.stops.any_([
            climin.stops.after_n_iterations(max_iter),
        ])

        pause = climin.stops.modulo_n_iterations(n_report)
        optimizer = 'gd', {'steprate': 0.01, 'momentum': 0.9}
        print "Zshape", Z.shape
        print "Xshape", X.shape
        print "VXshape", VX.shape
        print "VZshape", VZ.shape
        m = Cnn(3072, [64, 64, 64], [256],
                Z.shape[1], ['rectifier', 'rectifier', 'rectifier'],
                ['rectifier'],
                out_transfer='softmax',
                loss='nce',
                image_height=32,
                image_width=32,
                n_image_channel=3,
                optimizer=optimizer,
                batch_size=self.batch_size,
                max_iter=max_iter,
                pool_shapes=[[3, 3], [3, 3], [3, 3]],
                filter_shapes=[[5, 5], [5, 5], [5, 5]],
                pool_strides=[[2, 2], [2, 2], [2, 2]],
                padding=[2, 2, 2],
                lrnorm=[True, True, False],
                init_weights_stdev=[0.01, 0.1, 0.1, 0.1, 0.1])

        weight_decay = 0.4*((m.parameters.in_to_hidden**2).sum()) + \
                       0.4*((m.parameters.hidden_conv_to_hidden_conv_0**2)
                            .sum()) + \
                       0.4*((m.parameters.hidden_conv_to_hidden_conv_1**2)
                            .sum()) \
                       + 2*(m.parameters.hidden_conv_to_hidden_full**2).sum()
        weight_decay /= m.exprs['inpt'].shape[0]
        m.exprs['true_loss'] = m.exprs['loss']
        m.exprs['loss'] += weight_decay

        n_wrong = 1 - T.eq(T.argmax(m.exprs['output'], axis=1),
                           T.argmax(m.exprs['target'], axis=1)).mean()
        f_n_wrong = m.function(['inpt', 'target'], n_wrong)

        v_losses = []
        print 'max iter', max_iter

        start = time.time()
        keys = '#', 'val loss', 'seconds', 'train emp', 'val emp'
        header = '\t'.join(i for i in keys)
        print header
        print '-' * len(header)
        info = None
        for i, info in enumerate(
                m.powerfit((X, Z), (VX, VZ),
                           stop,
                           pause,
                           eval_train_loss=False)):
            if info['n_iter'] % n_report != 0:
                continue
            passed = time.time() - start
            v_losses.append(info['val_loss'])
            f_wrong_val = m.apply_minibatches_function(f_n_wrong, VX, VZ)
            f_wrong_val = f_wrong_val * VX.shape[0]
            f_wrong_train = m.apply_minibatches_function(
                f_n_wrong, X[:len(VX)], Z[:len(VZ)]) * len(VX)
            info.update({
                'time': passed,
                'val_emp': f_wrong_val,
                'train_emp': f_wrong_train
            })
            row = '%(n_iter)i\t%(val_loss)g\t%(time)g\t%(train_emp)g\t%(' \
                  'val_emp)g' % info
            print row
            if (i % 5) == 0:
                np.save(self.model_name, info['best_pars'])
        np.save(self.model_name, info['best_pars'])
        m.parameters.data[:] = info['best_pars']
        return m
Esempio n. 10
0
def convolutional_nets_on_CIFAR10():

    #### load data ####
    train_file = 'pylearn2_gcn_whitened/train.pkl'
    test_file = 'pylearn2_gcn_whitened/test.pkl'
    # Load data.

    f = open(train_file, 'rb')
    train_set = cPickle.load(f)
    f = open(test_file)
    test_set = cPickle.load(f)

    X, Z = train_set.get_data()
    VX, VZ = test_set.get_data()

    Z = one_hot(Z, 10)
    VZ = one_hot(VZ, 10)

    X = X[:128 * 390]  #390]
    Z = Z[:128 * 390]  #390]
    VX = VX[:128 * 78]  #*78]
    VZ = VZ[:128 * 78]  #*78]
    X = np.array(X, dtype=np.float32)
    Z = np.array(Z, dtype=np.float32)
    VZ = np.array(VZ, dtype=np.float32)
    VX = np.array(VX, dtype=np.float32)
    #### initialize model ####

    max_passes = 500
    batch_size = 128
    max_iter = max_passes * X.shape[0] / batch_size
    n_report = X.shape[0] / (5 * batch_size)

    stop = climin.stops.any_([
        climin.stops.after_n_iterations(max_iter),
    ])

    pause = climin.stops.modulo_n_iterations(n_report)
    #optimizer = 'rmsprop', {'steprate': 0.1, 'momentum': 0.8, 'decay': 0.9, 'step_adapt': 0.001}
    optimizer = 'gd', {'steprate': 0.01, 'momentum': 0.9}
    #optimizer = dropout_optimizer_conf(steprate_0=1, n_repeats=1)
    #m = Cnn(3072, [96, 192, 192], [500], 10, ['tanh', 'tanh', 'tanh'], ['tanh'], out_transfer='softmax',
    #loss='nce', image_height=32, image_width=32, n_image_channel=3, optimizer=optimizer,
    #batch_size=batch_size, max_iter=max_iter, pool_shapes=[[4, 4], [4, 4], [2, 2]],
    #filter_shapes=[[8, 8], [8, 8], [5, 5]], pool_strides=[[2, 2], [2, 2], [2, 2]],
    #padding=[4,3,3])
    m = Cnn(3072, [32, 64, 128], [50],
            10, ['rectifier', 'rectifier', 'rectifier'], ['rectifier'],
            out_transfer='softmax',
            loss='nce',
            image_height=32,
            image_width=32,
            n_image_channel=3,
            optimizer=optimizer,
            batch_size=batch_size,
            max_iter=max_iter,
            pool_shapes=[[3, 3], [3, 3], [3, 3]],
            filter_shapes=[[5, 5], [5, 5], [5, 5]],
            pool_strides=[[2, 2], [2, 2], [2, 2]],
            padding=[2, 2, 2],
            lrnorm=[True, True, False],
            init_weights_stdev=[0.01, 0.1, 0.1, 0.1, 0.1])

    #m = Cnn(3072, [32, 32, 64], [64, 10], 10, ['rectifier', 'rectifier', 'rectifier'],  ['rectifier', 'rectifier'], out_transfer='softmax',
    #            loss='nce', image_height=32, image_width=32, n_image_channel=3, optimizer=optimizer,
    #            batch_size=batch_size, max_iter=max_iter, pool_shapes=[[2, 2], [2, 2], [1, 1]],
    #            filter_shapes=[[5, 5], [5, 5], [5, 5]], pool_strides=[[2, 2], [2, 2], [1, 1]])

    #m.parameters.data[...] = np.random.normal(0, 0.1, m.parameters.data.shape)
    #inits = m.sample_conv_weights()
    #for name, val in inits:
    #    m.parameters[name] = val
    weight_decay = 0.04 * ((m.parameters.in_to_hidden**2).sum()) + 0.04 * (
        (m.parameters.hidden_conv_to_hidden_conv_0**2).sum()) + 0.04 * (
            (m.parameters.hidden_conv_to_hidden_conv_1**2).sum()) + 2 * (
                m.parameters.hidden_conv_to_hidden_full**2).sum()
    weight_decay /= m.exprs['inpt'].shape[0]
    m.exprs['true_loss'] = m.exprs['loss']
    m.exprs['loss'] = m.exprs['loss'] + weight_decay

    n_wrong = 1 - T.eq(T.argmax(m.exprs['output'], axis=1),
                       T.argmax(m.exprs['target'], axis=1)).mean()
    f_n_wrong = m.function(['inpt', 'target'], n_wrong)

    losses = []
    v_losses = []
    print 'max iter', max_iter

    #### train model ####

    start = time.time()
    # Set up a nice printout.
    keys = '#', 'val loss', 'seconds', 'train emp', 'val emp'
    max_len = max(len(i) for i in keys)
    header = '\t'.join(i for i in keys)
    print header
    print '-' * len(header)

    f_loss = m.function(['inpt', 'target'], ['loss'])
    for i, info in enumerate(
            m.powerfit((X, Z), (VX, VZ), stop, pause, eval_train_loss=False)):
        if info['n_iter'] % n_report != 0:
            continue
        passed = time.time() - start
        v_losses.append(info['val_loss'])

        #img = tile_raster_images(fe.parameters['in_to_hidden'].T, image_dims, feature_dims, (1, 1))
        #save_and_display(img, 'filters-%i.png' % i
        f_wrong_val = m.apply_minibatches_function(f_n_wrong, VX,
                                                   VZ) * VX.shape[0]
        f_wrong_train = m.apply_minibatches_function(f_n_wrong, X[:len(VX)],
                                                     Z[:len(VZ)]) * len(VX)
        info.update({
            'time': passed,
            'val_emp': f_wrong_val,
            'train_emp': f_wrong_train
        })
        row = '%(n_iter)i\t%(val_loss)g\t%(time)g\t%(train_emp)g\t%(val_emp)g' % info
        print row