class TestMNIST(unittest.TestCase): def setUp(self): skip_if_no_data() self.train = MNIST(which_set='train') self.test = MNIST(which_set='test') def test_range(self): """Tests that the data spans [0,1]""" for X in [self.train.X, self.test.X]: assert X.min() == 0.0 assert X.max() == 1.0 def test_topo(self): """Tests that a topological batch has 4 dimensions""" topo = self.train.get_batch_topo(1) assert topo.ndim == 4 def test_topo_c01b(self): """ Tests that a topological batch with axes ('c',0,1,'b') can be dimshuffled back to match the standard ('b',0,1,'c') format. """ batch_size = 100 c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b')) c01b_X = c01b_test.X[0:batch_size, :] c01b = c01b_test.get_topological_view(c01b_X) assert c01b.shape == (1, 28, 28, batch_size) b01c = c01b.transpose(3, 1, 2, 0) b01c_X = self.test.X[0:batch_size, :] assert c01b_X.shape == b01c_X.shape assert np.all(c01b_X == b01c_X) b01c_direct = self.test.get_topological_view(b01c_X) assert b01c_direct.shape == b01c.shape assert np.all(b01c_direct == b01c) def test_y_index_space(self): """ Tests that requesting the targets to be in IndexSpace and iterating over them works """ data_specs = (IndexSpace(max_labels=10, dim=1), 'targets') it = self.test.iterator(mode='sequential', data_specs=data_specs, batch_size=100) for y in it: pass def test_y_vector_space(self): """ Tests that requesting the targets to be in VectorSpace and iterating over them works """ data_specs = (VectorSpace(dim=10), 'targets') it = self.test.iterator(mode='sequential', data_specs=data_specs, batch_size=100) for y in it: pass
class TestMNIST(unittest.TestCase): def setUp(self): skip_if_no_data() self.train = MNIST(which_set = 'train') self.test = MNIST(which_set = 'test') def test_range(self): """Tests that the data spans [0,1]""" for X in [self.train.X, self.test.X ]: assert X.min() == 0.0 assert X.max() == 1.0 def test_topo(self): """Tests that a topological batch has 4 dimensions""" topo = self.train.get_batch_topo(1) assert topo.ndim == 4 def test_topo_c01b(self): """ Tests that a topological batch with axes ('c',0,1,'b') can be dimshuffled back to match the standard ('b',0,1,'c') format. """ batch_size = 100 c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b')) c01b_X = c01b_test.X[0:batch_size,:] c01b = c01b_test.get_topological_view(c01b_X) assert c01b.shape == (1, 28, 28, batch_size) b01c = c01b.transpose(3,1,2,0) b01c_X = self.test.X[0:batch_size,:] assert c01b_X.shape == b01c_X.shape assert np.all(c01b_X == b01c_X) b01c_direct = self.test.get_topological_view(b01c_X) assert b01c_direct.shape == b01c.shape assert np.all(b01c_direct == b01c)
def test_sample_ae(): """ Visualize some samples from the trained unsupervised GSN. """ with open("gsn_ae_example.pkl") as f: gsn = pickle.load(f) # random point to start at mb_data = MNIST(which_set='test').X[105:106, :] history = gsn.get_samples([(0, mb_data)], walkback=1000, symbolic=False, include_first=True) history = list(itertools.chain(*history)) history = np.vstack(history) tiled = image.tile_raster_images(history, img_shape=[28, 28], tile_shape=[50, 50], tile_spacing=(2, 2)) image.save("gsn_ae_example.png", tiled) # code to get log likelihood from kernel density estimator # this crashed on GPU (out of memory), but works on CPU pw = ParzenWindows(MNIST(which_set='test').X, .20) print(pw.get_ll(history))
def model1(): #pdb.set_trace() # train set X has dim (60,000, 784), y has dim (60,000, 10) train_set = MNIST(which_set='train', one_hot=True) # test set X has dim (10,000, 784), y has dim (10,000, 10) valid_set = MNIST(which_set='test', one_hot=True) test_set = MNIST(which_set='test', one_hot=True) #import pdb #pdb.set_trace() #print train_set.X.shape[1] # =====<Create the MLP Model>===== h2_layer = NoisyRELU(layer_name='h1', sparse_init=15, noise_factor=5, dim=1000, desired_active_rate=0.2, bias_factor=20, max_col_norm=1) #h2_layer = RectifiedLinear(layer_name='h2', dim=100, sparse_init=15, max_col_norm=1) #print h1_layer.get_params() #h2 = RectifiedLinear(layer_name='h2', dim=500, sparse_init=15, max_col_norm=1) y_layer = Softmax(layer_name='y', n_classes=10, irange=0., max_col_norm=1) mlp = MLP(batch_size=200, input_space=VectorSpace(dim=train_set.X.shape[1]), layers=[h2_layer, y_layer]) # =====<Create the SGD algorithm>===== sgd = SGD(init_momentum=0.1, learning_rate=0.01, monitoring_dataset={'valid': valid_set}, cost=MethodCost('cost_from_X'), termination_criterion=MonitorBased( channel_name='valid_y_misclass', prop_decrease=0.001, N=50)) #sgd.setup(model=mlp, dataset=train_set) # =====<Extensions>===== ext = [MomentumAdjustor(start=1, saturate=10, final_momentum=0.9)] # =====<Create Training Object>===== save_path = './mlp_model1.pkl' train_obj = Train(dataset=train_set, model=mlp, algorithm=sgd, extensions=ext, save_path=save_path, save_freq=0) #train_obj.setup_extensions() #import pdb #pdb.set_trace() train_obj.main_loop() # =====<Run the training>===== '''
def main(): #import pdb #pdb.set_trace() ################### #BUILD THE DATASET# ################### print 'build the dataset' train_set = MNIST(which_set='train', one_hot=False) test_set = MNIST(which_set='test', one_hot=False) train_setX, valid_setX = split(train_set.X, [50000], axis=0) train_sety, valid_sety = split(train_set.y, [50000], axis=0) #import pdb #pdb.set_trace() ################## #BUILD THE LAYERS# ################## print 'build the layers' input_size = len(train_setX[0]) h1 = Tanh(prev_layer_size=input_size, this_layer_size=2000) output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10, W_range=[0,0], b_range=[0,0]) #y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1]) mlp = MLP(input_size=input_size, layers=[h1, output_layer], train_set=[train_setX, train_sety], valid_set=[valid_setX, valid_sety], test_set=[test_set.X, test_set.y], error_function=loglikehood, batch_size=20, learning_rate=0.1) print 'start training' mlp.train() #p = plt.plot(mlp.epoch, mlp.valid_error) #plots.append(p) with open('batches_clean.pkl', 'wb') as bat: cPickle.dump(mlp.epoch, bat) with open('errors_clean.pkl', 'wb') as err: cPickle.dump(mlp.valid_error, err) with open('legends_clean.pkl', 'wb') as leg: cPickle.dump(['2000-tanh'], leg)
def model2(): #pdb.set_trace() # train set X has dim (60,000, 784), y has dim (60,000, 10) train_set = MNIST(which_set='train', one_hot=True) # test set X has dim (10,000, 784), y has dim (10,000, 10) test_set = MNIST(which_set='test', one_hot=True) # =====<Create the MLP Model>===== h1_layer = RectifiedLinear(layer_name='h1', dim=1000, irange=0.5) #print h1_layer.get_params() h2_layer = RectifiedLinear(layer_name='h2', dim=1000, sparse_init=15, max_col_norm=1) y_layer = Softmax(layer_name='y', n_classes=train_set.y.shape[1], irange=0.5) mlp = MLP(batch_size=100, input_space=VectorSpace(dim=train_set.X.shape[1]), layers=[h1_layer, h2_layer, y_layer]) # =====<Create the SGD algorithm>===== sgd = SGD(batch_size=100, init_momentum=0.1, learning_rate=0.01, monitoring_dataset={ 'valid': train_set, 'test': test_set }, cost=SumOfCosts(costs=[ MethodCost('cost_from_X'), WeightDecay(coeffs=[0.00005, 0.00005, 0.00005]) ]), termination_criterion=MonitorBased( channel_name='valid_y_misclass', prop_decrease=0.0001, N=5)) #sgd.setup(model=mlp, dataset=train_set) # =====<Extensions>===== ext = [MomentumAdjustor(start=1, saturate=10, final_momentum=0.99)] # =====<Create Training Object>===== save_path = './mlp_model2.pkl' train_obj = Train(dataset=train_set, model=mlp, algorithm=sgd, extensions=ext, save_path=save_path, save_freq=0) #train_obj.setup_extensions() train_obj.main_loop()
def mnist(): #import pdb #pdb.set_trace() ################### #BUILD THE DATASET# ################### print 'build the dataset' train_set = MNIST(which_set='train') test_set = MNIST(which_set='test') train_setX, valid_setX = split(train_set.X, [50000], axis=0) train_sety, valid_sety = split(train_set.y, [50000], axis=0) #import pdb #pdb.set_trace() ################## #BUILD THE LAYERS# ################## print 'build the layers' input_size = len(train_setX[0]) h1 = NoisyRELU(prev_layer_size=input_size, this_layer_size=1000, threshold=5, noise_factor=1) h2 = NoisyRELU(prev_layer_size=h1.get_size(), this_layer_size=1000, threshold=5, noise_factor=1) #h3 = NoisyRELU(prev_layer_size=h2.get_size(), this_layer_size=1000, threshold=5, noise_factor=1) output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10, W_range=[0,0], b_range=[0,0]) #y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1]) mlp = MLP(input_size=input_size, layers=[h1, h2, output_layer], train_set=[train_setX, train_sety], valid_set=[valid_setX, valid_sety], test_set=[test_set.X, test_set.y], error_function=loglikehood, batch_size=20, learning_rate=0.1) print 'start training' mlp.train(save_freq=1)
def setup(): """ Create pickle file with a simple model. """ # tearDown is guaranteed to run pop_load_data. control.push_load_data(False) with open('dbm.pkl', 'wb') as f: dataset = MNIST(which_set='train', start=0, stop=100, binarize=True) vis_layer = BinaryVector(nvis=784, bias_from_marginals=dataset) hid_layer1 = BinaryVectorMaxPool(layer_name='h1', pool_size=1, irange=.05, init_bias=-2., detector_layer_dim=50) hid_layer2 = BinaryVectorMaxPool(layer_name='h2', pool_size=1, irange=.05, init_bias=-2., detector_layer_dim=10) model = DBM(batch_size=20, niter=2, visible_layer=vis_layer, hidden_layers=[hid_layer1, hid_layer2]) model.dataset_yaml_src = """ !obj:pylearn2.datasets.binarizer.Binarizer { raw: !obj:pylearn2.datasets.mnist.MNIST { which_set: "train", start: 0, stop: 100 } } """ model.layer_to_chains = model.make_layer_to_state(1) cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
def test_sample_supervised(idxs=None, noisy=True): """ Visualize samples and labels produced by GSN. """ with open("gsn_sup_example.pkl") as f: gsn = pickle.load(f) gsn._corrupt_switch = noisy ds = MNIST(which_set='test') if idxs is None: data = ds.X[100:150] else: data = ds.X[idxs] # change the walkback parameter to make the data fill up rows in image samples = gsn.get_samples([(0, data)], indices=[0, 2], walkback=21, symbolic=False, include_first=True) stacked = vis_samples(samples) tiled = image.tile_raster_images(stacked, img_shape=[28, 28], tile_shape=[50, 50], tile_spacing=(2, 2)) image.save("gsn_sup_example.png", tiled)
def test_decision_function(self): """ Test DenseMulticlassSVM.decision_function. """ dataset = MNIST(which_set='train') X = dataset.X[0:20, :] y = dataset.y[0:20] for i in xrange(10): assert (y == i).sum() > 0 model = DenseMulticlassSVM(kernel='poly', C=1.0).fit(X, y) f = model.decision_function(X) print(f) yhat_f = np.argmax(f, axis=1) yhat = np.cast[yhat_f.dtype](model.predict(X)) print(yhat_f) print(yhat) assert (yhat_f != yhat).sum() == 0
def test_mnist(): """ Test the mnist.yaml file from the maxout paper on random input """ skip_if_no_gpu() train = load_train_file( os.path.join(pylearn2.__path__[0], "scripts/papers/maxout/mnist.yaml")) # Load fake MNIST data init_value = control.load_data control.load_data = [False] train.dataset = MNIST(which_set='train', one_hot=1, axes=['c', 0, 1, 'b'], start=0, stop=100) train.algorithm._set_monitoring_dataset(train.dataset) control.load_data = init_value # Train shortly and prevent saving train.algorithm.termination_criterion = EpochCounter(max_epochs=1) train.extensions.pop(0) train.save_freq = 0 train.main_loop()
def try_pylearn2_generator(): dataset = MNIST('train') dataset_generator = Pylearn2OldGenerator(dataset, batch_size=1000, num_batches=-1) for b in dataset_generator: print b.shape assert b.shape == (1000, 784)
def load_dataset(self): # TODO: we might need other variables for identifying what kind of # extra preprocessing was done such as features product and number # of features kept based on MI. #base_path = get_data_path(self.state) #self.base_path = base_path #import pdb #pdb.set_trace() if self.state.dataset == 'mnist': self.test_ddm = MNIST(which_set='test', one_hot=True) dataset = MNIST(which_set='train', shuffle=True, one_hot=True) train_X, valid_X = np.split(dataset.X, [50000]) train_y, valid_y = np.split(dataset.y, [50000]) self.train_ddm = DenseDesignMatrix(X=train_X, y=train_y) self.valid_ddm = DenseDesignMatrix(X=valid_X, y=valid_y) elif self.state.dataset == 'svhn': self.train_ddm = SVHN(which_set='splitted_train') self.test_ddm = SVHN(which_set='test') self.valid_ddm = SVHN(which_set='valid') elif self.state.dataset == 'cifar10': self.train_ddm = My_CIFAR10(which_set='train', one_hot=True) self.test_ddm = None self.valid_ddm = My_CIFAR10(which_set='test', one_hot=True) if self.train_ddm is not None: self.nvis = self.train_ddm.X.shape[1] self.nout = self.train_ddm.y.shape[1] print "nvis, nout :", self.nvis, self.nout self.ntrain = self.train_ddm.X.shape[0] print "ntrain :", self.ntrain if self.valid_ddm is not None: self.nvalid = self.valid_ddm.X.shape[0] print "nvalid :", self.nvalid if self.test_ddm is not None: self.ntest = self.test_ddm.X.shape[0] print "ntest :", self.ntest
def get_valid(ds, limit_size=-1, fold=0): if ds == 'mnist': data = MNIST('train', start=50000, stop=60000) return data.X[:limit_size] elif ds == 'tfd': data = TFD('valid', fold=fold, scale=True) return data.X else: raise ValueError("Unknow dataset: {}".format(args.dataet))
def test_topo_c01b(self): """ Tests that a topological batch with axes ('c',0,1,'b') can be dimshuffled back to match the standard ('b',0,1,'c') format. """ batch_size = 100 c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b')) c01b_X = c01b_test.X[0:batch_size,:] c01b = c01b_test.get_topological_view(c01b_X) assert c01b.shape == (1, 28, 28, batch_size) b01c = c01b.transpose(3,1,2,0) b01c_X = self.test.X[0:batch_size,:] assert c01b_X.shape == b01c_X.shape assert np.all(c01b_X == b01c_X) b01c_direct = self.test.get_topological_view(b01c_X) assert b01c_direct.shape == b01c.shape assert np.all(b01c_direct == b01c)
def MNISTmain(): # TODO: max_col_norm h0 = maxout.Maxout(layer_name='h0', num_units=1200, num_pieces=2, W_lr_scale=1.0, irange=0.005, b_lr_scale=1.0) h1 = maxout.Maxout(layer_name='h1', num_units=1200, num_pieces=2, W_lr_scale=1.0, irange=0.005, b_lr_scale=1.0) # h2 = maxout.Maxout(layer_name='h2, num_units=1200, num_pieces=2, W_lr_scale=1.0, irange=0.005, b_lr_scale=1.0) outlayer = mlp.Softmax(layer_name='y', n_classes=10, irange=0) # layers = [h0, h1, h2, outlayer] layers = [h0, h1, outlayer] model = mlp.MLP(layers, nvis=784) train = MNIST('train', one_hot=1, start=0, stop=50000) valid = MNIST('train', one_hot=1, start=50000, stop=60000) test = MNIST('test', one_hot=1, start=0, stop=10000) monitoring = dict(valid=valid) termination = MonitorBased(channel_name="valid_y_misclass", N=100) extensions = [best_params.MonitorBasedSaveBest(channel_name="valid_y_misclass", save_path="/data/mcr10/train_best.pkl")] algorithm = sgd.SGD(0.1, batch_size=100, cost=Dropout(), monitoring_dataset = monitoring, termination_criterion = termination) save_path = "/data/mcr10/train_best.pkl" if not args.train and os.path.exists(save_path): model = serial.load(save_path) else: print 'Running training' train_job = Train(train, model, algorithm, extensions=extensions, save_path="/data/mcr10/train.pkl", save_freq=1) train_job.main_loop() X = model.get_input_space().make_batch_theano() Y = model.fprop(X) y = T.argmax(Y, axis=1) f = function(inputs=[X], outputs=y) yhat = f(test.X) y = np.squeeze(test.get_targets()) print 'accuracy', (y==yhat).sum() / y.size
def main(): ################### #BUILD THE DATASET# ################### print 'build the dataset' train_set = MNIST(which_set='train', one_hot=False) test_set = MNIST(which_set='test', one_hot=False) train_setX, valid_setX = split(train_set.X, [50000], axis=0) train_sety, valid_sety = split(train_set.y, [50000], axis=0) #import pdb #pdb.set_trace() ################## #BUILD THE LAYERS# ################## print 'build the layers' input_size = len(train_setX[0]) h1 = NoisyRELU(prev_layer_size=input_size, this_layer_size=200, noise_factor=0, activation_threshold=0) output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10, W_range=[0, 0], b_range=[0, 0]) #y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1]) # import pdb # pdb.set_trace() print 'build the model' mlp = MLP(input_size=input_size, layers=[h1, output_layer], train_set=[train_setX, train_sety], valid_set=[valid_setX, valid_sety], test_set=[test_set.X, test_set.y], error_function=loglikehood, batch_size=20) print 'start training' mlp.train_batch(100000)
def test_topo_c01b(self): """ Tests that a topological batch with axes ('c',0,1,'b') can be dimshuffled back to match the standard ('b',0,1,'c') format. """ batch_size = 100 # TODO: the one_hot=True is only necessary because one_hot=False is # broken, remove it after one_hot=False is fixed. c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b'), one_hot=True) c01b_X = c01b_test.X[0:batch_size,:] c01b = c01b_test.get_topological_view(c01b_X) assert c01b.shape == (1, 28, 28, batch_size) b01c = c01b.transpose(3,1,2,0) b01c_X = self.test.X[0:batch_size,:] assert c01b_X.shape == b01c_X.shape assert np.all(c01b_X == b01c_X) b01c_direct = self.test.get_topological_view(b01c_X) assert b01c_direct.shape == b01c.shape assert np.all(b01c_direct == b01c)
class TestMNIST(unittest.TestCase): def setUp(self): skip_if_no_data() self.train = MNIST(which_set = 'train') self.test = MNIST(which_set = 'test') def test_range(self): """Tests that the data spans [0,1]""" for X in [self.train.X, self.test.X ]: assert X.min() == 0.0 assert X.max() == 1.0 def test_topo(self): """Tests that a topological batch has 4 dimensions""" topo = self.train.get_batch_topo(1) assert topo.ndim == 4
class TestMNIST(unittest.TestCase): def setUp(self): skip_if_no_data() self.train = MNIST(which_set='train') self.test = MNIST(which_set='test') def test_range(self): """Tests that the data spans [0,1]""" for X in [self.train.X, self.test.X]: assert X.min() == 0.0 assert X.max() == 1.0 def test_topo(self): """Tests that a topological batch has 4 dimensions""" topo = self.train.get_batch_topo(1) assert topo.ndim == 4
def setUp(self): skip_if_no_h5py() import h5py skip_if_no_data() from pylearn2.datasets.mnist import MNIST # save MNIST data to HDF5 train = MNIST(which_set='train', one_hot=1, start=0, stop=100) for name, dataset in [('train', train)]: with h5py.File("{}.h5".format(name), "w") as f: f.create_dataset('X', data=dataset.get_design_matrix()) f.create_dataset('topo_view', data=dataset.get_topological_view()) f.create_dataset('y', data=dataset.get_targets()) # instantiate Train object self.train = yaml_parse.load(trainer_yaml)
def get_dim_input(state): if state.dataset == 'mnist': dataset = MNIST(which_set='test') dim = dataset.X.shape[1] elif state.dataset == 'svhn': dataset = SVHN(which_set='test') dim = dataset.X.shape[1] elif state.dataset == 'cifar10': dataset = My_CIFAR10(which_set='test') dim = dataset.X.shape[1] else: raise ValueError( 'only mnist, cifar10 and svhn are supported for now in get_dim_input' ) del dataset return dim
def get_valid(ds, limit_size=-1, fold=0): if ds == 'mnist': data = MNIST('train', start=50000, stop=60000) return data.X[:limit_size] elif ds == 'tfd': data = TFD('valid', fold=fold, scale=True) return data.X elif ds == 'lfwcrop': # HACK return LFW( axes=('c', 0, 1, 'b'), gcn=55, lfw_path= '/afs/cs.stanford.edu/u/jgauthie/scr/lfwcrop_color/faces32', filelist_path= '/afs/cs.stanford.edu/u/jgauthie/scr/lfwcrop_color/filelist.dev.ids.txt', embedding_file= '/afs/cs.stanford.edu/u/jgauthie/scr/lfw-lsa/LFW_attributes_30d.npz', img_shape=(3, 32, 32)).X else: raise ValueError("Unknow dataset: {}".format(args.dataet))
def ais_data(fname, do_exact=True): rbm_params = load_rbm_params(fname) # load data to set visible biases to ML solution from pylearn2.datasets.mnist import MNIST dataset = MNIST(which_set='train', one_hot=True) data = numpy.asarray(dataset.X, dtype=config.floatX) # run ais using B=0 model with ML visible biases t1 = time.time() (logz, log_var_dz), aisobj = \ rbm_tools.rbm_ais(rbm_params, n_runs=100, seed=123, data=data) print 'AIS logZ : %f' % logz print ' log_variance : %f' % log_var_dz print 'Elapsed time: ', time.time() - t1 if do_exact: exact_logz = compute_logz(rbm_params) print 'Exact logZ = %f' % exact_logz numpy.testing.assert_almost_equal(exact_logz, logz, decimal=0)
def get_ddm_mnist(self, ddm_id): row = self.db.executeSQL( """ SELECT which_set, center, shuffle, one_hot, binarize, start, stop, axes FROM hps3.ddm_mnist WHERE ddm_id = %s """, (ddm_id, ), self.db.FETCH_ONE) if not row or row is None: raise HPSData("No cifar100 ddm for ddm_id="\ +str(ddm_id)) (which_set, center, shuffle, one_hot, binarize, start, stop, axes_char) = row axes = self.get_axes(axes_char) return MNIST(which_set=which_set, center=center, shuffle=shuffle, binarize=binarize, axes=axes, start=start, stop=stop, one_hot=one_hot)
def mnist_test(model_paths=["dae_l1.pkl", "dae_l2.pkl", "dae_l3.pkl"], sparse_coef=0, sparse_p=0.1): cost = yaml_parse.load( 'cost : !obj:pylearn2.costs.cost.SumOfCosts {costs: [!obj:pylearn2.costs.autoencoder.MeanSquaredReconstructionError {}, !obj:pylearn2.costs.autoencoder.SparseActivation {coeff: ' + str(sparse_coef) + ',p: ' + str(sparse_p) + '}]}')['cost'] model = load_model(model_paths, cost) DUconfig.dataset = MNIST(which_set='train', start=0, stop=50000) #reconstructed_dataset = represent(DUconfig.dataset,model) #print reconstructed_dataset[0] print recon_errors(DUconfig.dataset.X, model) #reconstruct(model,img_path="mnist7.png",out_path="out7.png") ipaths = ["mnist1.png", "mnist4.png", "mnist5.png", "mnist7.png"] imgs = [] for img in ipaths: i = Image.open(img) imgs.append(np.reshape(i, (1, i.size[0] * i.size[1]))) flatimgs = np.concatenate(imgs) out = decode(encode(flatimgs, model), model) scipy.misc.imsave("out_prime.png", out.reshape([flatimgs.shape[0] * 28, 28])) save_as_patches(show_weights(model), [28, 28], out_path="outpatches.png")
def _get_dataset(dataset, no_imgs): """ Dump machine learning dataset into C source file and create a header file with array declaration and few macro definition """ dataset = dataset.lower() if (dataset not in dataset_factory['name']): print('Dataset is not in the factory') sys.exit() if (dataset == 'mnist'): print('Reading MNIST dataset from pylearn2 database') data = MNIST(which_set='test') if (no_imgs > data.X.shape[0]): cprint( 'Only {:d} images are available in this dataset. Dumping only those many' .format(data.X.shape[0]), 'red') data_x = data.X[0:min(no_imgs, data.X.shape[0])] data_y = data.y[0:min(no_imgs, data.y.shape[0])] data_x = np.uint8(data_x * 255) # conversion from 0 -> 0.99 to 0->255 elif (dataset == 'cifar10'): print('Reading CIFAR-10 dataset from pylearn2 database') data = CIFAR10(which_set='test') if (no_imgs > data.X.shape[0]): cprint( 'Only {:d} images are available in this dataset. Dumping only those many' .format(data.X.shape[0]), 'red') data_x = data.X[0:min(no_imgs, data.X.shape[0])] data_y = data.y[0:min(no_imgs, data.y.shape[0])] data_x = np.uint8(data_x) # already in the range 0 -> 255 elif (dataset == 'svhn'): cprint('Not supported', 'red') return return data_x, data_y
def test_classify(): """ See how well a (supervised) GSN performs at classification. """ with open("gsn_sup_example.pkl") as f: gsn = pickle.load(f) gsn = JointGSN.convert(gsn) # turn off corruption gsn._corrupt_switch = False ds = MNIST(which_set='test') mb_data = ds.X y = ds.y for i in xrange(1, 10): y_hat = gsn.classify(mb_data, trials=i) errors = np.abs(y_hat - y).sum() / 2.0 # error indices #np.sum(np.abs(y_hat - y), axis=1) != 0 print(i, errors, errors / mb_data.shape[0])
def load_data(dataset, train_percent=0.8, val_percent=0.2): """ Load MNIST, CIFAR-10 dataset dataset: string: MNIST or CIFAR-10 train_percent: float: percentage of the dataset to be used for training val_per: string: percentage of the dataset to be used for the validation purpose Output: (train_x, val_x, test_x, train_y, val_y, test_y) """ zero_mean = False if(dataset.lower() == 'mnist'): print('Loading MNIST dataset from pylearn2') train_set_size = int(_DATASET_SIZE['mnist'] * train_percent) train_data = MNIST(which_set='train', start=0, stop=train_set_size, center=zero_mean) val_data = MNIST(which_set='train', start=train_set_size, stop=_DATASET_SIZE[dataset], center=zero_mean) test_data = MNIST(which_set='test', center=zero_mean) # convert labels into 1D array train_data.y = np.hstack(train_data.y) val_data.y = np.hstack(val_data.y) test_data.y = np.hstack(test_data.y) # create 10 dimensional vector corresponding to each label train_data.y = np.float32(np.eye(10))[train_data.y] val_data.y = np.float32(np.eye(10))[val_data.y] test_data.y = np.float32(np.eye(10))[test_data.y] # TODO: convert the data to range [-1,1] # reshape the data into image size(#images, channels, height, width). # Each row contains an image in the original dataset train_data.X = np.reshape(train_data.X, (-1, 1, 28, 28)) val_data.X = np.reshape(val_data.X, (-1, 1, 28, 28)) test_data.X = np.reshape(test_data.X, (-1, 1, 28, 28)) # convert to [-1 1] range train_data.X = train_data.X * 2.0 - 1.0 val_data.X = val_data.X * 2.0 - 1.0 test_data.X = test_data.X * 2.0 - 1.0 elif(dataset.lower() == 'cifar10'): print('Loading CIFAR-10 dataset from pylearn2') train_set_size = int(_DATASET_SIZE['cifar10'] * train_percent) train_data = CIFAR10(which_set='train', start=0, stop=train_set_size) val_data = CIFAR10(which_set='train', start=train_set_size, stop=50000) test_data = CIFAR10(which_set='test') # convert labels into 1D array train_data.y = np.hstack(train_data.y) val_data.y = np.hstack(val_data.y) test_data.y = np.hstack(test_data.y) # create 10 dimensional vector corresponding to each label train_data.y = np.float32(np.eye(10))[train_data.y] val_data.y = np.float32(np.eye(10))[val_data.y] test_data.y = np.float32(np.eye(10))[test_data.y] # TODO: convert the data to range [-1,1] # reshape the data into image size(#images, channels, height, width). # Each row contains an image in the original dataset train_data.X = np.reshape(train_data.X, (-1, 3, 32, 32)) val_data.X = np.reshape(val_data.X, (-1, 3, 32, 32)) test_data.X = np.reshape(test_data.X, (-1, 3, 32, 32)) # convert to [-1 1] range train_data.X = train_data.X * (2.0/255) - 1.0 val_data.X = val_data.X * (2.0/255) - 1.0 test_data.X = test_data.X * (2.0/255) - 1.0 elif(dataset.lower() == 'svhn'): train_data = SVHN(which_set= 'splitted_train', axes= ['b', 'c', 0, 1]) val_data = SVHN(which_set= 'valid', axes= ['b', 'c', 0, 1]) test_data = SVHN(which_set= 'test', axes= ['b', 'c', 0, 1]) # convert labels into 1D array train_data.y = np.hstack(train_data.y) val_data.y = np.hstack(val_data.y) test_data.y = np.hstack(test_data.y) # create 10 dimensional vector corresponding to each label train_data.y = np.float32(np.eye(10))[train_data.y] val_data.y = np.float32(np.eye(10))[val_data.y] test_data.y = np.float32(np.eye(10))[test_data.y] # convert to [-1, 1] range train_data.X = np.reshape(np.subtract(np.multiply(2.0/255, train_data.X), 1.0), (-1, 3, 32, 32)) val_data.X = np.reshape(np.subtract(np.multiply(2.0/255, val_data.X), 1.0), (-1, 3, 32, 32)) test_data.X = np.reshape(np.subtract(np.multiply(2.0/255, test_data.X), 1.0), (-1, 3, 32, 32)) else: print('This dataset is not supported. Only MNIST and CIFAR-10 are supported as of now.') raise ValueError('Dataset is not supported') print('Trainset shape = ', train_data.X.shape, train_data.y.shape) print('Valset shape = ', val_data.X.shape, val_data.y.shape) print('Testset shape = ', test_data.X.shape, test_data.y.shape) return train_data.X, val_data.X, test_data.X, train_data.y, val_data.y, test_data.y
def setUp(self): skip_if_no_data() self.train = MNIST(which_set = 'train') self.test = MNIST(which_set = 'test')
cols = 10 from pylearn2.models.dbm import load_matlab_dbm model = load_matlab_dbm('joint_trained_dbm_interm.mat', num_chains = rows * cols) dbm = model from theano import function import theano.tensor as T sample_func = function([],updates = model.get_sampling_updates()) render_func = function([],T.nnet.sigmoid(T.dot(dbm.H_chains[0],dbm.W[0].T)+dbm.bias_vis)) from pylearn2.datasets.mnist import MNIST dataset = MNIST(which_set = 'train') X = dataset.get_batch_design(rows*cols) model.V_chains.set_value(X) for i in xrange(200): print i sample_func() from pylearn2.gui.patch_viewer import make_viewer pv = make_viewer(dataset.adjust_for_viewer(render_func()))
except IndexError: print "Usage: predict.py <model file> <test file> <output file>" quit() try: model = serial.load(model_path) except Exception, e: print model_path + "doesn't seem to be a valid model path, I got this error when trying to load it: " print e # dataset = yaml_parse.load( model.dataset_yaml_src ) # dataset = dataset.get_test_set() # or maybe specify test in yaml dataset = MNIST(path=test_path, one_hot=True) # use smallish batches to avoid running out of memory batch_size = 100 model.set_batch_size(batch_size) # dataset must be multiple of batch size of some batches will have # different sizes. theano convolution requires a hard-coded batch size m = dataset.X.shape[0] extra = batch_size - m % batch_size assert (m + extra) % batch_size == 0 import numpy as np if extra > 0: dataset.X = np.concatenate((dataset.X, np.zeros((extra, dataset.X.shape[1]), dtype=dataset.X.dtype)), axis=0) assert dataset.X.shape[0] % batch_size == 0
from pylearn2.datasets.mnist import MNIST import os, sys import cv2 import numpy as np test_imgs = MNIST(which_set='test', center=False) test_imgs.X = np.reshape(test_imgs.X, (-1, 1, 28, 28)) dst_dir = os.path.join(os.getcwd(), 'mnist-testset') if(not os.path.isdir(dst_dir)): os.mkdir(dst_dir) print ('No of test images = {:d}'.format(test_imgs.X.shape[0])) print type(test_imgs.y[0,0]) for im in range(test_imgs.X.shape[0]): img_name = os.path.join(dst_dir, 'mnist_test_img_'+str(im)+'.pgm') print('Writing {:s}'.format(img_name)) cv2.imwrite(img_name, 255*test_imgs.X[im, 0, :, :]) with open(os.path.join(os.getcwd(), 'mnist_test_img_list.csv'), 'w') as lf: for im in range(test_imgs.y.shape[0]): lf.write('mnist_test_img_{:s}.pgm, {:d}\n'.format(str(im), int(test_imgs.y[im, :])))
print("LR_start = "+str(LR_start)) LR_fin = 0.0000003 print("LR_fin = "+str(LR_fin)) LR_decay = (LR_fin/LR_start)**(1./num_epochs) print("LR_decay = "+str(LR_decay)) # BTW, LR decay might good for the BN moving average... save_path = "mnist_parameters.npz" print("save_path = "+str(save_path)) shuffle_parts = 1 print("shuffle_parts = "+str(shuffle_parts)) print('Loading MNIST dataset...') train_set = MNIST(which_set= 'train', start=0, stop = 50000, center = False) valid_set = MNIST(which_set= 'train', start=50000, stop = 60000, center = False) test_set = MNIST(which_set= 'test', center = False) # bc01 format # Inputs in the range [-1,+1] # print("Inputs in the range [-1,+1]") train_set.X = 2* train_set.X.reshape(-1, 1, 28, 28) - 1. valid_set.X = 2* valid_set.X.reshape(-1, 1, 28, 28) - 1. test_set.X = 2* test_set.X.reshape(-1, 1, 28, 28) - 1. # flatten targets train_set.y = np.hstack(train_set.y) valid_set.y = np.hstack(valid_set.y) test_set.y = np.hstack(test_set.y)
nhid2 = 500 mf_iter = 10 batch_size = 100 lr = .1 lr_decay = 1.0001 min_lr = 5e-5 init_tv = 2. tv_mult = 1.001 max_tv = 10. l1wd = 0. l2wd = 0. l3wd = 0. sp_coeff = .000 sp_targ = .1 dataset = MNIST(which_set = 'train', one_hot = True) base_topo = dataset.get_topological_view() transformed_dataset = MNIST(which_set = 'train', one_hot = True) transformed_topo = base_topo[:,remove_border:-remove_border,remove_border:-remove_border,:].copy() transformed_dataset.set_topological_view(transformed_topo) transformed_topo X = sharedX(transformed_dataset.X) rng = np.random.RandomState([2012,7,24]) W1 = rng.uniform(-irange1,irange1,(nvis,nhid1)) b1 = np.zeros((nhid1,))-1.
from pylearn2.costs.cost import MethodCost from pylearn2.datasets.mnist import MNIST from pylearn2.models.mlp import MLP, Sigmoid, Softmax from pylearn2.train import Train from pylearn2.training_algorithms.sgd import SGD from pylearn2.training_algorithms.learning_rule import Momentum, MomentumAdjustor from pylearn2.termination_criteria import EpochCounter train_set = MNIST(which_set='train', start=0, stop=50000) valid_set = MNIST(which_set='train', start=50000, stop=60000) test_set = MNIST(which_set='test') model = MLP(nvis=784, layers=[Sigmoid(layer_name='h', dim=500, irange=0.01), Softmax(layer_name='y', n_classes=10, irange=0.01)]) algorithm = SGD(batch_size=100, learning_rate=0.01, learning_rule=Momentum(init_momentum=0.5), monitoring_dataset={'train': train_set, 'valid': valid_set, 'test': test_set}, cost=MethodCost('cost_from_X'), termination_criterion=EpochCounter(10)) train = Train(dataset=train_set, model=model, algorithm=algorithm, save_path="mnist_example.pkl", save_freq=1, extensions=[MomentumAdjustor(start=5, saturate=6, final_momentum=0.95)]) train.main_loop()
def test_ais(): """ Test ais computation by comparing the output of estimate_likelihood to Russ's code's output for the same parameters. """ try: trainset = MNIST(which_set='train') testset = MNIST(which_set='test') except NoDataPathError: raise SkipTest("PYLEARN2_DATA_PATH environment variable not defined") nvis = 784 nhid = 20 # Random initialization of RBM parameters numpy.random.seed(98734) w_hid = 10 * numpy.cast[theano.config.floatX](numpy.random.randn(nvis, nhid)) b_vis = 10 * numpy.cast[theano.config.floatX](numpy.random.randn(nvis)) b_hid = 10 * numpy.cast[theano.config.floatX](numpy.random.randn(nhid)) # Initialization of RBM visible_layer = BinaryVector(nvis) hidden_layer = BinaryVectorMaxPool(detector_layer_dim=nhid, pool_size=1, layer_name='h', irange=0.1) rbm = DBM(100, visible_layer, [hidden_layer], 1) rbm.visible_layer.set_biases(b_vis) rbm.hidden_layers[0].set_weights(w_hid) rbm.hidden_layers[0].set_biases(b_hid) rbm.nvis = nvis rbm.nhid = nhid # Compute real logz and associated train_ll and test_ll using rbm_tools v_sample = T.matrix('v_sample') h_sample = T.matrix('h_sample') W = theano.shared(rbm.hidden_layers[0].get_weights()) hbias = theano.shared(rbm.hidden_layers[0].get_biases()) vbias = theano.shared(rbm.visible_layer.get_biases()) wx_b = T.dot(v_sample, W) + hbias vbias_term = T.dot(v_sample, vbias) hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1) free_energy_v = -hidden_term - vbias_term free_energy_v_fn = theano.function(inputs=[v_sample], outputs=free_energy_v) wh_c = T.dot(h_sample, W.T) + vbias hbias_term = T.dot(h_sample, hbias) visible_term = T.sum(T.log(1 + T.exp(wh_c)), axis=1) free_energy_h = -visible_term - hbias_term free_energy_h_fn = theano.function(inputs=[h_sample], outputs=free_energy_h) real_logz = rbm_tools.compute_log_z(rbm, free_energy_h_fn) real_ais_train_ll = -rbm_tools.compute_nll(rbm, trainset.get_design_matrix(), real_logz, free_energy_v_fn) real_ais_test_ll = -rbm_tools.compute_nll(rbm, testset.get_design_matrix(), real_logz, free_energy_v_fn) # Compute train_ll, test_ll and logz using dbm_metrics train_ll, test_ll, logz = dbm_metrics.estimate_likelihood([W], [vbias, hbias], trainset, testset, pos_mf_steps=100) assert (real_logz - logz) < 2.0 assert (real_ais_train_ll - train_ll) < 2.0 assert (real_ais_test_ll - test_ll) < 2.0
num_pieces = 1, kernel_shape = (4, 4), pool_shape = (1, 1), pool_stride=(1, 1), irange = 0.05) deconv = Deconv(layer_name = 'deconv', num_channels = 1, kernel_shape = (4, 4), irange = 0.05) mlp = MLP(input_space =input_space, layers = [conv, deconv]) mlp.layers[1].transformer._filters.set_value(mlp.layers[0].transformer._filters.get_value()) x = input_space.get_theano_batch() out = mlp.fprop(x) f = theano.function([x], out) data = MNIST('test') data_specs = (input_space, 'features') iter = data.iterator(mode = 'sequential', batch_size = 2, data_specs = data_specs) pv = patch_viewer.PatchViewer((10, 10), (28, 28)) for item in iter: res = f(item) pv.add_patch(item[0,:,:,0]) pv.add_patch(res[0,:,:,0]) pv.show() break
def main(): data_name, model, pretrain, test, filter_channel, \ filter_size, activation, sparse_coeff, is_tied_weights, \ is_linear, do_zca, do_scale, do_maxpoool, n_epochs, batch_size = ProcessCommandLine() print '... Loading data and parameters' print 'dataset: ', data_name print 'filter_channel=', filter_channel print 'filter_size=', filter_size print 'activation=', activation print 'sparsity coefficient=', sparse_coeff print 'Max pooling=', do_maxpoool print 'tied weight=', is_tied_weights print 'linear CAE=', is_linear print 'ZCA whitening=', do_zca print 'Scale image=', do_scale print 'Batch size=', batch_size print 'number of epoch=', n_epochs # batch_size = 100 # number of images in each batch n_epochs = 100 # number of experiment epochs learning_rate = 0.1 # learning rate of SGD # filter_channel = 16 # number of feature maps in ConvAE # dataset = 'data/mnist.pkl.gz' # address of data # rng = np.random.RandomState(23455) # random generator # filter_size = 11 # n_images = 20 # sparse_coeff = 1 if sparse_coeff == 0: model_type = 'cae' else: model_type = 'spcae' model_save_name = model_type+'_'+data_name+'_'+ \ '[fn=%d,fs=%d,sp=%.3f,maxpool=%s,tied=%s,act=%s,linear=%s,ZCA=%s,scale=%s]' \ % (filter_channel, filter_size, sparse_coeff, do_maxpoool, is_tied_weights, activation, is_linear, do_zca, do_scale) results_dir = model_save_name+'/' # results_dir = data_name+'_'+model_name+'/' if not os.path.isdir(results_dir): os.mkdir(results_dir) if data_name == 'smallNORB': ### load smallNORB train set ### # norb = SmallNORB('train', True) # data_x = norb.adjust_for_viewer(norb.X) # data_y = norb.y # pickle.dump((data_x, data_y),open('smallNORB.pkl','wb'), -1) # results_dir = 'smallNORB_scae/' # results_dir = 'smallNORB_'+model_name+'/' # if not os.path.isdir(results_dir): # os.mkdir(results_dir) # f = open('smallNORB.pkl', 'r') # data, data_y = pickle.load(f) # _, feat = data.shape # f.close() # train = NORB(which_norb='small', which_set='train') # ipdb.set_trace() # window = preprocessing.CentralWindow(window_shape=(64,64)) # train.apply_preprocessor(preprocessor=window) # train.X = train.X.astype('float32') # zca = preprocessing.ZCA() # train.apply_preprocessor(preprocessor=zca, can_fit=True) # _, feat = train.X.shape # data_x = train.X[:, :feat/2] norb = sio.loadmat('smallNORB_matlab/smallNORB_train_32x32.mat') train_x = norb['trainData'].transpose(1, 0) if do_zca: zca = zca_whitening.ZCA() zca.fit(train_x) train_x = zca.transform(train_x) if do_scale: min_max_scaler = MinMaxScaler() train_x_T = min_max_scaler.fit_transform(train_x.T) train_x = train_x_T.T data_x = train_x idx = random.shuffle(range(data_x.shape[0])) data_x = data_x[idx, :][0,:,:] # ipdb.set_trace() im_channel = 1 im_height = 32 im_width = 32 elif data_name == 'cifar': ### load cifar10 ### # results_dir = 'cifar'+model_name+'/' # data_x = pickle.load(open('cifar10.pkl', 'r')) train = CIFAR10('train', gcn=55.) im_channel = 3 im_height = 32 im_width = 32 # min_max_scaler = MinMaxScaler() # data_x = min_max_scaler.fit_transform(data_x) # data_x = cifar10.X # elif data_name == 'cifarw': # ### load cifar10 ### # # results_dir = 'cifar'+model_name+'/' # # data_x = pickle.load(open('cifar10_whitened.pkl', 'r')) # train = CIFAR10('train', gcn=55.) # # zca = preprocessing.ZCA() # # cifar10.apply_preprocessor(preprocessor=zca, can_fit=True) # im_channel = 3 # im_height = 32 # im_width = 32 # min_max_scaler = MinMaxScaler() # data_x = min_max_scaler.fit_transform(cifar10.X) # data_x = cifar10.X # ipdb.set_trace() elif data_name == 'svhn': f = open('svhn.pkl', 'r') svhn = pickle.load(f) f.close() im_channel = 3 im_height = 32 im_width = 32 train_x, train_y = svhn[0] test_x, test_y = svhn[1] extra_x, extra_y = svhn[2] train_x = train_x.transpose([3, 2, 0, 1]) test_x = test_x.transpose([3, 2, 0, 1]) extra_x = extra_x.transpose([3, 2, 0, 1]) # Scale to [0,1] channel_scale_factor = train_x.max(axis=(2, 3)).astype('float32') train_x_scaled = train_x/channel_scale_factor.reshape(channel_scale_factor.shape[0], im_channel, 1, 1) data_x = train_x_scaled.reshape(train_x.shape[0], im_channel*im_height*im_width) elif data_name == 'mnist': # f = open('mnist.pkl', 'r') # mnist = pickle.load(f) # f.close() # train_x, train_y = mnist[0] # valid_x, valid_y = mnist[1] # test_x, test_y = mnist[2] # data_x = train_x train = MNIST('train') # zca = preprocessing.ZCA() # train.apply_preprocessor(preprocessor=zca, can_fit=True) # ipdb.set_trace() # min_max_scaler = MinMaxScaler() # data_x = min_max_scaler.fit_transform(train.X) # data_x = train.X # data_y = train.y im_channel = 1 im_height = 28 im_width = 28 elif data_name == 'bmnist': train = BinarizedMNIST(which_set='train') # zca = preprocessing.ZCA() # train.apply_preprocessor(preprocessor=zca, can_fit=True) # ipdb.set_trace() # min_max_scaler = MinMaxScaler() # data_x = min_max_scaler.fit_transform(train.X) # data_x = train.X # data_y = train.y im_channel = 1 im_height = 28 im_width = 28 if do_zca and data_name not in ['smallNORB', 'svhn']: zca = preprocessing.ZCA() train.apply_preprocessor(preprocessor=zca, can_fit=True) data_x = train.X pass if do_scale and data_name not in ['smallNORB', 'svhn']: min_max_scaler = MinMaxScaler() data_x_T = min_max_scaler.fit_transform(train.X.T) data_x = data_x_T.T pass if not do_zca and not do_scale and data_name not in ['smallNORB', 'svhn']: data_x = train.X # if data_name not in ['smallNORB']: # data_x = train.X n_samples, n_feat = data_x.shape data_x = data_x.reshape((n_samples, im_channel, im_height, im_width)) if data_name == 'mnist': data_x = data_x.transpose(0,1,3,2) train_set_x = theano.shared(np.asarray(data_x, dtype=np.float32), borrow=True) # image_shp = (batch_size, im_channel, data_x.shape[2], data_x.shape[3]) image_shp = (batch_size, im_channel, im_height, im_width) filter_shp = (filter_channel, im_channel, filter_size, filter_size) print 'building model' cae1 = CAE(image_shape=image_shp, data=train_set_x, filter_shape=filter_shp, poolsize=(2, 2), sparse_coeff=sparse_coeff, activation=activation, do_max_pool=do_maxpoool, tied_weight=is_tied_weights, is_linear=is_linear) print 'model built' sys.stdout.flush() if model: cae1.load(model) pass if pretrain: do_pretraining_cae(data_name=data_name, model=cae1, save_name=model_save_name, image_shape=image_shp, result_dir=results_dir, max_epoch=n_epochs) elif test: do_visualize(data_name=data_name, model=cae1, result_dir=results_dir)
# MAIN # stochastic learning rate works with continuous weights # multilayer does not work so far # discrete weights + continuous bias do work for one layer # batch and activations -> only update the most wrong weight if __name__ == "__main__": print 'Beginning of the program' start_time = time.clock() print 'Loading the dataset' train_set = MNIST(which_set= 'train', start=0, stop = 50000, center = True) valid_set = MNIST(which_set= 'train', start=50000, stop = 60000, center = True) test_set = MNIST(which_set= 'test', center = True) # for both datasets, onehot the target train_set.y = np.float32(onehot(train_set.y)) valid_set.y = np.float32(onehot(valid_set.y)) test_set.y = np.float32(onehot(test_set.y)) train_set.y = 2* train_set.y - 1. valid_set.y = 2* valid_set.y - 1. test_set.y = 2* test_set.y - 1. # print train_set.X # print np.shape(train_set.X) # print np.max(train_set.X)
job_name = 'cfa_test' from pylearn2.datasets.mnist import MNIST from pylearn2.pca import CovEigPCA import theano.tensor as T from theano import function from models import expand from pylearn2.utils import serial import time import SkyNet import gc import numpy as N print 'Loading MNIST train set' t1 = time.time() X = MNIST(which_set='train').get_design_matrix() t2 = time.time() print(t2 - t1), ' seconds' num_examples, input_dim = X.shape print 'Training PCA with %d dimensions' % pca_dim t1 = time.time() pca_model = CovEigPCA(num_components=pca_dim) pca_model.train(X) t2 = time.time() print(t2 - t1), ' seconds' print 'Compiling theano PCA function' t1 = time.time() pca_input = T.matrix()
print("batch_size = "+str(batch_size)) # MLP parameters num_units = 4096 print("num_units = "+str(num_units)) n_hidden_layers = 3 print("n_hidden_layers = "+str(n_hidden_layers)) # kernel = "baseline" kernel = "xnor" # kernel = "theano" print("kernel = "+ kernel) print('Loading MNIST dataset...') test_set = MNIST(which_set= 'test', center = False) # Inputs in the range [-1,+1] test_set.X = 2* test_set.X.reshape(-1, 784) - 1. # flatten targets test_set.y = test_set.y.reshape(-1) print('Building the MLP...') # Prepare Theano variables for inputs and targets input = T.matrix('inputs') target = T.vector('targets') mlp = lasagne.layers.InputLayer(shape=(None, 784),input_var=input) # Input layer is not binary -> use baseline kernel in first hidden layer mlp = binary_ops.DenseLayer(
def setUp(self): skip_if_no_data() # TODO: the one_hot=True is only necessary because one_hot=False is # broken, remove it after one_hot=False is fixed. self.train = MNIST(which_set = 'train', one_hot=True) self.test = MNIST(which_set = 'test', one_hot=True)
#gets #doing epoch 3999 # TRAIN acc [ 0.99998333] # terminal velocity multiplier: 10.0 # learning rate: 0.0670400485022 # test acc: [array(0.9832)] from pylearn2.datasets.mnist import MNIST from theano.printing import Print import numpy as np from cDBM import cDBM import sys name = sys.argv[0].replace('.py','') remove_border = 3 dataset = MNIST(which_set = 'train', one_hot = True) topo = dataset.get_topological_view() topo = topo[:,remove_border:-remove_border,remove_border:-remove_border,:] dataset.set_topological_view(topo) rng = np.random.RandomState([2012,07,24]) irange1 = .05 irange2 = .05 irange3 = .05 nvis = (28-2*remove_border)**2 nclass = 10 nhid1 = 500 nhid2 = 500 mf_iter = 10 batch_size = 100 lr = .1
return result # MAIN if __name__ == "__main__": print 'Beginning of the program' start_time = time.clock() print 'Loading the dataset' dataset = sys.argv[1] if dataset == "PI_MNIST" or dataset == "MNIST": train_set = MNIST(which_set= 'train',start=0, stop = 50000)#, center = True) valid_set = MNIST(which_set= 'train',start=50000, stop = 60000)#, center = True) test_set = MNIST(which_set= 'test')#, center = True) # for both datasets, onehot the target train_set.y = np.float32(onehot(train_set.y)) valid_set.y = np.float32(onehot(valid_set.y)) test_set.y = np.float32(onehot(test_set.y)) elif dataset == "CIFAR10": preprocessor = serial.load("${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/preprocessor.pkl") train_set = ZCA_Dataset( preprocessed_dataset=serial.load("${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/train.pkl"), preprocessor = preprocessor, start=0, stop = 45000)
from pylearn2.termination_criteria import MonitorBased from pylearn2.train_extensions import best_params from pylearn2.utils import serial from theano import function from theano import tensor as T import numpy as np import os h0 = mlp.Sigmoid(layer_name='h0', dim=500, sparse_init=15) ylayer = mlp.Softmax(layer_name='y', n_classes=10, irange=0) layers = [h0, ylayer] model = mlp.MLP(layers, nvis=784) train = MNIST('train', one_hot=1, start=0, stop=50000) valid = MNIST('train', one_hot=1, start=50000, stop=60000) test = MNIST('test', one_hot=1, start=0, stop=10000) monitoring = dict(valid=valid) termination = MonitorBased(channel_name="valid_y_misclass") extensions = [best_params.MonitorBasedSaveBest(channel_name="valid_y_misclass", save_path="train_best.pkl")] algorithm = bgd.BGD(batch_size=10000, line_search_mode = 'exhaustive', conjugate = 1, monitoring_dataset = monitoring, termination_criterion = termination) save_path = "train_best.pkl" if os.path.exists(save_path): model = serial.load(save_path) else: print 'Running training' train_job = Train(train, model, algorithm, extensions=extensions, save_path="train.pkl", save_freq=1) train_job.main_loop()
def main(): #import pdb #pdb.set_trace() ################### #BUILD THE DATASET# ################### print 'build the dataset' train_set = MNIST(which_set='train', one_hot=False) test_set = MNIST(which_set='test', one_hot=False) train_setX, valid_setX = split(train_set.X, [50000], axis=0) train_sety, valid_sety = split(train_set.y, [50000], axis=0) #import pdb #pdb.set_trace() ################## #BUILD THE LAYERS# ################## print 'build the layers' input_size = len(train_setX[0]) h1 = NoisyRELU(prev_layer_size=input_size, this_layer_size=2000, noise_factor=1, threshold=5) output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10, W_range=[0,0], b_range=[0,0]) #y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1]) # import pdb # pdb.set_trace() #, 0.1, 0.5, 0.001, 0.005 plots = [] legends = [] batches = [] errors = [] for learning_rate in [0.01, 0.05]: print 'build the model' print 'learning rate', learning_rate mlp = MLP(input_size=input_size, layers=[h1, output_layer], train_set=[train_setX, train_sety], valid_set=[valid_setX, valid_sety], test_set=[test_set.X, test_set.y], error_function=loglikehood, batch_size=20, learning_rate=learning_rate) print 'start training' mlp.train() #p = plt.plot(mlp.epoch, mlp.valid_error) #plots.append(p) batches.append(mlp.epoch) errors.append(mlp.valid_error) legends.append(learning_rate) with open('batches.pkl', 'wb') as bat: cPickle.dump(batches, bat) with open('errors.pkl', 'wb') as err: cPickle.dump(errors, err) with open('legends.pkl', 'wb') as leg: cPickle.dump(legends, leg)
# Old hyperparameters binary_training=False stochastic_training=False binary_test=False stochastic_test=False if BinaryConnect == True: binary_training=True if stochastic == True: stochastic_training=True else: binary_test=True print 'Loading the dataset' train_set = MNIST(which_set= 'train', start=0, stop = train_set_size, center = True) valid_set = MNIST(which_set= 'train', start=50000, stop = 60000, center = True) test_set = MNIST(which_set= 'test', center = True) # bc01 format train_set.X = train_set.X.reshape(train_set_size,1,28,28) valid_set.X = valid_set.X.reshape(10000,1,28,28) test_set.X = test_set.X.reshape(10000,1,28,28) # flatten targets train_set.y = np.hstack(train_set.y) valid_set.y = np.hstack(valid_set.y) test_set.y = np.hstack(test_set.y) # Onehot the targets train_set.y = np.float32(np.eye(10)[train_set.y])