def AutoEncoder_demo(learning_rate=0.1, training_epochs=2, dataset='mnist.pkl.gz', batch_size=20, output_folder='dA_plots'): datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size index = T.lscalar() # index to a [mini]batch x = T.matrix('x') if not os.path.isdir(output_folder): os.makedirs(output_folder) os.chdir(output_folder) ##################################### # BUILDING THE MODEL CORRUPTION 0% # ##################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2**30)) da = AutoEncoder(np_rng=rng, theano_rng=theano_rng, input=x, n_vis=28 * 28, n_hid=500) cost, updates = da.get_cost_updates(corruption_level=0., learning_rate=learning_rate) train_da = theano.function( inputs=[index], outputs=[cost], updates=updates, givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]}) start_time = time.clock() for epoch in xrange(training_epochs): # go through trainng set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> sys.stderr, ('The no corruption code ran for %.2fm' % ((training_time) / 60.)) image = PIL.Image.fromarray( tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_0.jpg') ##################################### # BUILDING THE MODEL CORRUPTION 30% # ##################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2**30)) da = AutoEncoder(np_rng=rng, theano_rng=theano_rng, input=x, n_vis=28 * 28, n_hid=500) cost, updates = da.get_cost_updates(corruption_level=0.3, learning_rate=learning_rate) train_da = theano.function( inputs=[index], outputs=[cost], updates=updates, givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]}) start_time = time.clock() for epoch in xrange(training_epochs): c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> sys.stderr, ('The 30 percent corruption code ran for %.2fm' % ((training_time) / 60.)) image = PIL.Image.fromarray( tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_30.jpg') os.chdir('../')
def test_pickled_sqe_dA(learning_rate=0.001, pickle_file='/scratch/z/zhaolei/lzamparo/gpu_tests/dA_results/dA_sqe_pickle.save', corruption=0.1, training_epochs=3, batch_size=20): """ Test creating a dA model from scratch, training for a set number of epochs, pickle the model, unpickle, continue. """ current_dir = os.getcwd() os.chdir(options.dir) today = datetime.today() day = str(today.date()) hour = str(today.time()) output_filename = "test_dA_squarederror_pickle." + day + "." + hour output_file = open(output_filename,'w') print >> output_file, "Run on " + str(datetime.now()) os.chdir(current_dir) data_set_file = openFile(str(options.inputfile), mode = 'r') datafiles, labels = extract_labeled_chunkrange(data_set_file, num_files = 10) datasets = load_data_labeled(datafiles, labels) train_set_x, train_set_y = datasets[0] data_set_file.close() # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size n_cols = train_set_x.get_value(borrow=True).shape[1] # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data matrix #################################### # BUILDING THE MODEL # #################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = AutoEncoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=n_cols, n_hidden=1000, loss='squared') cost, updates = da.get_cost_updates(corruption_level=float(options.corruption), learning_rate=learning_rate) train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) start_time = time.clock() ############ # TRAINING # ############ # go through training epochs for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print >> output_file, 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> output_file, ('The 0 corruption code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((training_time) / 60.)) ############ # Pickle # ############ f = file(pickle_file, 'wb') cPickle.dump(da, f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() ############ # Unpickle the model, try to recover # ############ f = file(pickle_file, 'rb') pickled_dA = cPickle.load(f) f.close() x = T.matrix('x') pickled_dA.set_input(x) ############ # Resume training # ############ cost, updates = pickled_dA.get_cost_updates(corruption_level=float(options.corruption), learning_rate=learning_rate) train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) start_time = time.clock() ############ # TRAINING # ############ # go through training epochs for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print >> output_file, 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> output_file, ('The 0 corruption code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((training_time) / 60.)) output_file.close()
def test_pickled_dA(learning_rate=0.1, dataset='../data/mnist.pkl.gz', pickle_file='/scratch/z/zhaolei/lzamparo/gpu_tests/dA_results/dA_pickle.save', corruption=0.1, training_epochs=3, batch_size=20): """ Test pickling, unpickling code for the dA class. Start up a model, train, pickle, unpickle, and continue to train """ datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images #################################### # Build the model # #################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = AutoEncoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=28 * 28, n_hidden=500, loss='xent') cost, updates = da.get_cost_updates(corruption_level=0., learning_rate=learning_rate) train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) ############ # Train the model for 3 epochs # ############ # go through training epochs for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) ############ # Pickle the model # ############ f = file(pickle_file, 'wb') cPickle.dump(da, f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() ############ # Unpickle the model, try to recover # ############ f = file(pickle_file, 'rb') pickled_dA = cPickle.load(f) f.close() ############ # Compare the two models # ########### dA_params = da.get_params() pickled_params = pickled_dA.get_params() if not numpy.allclose(dA_params[0].get_value(), pickled_params[0].get_value()): print "numpy says that Ws are not close" if not numpy.allclose(dA_params[1].get_value(), pickled_params[1].get_value()): print "numpy says that the bvis are not close" if not numpy.allclose(dA_params[2].get_value(), pickled_params[2].get_value()): print "numpy says that the bhid are not close" ############ # Compare the two models # ########## pickled_dA.set_input(x) cost, updates = pickled_dA.get_cost_updates(corruption_level=0.1, learning_rate=learning_rate) pickle_train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) ############ # Train the model for 3 epochs # ############ # go through training epochs for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print "Passed create, pickle, unpickle, train test"
def drive_dA(learning_rate=0.1, training_epochs=15, dataset='../data/mnist.pkl.gz', batch_size=20): """ This demo is tested on MNIST :type learning_rate: float :param learning_rate: learning rate used for training the DeNosing AutoEncoder :type training_epochs: int :param training_epochs: number of epochs used for training :type dataset: string :param dataset: path to the picked dataset """ parser = OptionParser() parser.add_option("-d", "--dir", dest="dir", help="test output directory") parser.add_option("-c", "--corruption", dest="corruption", help="use this amount of corruption for the denoising AE") (options, args) = parser.parse_args() current_dir = os.getcwd() os.chdir(options.dir) today = datetime.today() day = str(today.date()) hour = str(today.time()) output_filename = "denoising_autoencoder_mnist." + day + "." + hour output_file = open(output_filename,'w') print >> output_file, "Run on " + str(datetime.now()) os.chdir(current_dir) datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images #################################### # BUILDING THE MODEL NO CORRUPTION # #################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = AutoEncoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=28 * 28, n_hidden=500) cost, updates = da.get_cost_updates(corruption_level=0., learning_rate=learning_rate) train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) start_time = time.clock() ############ # TRAINING # ############ # go through training epochs for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print >> output_file, 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> output_file, ('The 0 corruption code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((training_time) / 60.)) ########## # Build the model, with corruption ########## rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = AutoEncoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=28 * 28, n_hidden=500) cost, updates = da.get_cost_updates(corruption_level=float(options.corruption), learning_rate=learning_rate) train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) start_time = time.clock() ########## # Train the model ########## # go through training epochs for epoch in xrange(training_epochs): # go through trainng set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print >> output_file, 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> output_file, ('The ' + str(options.corruption) + '% corruption code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((training_time) / 60.)) output_file.close()
def test_AutoEncoder(learning_rate=0.1, training_epochs=15, batch_size=20): """ :type learning_rate: float :param learning_rate: learning rate used for training the DeNosing AutoEncoder :type training_epochs: int :param training_epochs: number of epochs used for training """ x_scipySparse = None; train_set_x = None; numInstances = 0; numFeatures = 0; if((os.path.exists("input_scipySparse.obj"))): print "loading sparse data from pickled file..." f = open("input_scipySparse.obj", 'r') x_scipySparse = cPickle.load(f) f.close() numInstances, numFeatures = x_scipySparse.shape else: print "extracting features and building sparse data..." fe = FeatureExtractor() fe.extractFeatures() train_set_x = fe.instanceList featureDict = fe.featDict numInstances = len(train_set_x) numFeatures = len(featureDict) x_lil = sp.lil_matrix((numInstances,numFeatures), dtype='float32') # the data is presented as a sparse matrix i = -1; v = -1; try: for i,instance in enumerate(train_set_x): for v in instance.input: x_lil[i, v] = 1 except: print "i=",i," v=",v x_scipySparse = x_lil.tocsc() f = open("input_scipySparse.obj", 'w') cPickle.dump(x_scipySparse, f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # compute number of mini-batches for training, validation and testing n_train_batches = numInstances / batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch #x = sparse.basic.as_sparse_variable(x_scipySparse, 'x') x = theano.shared(x_scipySparse, borrow=True) #################################### # BUILDING THE MODEL # #################################### print "building the model..." rng = numpy.random.RandomState(123) ae = AutoEncoder(numpy_rng=rng, input=x, n_visible=numFeatures, n_hidden=10, n_trainExs=numInstances) cost, updates = ae.get_cost_updates(corruption_level=0., learning_rate=learning_rate) train_ae = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) start_time = time.clock() ############ # TRAINING # ############ # go through training epochs print "starting training..." for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_ae(batch_index)) print 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print "training completed in : ", training_time
def run_generative_model( learning_rate=0.1, dataset='mnist.pkl.gz', n_epochs=5, batch_size=20, display_step=1000, n_visible=28 * 28, # MNIST Pixels n_hidden=500, corruption_level=0.3, # DA contraction_level=0.1, # CA k=5, # RBM chains=10, # RBM output_folder='Generative_plots', img_shape=(28, 28), # image shape of MNIST for tile_raster_images model_name='AutoEncoder', ): """ This demo is tested on MNIST :type learning_rate: float :param learning_rate: learning rate used for training the DeNosing AutoEncoder :type n_epochs: int :param n_epochs: number of epochs used for training :type dataset: string :param dataset: path to the picked dataset """ # numpy random generator rng = np.random.RandomState(123) # create a Theano random generator that gives symbolic random values theano_rng = RandomStreams(rng.randint(2**30)) if not os.path.isdir(output_folder): os.makedirs(output_folder) os.chdir(output_folder) ############# # Load Data # ############# datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] # valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] ################################### # Calculate number of Minibatches # ################################### n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size # n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size ############################################ # allocate symbolic variables for the data # ############################################ # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images ############### # BUILD MODEL # ############### print('... building the model') if model_name == 'AutoEncoder': model = AutoEncoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=n_visible, n_hidden=n_hidden) elif model_name == 'DA': model = DA(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=n_visible, n_hidden=n_hidden) elif model_name == 'CA': model = CA(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=n_visible, n_hidden=n_hidden, batch_size=batch_size) elif model_name == 'RBM': model = RBM(input=x, numpy_rng=rng, theano_rng=theano_rng, n_visible=n_visible, n_hidden=n_hidden) ##################### # Training Function # ##################### # COST & UPDATES if model_name == 'AutoEncoder': cost, updates = model.get_cost_updates(learning_rate=learning_rate) elif model_name == 'DA': cost, updates = model.get_cost_updates( corruption_level=corruption_level, learning_rate=learning_rate) elif model_name == 'CA': cost, updates = model.get_cost_updates( contraction_level=contraction_level, learning_rate=learning_rate) elif model_name == 'RBM': # initialize storage for the persistent chain (state = hidden layer of chain) persistent_chain = theano.shared(np.zeros(shape=(batch_size, model.n_hidden), dtype=theano.config.floatX), borrow=True) # get the cost and the gradient corresponding to one step of CD-15 cost, updates = model.get_cost_updates(learning_rate=learning_rate, persistent=persistent_chain, k=k) # TRAINING FUNCTION train_model = theano.function( inputs=[index], outputs=cost, updates=updates, givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]}) ############### # TRAIN MODEL # ############### print('... training') plotting_time = 0. start_time = timeit.default_timer() # go through training epochs for epoch in range(n_epochs): minibatch_avg_cost = [] for minibatch_index in range(n_train_batches): minibatch_avg_cost.append(train_model(minibatch_index)) # iteration number iter = epoch * n_train_batches + minibatch_index if iter % display_step == 0: print('training @ iter = ', iter) print('Training epoch %d, cost ' % epoch, np.mean(minibatch_avg_cost, dtype='float64')) # Plot filters after each training epoch plotting_start = timeit.default_timer() # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=model.W.get_value(borrow=True).T, img_shape=img_shape, tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_at_epoch_%i.png' % epoch) plotting_stop = timeit.default_timer() plotting_time += (plotting_stop - plotting_start) end_time = timeit.default_timer() pretraining_time = (end_time - start_time) - plotting_time print('Training took %f minutes' % (pretraining_time / 60.)) print( ('The code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr) image = Image.fromarray( tile_raster_images(X=model.W.get_value(borrow=True).T, img_shape=img_shape, tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('trained_filters.png') ################################# # Sampling from the Model # ################################# #if model_name == 'RBM': # sample_RBM(model=model, test_set_x=test_set_x, chains=20) #################### # Change Directory # #################### os.chdir('../')
def AutoEncoder_demo(learning_rate=0.1, training_epochs=2, dataset='mnist.pkl.gz', batch_size=20, output_folder='dA_plots'): datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size index = T.lscalar() # index to a [mini]batch x = T.matrix('x') if not os.path.isdir(output_folder): os.makedirs(output_folder) os.chdir(output_folder) ##################################### # BUILDING THE MODEL CORRUPTION 0% # ##################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = AutoEncoder(np_rng=rng, theano_rng=theano_rng, input=x, n_vis=28 * 28, n_hid=500) cost, updates = da.get_cost_updates(corruption_level=0., learning_rate=learning_rate) train_da = theano.function(inputs=[index], outputs=[cost], updates=updates, givens={x: train_set_x[index * batch_size: (index + 1) * batch_size]}) start_time = time.clock() for epoch in xrange(training_epochs): # go through trainng set c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> sys.stderr, ('The no corruption code ran for %.2fm' % ((training_time) / 60.)) image = PIL.Image.fromarray(tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_0.jpg') ##################################### # BUILDING THE MODEL CORRUPTION 30% # ##################################### rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = AutoEncoder(np_rng=rng, theano_rng=theano_rng, input=x, n_vis=28 * 28, n_hid=500) cost, updates = da.get_cost_updates(corruption_level=0.3, learning_rate=learning_rate) train_da = theano.function(inputs=[index], outputs=[cost], updates=updates, givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]}) start_time = time.clock() for epoch in xrange(training_epochs): c = [] for batch_index in xrange(n_train_batches): c.append(train_da(batch_index)) print 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print >> sys.stderr, ('The 30 percent corruption code ran for %.2fm' % ((training_time) / 60.)) image = PIL.Image.fromarray(tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_30.jpg') os.chdir('../')
def test_AutoEncoder(learning_rate=0.1, training_epochs=15, batch_size=20): """ :type learning_rate: float :param learning_rate: learning rate used for training the DeNosing AutoEncoder :type training_epochs: int :param training_epochs: number of epochs used for training """ x_scipySparse = None train_set_x = None numInstances = 0 numFeatures = 0 if ((os.path.exists("input_scipySparse.obj"))): print "loading sparse data from pickled file..." f = open("input_scipySparse.obj", 'r') x_scipySparse = cPickle.load(f) f.close() numInstances, numFeatures = x_scipySparse.shape else: print "extracting features and building sparse data..." fe = FeatureExtractor() fe.extractFeatures() train_set_x = fe.instanceList featureDict = fe.featDict numInstances = len(train_set_x) numFeatures = len(featureDict) x_lil = sp.lil_matrix( (numInstances, numFeatures), dtype='float32') # the data is presented as a sparse matrix i = -1 v = -1 try: for i, instance in enumerate(train_set_x): for v in instance.input: x_lil[i, v] = 1 except: print "i=", i, " v=", v x_scipySparse = x_lil.tocsc() f = open("input_scipySparse.obj", 'w') cPickle.dump(x_scipySparse, f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # compute number of mini-batches for training, validation and testing n_train_batches = numInstances / batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch #x = sparse.basic.as_sparse_variable(x_scipySparse, 'x') x = theano.shared(x_scipySparse, borrow=True) #################################### # BUILDING THE MODEL # #################################### print "building the model..." rng = numpy.random.RandomState(123) ae = AutoEncoder(numpy_rng=rng, input=x, n_visible=numFeatures, n_hidden=10, n_trainExs=numInstances) cost, updates = ae.get_cost_updates(corruption_level=0., learning_rate=learning_rate) train_ae = theano.function( [index], cost, updates=updates, givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]}) start_time = time.clock() ############ # TRAINING # ############ # go through training epochs print "starting training..." for epoch in xrange(training_epochs): # go through training set c = [] for batch_index in xrange(n_train_batches): c.append(train_ae(batch_index)) print 'Training epoch %d, cost ' % epoch, numpy.mean(c) end_time = time.clock() training_time = (end_time - start_time) print "training completed in : ", training_time