def get_representations_for_joint_layer(yaml_file_path, save_path, batch_size): """ The purpose of doing this is to test the compatibility of DBM with StackBlocks and TransformerDatasets Of course one can instead take use the "get_represenations.py" for data preparation for the next step. """ hyper_params = {'save_path':save_path} yaml = open("{0}/stacked_image_unimodaliy.yaml".format(yaml_file_path), 'r').read() yaml = yaml % (hyper_params) image_stacked_blocks = yaml_parse.load(yaml) yaml = open("{0}/stacked_text_unimodaliy.yaml".format(yaml_file_path), 'r').read() yaml = yaml % (hyper_params) text_stacked_blocks = yaml_parse.load(yaml) image_raw = Flickr_Image_Toronto(which_cat = 'unlabelled',which_sub='nnz', using_statisfile = True) image_rep = TransformerDataset( raw = image_raw, transformer = image_stacked_blocks ) m, n = image_raw.get_data().shape() dw = data_writer.DataWriter(['image_h2_rep'], save_path + 'image/', '10G', [n], m) image_iterator = image_rep.iterator(batch_size= batch_size) for data in image_iterator: dw.Submit(data) dw.Commit() text_raw = Flickr_Text_Toronto(which_cat='unlabelled') text_rep = TransformerDataset( raw = text_raw, transformer = text_stacked_blocks ) m, n = text_raw.get_data().shape() dw = data_writer.DataWriter(['text_h2_rep'], save_path + 'text/', '10G', [n], m) text_iterator = text_rep.iterator(batch_size= batch_size) for data in text_iterator: dw.Submit(data) dw.Commit()
def main(): parser = OptionParser() parser.add_option("-d", "--data", dest="dataset", default="toy", help="specify the dataset, either cifar10 or toy") (options, args) = parser.parse_args() global SAVE_MODEL if options.dataset == 'toy': trainset, testset = get_dataset_toy() SAVE_MODEL = False elif options.dataset == 'cifar10': trainset, testset, = get_dataset_cifar10() SAVE_MODEL = True design_matrix = trainset.get_design_matrix() n_input = design_matrix.shape[1] # build layers layers = [] structure = [[n_input, 400], [400, 50], [50, 100], [100, 2]] # layer 0: gaussianRBM layers.append(get_grbm(structure[0])) # layer 1: denoising AE layers.append(get_denoising_autoencoder(structure[1])) # layer 2: AE layers.append(get_autoencoder(structure[2])) # layer 3: logistic regression used in supervised training layers.append(get_logistic_regressor(structure[3])) #construct training sets for different layers trainset = [ trainset, TransformerDataset(raw=trainset, transformer=layers[0]), TransformerDataset(raw=trainset, transformer=StackedBlocks(layers[0:2])), TransformerDataset(raw=trainset, transformer=StackedBlocks(layers[0:3])) ] # construct layer trainers layer_trainers = [] layer_trainers.append(get_layer_trainer_sgd_rbm(layers[0], trainset[0])) layer_trainers.append( get_layer_trainer_sgd_autoencoder(layers[1], trainset[1])) layer_trainers.append( get_layer_trainer_sgd_autoencoder(layers[2], trainset[2])) layer_trainers.append(get_layer_trainer_logistic(layers[3], trainset[3])) #unsupervised pretraining for layer_trainer in layer_trainers[0:3]: layer_trainer.main_loop() #supervised training layer_trainers[-1].main_loop()
def __init__(self, raw, transformer, cpu_only = False): TransformerDataset.__init__(self,raw, transformer, cpu_only = cpu_only) N = self.transformer.nhid r = int(np.sqrt(N)) c = N / r if N == r * c: shape = (r,c,1) else: shape = (N,1,1) self.view_converter = DefaultViewConverter(shape=shape)
def test_transformer_iterator(): """ Tests whether TransformerIterator is iterable """ test_path = os.path.join(pylearn2.__path__[0], 'datasets', 'tests', 'test.csv') raw = CSVDataset(path=test_path, expect_headers=False) block = Block() dataset = TransformerDataset(raw, block) iterator = dataset.iterator('shuffled_sequential', 3) try: iter(iterator) except TypeError: assert False, "TransformerIterator isn't iterable"
def test_transformer_iterator(): """ Tests whether TransformerIterator is iterable """ test_path = os.path.join(pylearn2.__path__[0], 'datasets', 'tests', 'test.csv') raw = CSVDataset(path=test_path, expect_headers=False) block = Block() dataset = TransformerDataset(raw, block) iterator = dataset.iterator('shuffled_sequential', 3) try: iter(iterator) except TypeError: assert False, "TransformerIterator isn't iterable"
def __iter__(self): """ Construct a Transformer dataset for each partition. """ for k, datasets in enumerate(self.dataset_iterator): if isinstance(self.transformers, list): transformer = self.transformers[k] elif isinstance(self.transformers, StackedBlocksCV): transformer = self.transformers.select_fold(k) else: transformer = self.transformers if isinstance(datasets, list): for i, dataset in enumerate(datasets): datasets[i] = TransformerDataset(dataset, transformer) else: for key, dataset in datasets.items(): datasets[key] = TransformerDataset(dataset, transformer) yield datasets
def main(): test = len(sys.argv)>1 and sys.argv[1]=='test' print "loading pretraining sets" pretrainset,prevalidset = load_predatasets(test) # print "loading training sets" # trainset,validset = load_traindatasets() # print "loading test sets" # testset = load_testdatasets() # for lr in [1e-4,1e-5,1e-6,1e-7]: if test: layer_trainers = build_conv(1e-5,pretrainset,prevalidset,None,None)#trainset,validset) print '-----------------------------------' print ' Supervised pretraining' print '-----------------------------------' layer_trainers[0].main_loop() sys.exit(0) for lr in [1e-4,1e-5,1e-6,1e-7,1e-8,1e-9]: # layer_trainers = build_mlp(pretrainset,prevalidset,trainset,validset) try: layer_trainers = build_conv(lr,pretrainset,prevalidset,None,None)#trainset,validset) # import theano # theano.config.compute_test_value = 'warn' print '-----------------------------------' print ' Supervised pretraining' print '-----------------------------------' layer_trainers[0].main_loop() # layer_trainers[0] = serial.load('pretrained.pkl') # X = model.get_input_space().make_theano_batch(name="%s[X]" % self.__class__.__name__) # self.f = function([X],model.fprop(X),on_unused_input='ignore') # dX = dataset.adjust_for_viewer(dataset.get_batch_topo(2)) # y_hat = self.f(dX[0:1].astype('float32')) # y_hat = y_hat.reshape([1]+dataset.output_shape) # y = dataset.y[0:1].reshape([1]+dataset.output_shape) # pylab.imshow(y_hat[0],cmap="gray") # pylab.show() except BaseException as e: print e sys.exit(0) print '-----------------------------------' print ' Supervised training' print '-----------------------------------' layer_trainers[-1].main_loop() # premodel = serial.load('pretrained.pkl') # model = serial.load('best_full.pkl') premodel = layer_trainers[0].model model = layer_trainers[1].model premodel._params = [] model._params = [] X = tensor.matrix() y = model.fprop(X) f = function([X],y) # base_path = "/home/xavier/data/ContestDataset" # testset = FacialKeypointDataset( # base_path = base_path, # which_set = 'train', # start = 1001, # stop = 1500) T_testset = TransformerDataset( raw = testset, transformer = StackedMLPs([premodel]) ) Xs = np.concatenate([batch for batch in T_testset.iterator(mode='sequential',batch_size=50,targets=False)]) print Xs.shape # Xs = testset.X Y_hat = f(np.float32(Xs)) Y = testset.y # Y_hat += 100. for i in range(10): print "Y: ",Y[i] print "Yh: ",Y_hat[i] D = np.sqrt(np.sum(Y_hat**2 -2*Y_hat*Y + Y**2, axis=1)) print D.shape print D.mean(0), D.std(0) stacked = StackedMLPs([premodel,model]) serial.save("stacked.pkl", stacked, on_overwrite = 'backup')
def main(): trainset, validset, testset, extraset = get_dataset_icml() #trainset,testset = get_dataset_mnist() design_matrix = trainset.get_design_matrix() n_input = design_matrix.shape[1] n_output = 9 #10 # build layers layers = [] structure = [[n_input, 1000], [1000,1000],[1000,1000], [1000, n_output]] #layers.append(get_grbm(structure[0])) # layer 0: denoising AE layers.append(get_grbm(structure[0])) # layer 1: denoising AE layers.append(get_grbm(structure[1])) # layer 1: denoising AE layers.append(get_grbm(structure[2])) # layer 2: logistic regression used in supervised training #layers.append(get_logistic_regressor(structure[3])) #construct training sets for different layers traindata = [ extraset , TransformerDataset( raw = extraset, transformer = layers[0] ), TransformerDataset( raw = extraset, transformer = StackedBlocks( layers[0:2] )), TransformerDataset( raw = extraset, transformer = StackedBlocks( layers[0:3] )) ] #valid = TransformerDataset( raw = validset, transformer = StackedBlocks( layers[0:2] )) #valid = trainset # construct layer trainers layer_trainers = [] #layer_trainers.append(get_layer_trainer_sgd_rbm(layers[0], trainset[0])) layer_trainers.append(get_layer_trainer_sgd_rbm(layers[0], traindata[0],'db1.pkl')) layer_trainers.append(get_layer_trainer_sgd_rbm(layers[1], traindata[1],'db2.pkl')) layer_trainers.append(get_layer_trainer_sgd_rbm(layers[2], traindata[2],'db3.pkl')) #layer_trainers.append(get_layer_trainer_logistic(layers[2], trainset[2], valid)) #unsupervised pretraining for i, layer_trainer in enumerate(layer_trainers[0:3]): print '-----------------------------------' print ' Unsupervised training (pretraining) layer %d, %s'%(i, layers[i].__class__) print '-----------------------------------' layer_trainer.main_loop() print '\n' print '------------------------------------------------------' print ' Unsupervised training done! Start supervised training (fine-tuning)...' print '------------------------------------------------------' print '\n' mlp_layers = [] mlp_layers.append(PretrainedLayer(layer_name = 'h0', layer_content = serial.load('db1.pkl'))) mlp_layers.append(PretrainedLayer(layer_name = 'h1', layer_content = serial.load('db2.pkl'))) mlp_layers.append(PretrainedLayer(layer_name = 'h2', layer_content = serial.load('db3.pkl'))) #supervised training #layer_trainers[-1].main_loop() mlp_model = get_layer_MLP(mlp_layers,trainset,validset) mlp_model.main_loop()
#if __name__ == '__main__': # _, fold_file, model_file = sys.argv fold_file = 'GTZAN_1024-fold-1_of_4.pkl' model_file = './saved-rlu-505050/mlp_rlu_fold1_best.pkl' # get model model = serial.load(model_file) # get stanardized dictionary which_set = 'test' with open(fold_file) as f: config = cPickle.load(f) dataset = TransformerDataset( raw=GTZAN_dataset.GTZAN_dataset(config, which_set), transformer=GTZAN_dataset.GTZAN_standardizer(config)) # test error #err, conf = frame_misclass_error(model, dataset) hist = class_histogram(model, dataset) hist = np.vstack(hist) test_files = np.array(config['test_files']) test_labels = test_files // 100 most_votes = np.argmax(hist, axis=0) most_rep_files = test_files[most_votes] most_rep_hist = hist[most_votes, :]
help='Full path and prefix for saving output models') parser.add_argument('--use_autoencoder', action='store_true') args = parser.parse_args() if args.epochs is None: args.epochs = 5 arch = [(i, j) for i, j in zip(args.arch[:-1], args.arch[1:])] with open(args.fold_config) as f: config = cPickle.load(f) preproc_layer = PreprocLayer(config=config, proc_type='standardize') dataset = TransformerDataset(raw=AudioDataset(which_set='train', config=config), transformer=preproc_layer.layer_content) # transformer_yaml = '''!obj:pylearn2.datasets.transformer_dataset.TransformerDataset { # raw : %(raw)s, # transformer : %(transformer)s # }''' # # dataset_yaml = transformer_yaml % { # 'raw' : '''!obj:audio_dataset.AudioDataset { # which_set : 'train', # config : !pkl: "%(fold_config)s" # }''' % {'fold_config' : args.fold_config}, # 'transformer' : '''!obj:pylearn2.models.mlp.MLP { # nvis : %(nvis)i, # layers :
for debugging perpose for learning pylearn2 """ from dataclassraw import CLICK4DAY from transformer import Transformer from pylearn2.datasets.transformer_dataset import TransformerDataset from pylearn2.models import mlp from pylearn2.training_algorithms import sgd from pylearn2.termination_criteria import EpochCounter raw_ds = CLICK4DAY(which_set='train', which_day=21) transformer = Transformer(raw=raw_ds, nfeatures=1024, rng=None) ds = TransformerDataset(raw=raw_ds, transformer=transformer, cpu_only=False, \ space_preserving=False) hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=256, irange=.1, init_bias=1.) output_layer = mlp.Softmax(2, 'output', irange=.1) trainer = sgd.SGD(learning_rate=.05, batch_size=1024, \ train_iteration_mode='even_sequential',termination_criterion=EpochCounter(400)) layers = [hidden_layer, output_layer] ann = mlp.MLP(layers, nvis=1024) trainer.setup(ann, ds)
def main(args=None): """ args is the list of arguments that will be passed to the option parser. The default (None) means use sys.argv[1:]. """ # parser = OptionParser() # parser.add_option("-d", "--data", dest="dataset", default="toy", # help="specify the dataset, either cifar10, mnist or toy") # (options, args) = parser.parse_args(args=args) # # if options.dataset == 'toy': # trainset, testset = get_dataset_toy() # n_output = 2 # elif options.dataset == 'cifar10': # trainset, testset, = get_dataset_cifar10() # n_output = 10 # # elif options.dataset == 'mnist': # trainset, testset, = get_dataset_mnist() # n_output = 10 # # else: # NotImplementedError() trainset = get_dataset_timitCons() n_output = 32 design_matrix = trainset.get_design_matrix() n_input = design_matrix.shape[1] # build layers layers = [] structure = [[n_input, 500], [500, 500], [500, 500], [500, n_output]] # layer 0: gaussianRBM layers.append(get_grbm(structure[0])) # # layer 1: denoising AE # layers.append(get_denoising_autoencoder(structure[1])) # # layer 2: AE # layers.append(get_autoencoder(structure[2])) # # layer 3: logistic regression used in supervised training # layers.append(get_logistic_regressor(structure[3])) # layer 1: gaussianRBM layers.append(get_grbm(structure[1])) # layer 2: gaussianRBM layers.append(get_grbm(structure[2])) # layer 3: logistic regression used in supervised training # layers.append(get_logistic_regressor(structure[3])) layers.append(get_mlp_softmax(structure[3])) # construct training sets for different layers trainset = [ trainset, TransformerDataset(raw=trainset, transformer=layers[0]), TransformerDataset(raw=trainset, transformer=StackedBlocks(layers[0:2])), TransformerDataset(raw=trainset, transformer=StackedBlocks(layers[0:3])) ] # construct layer trainers layer_trainers = [] layer_trainers.append(get_layer_trainer_sgd_rbm0(layers[0], trainset[0])) # layer_trainers.append(get_layer_trainer_sgd_autoencoder(layers[1], trainset[1])) # layer_trainers.append(get_layer_trainer_sgd_autoencoder(layers[2], trainset[2])) layer_trainers.append(get_layer_trainer_sgd_rbm1(layers[1], trainset[1])) layer_trainers.append(get_layer_trainer_sgd_rbm2(layers[2], trainset[2])) # layer_trainers.append(get_layer_trainer_logistic(layers[3], trainset[3])) layer_trainers.append(get_layer_trainer_softmax(layers[3], trainset[3])) # unsupervised pretraining for i, layer_trainer in enumerate(layer_trainers[0:3]): print('-----------------------------------') print(' Unsupervised training layer %d, %s' % (i, layers[i].__class__)) print('-----------------------------------') layer_trainer.main_loop() print('\n') print('------------------------------------------------------') print(' Unsupervised training done! Start supervised training...') print('------------------------------------------------------') print('\n') # supervised training # layer_trainers[-1].main_loop() layer1_yaml = open('MachineLearning.yaml', 'r').read() train = yaml_parse.load(layer1_yaml) train.main_loop()
def train_SdA(config, dataset): ## load config hidden_layers_sizes = config.get('hidden_layers_sizes', [10, 10]) corruption_levels = config.get('corruption_levels', [0.1, 0.2]) stage2_corruption_levels = config.get('stage2_corruption_levels', [0.1, 0.1]) pretrain_epochs = config.get('pretrain_epochs', 10) pretrain_lr = config.get('pretrain_learning_rate', 0.001) finetune_epochs = config.get('finetune_epochs', 10) finetune_lr = config.get('finetune_learning_rate', 0.01) batch_size = config.get('batch_size', 10) monitoring_batches = config.get('monitoring_batches', 5) output_path = config.get('output_path', './') input_trainset = dataset design_matrix = input_trainset.get_design_matrix() # print design_matrix.shape; n_input = design_matrix.shape[1] log.info('done') log.debug('input dimensions : {0}'.format(n_input)) log.debug('training examples: {0}'.format(design_matrix.shape[0])) # numpy random generator # numpy_rng = numpy.random.RandomState(89677) log.info('... building the model') # build layers layer_dims = [n_input] layer_dims.extend(hidden_layers_sizes) layers = [] for i in xrange(1, len(layer_dims)): structure = [layer_dims[i - 1], layer_dims[i]] layers.append( create_denoising_autoencoder(structure, corruption=corruption_levels[i - 1])) # unsupervised pre-training log.info('... pre-training the model') start_time = time.clock() for i in xrange(len(layers)): # reset corruption to make sure input is not corrupted for layer in layers: layer.set_corruption_level(0) if i == 0: trainset = input_trainset elif i == 1: trainset = TransformerDataset(raw=input_trainset, transformer=layers[0]) else: trainset = TransformerDataset(raw=input_trainset, transformer=StackedBlocks( layers[0:i])) # set corruption for layer to train layers[i].set_corruption_level(corruption_levels[i]) trainer = get_layer_trainer_sgd_autoencoder( layers[i], trainset, learning_rate=pretrain_lr, max_epochs=pretrain_epochs, batch_size=batch_size, monitoring_batches=monitoring_batches, name='pre-train' + str(i)) log.info('unsupervised training layer %d, %s ' % (i, layers[i].__class__)) trainer.main_loop() # theano.printing.pydotprint_variables( # layer_trainer.algorithm.sgd_update.maker.fgraph.outputs[0], # outfile='pylearn2-sgd_update.png', # var_with_name_simple=True); end_time = time.clock() log.info('pre-training code ran for {0:.2f}m'.format( (end_time - start_time) / 60.)) # now untie the decoder weights log.info('untying decoder weights') for layer in layers: layer.untie_weights() # construct multi-layer training fuctions # unsupervised training log.info('... training the model') sdae = None for depth in xrange(1, len(layers) + 1): first_layer_i = len(layers) - depth log.debug('training layers {}..{}'.format(first_layer_i, len(layers) - 1)) group = layers[first_layer_i:len(layers)] # log.debug(group); # reset corruption for layer in layers: layer.set_corruption_level(0) if first_layer_i == 0: trainset = input_trainset elif first_layer_i == 1: trainset = TransformerDataset(raw=input_trainset, transformer=layers[0]) else: trainset = TransformerDataset(raw=input_trainset, transformer=StackedBlocks( layers[0:first_layer_i])) # set corruption for input layer of stack to train # layers[first_layer_i].set_corruption_level(stage2_corruption_levels[first_layer_i]); corruptor = LoggingCorruptor(BinomialCorruptor( corruption_level=stage2_corruption_levels[first_layer_i]), name='depth {}'.format(depth)) sdae = StackedDenoisingAutoencoder(group, corruptor) trainer = get_layer_trainer_sgd_autoencoder( sdae, trainset, learning_rate=finetune_lr, max_epochs=finetune_epochs, batch_size=batch_size, monitoring_batches=monitoring_batches, name='multi-train' + str(depth)) log.info('unsupervised multi-layer training %d' % (i)) trainer.main_loop() end_time = time.clock() log.info('full training code ran for {0:.2f}m'.format( (end_time - start_time) / 60.)) # save the model model_file = os.path.join(output_path, 'sdae-model.pkl') with log_timing(log, 'saving SDA model to {}'.format(model_file)): serial.save(model_file, sdae) # TODO: pylearn2.train_extensions.best_params.KeepBestParams(model, cost, monitoring_dataset, batch_size) # pylearn2.train_extensions.best_params.MonitorBasedSaveBest log.info('done') return sdae