def get_denoising_autoencoder(structure): n_input, n_output = structure curruptor = BinomialCorruptor(corruption_level=0.5) config = { 'corruptor': curruptor, 'nhid': n_output, 'nvis': n_input, 'tied_weights': Trueg, 'act_enc': 'tanh', 'act_dec': 'sigmoid', 'irange': 0.001, } return DenoisingAutoencoder(**config)
'act_enc': 'tanh', 'act_dec': None, #'lr_hb': 0.10, #'lr_vb': 0.10, 'irange': 0.001, #note the kmean hyper-parameter here 'kmeans_k': 2 } print '== training ==' # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(corruption_level=conf['corruption_level']) da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'], conf['act_enc'], conf['act_dec'], tied_weights=conf['tied_weights'], irange=conf['irange']) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da.params(), conf['base_lr'], conf['anneal_start']) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=trainer.cost_updates(cost)) # Suppose we want minibatches of size 10 batchsize = 10 # Here's a manual training loop. I hope to have some classes that
def main_train(epochs, batchsize, solution='',sparse_penalty=0,sparsityTarget=0,sparsityTargetPenalty=0): # Experiment specific arguments conf_dataset = {'dataset' : 'avicenna', 'expname' : 'dummy', # Used to create the submission file 'transfer' : True, 'normalize' : True, # (Default = True) 'normalize_on_the_fly' : False, # (Default = False) 'randomize_valid' : True, # (Default = True) 'randomize_test' : True, # (Default = True) 'saving_rate': 0, # (Default = 0) 'savedir' : './outputs', } # First layer = PCA-75 whiten pca_layer = {'name' : '1st-PCA', 'num_components': 75, 'min_variance': -50, 'whiten': True, 'pca_class' : 'CovEigPCA', # Training properties 'proba' : [1, 0, 0], 'savedir' : './outputs', } # Load the dataset data = utils.load_data(conf_dataset) if conf_dataset['transfer']: # Data for the ALC proxy label = data[3] data = data[:3] # First layer : train or load a PCA pca = create_pca(conf_dataset, pca_layer, data, model=pca_layer['name']) data = [utils.sharedX(pca.function()(set.get_value(borrow=True)),borrow=True) for set in data] ''' if conf_dataset['transfer']: data_train, label_train = utils.filter_labels(data[0], label) alc = embed.score(data_train, label_train) print '... resulting ALC on train (for PCA) is', alc ''' nvis = utils.get_constant(data[0].shape[1]).item() conf = { 'corruption_level': 0.1, 'nhid': 200, 'nvis': nvis, 'anneal_start': 100, 'base_lr': 0.001, 'tied_weights': True, 'act_enc': 'sigmoid', 'act_dec': None, #'lr_hb': 0.10, #'lr_vb': 0.10, 'tied_weights': True , 'solution': solution, 'sparse_penalty': sparse_penalty, 'sparsityTarget': sparsityTarget , 'sparsityTargetPenalty': sparsityTargetPenalty, 'irange': 0, } # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(conf['corruption_level']) da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'], conf['act_enc'], conf['act_dec'], conf['tied_weights'], conf['solution'], conf['sparse_penalty'], conf['sparsityTarget'], conf['sparsityTargetPenalty']) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da, conf['base_lr'], conf['anneal_start']) updates = trainer.cost_updates(cost) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=updates) # Suppose we want minibatches of size 10 proba = utils.getboth(conf, pca_layer, 'proba') iterator = BatchIterator(data, proba, batchsize) # Here's a manual training loop. I hope to have some classes that # automate this a litle bit. final_cost = 0 for epoch in xrange(epochs): c = [] for minibatch_data in iterator: minibatch_err = train_fn(minibatch_data) c.append(minibatch_err) final_cost = numpy.mean(c) print "epoch %d, cost : %f" % (epoch , final_cost) print '############################## Fin de l\'experience ############################' print 'Calcul de l\'ALC : ' if conf_dataset['transfer']: data_train, label_train = utils.filter_labels(data[0], label) alc = embed.score(data_train, label_train) print 'Solution : ',solution print 'sparse_penalty = ',sparse_penalty print 'sparsityTarget = ',sparsityTarget print 'sparsityTargetPenalty = ',sparsityTargetPenalty print 'Final denoising error is : ',final_cost print '... resulting ALC on train is', alc return (alc,final_cost)
'tied_weights': False , 'solution': 'l1_penalty', 'sparse_penalty': 0.01, 'sparsity_target': 0.1, 'sparsity_target_penalty': 0.001, 'irange': 0.001, } # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(conf['corruption_level']) da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'], conf['act_enc'], conf['act_dec'], conf['tied_weights'], conf['solution'], conf['sparse_penalty'], conf['sparsity_target'], conf['sparsity_target_penalty']) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da, conf['base_lr'], conf['anneal_start']) updates = trainer.cost_updates(cost) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=updates) # Suppose we want minibatches of size 20 batchsize = 20
def main_train(epochs, batchsize, solution='', sparse_penalty=0, sparsityTarget=0, sparsityTargetPenalty=0): # Experiment specific arguments conf_dataset = { 'dataset': 'avicenna', 'expname': 'dummy', # Used to create the submission file 'transfer': True, 'normalize': True, # (Default = True) 'normalize_on_the_fly': False, # (Default = False) 'randomize_valid': True, # (Default = True) 'randomize_test': True, # (Default = True) 'saving_rate': 0, # (Default = 0) 'savedir': './outputs', } # First layer = PCA-75 whiten pca_layer = { 'name': '1st-PCA', 'num_components': 75, 'min_variance': -50, 'whiten': True, 'pca_class': 'CovEigPCA', # Training properties 'proba': [1, 0, 0], 'savedir': './outputs', } # Load the dataset data = utils.load_data(conf_dataset) if conf_dataset['transfer']: # Data for the ALC proxy label = data[3] data = data[:3] # First layer : train or load a PCA pca = create_pca(conf_dataset, pca_layer, data, model=pca_layer['name']) data = [ utils.sharedX(pca.function()(set.get_value(borrow=True)), borrow=True) for set in data ] ''' if conf_dataset['transfer']: data_train, label_train = utils.filter_labels(data[0], label) alc = embed.score(data_train, label_train) print '... resulting ALC on train (for PCA) is', alc ''' nvis = utils.get_constant(data[0].shape[1]).item() conf = { 'corruption_level': 0.1, 'nhid': 200, 'nvis': nvis, 'anneal_start': 100, 'base_lr': 0.001, 'tied_weights': True, 'act_enc': 'sigmoid', 'act_dec': None, #'lr_hb': 0.10, #'lr_vb': 0.10, 'tied_weights': True, 'solution': solution, 'sparse_penalty': sparse_penalty, 'sparsityTarget': sparsityTarget, 'sparsityTargetPenalty': sparsityTargetPenalty, 'irange': 0, } # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(conf['corruption_level']) da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'], conf['act_enc'], conf['act_dec'], conf['tied_weights'], conf['solution'], conf['sparse_penalty'], conf['sparsityTarget'], conf['sparsityTargetPenalty']) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da, conf['base_lr'], conf['anneal_start']) updates = trainer.cost_updates(cost) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=updates) # Suppose we want minibatches of size 10 proba = utils.getboth(conf, pca_layer, 'proba') iterator = BatchIterator(data, proba, batchsize) # Here's a manual training loop. I hope to have some classes that # automate this a litle bit. final_cost = 0 for epoch in xrange(epochs): c = [] for minibatch_data in iterator: minibatch_err = train_fn(minibatch_data) c.append(minibatch_err) final_cost = numpy.mean(c) print "epoch %d, cost : %f" % (epoch, final_cost) print '############################## Fin de l\'experience ############################' print 'Calcul de l\'ALC : ' if conf_dataset['transfer']: data_train, label_train = utils.filter_labels(data[0], label) alc = embed.score(data_train, label_train) print 'Solution : ', solution print 'sparse_penalty = ', sparse_penalty print 'sparsityTarget = ', sparsityTarget print 'sparsityTargetPenalty = ', sparsityTargetPenalty print 'Final denoising error is : ', final_cost print '... resulting ALC on train is', alc return (alc, final_cost)
'tied_weights': False, 'solution': 'l1_penalty', 'sparse_penalty': 0.01, 'sparsity_target': 0.1, 'sparsity_target_penalty': 0.001, 'irange': 0.001, } # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(conf['corruption_level']) da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'], conf['act_enc'], conf['act_dec'], conf['tied_weights'], conf['solution'], conf['sparse_penalty'], conf['sparsity_target'], conf['sparsity_target_penalty']) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da, conf['base_lr'], conf['anneal_start']) updates = trainer.cost_updates(cost) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=updates) # Suppose we want minibatches of size 20 batchsize = 20
def main_train(epochs, batchsize, solution="", sparse_penalty=0, sparsityTarget=0, sparsityTargetPenalty=0): # Experiment specific arguments conf_dataset = { "dataset": "avicenna", "expname": "dummy", # Used to create the submission file "transfer": True, "normalize": True, # (Default = True) "normalize_on_the_fly": False, # (Default = False) "randomize_valid": True, # (Default = True) "randomize_test": True, # (Default = True) "saving_rate": 0, # (Default = 0) "savedir": "./outputs", } # First layer = PCA-75 whiten pca_layer = { "name": "1st-PCA", "num_components": 75, "min_variance": -50, "whiten": True, "pca_class": "CovEigPCA", # Training properties "proba": [1, 0, 0], "savedir": "./outputs", } # Load the dataset data = utils.load_data(conf_dataset) if conf_dataset["transfer"]: # Data for the ALC proxy label = data[3] data = data[:3] # First layer : train or load a PCA pca = create_pca(conf_dataset, pca_layer, data, model=pca_layer["name"]) data = [utils.sharedX(pca.function()(set.get_value(borrow=True)), borrow=True) for set in data] """ if conf_dataset['transfer']: data_train, label_train = utils.filter_labels(data[0], label) alc = embed.score(data_train, label_train) print '... resulting ALC on train (for PCA) is', alc """ nvis = utils.get_constant(data[0].shape[1]).item() conf = { "corruption_level": 0.1, "nhid": 200, "nvis": nvis, "anneal_start": 100, "base_lr": 0.001, "tied_weights": True, "act_enc": "sigmoid", "act_dec": None, #'lr_hb': 0.10, #'lr_vb': 0.10, "tied_weights": True, "solution": solution, "sparse_penalty": sparse_penalty, "sparsityTarget": sparsityTarget, "sparsityTargetPenalty": sparsityTargetPenalty, "irange": 0, } # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(conf["corruption_level"]) da = DenoisingAutoencoder( corruptor, conf["nvis"], conf["nhid"], conf["act_enc"], conf["act_dec"], conf["tied_weights"], conf["solution"], conf["sparse_penalty"], conf["sparsityTarget"], conf["sparsityTargetPenalty"], ) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da, conf["base_lr"], conf["anneal_start"]) updates = trainer.cost_updates(cost) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=updates) # Suppose we want minibatches of size 10 proba = utils.getboth(conf, pca_layer, "proba") iterator = BatchIterator(data, proba, batchsize) # Here's a manual training loop. I hope to have some classes that # automate this a litle bit. final_cost = 0 for epoch in xrange(epochs): c = [] for minibatch_data in iterator: minibatch_err = train_fn(minibatch_data) c.append(minibatch_err) final_cost = numpy.mean(c) print "epoch %d, cost : %f" % (epoch, final_cost) print "############################## Fin de l'experience ############################" print "Calcul de l'ALC : " if conf_dataset["transfer"]: data_train, label_train = utils.filter_labels(data[0], label) alc = embed.score(data_train, label_train) print "Solution : ", solution print "sparse_penalty = ", sparse_penalty print "sparsityTarget = ", sparsityTarget print "sparsityTargetPenalty = ", sparsityTargetPenalty print "Final denoising error is : ", final_cost print "... resulting ALC on train is", alc return (alc, final_cost)
def main_train(work_dir="../results/avicenna/", corruption_level=0.3, nvis=75, nhid=600, tied_weights=True, act_enc="sigmoid", act_dec=None, max_epochs=2, learning_rate=0.001, batch_size=20, monitoring_batches=5, save_freq=1, n_components_trans_pca=7): conf = { 'corruption_level': corruption_level, 'nvis': nvis, 'nhid': nhid, 'tied_weights': tied_weights, 'act_enc': act_enc, 'act_dec': act_dec, 'max_epochs': max_epochs, 'learning_rate': learning_rate, 'batch_size': batch_size, 'monitoring_batches': monitoring_batches, 'save_freq': save_freq, 'n_components_trans_pca': n_components_trans_pca } start = time.clock() ############### TRAIN THE DAE train_file = work_dir + "train_pca" + str(conf['nvis']) + ".npy" save_path = work_dir + "train_pca" + str(conf['nvis']) + "_dae" + str( conf['nhid']) + "_model.pkl" trainset = NpyDataset(file=train_file) trainset.yaml_src = 'script' corruptor = BinomialCorruptor(corruption_level=conf['corruption_level']) dae = DenoisingAutoencoder(nvis=conf['nvis'], nhid=conf['nhid'], tied_weights=conf['tied_weights'], corruptor=corruptor, act_enc=conf['act_enc'], act_dec=conf['act_dec']) cost = MeanSquaredReconstructionError() termination_criterion = EpochCounter(max_epochs=conf['max_epochs']) algorithm = UnsupervisedExhaustiveSGD( learning_rate=conf['learning_rate'], batch_size=conf['batch_size'], monitoring_batches=conf['monitoring_batches'], monitoring_dataset=trainset, cost=cost, termination_criterion=termination_criterion) train_obj = Train(dataset=trainset, model=dae, algorithm=algorithm, save_freq=conf['save_freq'], save_path=save_path) train_obj.main_loop() ############### APPLY THE MODEL ON THE TRAIN DATASET print("Applying the model on the train dataset...") model = load(save_path) save_train_path = work_dir + "train_pca" + str( conf['nvis']) + "_dae" + str(conf['nhid']) + ".npy" dump_obj = FeatureDump(encoder=model, dataset=trainset, path=save_train_path) dump_obj.main_loop() ############### APPLY THE MODEL ON THE VALID DATASET print("Applying the model on the valid dataset...") valid_file = work_dir + "valid_pca" + str(conf['nvis']) + ".npy" validset = NpyDataset(file=valid_file) validset.yaml_src = 'script' save_valid_path = work_dir + "valid_pca" + str( conf['nvis']) + "_dae" + str(conf['nhid']) + ".npy" dump_obj = FeatureDump(encoder=model, dataset=validset, path=save_valid_path) dump_obj.main_loop() ############### APPLY THE MODEL ON THE TEST DATASET print("Applying the model on the test dataset...") test_file = work_dir + "test_pca" + str(conf['nvis']) + ".npy" testset = NpyDataset(file=test_file) testset.yaml_src = 'script' save_test_path = work_dir + "test_pca" + str(conf['nvis']) + "_dae" + str( conf['nhid']) + ".npy" dump_obj = FeatureDump(encoder=model, dataset=testset, path=save_test_path) dump_obj.main_loop() ############### COMPUTE THE ALC SCORE ON VALIDATION SET valid_data = ift6266h12.load_npy(save_valid_path) label_data = ift6266h12.load_npy( '/data/lisa/data/UTLC/numpy_data/avicenna_valid_y.npy') alc_1 = score(valid_data, label_data) ############### APPLY THE TRANSDUCTIVE PCA test_data = ift6266h12.load_npy(save_test_path) trans_pca = PCA(n_components=conf['n_components_trans_pca']) final_valid = trans_pca.fit_transform(valid_data) final_test = trans_pca.fit_transform(test_data) save_valid_path = work_dir + "valid_pca" + str( conf['nvis']) + "_dae" + str(conf['nhid']) + "_tpca" + str( conf['n_components_trans_pca']) + ".npy" save_test_path = work_dir + "test_pca" + str(conf['nvis']) + "_dae" + str( conf['nhid']) + "_tpca" + str(conf['n_components_trans_pca']) + ".npy" np.save(save_valid_path, final_valid) np.save(save_test_path, final_test) ############### COMPUTE THE NEW ALC SCORE ON VALIDATION SET alc_2 = score(final_valid, label_data) ############### OUTPUT AND RETURN THE RESULTS timeSpent = ((time.clock() - start) / 60.) print 'FINAL RESULTS (PCA-' + str(conf['nvis']) + ' DAE-' + str(conf['nhid']) + ' TransPCA-' + str(conf['n_components_trans_pca']) + ') ALC after DAE: ', alc_1, ' FINAL ALC: ', alc_2, \ ' Computed in %5.2f min' % (timeSpent) return timeSpent, alc_1, alc_2
#'lr_hb': 0.10, #'lr_vb': 0.10, 'irange': 0.001, #note the kmean hyper-parameter here 'kmeans_k': 2 } print '== training ==' # A symbolic input representing your minibatch. minibatch = tensor.matrix() # Allocate a denoising autoencoder with binomial noise corruption. corruptor = GaussianCorruptor(corruption_level=conf['corruption_level']) da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'], conf['act_enc'], conf['act_dec'], tied_weights=conf['tied_weights'], irange=conf['irange']) # Allocate an optimizer, which tells us how to update our model. # TODO: build the cost another way cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean() trainer = SGDOptimizer(da.params(), conf['base_lr'], conf['anneal_start']) # Finally, build a Theano function out of all this. train_fn = theano.function([minibatch], cost, updates=trainer.cost_updates(cost)) # Suppose we want minibatches of size 10