def test_pretrain(self): n_in = self.n_in mini_batch_size = 200 net = Network([ AutoencoderLayer(n_in=n_in, n_hidden=n_in-3, rnd=rnd), AutoencoderLayer(n_in=n_in-3, n_hidden=n_in-6, corruption_level=0.1, rnd=rnd), FullyConnectedLayer(n_in=n_in-6, n_out=1, rnd=rnd), ], mini_batch_size) net.pretrain_autoencoders( tdata=self.pretrain, mbs=mini_batch_size, eta=0.025, epochs=1) pass
def train(job_id, params): print "Job ID: %d" % job_id border = 2 n_hidden_layer = params['hidden'] metric_recorder = MetricRecorder(config_dir_path='./config.json', job_id=job_id) C = { 'X_dirpath' : '../../../data/onetext_train_small/*', 'X_valid_dirpath' : '../../../data/onetext_valid_small/*', 'y_dirpath' : '../../../data/train_cleaned/', 'batchsize' : 2000000, 'limit' : None, 'epochs' : 15, 'patience' : 70000, 'patience_increase' : 2, 'improvement_threshold' : 0.995, 'validation_frequency' : 2, 'lmbda' : 0.0, 'dropout' : 0.0, 'training_size' : None, 'validation_size' : None, 'algorithm' : 'RMSProp', 'eta' : float(params['eta'][0]), 'eta_min': float(params['eta_min'][0]), 'eta_pre' : float(params['eta_pre'][0]), 'corruption_level' : float(params['corruption_level'][0]), 'border' : 2, 'hidden' : int(params['hidden'][0]), 'mini_batch_size': 500 } training_data = BatchProcessor( X_dirpath=C['X_dirpath'], y_dirpath=C['y_dirpath'], batchsize=C['batchsize'], border=C['border'], limit=C['limit'], random=True, random_mode='fully', dtype=theano.config.floatX, rnd=rnd) validation_data = BatchProcessor( X_dirpath=C['X_valid_dirpath'], y_dirpath=C['y_dirpath'], batchsize=C['batchsize'], border=C['border'], limit=C['limit'], random=False, dtype=theano.config.floatX, rnd=rnd) pretrain_data = BatchProcessor( X_dirpath='../../../data/onetext_pretrain_small/*', y_dirpath='../../../data/train_cleaned/', batchsize=50000, border=border, limit=None, random=True, random_mode='fully', rnd=rnd, dtype=theano.config.floatX) C['training_size'] = training_data.size() C['validation_size'] = validation_data.size() print "Training size: %d" % C['training_size'] print "Validation size: %d" % C['validation_size'] metric_recorder.add_experiment_metainfo(constants=C) metric_recorder.start() n_in = (2*border+1)**2 net = Network([ AutoencoderLayer(n_in=n_in, n_hidden=C['hidden'], rnd=rnd, corruption_level=C['corruption_level']), FullyConnectedLayer(n_in=C['hidden'], n_out=1, rnd=rnd)], C['mini_batch_size']) print '...start pretraining' net.pretrain_autoencoders(training_data=pretrain_data, mbs=C['mini_batch_size'], eta=C['eta_pre'], epochs=15, metric_recorder=metric_recorder) result = net.train(tdata=training_data, epochs=C['epochs'], mbs=C['mini_batch_size'], eta=C['eta'], eta_min=C['eta_min'], vdata=validation_data, lmbda=C['lmbda'], momentum=None, patience_increase=C['patience_increase'], improvement_threshold=C['improvement_threshold'], validation_frequency=C['validation_frequency'], metric_recorder=metric_recorder, save_dir='./models/%d_' % metric_recorder.job_id, early_stoping=False) print 'Time = %f' % metric_recorder.stop() print 'Result = %f' % result return float(result)
pretrain_save_dir = save_dir + "pretrain_" n_in = (2*C['border']+1)**2 net = Network([ AutoencoderLayer(n_in=n_in, n_hidden=C['hidden_1'], rnd=rnd, corruption_level=C['corruption_level']), AutoencoderLayer(n_in=C['hidden_1'], n_hidden=C['hidden_2'], rnd=rnd, corruption_level=C['corruption_level'], p_dropout=C['dropout']), AutoencoderLayer(n_in=C['hidden_2'], n_hidden=C['hidden_3'], rnd=rnd, corruption_level=C['corruption_level']), FullyConnectedLayer(n_in=C['hidden_3'], n_out=1, rnd=rnd)], C['mini_batch_size']) print '...start pretraining' net.pretrain_autoencoders(tdata=pretrain_data, mbs=C['mini_batch_size'], eta=C['eta_pre'], epochs=10, metric_recorder=mr, save_dir=pretrain_save_dir) print '...start training' result = net.train(tdata=training_data, epochs=C['epochs'], mbs=C['mini_batch_size'], eta=C['eta'], eta_min=C['eta_min'], vdata=validation_data, lmbda=C['lmbda'], momentum=None, patience_increase=C['patience_increase'], improvement_threshold=C['improvement_threshold'], validation_frequency=C['validation_frequency'], metric_recorder=mr, save_dir=save_dir, early_stoping=False) print 'Time = %f' % mr.stop()
n_in = (2*border+1)**2 mr.start() mbs = 500 net = Network([ AutoencoderLayer(n_in=n_in, n_hidden=80, corruption_level=0.2), AutoencoderLayer(n_in=80, n_hidden=50, corruption_level=0.2), AutoencoderLayer(n_in=50, n_hidden=20, corruption_level=0.2), FullyConnectedLayer(n_in=20, n_out=1), ], mbs) print '...start pretraining' net.pretrain_autoencoders(tdata=pretrain_data, mbs=mbs, metric_recorder=mr, save_dir='./models/sea_test_pretrain_', eta=0.01, epochs=10) #image = PIL.Image.fromarray(tile_raster_images( # X=net.layers[0].w.get_value(borrow=True).T, # img_shape=(5, 5), tile_shape=(10, 10), # tile_spacing=(2, 2))) #image.show() training_data.reset() print '...start training' cost = net.SGD(tdata=training_data, epochs=10, mbs=mbs, eta=0.025, eta_min=0.01, vdata=validation_data, lmbda=0.0, momentum=None, patience_increase=2, improvement_threshold=0.995, validation_frequency=2,