def test_pretrain(self):
n_in = self.n_in
mini_batch_size = 200
net = Network([
AutoencoderLayer(n_in=n_in, n_hidden=n_in-3, rnd=rnd),
AutoencoderLayer(n_in=n_in-3, n_hidden=n_in-6,
corruption_level=0.1, rnd=rnd),
FullyConnectedLayer(n_in=n_in-6, n_out=1, rnd=rnd),
], mini_batch_size)
net.pretrain_autoencoders(
tdata=self.pretrain,
mbs=mini_batch_size,
eta=0.025, epochs=1)
pass
def train(job_id, params):
print "Job ID: %d" % job_id
border = 2
n_hidden_layer = params['hidden']
metric_recorder = MetricRecorder(config_dir_path='./config.json',
job_id=job_id)
C = {
'X_dirpath' : '../../../data/onetext_train_small/*',
'X_valid_dirpath' : '../../../data/onetext_valid_small/*',
'y_dirpath' : '../../../data/train_cleaned/',
'batchsize' : 2000000,
'limit' : None,
'epochs' : 15,
'patience' : 70000,
'patience_increase' : 2,
'improvement_threshold' : 0.995,
'validation_frequency' : 2,
'lmbda' : 0.0,
'dropout' : 0.0,
'training_size' : None,
'validation_size' : None,
'algorithm' : 'RMSProp',
'eta' : float(params['eta'][0]),
'eta_min': float(params['eta_min'][0]),
'eta_pre' : float(params['eta_pre'][0]),
'corruption_level' : float(params['corruption_level'][0]),
'border' : 2,
'hidden' : int(params['hidden'][0]),
'mini_batch_size': 500
}
training_data = BatchProcessor(
X_dirpath=C['X_dirpath'],
y_dirpath=C['y_dirpath'],
batchsize=C['batchsize'],
border=C['border'],
limit=C['limit'],
random=True,
random_mode='fully',
dtype=theano.config.floatX,
rnd=rnd)
validation_data = BatchProcessor(
X_dirpath=C['X_valid_dirpath'],
y_dirpath=C['y_dirpath'],
batchsize=C['batchsize'],
border=C['border'],
limit=C['limit'],
random=False,
dtype=theano.config.floatX,
rnd=rnd)
pretrain_data = BatchProcessor(
X_dirpath='../../../data/onetext_pretrain_small/*',
y_dirpath='../../../data/train_cleaned/',
batchsize=50000, border=border, limit=None,
random=True, random_mode='fully', rnd=rnd,
dtype=theano.config.floatX)
C['training_size'] = training_data.size()
C['validation_size'] = validation_data.size()
print "Training size: %d" % C['training_size']
print "Validation size: %d" % C['validation_size']
metric_recorder.add_experiment_metainfo(constants=C)
metric_recorder.start()
n_in = (2*border+1)**2
net = Network([
AutoencoderLayer(n_in=n_in, n_hidden=C['hidden'], rnd=rnd,
corruption_level=C['corruption_level']),
FullyConnectedLayer(n_in=C['hidden'], n_out=1, rnd=rnd)],
C['mini_batch_size'])
print '...start pretraining'
net.pretrain_autoencoders(training_data=pretrain_data,
mbs=C['mini_batch_size'], eta=C['eta_pre'], epochs=15, metric_recorder=metric_recorder)
result = net.train(tdata=training_data, epochs=C['epochs'],
mbs=C['mini_batch_size'], eta=C['eta'],
eta_min=C['eta_min'],
vdata=validation_data, lmbda=C['lmbda'],
momentum=None,
patience_increase=C['patience_increase'],
improvement_threshold=C['improvement_threshold'],
validation_frequency=C['validation_frequency'],
metric_recorder=metric_recorder,
save_dir='./models/%d_' % metric_recorder.job_id,
early_stoping=False)
print 'Time = %f' % metric_recorder.stop()
print 'Result = %f' % result
return float(result)
pretrain_save_dir = save_dir + "pretrain_"
n_in = (2*C['border']+1)**2
net = Network([
AutoencoderLayer(n_in=n_in, n_hidden=C['hidden_1'], rnd=rnd,
corruption_level=C['corruption_level']),
AutoencoderLayer(n_in=C['hidden_1'], n_hidden=C['hidden_2'], rnd=rnd,
corruption_level=C['corruption_level'], p_dropout=C['dropout']),
AutoencoderLayer(n_in=C['hidden_2'], n_hidden=C['hidden_3'], rnd=rnd,
corruption_level=C['corruption_level']),
FullyConnectedLayer(n_in=C['hidden_3'], n_out=1, rnd=rnd)],
C['mini_batch_size'])
print '...start pretraining'
net.pretrain_autoencoders(tdata=pretrain_data,
mbs=C['mini_batch_size'], eta=C['eta_pre'],
epochs=10, metric_recorder=mr,
save_dir=pretrain_save_dir)
print '...start training'
result = net.train(tdata=training_data, epochs=C['epochs'],
mbs=C['mini_batch_size'], eta=C['eta'],
eta_min=C['eta_min'],
vdata=validation_data, lmbda=C['lmbda'],
momentum=None,
patience_increase=C['patience_increase'],
improvement_threshold=C['improvement_threshold'],
validation_frequency=C['validation_frequency'],
metric_recorder=mr, save_dir=save_dir,
early_stoping=False)
print 'Time = %f' % mr.stop()
n_in = (2*border+1)**2
mr.start()
mbs = 500
net = Network([
AutoencoderLayer(n_in=n_in, n_hidden=80, corruption_level=0.2),
AutoencoderLayer(n_in=80, n_hidden=50, corruption_level=0.2),
AutoencoderLayer(n_in=50, n_hidden=20, corruption_level=0.2),
FullyConnectedLayer(n_in=20, n_out=1),
], mbs)
print '...start pretraining'
net.pretrain_autoencoders(tdata=pretrain_data,
mbs=mbs, metric_recorder=mr,
save_dir='./models/sea_test_pretrain_',
eta=0.01, epochs=10)
#image = PIL.Image.fromarray(tile_raster_images(
# X=net.layers[0].w.get_value(borrow=True).T,
# img_shape=(5, 5), tile_shape=(10, 10),
# tile_spacing=(2, 2)))
#image.show()
training_data.reset()
print '...start training'
cost = net.SGD(tdata=training_data, epochs=10,
mbs=mbs, eta=0.025, eta_min=0.01,
vdata=validation_data, lmbda=0.0,
momentum=None, patience_increase=2,
improvement_threshold=0.995, validation_frequency=2,
