def train_example(dataset=None):
model = GaussianBinaryRBM(nvis=1296,
nhid=61,
irange=0.5,
energy_function_class=grbm_type_1(),
learn_sigma=True,
init_sigma=.4,
init_bias_hid=2.,
mean_vis=False,
sigma_lr_scale=1e-3)
cost = SMD(corruptor=GaussianCorruptor(stdev=0.4))
algorithm = SGD(learning_rate=.1,
batch_size=5,
monitoring_batches=20,
monitoring_dataset=dataset,
cost=cost,
termination_criterion=MonitorBased(prop_decrease=0.01,
N=1))
train = Train(dataset=dataset,
model=model,
save_path="./experiment/training.pkl",
save_freq=10,
algorithm=algorithm,
extensions=[])
train.main_loop()
def test_train_supervised():
"""
Train a supervised GSN.
"""
# initialize the GSN
gsn = GSN.new(
layer_sizes=[ds.X.shape[1], 1000, ds.y.shape[1]],
activation_funcs=["sigmoid", "tanh", rescaled_softmax],
pre_corruptors=[GaussianCorruptor(0.5)] * 3,
post_corruptors=[
SaltPepperCorruptor(.3), None,
SmoothOneHotCorruptor(.5)
],
layer_samplers=[BinomialSampler(), None,
MultinomialSampler()],
tied=False)
# average over costs rather than summing
_rcost = MeanBinaryCrossEntropy()
reconstruction_cost = lambda a, b: _rcost.cost(a, b) / ds.X.shape[1]
_ccost = MeanBinaryCrossEntropy()
classification_cost = lambda a, b: _ccost.cost(a, b) / ds.y.shape[1]
# combine costs into GSNCost object
c = GSNCost(
[
# reconstruction on layer 0 with weight 1.0
(0, 1.0, reconstruction_cost),
# classification on layer 2 with weight 2.0
(2, 2.0, classification_cost)
],
walkback=WALKBACK,
mode="supervised")
alg = SGD(
LEARNING_RATE,
init_momentum=MOMENTUM,
cost=c,
termination_criterion=EpochCounter(MAX_EPOCHS),
batches_per_iter=BATCHES_PER_EPOCH,
batch_size=BATCH_SIZE,
monitoring_dataset=ds,
monitoring_batches=10,
)
trainer = Train(ds,
gsn,
algorithm=alg,
save_path="gsn_sup_example.pkl",
save_freq=10,
extensions=[MonitorBasedLRAdjuster()])
trainer.main_loop()
print("done training")
def get_denoising_autoencoder(structure,corr_val):
n_input, n_output = structure
#corruptor = BinomialCorruptor(corruption_level=corr_val)
corruptor = GaussianCorruptor(stdev=0.25)
config = {
'corruptor': corruptor,
'nhid': n_output,
'nvis': n_input,
'tied_weights': True,
'act_enc': 'sigmoid',
'act_dec': 'sigmoid',
'irange': 4*np.sqrt(6. / (n_input + n_output)),
}
return DenoisingAutoencoder(**config)
def get_layer_trainer_sgd_rbm(layer, trainset):
train_algo = SGD(
learning_rate = 1e-1,
batch_size = 5,
#"batches_per_iter" : 2000,
monitoring_batches = 20,
monitoring_dataset = trainset,
cost = SMD(corruptor=GaussianCorruptor(stdev=0.4)),
termination_criterion = EpochCounter(max_epochs=MAX_EPOCHS_UNSUPERVISED),
)
model = layer
extensions = [MonitorBasedLRAdjuster()]
return Train(model = model, algorithm = train_algo,
save_path='grbm.pkl',save_freq=1,
extensions = extensions, dataset = trainset)
def get_layer_trainer_sgd_rbm(layer, trainset):
train_algo = SGD(
learning_rate = 1e-1,
batch_size = 5,
#"batches_per_iter" : 2000,
monitoring_batches = 20,
monitoring_dataset = trainset,
cost = SMD(corruptor=GaussianCorruptor(stdev=0.4)),
termination_criterion = EpochCounter(max_epochs=MAX_EPOCHS),
# another option:
# MonitorBasedTermCrit(prop_decrease=0.01, N=10),
)
model = layer
callbacks = [MonitorBasedLRAdjuster(), ModelSaver()]
return Train(model = model, algorithm = train_algo,
callbacks = callbacks, dataset = trainset)
def get_reconstruction_func():
V = model.get_input_space().make_theano_batch(name="V")
assert V.dtype == 'float32'
if hasattr(model, 'e_step'):
#S3C
mf = model.e_step.variational_inference(V)
H = mf['H_hat']
S = mf['S_hat']
Z = H * S
recons = T.dot(Z, model.W.T)
elif hasattr(model, 's3c'):
#PDDBM
mf = model.inference_procedure.infer(V)
H = mf['H_hat']
S = mf['S_hat']
Z = H * S
recons = T.dot(Z, model.s3c.W.T)
else:
#RBM
if corrupt > -1.:
from pylearn2.corruption import GaussianCorruptor
c = GaussianCorruptor(stdev=corrupt)
corrupted_V = c(V)
H = model.mean_h_given_v(corrupted_V)
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
H_sample = theano_rng.binomial(size=H.shape, p=H)
from theano.printing import Print
H_sample = Print('H_sample', attrs=['mean'])(H_sample)
H_sample = T.cast(H, 'float32')
recons = model.mean_v_given_h(H_sample)
recons = Print('recons', attrs=['min', 'mean', 'max'])(recons)
rval = function([V], recons)
return rval
def __init__(self,
energy_function_class=GRBM_Type_1,
nhid=10,
irange=0.5,
rng=None,
mean_vis=False,
init_sigma=.4,
learn_sigma=True,
sigma_lr_scale=1.,
init_bias_hid=-2.,
min_sigma=.1,
max_sigma=10.,
dataset_adaptor=VectorDataset(),
trainer=SGDTrainer(
SMD(corruptor=GaussianCorruptor(stdev=0.00)))):
# trainer = SGDTrainer(SMD() )):
self.config = {
'energy_function_class': energy_function_class,
'nhid': nhid,
'irange': irange,
'rng': rng,
'mean_vis': mean_vis,
'init_sigma': init_sigma,
'learn_sigma': True,
'sigma_lr_scale': sigma_lr_scale,
'init_bias_hid': init_bias_hid,
'min_sigma': min_sigma,
'max_sigma': max_sigma,
'learn_sigma': learn_sigma
}
self.dataset_adaptor = dataset_adaptor
self.trainer = trainer
return
def main_train(epochs,
batchsize,
solution='',
sparse_penalty=0,
sparsityTarget=0,
sparsityTargetPenalty=0):
# Experiment specific arguments
conf_dataset = {
'dataset': 'avicenna',
'expname': 'dummy', # Used to create the submission file
'transfer': True,
'normalize': True, # (Default = True)
'normalize_on_the_fly': False, # (Default = False)
'randomize_valid': True, # (Default = True)
'randomize_test': True, # (Default = True)
'saving_rate': 0, # (Default = 0)
'savedir': './outputs',
}
# First layer = PCA-75 whiten
pca_layer = {
'name': '1st-PCA',
'num_components': 75,
'min_variance': -50,
'whiten': True,
'pca_class': 'CovEigPCA',
# Training properties
'proba': [1, 0, 0],
'savedir': './outputs',
}
# Load the dataset
data = utils.load_data(conf_dataset)
if conf_dataset['transfer']:
# Data for the ALC proxy
label = data[3]
data = data[:3]
# First layer : train or load a PCA
pca = create_pca(conf_dataset, pca_layer, data, model=pca_layer['name'])
data = [
utils.sharedX(pca.function()(set.get_value(borrow=True)), borrow=True)
for set in data
]
'''
if conf_dataset['transfer']:
data_train, label_train = utils.filter_labels(data[0], label)
alc = embed.score(data_train, label_train)
print '... resulting ALC on train (for PCA) is', alc
'''
nvis = utils.get_constant(data[0].shape[1]).item()
conf = {
'corruption_level': 0.1,
'nhid': 200,
'nvis': nvis,
'anneal_start': 100,
'base_lr': 0.001,
'tied_weights': True,
'act_enc': 'sigmoid',
'act_dec': None,
#'lr_hb': 0.10,
#'lr_vb': 0.10,
'tied_weights': True,
'solution': solution,
'sparse_penalty': sparse_penalty,
'sparsityTarget': sparsityTarget,
'sparsityTargetPenalty': sparsityTargetPenalty,
'irange': 0,
}
# A symbolic input representing your minibatch.
minibatch = tensor.matrix()
# Allocate a denoising autoencoder with binomial noise corruption.
corruptor = GaussianCorruptor(conf['corruption_level'])
da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'],
conf['act_enc'], conf['act_dec'],
conf['tied_weights'], conf['solution'],
conf['sparse_penalty'], conf['sparsityTarget'],
conf['sparsityTargetPenalty'])
# Allocate an optimizer, which tells us how to update our model.
# TODO: build the cost another way
cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean()
trainer = SGDOptimizer(da, conf['base_lr'], conf['anneal_start'])
updates = trainer.cost_updates(cost)
# Finally, build a Theano function out of all this.
train_fn = theano.function([minibatch], cost, updates=updates)
# Suppose we want minibatches of size 10
proba = utils.getboth(conf, pca_layer, 'proba')
iterator = BatchIterator(data, proba, batchsize)
# Here's a manual training loop. I hope to have some classes that
# automate this a litle bit.
final_cost = 0
for epoch in xrange(epochs):
c = []
for minibatch_data in iterator:
minibatch_err = train_fn(minibatch_data)
c.append(minibatch_err)
final_cost = numpy.mean(c)
print "epoch %d, cost : %f" % (epoch, final_cost)
print '############################## Fin de l\'experience ############################'
print 'Calcul de l\'ALC : '
if conf_dataset['transfer']:
data_train, label_train = utils.filter_labels(data[0], label)
alc = embed.score(data_train, label_train)
print 'Solution : ', solution
print 'sparse_penalty = ', sparse_penalty
print 'sparsityTarget = ', sparsityTarget
print 'sparsityTargetPenalty = ', sparsityTargetPenalty
print 'Final denoising error is : ', final_cost
print '... resulting ALC on train is', alc
return (alc, final_cost)
'act_dec': None,
#'lr_hb': 0.10,
#'lr_vb': 0.10,
'tied_weights': False,
'solution': 'l1_penalty',
'sparse_penalty': 0.01,
'sparsity_target': 0.1,
'sparsity_target_penalty': 0.001,
'irange': 0.001,
}
# A symbolic input representing your minibatch.
minibatch = tensor.matrix()
# Allocate a denoising autoencoder with binomial noise corruption.
corruptor = GaussianCorruptor(conf['corruption_level'])
da = DenoisingAutoencoder(corruptor, conf['nvis'], conf['nhid'],
conf['act_enc'], conf['act_dec'],
conf['tied_weights'], conf['solution'],
conf['sparse_penalty'], conf['sparsity_target'],
conf['sparsity_target_penalty'])
# Allocate an optimizer, which tells us how to update our model.
# TODO: build the cost another way
cost = SquaredError(da)(minibatch, da.reconstruct(minibatch)).mean()
trainer = SGDOptimizer(da, conf['base_lr'], conf['anneal_start'])
updates = trainer.cost_updates(cost)
# Finally, build a Theano function out of all this.
train_fn = theano.function([minibatch], cost, updates=updates)