def test_train_example():
# path definition
train_path = cwd = os.getcwd()
data_path = os.path.join('${PYLEARN2_DATA_PATH}', 'gtsrb', 'preprocessed')
data_path = serial.preprocess(data_path)
NVIS = 32 * 32 * 3
try:
os.chdir(train_path)
if TRAINING:
# load and train first layer
train_yaml_path = os.path.join(train_path, 'mpdbm_gtsrb.yaml')
layer1_yaml = open(train_yaml_path, 'r').read()
hyper_params_mpdbm = {'batch_size' : 100,
'nvis' : NVIS,
'detector_layer_0_dim' : N_HIDDEN_0,
'detector_layer_1_dim' : N_HIDDEN_1,
'max_epochs' : MAX_EPOCHS_MPDBM,
'data_path' : data_path
}
layer1_yaml = layer1_yaml % (hyper_params_mpdbm)
train = yaml_parse.load(layer1_yaml)
print '\nTraining...\n'
train.main_loop()
if FINETUNING:
dbm = serial.load(os.path.join(train_path,
'mpdbm_gtsrb.pkl'))
if AUGMENT_INPUT:
# data augmentation
train_set = serial.load(os.path.join(data_path,
'preprocessed_train.pkl'))
valid_set = serial.load(os.path.join(data_path,
'preprocessed_valid.pkl'))
test_set = serial.load(os.path.join(data_path,
'preprocessed_test.pkl'))
augmented_train = augment_input(X=train_set.X, model=dbm,
mf_steps=MF_STEPS,
which_set='training')
augmented_valid = augment_input(X=valid_set.X, model=dbm,
mf_steps=MF_STEPS,
which_set='validation')
augmented_test = augment_input(X=test_set.X, model=dbm,
mf_steps=MF_STEPS,
which_set='test')
augmented_train_design_matrix = DenseDesignMatrix(
X=augmented_train,
y=train_set.y)
augmented_valid_design_matrix = DenseDesignMatrix(
X=augmented_valid,
y=valid_set.y)
augmented_test_design_matrix = DenseDesignMatrix(
X=augmented_test,
y=test_set.y)
serial.save(os.path.join(data_path, 'augmented_train.pkl'),
augmented_train_design_matrix)
serial.save(os.path.join(data_path, 'augmented_valid.pkl'),
augmented_valid_design_matrix)
serial.save(os.path.join(data_path, 'augmented_test.pkl'),
augmented_test_design_matrix)
# START SUPERVISED TRAINING WITH BACKPROPAGATION
print("\n-----------------------------------"
"\n Supervised training "
"\n-----------------------------------\n")
# load dbm as a mlp
if DROPOUT:
train_yaml_path = os.path.join(train_path,
'mpdbm_gtsrb_mlp_dropout.yaml')
else:
train_yaml_path = os.path.join(train_path,
'mpdbm_gtsrb_mlp.yaml')
mlp_yaml = open(train_yaml_path, 'r').read()
hyper_params_mlp = {'batch_size' : 100,
'nvis' : NVIS + hyper_params_mpdbm['detector_layer_1_dim'],
'n_h0' : hyper_params_mpdbm['detector_layer_0_dim'],
'n_h1' : hyper_params_mpdbm['detector_layer_1_dim'],
'max_epochs' : MAX_EPOCHS_MLP,
'data_path' : data_path,
'save_path' : train_path,
}
mlp_yaml = mlp_yaml % (hyper_params_mlp)
train = yaml_parse.load(mlp_yaml)
# DBM TRAINED WEIGHTS CLAMPED FOR
# FINETUNING AS EXPLAINED BY HINTON
# concatenate weights between first and second hidden layer
# & weights between visible and first hidden layer
train.model.layers[0].set_weights(
numpy.concatenate(
(dbm.hidden_layers[1].get_weights().transpose(),
dbm.hidden_layers[0].get_weights())
)
)
# then clamp all the others normally
for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]):
l.set_weights(h.get_weights())
# clamp biases
for l, h in zip(train.model.layers, dbm.hidden_layers):
l.set_biases(h.get_biases())
print("\nDBM trained weights and biases have been clamped"
" in the MLP.")
print("\n...Finetuning...\n")
train.main_loop()
finally:
os.chdir(cwd)
def test_train_example():
# path definition
train_path = cwd = os.getcwd()
data_path = os.path.join('${PYLEARN2_DATA_PATH}', 'gtsrb', 'preprocessed')
data_path = serial.preprocess(data_path)
grbm_path = os.path.join(train_path, '..', 'grbm')
grbm = serial.load(os.path.join(grbm_path, 'grbm_gtsrb.pkl'))
NVIS = grbm.nhid
try:
os.chdir(train_path)
# START PRETRAINING
# load and train first layer
train_yaml_path = os.path.join(train_path, 'dbm_gtsrb_l1.yaml')
layer1_yaml = open(train_yaml_path, 'r').read()
hyper_params_l1 = {'batch_size' : 100,
'monitoring_batches' : 5,
'nvis' : NVIS,
'nhid' : N_HIDDEN_0,
'max_epochs' : MAX_EPOCHS_L1,
'data_path' : data_path,
'grbm_path' : grbm_path,
'save_path' : train_path,
}
if PRETRAINING_1:
layer1_yaml = layer1_yaml % (hyper_params_l1)
train = yaml_parse.load(layer1_yaml)
print("\n-----------------------------------"
"\n Unsupervised pre-training "
"\n-----------------------------------\n")
print("\nPre-Training first layer...\n")
train.main_loop()
# load and train second layer
train_yaml_path = os.path.join(train_path, 'dbm_gtsrb_l2.yaml')
layer2_yaml = open(train_yaml_path, 'r').read()
hyper_params_l2 = {'batch_size' : 100,
'monitoring_batches' : 5,
'nvis' : hyper_params_l1['nhid'],
'nhid' : N_HIDDEN_1,
'max_epochs' : MAX_EPOCHS_L2,
'data_path' : data_path,
'grbm_path' : grbm_path,
'save_path' : train_path,
}
if PRETRAINING_2:
layer2_yaml = layer2_yaml % (hyper_params_l2)
train = yaml_parse.load(layer2_yaml)
print("\n...Pre-training second layer...\n")
train.main_loop()
if TRAINING:
# START TRAINING
if SOFTMAX:
train_yaml_path = os.path.join(train_path,
'dbm_gtsrb_softmax.yaml')
else:
train_yaml_path = os.path.join(train_path, 'dbm_gtsrb.yaml')
yaml = open(train_yaml_path, 'r').read()
hyper_params_dbm = {
'batch_size' : 100,
'nvis' : NVIS,
'detector_layer_1_dim' : hyper_params_l1['nhid'],
'detector_layer_2_dim' : hyper_params_l2['nhid'],
'max_epochs' : MAX_EPOCHS_DBM,
'data_path' : data_path,
'grbm_path' : grbm_path,
'save_path' : train_path,
}
yaml = yaml % (hyper_params_dbm)
train = yaml_parse.load(yaml)
rbm1 = serial.load(os.path.join(train_path, 'dbm_gtsrb_l1.pkl'))
rbm2 = serial.load(os.path.join(train_path, 'dbm_gtsrb_l2.pkl'))
pretrained_rbms = [rbm1, rbm2]
# clamp pretrained weights into respective dbm layers
for h, l in zip(train.model.hidden_layers, pretrained_rbms):
h.set_weights(l.get_weights())
# clamp pretrained biases into respective dbm layers
bias_param = pretrained_rbms[0].get_params()[1]
fun = function([], bias_param)
cuda_bias = fun()
bias = numpy.asarray(cuda_bias)
train.model.visible_layer.set_biases(bias)
bias_param = pretrained_rbms[-1].get_params()[1]
fun = function([], bias_param)
cuda_bias = fun()
bias = numpy.asarray(cuda_bias)
train.model.hidden_layers[0].set_biases(bias)
bias_param = pretrained_rbms[-1].get_params()[2]
fun = function([], bias_param)
cuda_bias = fun()
bias = numpy.asarray(cuda_bias)
train.model.hidden_layers[1].set_biases(bias)
print("\nAll layers weights and biases have been clamped to"
" the respective layers of the DBM")
print("\n-----------------------------------"
"\n Unsupervised training "
"\n-----------------------------------\n")
print("\nTraining phase...")
train.main_loop()
if FINETUNING:
if SOFTMAX:
dbm = serial.load(os.path.join(train_path,
'dbm_gtsrb_softmax.pkl'))
else:
dbm = serial.load(os.path.join(train_path,
'dbm_gtsrb.pkl'))
if AUGMENT_INPUT:
# data augmentation
train_set = serial.load(os.path.join(data_path,
'preprocessed_train.pkl'))
valid_set = serial.load(os.path.join(data_path,
'preprocessed_valid.pkl'))
test_set = serial.load(os.path.join(data_path,
'preprocessed_test.pkl'))
augmented_train = augment_input(X=train_set.X, model=dbm,
mf_steps=MF_STEPS,
which_set='training',
transformer=grbm)
augmented_valid = augment_input(X=valid_set.X, model=dbm,
mf_steps=MF_STEPS,
which_set='validation',
transformer=grbm)
augmented_test = augment_input(X=test_set.X, model=dbm,
mf_steps=MF_STEPS,
which_set='test',
transformer=grbm)
augmented_train_design_matrix = DenseDesignMatrix(
X=augmented_train,
y=train_set.y)
augmented_valid_design_matrix = DenseDesignMatrix(
X=augmented_valid,
y=valid_set.y)
augmented_test_design_matrix = DenseDesignMatrix(
X=augmented_test,
y=test_set.y)
serial.save(os.path.join(data_path, 'augmented_train.pkl'),
augmented_train_design_matrix)
serial.save(os.path.join(data_path, 'augmented_valid.pkl'),
augmented_valid_design_matrix)
serial.save(os.path.join(data_path, 'augmented_test.pkl'),
augmented_test_design_matrix)
# START SUPERVISED TRAINING WITH BACKPROPAGATION
print("\n-----------------------------------"
"\n Supervised training "
"\n-----------------------------------\n")
# load dbm as a mlp
if DROPOUT:
train_yaml_path = os.path.join(train_path,
'dbm_gtsrb_mlp_dropout.yaml')
else:
train_yaml_path = os.path.join(train_path,
'dbm_gtsrb_mlp.yaml')
mlp_yaml = open(train_yaml_path, 'r').read()
hyper_params_mlp = {'batch_size' : 100,
'nvis' : NVIS + hyper_params_l2['nhid'],
'n_h0' : hyper_params_l1['nhid'],
'n_h1' : hyper_params_l2['nhid'],
'max_epochs' : MAX_EPOCHS_MLP,
'data_path' : data_path,
'grbm_path' : grbm_path,
'save_path' : train_path,
}
mlp_yaml = mlp_yaml % (hyper_params_mlp)
train = yaml_parse.load(mlp_yaml)
# DBM TRAINED WEIGHTS CLAMPED FOR
# FINETUNING AS EXPLAINED BY HINTON
# concatenate weights between first and second hidden layer
# & weights between visible and first hidden layer
train.model.layers[0].set_weights(
numpy.concatenate(
(dbm.hidden_layers[1].get_weights().transpose(),
dbm.hidden_layers[0].get_weights())
)
)
# then clamp all the others normally
for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]):
l.set_weights(h.get_weights())
# clamp biases
for l, h in zip(train.model.layers, dbm.hidden_layers):
l.set_biases(h.get_biases())
print("\nDBM trained weights and biases have been clamped"
" in the MLP.")
print("\n...Finetuning...\n")
train.main_loop()
finally:
os.chdir(cwd)
