def load_from_np(dataset_path):
if dataset_path != '':
return np.load(dataset_path)
else:
return None
trX = load_from_np(FLAGS.train_dataset)
vlX = load_from_np(FLAGS.valid_dataset)
teX = load_from_np(FLAGS.test_dataset)
else:
trX, vlX, teX, width, height = None, None, None, None, None
# Create the object
r = rbm.RBM(num_hidden=FLAGS.num_hidden,
visible_unit_type=FLAGS.visible_unit_type, learning_rate=FLAGS.learning_rate,
num_epochs=FLAGS.num_epochs, batch_size=FLAGS.batch_size, stddev=FLAGS.stddev,
gibbs_sampling_steps=FLAGS.gibbs_sampling_steps, name=FLAGS.name)
# Fit the model
print('Start training...')
r.fit(trX, teX)
# Save the model paramenters
if FLAGS.save_parameters:
print('Saving the parameters of the model...')
params = r.get_parameters()
for p in params:
np.save(FLAGS.save_parameters + '-' + p, params[p])
# Save the reconstructions of the model
if FLAGS.save_reconstructions:
def __init__(self,
rbm_layers,
name='dbn',
do_pretrain=False,
rbm_num_epochs=[10],
rbm_gibbs_k=[1],
rbm_gauss_visible=False,
rbm_stddev=0.1,
rbm_batch_size=[10],
rbm_learning_rate=[0.01],
finetune_dropout=1,
finetune_loss_func='softmax_cross_entropy',
finetune_act_func=tf.nn.sigmoid,
finetune_opt='sgd',
finetune_learning_rate=0.001,
finetune_num_epochs=10,
finetune_batch_size=20,
momentum=0.5):
"""Constructor.
:param rbm_layers: list containing the hidden units for each layer
:param finetune_loss_func: Loss function for the softmax layer.
string, default ['softmax_cross_entropy', 'mse']
:param finetune_dropout: dropout parameter
:param finetune_learning_rate: learning rate for the finetuning.
float, default 0.001
:param finetune_act_func: activation function for the finetuning phase
:param finetune_opt: optimizer for the finetuning phase
:param finetune_num_epochs: Number of epochs for the finetuning.
int, default 20
:param finetune_batch_size: Size of each mini-batch for the finetuning.
int, default 20
:param do_pretrain: True: uses variables from pretraining,
False: initialize new variables.
"""
SupervisedModel.__init__(self, name)
self.loss_func = finetune_loss_func
self.learning_rate = finetune_learning_rate
self.opt = finetune_opt
self.num_epochs = finetune_num_epochs
self.batch_size = finetune_batch_size
self.momentum = momentum
self.dropout = finetune_dropout
self.loss = Loss(self.loss_func)
self.trainer = Trainer(finetune_opt,
learning_rate=finetune_learning_rate,
momentum=momentum)
self.do_pretrain = do_pretrain
self.layers = rbm_layers
self.finetune_act_func = finetune_act_func
# Model parameters
self.encoding_w_ = [] # list of matrices of encoding weights per layer
self.encoding_b_ = [] # list of arrays of encoding biases per layer
self.softmax_W = None
self.softmax_b = None
rbm_params = {
'num_epochs': rbm_num_epochs,
'gibbs_k': rbm_gibbs_k,
'batch_size': rbm_batch_size,
'learning_rate': rbm_learning_rate
}
for p in rbm_params:
if len(rbm_params[p]) != len(rbm_layers):
# The current parameter is not specified by the user,
# should default it for all the layers
rbm_params[p] = [rbm_params[p][0] for _ in rbm_layers]
self.rbms = []
self.rbm_graphs = []
for l, layer in enumerate(rbm_layers):
rbm_str = 'rbm-' + str(l + 1)
if l == 0 and rbm_gauss_visible:
self.rbms.append(
rbm.RBM(name=self.name + '-' + rbm_str,
num_hidden=layer,
learning_rate=rbm_params['learning_rate'][l],
num_epochs=rbm_params['num_epochs'][l],
batch_size=rbm_params['batch_size'][l],
gibbs_sampling_steps=rbm_params['gibbs_k'][l],
visible_unit_type='gauss',
stddev=rbm_stddev))
else:
self.rbms.append(
rbm.RBM(name=self.name + '-' + rbm_str,
num_hidden=layer,
learning_rate=rbm_params['learning_rate'][l],
num_epochs=rbm_params['num_epochs'][l],
batch_size=rbm_params['batch_size'][l],
gibbs_sampling_steps=rbm_params['gibbs_k'][l]))
self.rbm_graphs.append(tf.Graph())
def __init__(self,
layers,
name='srbm',
num_epochs=[10],
batch_size=[10],
learning_rate=[0.01],
gibbs_k=[1],
loss_func=['mse'],
momentum=0.5,
finetune_dropout=1,
finetune_loss_func='cross_entropy',
finetune_enc_act_func=[tf.nn.relu],
finetune_dec_act_func=[tf.nn.sigmoid],
finetune_opt='sgd',
finetune_learning_rate=0.001,
regcoef=5e-4,
finetune_num_epochs=10,
noise=['gauss'],
stddev=0.1,
finetune_batch_size=20,
do_pretrain=False,
tied_weights=False,
regtype=['none'],
finetune_regtype='none'):
"""Constructor.
:param layers: list containing the hidden units for each layer
:param finetune_loss_func: Loss function for the softmax layer.
string, default ['cross_entropy', 'mse']
:param finetune_dropout: dropout parameter
:param finetune_learning_rate: learning rate for the finetuning.
float, default 0.001
:param finetune_enc_act_func: activation function for the encoder
finetuning phase
:param finetune_dec_act_func: activation function for the decoder
finetuning phase
:param finetune_opt: optimizer for the finetuning phase
:param finetune_num_epochs: Number of epochs for the finetuning.
int, default 20
:param finetune_batch_size: Size of each mini-batch for the finetuning.
int, default 20
:param do_pretrain: True: uses variables from pretraining,
False: initialize new variables.
"""
# WARNING! This must be the first expression in the function or else it
# will send other variables to expanded_args()
# This function takes all the passed parameters that are lists and
# expands them to the number of layers, if the number
# of layers is more than the list of the parameter.
expanded_args = utilities.expand_args(**locals())
UnsupervisedModel.__init__(self, name)
self.loss_func = finetune_loss_func
self.learning_rate = finetune_learning_rate
self.opt = finetune_opt
self.num_epochs = finetune_num_epochs
self.batch_size = finetune_batch_size
self.momentum = momentum
self.dropout = finetune_dropout
self.regtype = finetune_regtype
self.regcoef = regcoef
self.loss = Loss(self.loss_func)
self.trainer = Trainer(finetune_opt,
learning_rate=finetune_learning_rate,
momentum=momentum)
self.do_pretrain = do_pretrain
self.layers = layers
self.tied_weights = tied_weights
self.finetune_enc_act_func = expanded_args['finetune_enc_act_func']
self.finetune_dec_act_func = expanded_args['finetune_dec_act_func']
self.input_ref = None
# Model parameters
self.encoding_w_ = [] # list of matrices of encoding weights per layer
self.encoding_b_ = [] # list of arrays of encoding biases per layer
self.decoding_w = [] # list of matrices of decoding weights per layer
self.decoding_b = [] # list of arrays of decoding biases per layer
self.reconstruction = None
self.rbms = []
self.rbm_graphs = []
for l, layer in enumerate(layers):
rbm_str = 'rbm-' + str(l + 1)
new_rbm = rbm.RBM(name=self.name + '-' + rbm_str,
loss_func=expanded_args['loss_func'][l],
visible_unit_type=expanded_args['noise'][l],
stddev=stddev,
num_hidden=expanded_args['layers'][l],
learning_rate=expanded_args['learning_rate'][l],
gibbs_sampling_steps=expanded_args['gibbs_k'][l],
num_epochs=expanded_args['num_epochs'][l],
batch_size=expanded_args['batch_size'][l],
regtype=expanded_args['regtype'][l])
self.rbms.append(new_rbm)
self.rbm_graphs.append(tf.Graph())