def load_from_np(dataset_path): if dataset_path != '': return np.load(dataset_path) else: return None trX = load_from_np(FLAGS.train_dataset) vlX = load_from_np(FLAGS.valid_dataset) teX = load_from_np(FLAGS.test_dataset) else: trX, vlX, teX, width, height = None, None, None, None, None # Create the object r = rbm.RBM(num_hidden=FLAGS.num_hidden, visible_unit_type=FLAGS.visible_unit_type, learning_rate=FLAGS.learning_rate, num_epochs=FLAGS.num_epochs, batch_size=FLAGS.batch_size, stddev=FLAGS.stddev, gibbs_sampling_steps=FLAGS.gibbs_sampling_steps, name=FLAGS.name) # Fit the model print('Start training...') r.fit(trX, teX) # Save the model paramenters if FLAGS.save_parameters: print('Saving the parameters of the model...') params = r.get_parameters() for p in params: np.save(FLAGS.save_parameters + '-' + p, params[p]) # Save the reconstructions of the model if FLAGS.save_reconstructions:
def __init__(self, rbm_layers, name='dbn', do_pretrain=False, rbm_num_epochs=[10], rbm_gibbs_k=[1], rbm_gauss_visible=False, rbm_stddev=0.1, rbm_batch_size=[10], rbm_learning_rate=[0.01], finetune_dropout=1, finetune_loss_func='softmax_cross_entropy', finetune_act_func=tf.nn.sigmoid, finetune_opt='sgd', finetune_learning_rate=0.001, finetune_num_epochs=10, finetune_batch_size=20, momentum=0.5): """Constructor. :param rbm_layers: list containing the hidden units for each layer :param finetune_loss_func: Loss function for the softmax layer. string, default ['softmax_cross_entropy', 'mse'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_act_func: activation function for the finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ SupervisedModel.__init__(self, name) self.loss_func = finetune_loss_func self.learning_rate = finetune_learning_rate self.opt = finetune_opt self.num_epochs = finetune_num_epochs self.batch_size = finetune_batch_size self.momentum = momentum self.dropout = finetune_dropout self.loss = Loss(self.loss_func) self.trainer = Trainer(finetune_opt, learning_rate=finetune_learning_rate, momentum=momentum) self.do_pretrain = do_pretrain self.layers = rbm_layers self.finetune_act_func = finetune_act_func # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.softmax_W = None self.softmax_b = None rbm_params = { 'num_epochs': rbm_num_epochs, 'gibbs_k': rbm_gibbs_k, 'batch_size': rbm_batch_size, 'learning_rate': rbm_learning_rate } for p in rbm_params: if len(rbm_params[p]) != len(rbm_layers): # The current parameter is not specified by the user, # should default it for all the layers rbm_params[p] = [rbm_params[p][0] for _ in rbm_layers] self.rbms = [] self.rbm_graphs = [] for l, layer in enumerate(rbm_layers): rbm_str = 'rbm-' + str(l + 1) if l == 0 and rbm_gauss_visible: self.rbms.append( rbm.RBM(name=self.name + '-' + rbm_str, num_hidden=layer, learning_rate=rbm_params['learning_rate'][l], num_epochs=rbm_params['num_epochs'][l], batch_size=rbm_params['batch_size'][l], gibbs_sampling_steps=rbm_params['gibbs_k'][l], visible_unit_type='gauss', stddev=rbm_stddev)) else: self.rbms.append( rbm.RBM(name=self.name + '-' + rbm_str, num_hidden=layer, learning_rate=rbm_params['learning_rate'][l], num_epochs=rbm_params['num_epochs'][l], batch_size=rbm_params['batch_size'][l], gibbs_sampling_steps=rbm_params['gibbs_k'][l])) self.rbm_graphs.append(tf.Graph())
def __init__(self, layers, name='srbm', num_epochs=[10], batch_size=[10], learning_rate=[0.01], gibbs_k=[1], loss_func=['mse'], momentum=0.5, finetune_dropout=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], finetune_dec_act_func=[tf.nn.sigmoid], finetune_opt='sgd', finetune_learning_rate=0.001, regcoef=5e-4, finetune_num_epochs=10, noise=['gauss'], stddev=0.1, finetune_batch_size=20, do_pretrain=False, tied_weights=False, regtype=['none'], finetune_regtype='none'): """Constructor. :param layers: list containing the hidden units for each layer :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mse'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) UnsupervisedModel.__init__(self, name) self.loss_func = finetune_loss_func self.learning_rate = finetune_learning_rate self.opt = finetune_opt self.num_epochs = finetune_num_epochs self.batch_size = finetune_batch_size self.momentum = momentum self.dropout = finetune_dropout self.regtype = finetune_regtype self.regcoef = regcoef self.loss = Loss(self.loss_func) self.trainer = Trainer(finetune_opt, learning_rate=finetune_learning_rate, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.tied_weights = tied_weights self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.decoding_w = [] # list of matrices of decoding weights per layer self.decoding_b = [] # list of arrays of decoding biases per layer self.reconstruction = None self.rbms = [] self.rbm_graphs = [] for l, layer in enumerate(layers): rbm_str = 'rbm-' + str(l + 1) new_rbm = rbm.RBM(name=self.name + '-' + rbm_str, loss_func=expanded_args['loss_func'][l], visible_unit_type=expanded_args['noise'][l], stddev=stddev, num_hidden=expanded_args['layers'][l], learning_rate=expanded_args['learning_rate'][l], gibbs_sampling_steps=expanded_args['gibbs_k'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l], regtype=expanded_args['regtype'][l]) self.rbms.append(new_rbm) self.rbm_graphs.append(tf.Graph())