def __init__(self, layers, model_name='sdae', main_dir='sdae/', models_dir='models/', data_dir='data/', summary_dir='logs/', num_epochs=[10], batch_size=[10], dataset='mnist', learning_rate=[0.01], gibbs_k=[1], momentum=0.5, finetune_dropout=1, verbose=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], finetune_dec_act_func=[tf.nn.sigmoid], finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10, noise=['gauss'], stddev=0.1, finetune_batch_size=20, do_pretrain=True): """ :param layers: list containing the hidden units for each layer :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mean_squared'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it will send other variables to expanded_args() # This function takes all the passed parameters that are lists and expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(layers, **locals()) UnsupervisedModel.__init__(self, model_name, main_dir, models_dir, data_dir, summary_dir) self._initialize_training_parameters(loss_func=finetune_loss_func, learning_rate=finetune_learning_rate, num_epochs=finetune_num_epochs, batch_size=finetune_batch_size, dropout=finetune_dropout, dataset=dataset, opt=finetune_opt, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.verbose = verbose self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights (one per layer) self.encoding_b_ = [] # list of arrays of encoding biases (one per layer) self.decoding_w = [] # list of matrices of decoding weights (one per layer) self.decoding_b = [] # list of arrays of decoding biases (one per layer) self.reconstruction = None self.rbms = [] self.rbm_graphs = [] for l, layer in enumerate(layers): rbm_str = 'rbm-' + str(l + 1) new_rbm = rbm.RBM(model_name=self.model_name + '-' + rbm_str, models_dir=os.path.join(self.models_dir, rbm_str), data_dir=os.path.join(self.data_dir, rbm_str), summary_dir=os.path.join(self.tf_summary_dir, rbm_str), visible_unit_type=expanded_args['noise'][l], stddev=stddev, num_hidden=expanded_args['layers'][l], main_dir=self.main_dir, learning_rate=expanded_args['learning_rate'][l], gibbs_sampling_steps=expanded_args['gibbs_k'][l], verbose=self.verbose, num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l]) self.rbms.append(new_rbm) self.rbm_graphs.append(tf.Graph())
def __init__( self, layers, name='sdae', enc_act_func=[tf.nn.tanh], dec_act_func=[None], loss_func=['cross_entropy'], num_epochs=[10], batch_size=[10], opt=['sgd'], regcoef=[5e-4], learning_rate=[0.01], momentum=0.5, finetune_dropout=1, corr_type=['none'], corr_frac=[0.], finetune_loss_func='softmax_cross_entropy', finetune_act_func=tf.nn.relu, finetune_opt='sgd', finetune_learning_rate=0.001, finetune_num_epochs=10, finetune_batch_size=20, do_pretrain=False): """Constructor. :param layers: list containing the hidden units for each layer :param enc_act_func: Activation function for the encoder. [tf.nn.tanh, tf.nn.sigmoid] :param dec_act_func: Activation function for the decoder. [tf.nn.tanh, tf.nn.sigmoid, None] :param finetune_loss_func: Loss function for the softmax layer. string, default ['softmax_cross_entropy', 'mse'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_act_func: activation function for the finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"] :param corr_frac: Fraction of the input to corrupt. float, default 0.0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) SupervisedModel.__init__(self, name) self.loss_func = finetune_loss_func self.learning_rate = finetune_learning_rate self.opt = finetune_opt self.num_epochs = finetune_num_epochs self.batch_size = finetune_batch_size self.momentum = momentum self.dropout = finetune_dropout self.loss = Loss(self.loss_func) self.trainer = Trainer( finetune_opt, learning_rate=finetune_learning_rate, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.finetune_act_func = finetune_act_func # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.last_W = None self.last_b = None self.autoencoders = [] self.autoencoder_graphs = [] for l, layer in enumerate(layers): dae_str = 'dae-' + str(l + 1) self.autoencoders.append( denoising_autoencoder.DenoisingAutoencoder( n_components=layer, name=self.name + '-' + dae_str, enc_act_func=expanded_args['enc_act_func'][l], dec_act_func=expanded_args['dec_act_func'][l], loss_func=expanded_args['loss_func'][l], opt=expanded_args['opt'][l], regcoef=expanded_args['regcoef'], learning_rate=expanded_args['learning_rate'][l], momentum=self.momentum, corr_type=expanded_args['corr_type'][l], corr_frac=expanded_args['corr_frac'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l])) self.autoencoder_graphs.append(tf.Graph())
def __init__(self, layers, model_name='srbm', main_dir='srbm/', models_dir='models/', data_dir='data/', summary_dir='logs/', num_epochs=[10], batch_size=[10], dataset='mnist', learning_rate=[0.01], gibbs_k=[1], loss_func=['mean_squared'], momentum=0.5, finetune_dropout=1, verbose=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], finetune_dec_act_func=[tf.nn.sigmoid], finetune_opt='gradient_descent', finetune_learning_rate=0.001, l2reg=5e-4, finetune_num_epochs=10, noise=['gauss'], stddev=0.1, finetune_batch_size=20, do_pretrain=False, tied_weights=False, regtype=['none'], finetune_reg_type='none'): """Constructor. :param layers: list containing the hidden units for each layer :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mean_squared'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) UnsupervisedModel.__init__(self, model_name, main_dir, models_dir, data_dir, summary_dir) self._initialize_training_parameters( loss_func=finetune_loss_func, learning_rate=finetune_learning_rate, regtype=finetune_reg_type, num_epochs=finetune_num_epochs, batch_size=finetune_batch_size, l2reg=l2reg, dropout=finetune_dropout, dataset=dataset, opt=finetune_opt, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.tied_weights = tied_weights self.verbose = verbose self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.decoding_w = [] # list of matrices of decoding weights per layer self.decoding_b = [] # list of arrays of decoding biases per layer self.reconstruction = None self.rbms = [] self.rbm_graphs = [] for l, layer in enumerate(layers): rbm_str = 'rbm-' + str(l + 1) new_rbm = rbm.RBM(model_name=self.model_name + '-' + rbm_str, loss_func=expanded_args['loss_func'][l], models_dir=os.path.join(self.models_dir, rbm_str), data_dir=os.path.join(self.data_dir, rbm_str), summary_dir=os.path.join(self.tf_summary_dir, rbm_str), visible_unit_type=expanded_args['noise'][l], stddev=stddev, num_hidden=expanded_args['layers'][l], main_dir=self.main_dir, learning_rate=expanded_args['learning_rate'][l], gibbs_sampling_steps=expanded_args['gibbs_k'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l], verbose=self.verbose, regtype=expanded_args['regtype'][l]) self.rbms.append(new_rbm) self.rbm_graphs.append(tf.Graph())
def __init__( self, layers, name='srbm', num_epochs=[10], batch_size=[10], learning_rate=[0.01], gibbs_k=[1], loss_func=['mse'], momentum=0.5, finetune_dropout=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], finetune_dec_act_func=[tf.nn.sigmoid], finetune_opt='sgd', finetune_learning_rate=0.001, regcoef=5e-4, finetune_num_epochs=10, noise=['gauss'], stddev=0.1, finetune_batch_size=20, do_pretrain=False, tied_weights=False, regtype=['none'], finetune_regtype='none'): """Constructor. :param layers: list containing the hidden units for each layer :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mse'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) UnsupervisedModel.__init__(self, name) self.loss_func = finetune_loss_func self.learning_rate = finetune_learning_rate self.opt = finetune_opt self.num_epochs = finetune_num_epochs self.batch_size = finetune_batch_size self.momentum = momentum self.dropout = finetune_dropout self.regtype = finetune_regtype self.regcoef = regcoef self.loss = Loss(self.loss_func) self.trainer = Trainer( finetune_opt, learning_rate=finetune_learning_rate, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.tied_weights = tied_weights self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.decoding_w = [] # list of matrices of decoding weights per layer self.decoding_b = [] # list of arrays of decoding biases per layer self.reconstruction = None self.rbms = [] self.rbm_graphs = [] for l, layer in enumerate(layers): rbm_str = 'rbm-' + str(l + 1) new_rbm = rbm.RBM( name=self.name + '-' + rbm_str, loss_func=expanded_args['loss_func'][l], visible_unit_type=expanded_args['noise'][l], stddev=stddev, num_hidden=expanded_args['layers'][l], learning_rate=expanded_args['learning_rate'][l], gibbs_sampling_steps=expanded_args['gibbs_k'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l], regtype=expanded_args['regtype'][l]) self.rbms.append(new_rbm) self.rbm_graphs.append(tf.Graph())
def __init__(self, layers, model_name='sdae', main_dir='sdae/', models_dir='models/', data_dir='data/', summary_dir='logs/', enc_act_func=[tf.nn.tanh], dec_act_func=[None], loss_func=['cross_entropy'], num_epochs=[10], batch_size=[10], dataset='mnist', opt=['gradient_descent'], l2reg=[5e-4], learning_rate=[0.01], momentum=0.5, finetune_dropout=1, corr_type=['none'], corr_frac=[0.], verbose=1, finetune_loss_func='softmax_cross_entropy', finetune_act_func=tf.nn.relu, finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10, finetune_batch_size=20, do_pretrain=False): """ :param layers: list containing the hidden units for each layer :param enc_act_func: Activation function for the encoder. [tf.nn.tanh, tf.nn.sigmoid] :param dec_act_func: Activation function for the decoder. [[tf.nn.tanh, tf.nn.sigmoid, None] :param finetune_loss_func: Loss function for the softmax layer. string, default ['softmax_cross_entropy', 'mean_squared'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_act_func: activation function for the finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"] :param corr_frac: Fraction of the input to corrupt. float, default 0.0 :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it will send other variables to expanded_args() # This function takes all the passed parameters that are lists and expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) SupervisedModel.__init__(self, model_name, main_dir, models_dir, data_dir, summary_dir) self._initialize_training_parameters(loss_func=finetune_loss_func, learning_rate=finetune_learning_rate, dropout=finetune_dropout, num_epochs=finetune_num_epochs, batch_size=finetune_batch_size, dataset=dataset, opt=finetune_opt, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.finetune_act_func = finetune_act_func self.verbose = verbose # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights (one per layer) self.encoding_b_ = [] # list of arrays of encoding biases (one per layer) self.last_W = None self.last_b = None self.autoencoders = [] self.autoencoder_graphs = [] for l, layer in enumerate(layers): dae_str = 'dae-' + str(l + 1) self.autoencoders.append(denoising_autoencoder.DenoisingAutoencoder( n_components=layer, main_dir=self.main_dir, model_name=self.model_name + '-' + dae_str, models_dir=os.path.join(self.models_dir, dae_str), data_dir=os.path.join(self.data_dir, dae_str), summary_dir=os.path.join(self.tf_summary_dir, dae_str), enc_act_func=expanded_args['enc_act_func'][l], dec_act_func=expanded_args['dec_act_func'][l], loss_func=expanded_args['loss_func'][l], opt=expanded_args['opt'][l], learning_rate=expanded_args['learning_rate'][l], l2reg=expanded_args['l2reg'], momentum=self.momentum, corr_type=expanded_args['corr_type'][l], corr_frac=expanded_args['corr_frac'][l], verbose=self.verbose, num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l], dataset=self.dataset)) self.autoencoder_graphs.append(tf.Graph())
def __init__(self, layers, model_name='sdae', main_dir='sdae/', models_dir='models/', data_dir='data/', summary_dir='logs/', enc_act_func=[tf.nn.tanh], dec_act_func=[None], loss_func=['cross_entropy'], num_epochs=[10], batch_size=[10], dataset='mnist', opt=['gradient_descent'], regtype=['none'], learning_rate=[0.01], momentum=0.5, finetune_dropout=1, corr_type=['none'], finetune_regtype='none', corr_frac=[0.], verbose=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], tied_weights=True, finetune_dec_act_func=[tf.nn.sigmoid], l2reg=[5e-4], finetune_batch_size=20, do_pretrain=False, finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10): """Constructor. :param layers: list containing the hidden units for each layer :param enc_act_func: Activation function for the encoder. [tf.nn.tanh, tf.nn.sigmoid] :param dec_act_func: Activation function for the decoder. [tf.nn.tanh, tf.nn.sigmoid, None] :param regtype: regularization type, can be 'l2','l1' and 'none' :param finetune_regtype: regularization type for finetuning :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mean_squared'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param tied_weights: if True, the decoder layers weights are constrained to be the transpose of the encoder layers :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"] :param corr_frac: Fraction of the input to corrupt. float, default 0.0 :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) UnsupervisedModel.__init__(self, model_name, main_dir, models_dir, data_dir, summary_dir) self._initialize_training_parameters( loss_func=finetune_loss_func, learning_rate=finetune_learning_rate, num_epochs=finetune_num_epochs, batch_size=finetune_batch_size, l2reg=l2reg, regtype=finetune_regtype, dropout=finetune_dropout, dataset=dataset, opt=finetune_opt, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.tied_weights = tied_weights self.verbose = verbose self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.decoding_w = [] # list of matrices of decoding weights per layer self.decoding_b = [] # list of arrays of decoding biases per layer self.reconstruction = None self.autoencoders = [] self.autoencoder_graphs = [] for l, layer in enumerate(layers): dae_str = 'dae-' + str(l + 1) self.autoencoders.append( denoising_autoencoder.DenoisingAutoencoder( n_components=layer, main_dir=self.main_dir, model_name=self.model_name + '-' + dae_str, models_dir=os.path.join(self.models_dir, dae_str), data_dir=os.path.join(self.data_dir, dae_str), summary_dir=os.path.join(self.tf_summary_dir, dae_str), enc_act_func=expanded_args['enc_act_func'][l], dec_act_func=expanded_args['dec_act_func'][l], loss_func=expanded_args['loss_func'][l], regtype=expanded_args['regtype'][l], opt=expanded_args['opt'][l], learning_rate=expanded_args['learning_rate'][l], l2reg=expanded_args['l2reg'], dataset=self.dataset, momentum=self.momentum, verbose=self.verbose, corr_type=expanded_args['corr_type'][l], corr_frac=expanded_args['corr_frac'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l])) self.autoencoder_graphs.append(tf.Graph())
def __init__(self, layers, name='sdae', enc_act_func=[tf.nn.tanh], dec_act_func=[None], loss_func=['cross_entropy'], num_epochs=[10], batch_size=[10], opt=['sgd'], regcoef=[5e-4], learning_rate=[0.01], momentum=0.5, finetune_dropout=1, corr_type=['none'], corr_frac=[0.], finetune_loss_func='softmax_cross_entropy', finetune_act_func=tf.nn.relu, finetune_opt='sgd', finetune_learning_rate=0.001, finetune_num_epochs=10, finetune_batch_size=20, do_pretrain=False): """Constructor. :param layers: list containing the hidden units for each layer :param enc_act_func: Activation function for the encoder. [tf.nn.tanh, tf.nn.sigmoid] :param dec_act_func: Activation function for the decoder. [tf.nn.tanh, tf.nn.sigmoid, None] :param finetune_loss_func: Loss function for the softmax layer. string, default ['softmax_cross_entropy', 'mse'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_act_func: activation function for the finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"] :param corr_frac: Fraction of the input to corrupt. float, default 0.0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) SupervisedModel.__init__(self, name) self.loss_func = finetune_loss_func self.learning_rate = finetune_learning_rate self.opt = finetune_opt self.num_epochs = finetune_num_epochs self.batch_size = finetune_batch_size self.momentum = momentum self.dropout = finetune_dropout self.loss = Loss(self.loss_func) self.trainer = Trainer(finetune_opt, learning_rate=finetune_learning_rate, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.finetune_act_func = finetune_act_func # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.last_W = None self.last_b = None self.autoencoders = [] self.autoencoder_graphs = [] for l, layer in enumerate(layers): dae_str = 'dae-' + str(l + 1) self.autoencoders.append( denoising_autoencoder.DenoisingAutoencoder( n_components=layer, name=self.name + '-' + dae_str, enc_act_func=expanded_args['enc_act_func'][l], dec_act_func=expanded_args['dec_act_func'][l], loss_func=expanded_args['loss_func'][l], opt=expanded_args['opt'][l], regcoef=expanded_args['regcoef'], learning_rate=expanded_args['learning_rate'][l], momentum=self.momentum, corr_type=expanded_args['corr_type'][l], corr_frac=expanded_args['corr_frac'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l])) self.autoencoder_graphs.append(tf.Graph())
def __init__(self, layers, name='srbm', num_epochs=[10], batch_size=[10], learning_rate=[0.01], gibbs_k=[1], loss_func=['mse'], momentum=0.5, finetune_dropout=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], finetune_dec_act_func=[tf.nn.sigmoid], finetune_opt='sgd', finetune_learning_rate=0.001, regcoef=5e-4, finetune_num_epochs=10, noise=['gauss'], stddev=0.1, finetune_batch_size=20, do_pretrain=False, tied_weights=False, regtype=['none'], finetune_regtype='none'): """Constructor. :param layers: list containing the hidden units for each layer :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mse'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) UnsupervisedModel.__init__(self, name) self.loss_func = finetune_loss_func self.learning_rate = finetune_learning_rate self.opt = finetune_opt self.num_epochs = finetune_num_epochs self.batch_size = finetune_batch_size self.momentum = momentum self.dropout = finetune_dropout self.regtype = finetune_regtype self.regcoef = regcoef self.loss = Loss(self.loss_func) self.trainer = Trainer(finetune_opt, learning_rate=finetune_learning_rate, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.tied_weights = tied_weights self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.decoding_w = [] # list of matrices of decoding weights per layer self.decoding_b = [] # list of arrays of decoding biases per layer self.reconstruction = None self.rbms = [] self.rbm_graphs = [] for l, layer in enumerate(layers): rbm_str = 'rbm-' + str(l + 1) new_rbm = rbm.RBM(name=self.name + '-' + rbm_str, loss_func=expanded_args['loss_func'][l], visible_unit_type=expanded_args['noise'][l], stddev=stddev, num_hidden=expanded_args['layers'][l], learning_rate=expanded_args['learning_rate'][l], gibbs_sampling_steps=expanded_args['gibbs_k'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l], regtype=expanded_args['regtype'][l]) self.rbms.append(new_rbm) self.rbm_graphs.append(tf.Graph())