def __init__(self, dae_layers, model_name='sdae', main_dir='sdae/', models_dir='models/', data_dir='data/', summary_dir='logs/', dae_enc_act_func=list(['tanh']), dae_dec_act_func=list(['none']), dae_loss_func=list(['cross_entropy']), dae_num_epochs=list([10]), dae_batch_size=list([10]), dataset='mnist', dae_opt=list(['gradient_descent']), dae_l2reg=list([5e-4]), dae_learning_rate=list([0.01]), momentum=0.5, finetune_dropout=1, dae_corr_type=list(['none']), dae_corr_frac=list([0.]), verbose=1, finetune_loss_func='softmax_cross_entropy', finetune_act_func='relu', finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10, finetune_batch_size=20, do_pretrain=True): """ :param dae_layers: list containing the hidden units for each layer :param dae_enc_act_func: Activation function for the encoder. ['sigmoid', 'tanh'] :param dae_dec_act_func: Activation function for the decoder. ['sigmoid', 'tanh', 'none'] :param finetune_loss_func: Loss function for the softmax layer. string, default ['softmax_cross_entropy', 'mean_squared'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_act_func: activation function for the finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param dae_corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"] :param dae_corr_frac: Fraction of the input to corrupt. float, default 0.0 :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ model.Model.__init__(self, model_name, main_dir, models_dir, data_dir, summary_dir) self._initialize_training_parameters( loss_func=finetune_loss_func, learning_rate=finetune_learning_rate, dropout=finetune_dropout, num_epochs=finetune_num_epochs, batch_size=finetune_batch_size, dataset=dataset, opt=finetune_opt, momentum=momentum) self.do_pretrain = do_pretrain self.layers = dae_layers self.finetune_act_func = finetune_act_func self.verbose = verbose # Model parameters self.encoding_w_ = [ ] # list of matrices of encoding weights (one per layer) self.encoding_b_ = [ ] # list of arrays of encoding biases (one per layer) self.last_W = None self.last_b = None dae_params = { 'enc_act_func': dae_enc_act_func, 'dec_act_func': dae_dec_act_func, 'loss_func': dae_loss_func, 'opt': dae_opt, 'learning_rate': dae_learning_rate, 'l2reg': dae_l2reg, 'corr_frac': dae_corr_frac, 'corr_type': dae_corr_type, 'num_epochs': dae_num_epochs, 'batch_size': dae_batch_size } print 'here' for p in dae_params: if len(dae_params[p]) != len(dae_layers): # The current parameter is not specified by the user, should default it for all the layers dae_params[p] = [dae_params[p][0] for _ in dae_layers] self.autoencoders = [] self.autoencoder_graphs = [] for l, layer in enumerate(dae_layers): dae_str = 'dae-' + str(l + 1) self.autoencoders.append( denoising_autoencoder.DenoisingAutoencoder( n_components=layer, main_dir=self.main_dir, model_name=self.model_name + '-' + dae_str, models_dir=os.path.join(self.models_dir, dae_str), data_dir=os.path.join(self.data_dir, dae_str), summary_dir=os.path.join(self.tf_summary_dir, dae_str), enc_act_func=dae_params['enc_act_func'][l], dec_act_func=dae_params['dec_act_func'][l], loss_func=dae_params['loss_func'][l], opt=dae_params['opt'][l], learning_rate=dae_params['learning_rate'][l], l2reg=dae_params['l2reg'], momentum=self.momentum, corr_type=dae_params['corr_type'][l], corr_frac=dae_params['corr_frac'][l], verbose=self.verbose, num_epochs=dae_params['num_epochs'][l], batch_size=dae_params['batch_size'][l], dataset=self.dataset)) self.autoencoder_graphs.append(tf.Graph())
def __init__(self, layers, model_name='sdae', main_dir='sdae/', models_dir='models/', data_dir='data/', summary_dir='logs/', enc_act_func=[tf.nn.tanh], dec_act_func=[None], loss_func=['cross_entropy'], num_epochs=[10], batch_size=[10], dataset='mnist', opt=['gradient_descent'], regtype=['none'], learning_rate=[0.01], momentum=0.5, finetune_dropout=1, corr_type=['none'], finetune_regtype='none', corr_frac=[0.], verbose=1, finetune_loss_func='cross_entropy', finetune_enc_act_func=[tf.nn.relu], tied_weights=True, finetune_dec_act_func=[tf.nn.sigmoid], l2reg=[5e-4], finetune_batch_size=20, do_pretrain=False, finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10): """Constructor. :param layers: list containing the hidden units for each layer :param enc_act_func: Activation function for the encoder. [tf.nn.tanh, tf.nn.sigmoid] :param dec_act_func: Activation function for the decoder. [tf.nn.tanh, tf.nn.sigmoid, None] :param regtype: regularization type, can be 'l2','l1' and 'none' :param finetune_regtype: regularization type for finetuning :param finetune_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mean_squared'] :param finetune_dropout: dropout parameter :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001 :param finetune_enc_act_func: activation function for the encoder finetuning phase :param finetune_dec_act_func: activation function for the decoder finetuning phase :param finetune_opt: optimizer for the finetuning phase :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20 :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20 :param tied_weights: if True, the decoder layers weights are constrained to be the transpose of the encoder layers :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"] :param corr_frac: Fraction of the input to corrupt. float, default 0.0 :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0 :param do_pretrain: True: uses variables from pretraining, False: initialize new variables. """ # WARNING! This must be the first expression in the function or else it # will send other variables to expanded_args() # This function takes all the passed parameters that are lists and # expands them to the number of layers, if the number # of layers is more than the list of the parameter. expanded_args = utilities.expand_args(**locals()) UnsupervisedModel.__init__(self, model_name, main_dir, models_dir, data_dir, summary_dir) self._initialize_training_parameters( loss_func=finetune_loss_func, learning_rate=finetune_learning_rate, num_epochs=finetune_num_epochs, batch_size=finetune_batch_size, l2reg=l2reg, regtype=finetune_regtype, dropout=finetune_dropout, dataset=dataset, opt=finetune_opt, momentum=momentum) self.do_pretrain = do_pretrain self.layers = layers self.tied_weights = tied_weights self.verbose = verbose self.finetune_enc_act_func = expanded_args['finetune_enc_act_func'] self.finetune_dec_act_func = expanded_args['finetune_dec_act_func'] self.input_ref = None # Model parameters self.encoding_w_ = [] # list of matrices of encoding weights per layer self.encoding_b_ = [] # list of arrays of encoding biases per layer self.decoding_w = [] # list of matrices of decoding weights per layer self.decoding_b = [] # list of arrays of decoding biases per layer self.reconstruction = None self.autoencoders = [] self.autoencoder_graphs = [] for l, layer in enumerate(layers): dae_str = 'dae-' + str(l + 1) self.autoencoders.append( denoising_autoencoder.DenoisingAutoencoder( n_components=layer, main_dir=self.main_dir, model_name=self.model_name + '-' + dae_str, models_dir=os.path.join(self.models_dir, dae_str), data_dir=os.path.join(self.data_dir, dae_str), summary_dir=os.path.join(self.tf_summary_dir, dae_str), enc_act_func=expanded_args['enc_act_func'][l], dec_act_func=expanded_args['dec_act_func'][l], loss_func=expanded_args['loss_func'][l], regtype=expanded_args['regtype'][l], opt=expanded_args['opt'][l], learning_rate=expanded_args['learning_rate'][l], l2reg=expanded_args['l2reg'], dataset=self.dataset, momentum=self.momentum, verbose=self.verbose, corr_type=expanded_args['corr_type'][l], corr_frac=expanded_args['corr_frac'][l], num_epochs=expanded_args['num_epochs'][l], batch_size=expanded_args['batch_size'][l])) self.autoencoder_graphs.append(tf.Graph())
summary_dir = os.path.join(config.summary_dir, FLAGS.main_dir) # Create the object enc_act_func = utilities.str2actfunc(FLAGS.enc_act_func) dec_act_func = utilities.str2actfunc(FLAGS.dec_act_func) dae = denoising_autoencoder.DenoisingAutoencoder( model_name=FLAGS.model_name, n_components=FLAGS.n_components, models_dir=models_dir, data_dir=data_dir, summary_dir=summary_dir, enc_act_func=enc_act_func, dec_act_func=dec_act_func, corr_type=FLAGS.corr_type, corr_frac=FLAGS.corr_frac, dataset=FLAGS.dataset, loss_func=FLAGS.loss_func, main_dir=FLAGS.main_dir, opt=FLAGS.opt, learning_rate=FLAGS.learning_rate, momentum=FLAGS.momentum, l2reg=FLAGS.l2reg, verbose=FLAGS.verbose, num_epochs=FLAGS.num_epochs, batch_size=FLAGS.batch_size) # Fit the model W = None if FLAGS.weights: W = np.load(FLAGS.weights)