def def_network(self, lr): next_layer_input = self.x for i in range(len(self.layer_sizes)): dim = self.layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) W = tf.Variable(xavier_init(input_dim, dim, const=1.0), dtype=tf.float32) b = tf.Variable(tf.zeros([dim]), dtype=tf.float32) self.encoding_matrices.append(W) self.encoding_biases.append(b) output = tf.nn.sigmoid(tf.matmul(next_layer_input, W) + b) next_layer_input = output self.encoded_x = next_layer_input # FFNN self.W = tf.Variable(tf.zeros([self.layer_sizes[-1], self.FFNN_layer], np.float32), name='Weight_FFNN') self.b = tf.Variable(tf.zeros([self.FFNN_layer], np.float32), name='bias_FFNN') self.W_out = tf.Variable(tf.zeros([self.FFNN_layer, self.n_class], np.float32), name='Weight_output') self.b_out = tf.Variable(tf.zeros([self.n_class], np.float32), name='bias_output') # compute cost # self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.x - self.reconstructed_x))) # self.cost = tf.sqrt(tf.reduce_mean(tf.square((tf.matmul(self.x, self.W) + self.b) - self.y))) self.y_ = tf.matmul(self.encoded_x, self.W) + self.b self.y_logits = tf.matmul(self.y_, self.W_out) + self.b_out tf.add_to_collection('pred_network', self.y_logits) self.cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=self.y_logits, labels=self.y, name='cross_entropy')) self.optimizer = tf.train.AdamOptimizer(lr).minimize(self.cost) init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def __init__(self, input_size, layer_sizes, layer_names, tied_weights=False, optimizer=tf.train.AdamOptimizer(), transfer_function=tf.nn.sigmoid): self.layer_names = layer_names self.tied_weights = tied_weights # Build the encoding layers self.x = tf.placeholder("float", [None, input_size]) next_layer_input = self.x assert len(layer_sizes) == len(layer_names) self.encoding_matrices = [] self.encoding_biases = [] for i in range(len(layer_sizes)): dim = layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) # Initialize W using xavier initialization W = tf.Variable(xavier_init(input_dim, dim, transfer_function), name=layer_names[i][0]) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1]) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) # the input into the next layer is the output of this layer next_layer_input = output # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input # build the reconstruction layers by reversing the reductions layer_sizes.reverse() self.encoding_matrices.reverse() self.decoding_matrices = [] self.decoding_biases = [] for i, dim in enumerate(layer_sizes[1:] + [int(self.x.get_shape()[1])]): W = None # if we are using tied weights, so just lookup the encoding matrix for this step and transpose it if tied_weights: W = tf.identity(tf.transpose(self.encoding_matrices[i])) else: W = tf.Variable(xavier_init(self.encoding_matrices[i].get_shape()[1].value,self.encoding_matrices[i].get_shape()[0].value, transfer_function)) b = tf.Variable(tf.zeros([dim])) self.decoding_matrices.append(W) self.decoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) next_layer_input = output # i need to reverse the encoding matrices back for loading weights self.encoding_matrices.reverse() self.decoding_matrices.reverse() # the fully encoded and reconstructed value of x is here: self.reconstructed_x = next_layer_input # compute cost self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.x - self.reconstructed_x))) self.optimizer = optimizer.minimize(self.cost) # initalize variables init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def __init__(self, input_size, layer_sizes, layer_names, tied_weights=False, optimizer=tf.train.AdamOptimizer(), transfer_function=tf.nn.sigmoid): self.layer_names = layer_names self.tied_weights = tied_weights # Build the encoding layers self.x = tf.placeholder("float", [None, input_size]) next_layer_input = self.x assert len(layer_sizes) == len(layer_names) self.encoding_matrices = [] self.encoding_biases = [] for i in range(len(layer_sizes)): dim = layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) # Initialize W using xavier initialization W = tf.Variable(xavier_init(input_dim, dim, transfer_function), name=layer_names[i][0]) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1]) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) # the input into the next layer is the output of this layer next_layer_input = output # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input # build the reconstruction layers by reversing the reductions layer_sizes.reverse() self.encoding_matrices.reverse() self.decoding_matrices = [] self.decoding_biases = [] for i, dim in enumerate(layer_sizes[1:] + [int(self.x.get_shape()[1])]): W = None # if we are using tied weights, so just lookup the encoding matrix for this step and transpose it if tied_weights: W = tf.identity(tf.transpose(self.encoding_matrices[i])) else: W = tf.Variable( xavier_init(self.encoding_matrices[i].get_shape()[1].value, self.encoding_matrices[i].get_shape()[0].value, transfer_function)) b = tf.Variable(tf.zeros([dim])) self.decoding_matrices.append(W) self.decoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) next_layer_input = output # i need to reverse the encoding matrices back for loading weights self.encoding_matrices.reverse() self.decoding_matrices.reverse() # the fully encoded and reconstructed value of x is here: self.reconstructed_x = next_layer_input # compute cost self.cost = tf.sqrt( tf.reduce_mean(tf.square(self.x - self.reconstructed_x))) self.optimizer = optimizer.minimize(self.cost) # initalize variables init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def __init__(self, architecture, layer_names, tied_weights=False, optimizer=tf.train.AdamOptimizer()): DEFAULT_ACTIVATION = tf.nn.sigmoid #input_size = architecture[0]['nodes'] #layer_sizes = [d['nodes'] for d in architecture[1:]] self.layer_names = layer_names self.tied_weights = tied_weights self.datetime = "000" #datetime.now().strftime(r"%y%m%d_%H%M") self.step = 0 assert len(architecture[1:]) == len(layer_names) # Build the encoding layers self.x = tf.placeholder("float", [None, architecture[0]['nodes']], name="x_in") next_layer_input = self.x # Build encoder (hidden layers) self.encoding_matrices = [] self.encoding_biases = [] for i, layer in enumerate(architecture[1:]): input_dim = int(next_layer_input.get_shape()[1]) dim = layer['nodes'] transfer_function = (layer['activation'] if 'activation' in layer else DEFAULT_ACTIVATION) # Initialize W using xavier initialization W = tf.Variable(initial_value=xavier_init(input_dim, dim, transfer_function), name=layer_names[i][0]) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1]) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) # the input into the next layer is the output of this layer next_layer_input = output # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input # build the reconstruction layers by reversing the reductions architecture.reverse() self.encoding_matrices.reverse() self.decoding_matrices = [] self.decoding_biases = [] for i, layer in enumerate(architecture[1:]): W = None transfer_function = (layer['activation'] if 'activation' in layer else DEFAULT_ACTIVATION) # if we are using tied weights, so just lookup the encoding matrix for this step and transpose it if tied_weights: W = tf.identity(tf.transpose(self.encoding_matrices[i])) else: W = tf.Variable( xavier_init(self.encoding_matrices[i].get_shape()[1].value, self.encoding_matrices[i].get_shape()[0].value, transfer_function)) b = tf.Variable(tf.zeros([layer['nodes']])) self.decoding_matrices.append(W) self.decoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) next_layer_input = output # need to reverse the encoding matrices back for loading weights self.encoding_matrices.reverse() self.decoding_matrices.reverse() # also reverse back the original architecture design architecture.reverse() # the fully encoded and reconstructed value of x is here: self.reconstructed_x = next_layer_input # compute cost and run optimizer self.total_updates = tf.Variable(0, trainable=False) #self.cost = tf.losses.mean_squared_error(self.reconstructed_x, self.x) self.cost = tf.reduce_mean( self.crossEntropy(self.reconstructed_x, self.x)) self.optimizer = optimizer.minimize(self.cost, global_step=self.total_updates) # compute MSE and cosine similarity self.mse = tf.losses.mean_squared_error(self.reconstructed_x, self.x) self.cosSim = self.cosSim(self.reconstructed_x, self.x) # fill summary charts & histograms with tf.name_scope("Finetuning"): self.summary_cost = tf.summary.scalar('cost', self.cost) self.summary_mse = tf.summary.scalar('MSE', self.mse) self.summary_cossim = tf.summary.scalar('cosine_similarity', self.cosSim) # initalize variables init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init) # for TensorBoard #self.merged_summaries = tf.summary.merge_all() # merge all the summaries and write them out self.logger = tf.summary.FileWriter("log/", self.sess.graph)
def __init__(self, input_size, layer_sizes, layer_names, tied_weights=False, keep_prob=1, optimizer=tf.train.AdamOptimizer(), transfer_function_enc=tf.nn.relu, transfer_function_dec=tf.nn.sigmoid, l2reg=5e-4, regtype='none', loss_func='mean_squared'): self.layer_names = layer_names self.tied_weights = tied_weights self.keep_prob = tf.placeholder(tf.float32) self.keep_prob_value = keep_prob self.l2reg = l2reg self.regtype = regtype self.loss_func = loss_func # Build the encoding layers self.x = tf.placeholder("float", [None, input_size]) next_layer_input = self.x assert len(layer_sizes) == len(layer_names) self.encoding_matrices = [] self.encoding_biases = [] for i in range(len(layer_sizes)): dim = layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) # Initialize W using xavier initialization W = tf.Variable(xavier_init(input_dim, dim, transfer_function_enc), name=layer_names[i][0]) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1]) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) output = transfer_function_enc(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) # the input into the next layer is the output of this layer next_layer_input = output # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input # build the reconstruction layers by reversing the reductions layer_sizes.reverse() self.encoding_matrices.reverse() self.decoding_matrices = [] self.decoding_biases = [] for i, dim in enumerate(layer_sizes[1:] + [int(self.x.get_shape()[1])]): W = None # if we are using tied weights, so just lookup the encoding matrix for this step and transpose it if tied_weights: W = tf.transpose(self.encoding_matrices[i]) else: #W = tf.Variable(tf.transpose(self.encoding_matrices[i].initialized_value())) W = tf.Variable( xavier_init(self.encoding_matrices[i].get_shape()[1].value, self.encoding_matrices[i].get_shape()[0].value, transfer_function_dec)) b = tf.Variable(tf.zeros([dim])) self.decoding_matrices.append(W) self.decoding_biases.append(b) output = transfer_function_dec(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) next_layer_input = output # i need to reverse the encoding matrices back for loading weights self.encoding_matrices.reverse() self.decoding_matrices.reverse() # the fully encoded and reconstructed value of x is here: self.reconstructed_x = next_layer_input # compute cost vars = [] vars.extend(self.encoding_matrices) vars.extend(self.encoding_biases) regterm = self.compute_regularization(vars) if self.loss_func == 'cross_entropy': clip_inf = tf.clip_by_value(self.reconstructed_x, 1e-10, float('inf')) clip_sup = tf.clip_by_value(1 - self.reconstructed_x, 1e-10, float('inf')) cost = -tf.reduce_mean(self.x * tf.log(clip_inf) + (1 - self.x) * tf.log(clip_sup)) elif self.loss_func == 'softmax_cross_entropy': softmax = tf.nn.softmax(self.reconstructed_x) cost = -tf.reduce_mean(self.x * tf.log(softmax) + (1 - self.x) * tf.log(1 - softmax)) else: #mean_squared cost = tf.sqrt( tf.reduce_mean(tf.square(self.x - self.reconstructed_x))) self.cost = (cost + regterm) if regterm is not None else (cost) #_ = tf.scalar_summary(self.loss_func, self.cost) self.optimizer = optimizer.minimize(self.cost) # initalize variables init = tf.initialize_all_variables() self.sess = tf.Session() self.sess.run(init)
def __init__(self, input_size, n_classes, layer_sizes, layer_names, finetune_learning_rate=0.001, momentum=0.5, keep_prob=1, transfer_function=tf.nn.sigmoid, l2reg=5e-4, regtype='none', loss_func='softmax_cross_entropy', opt='gradient_descent', dir_='dbn'): self.layer_names = layer_names self.keep_prob = tf.placeholder(tf.float32, name='keep_prob-input') self.keep_prob_value = keep_prob self.l2reg = l2reg self.regtype = regtype self.loss_func = loss_func self.opt = opt self.momentum = momentum # Build the encoding layers self.x = tf.placeholder("float", [None, input_size], name='x-input') self.y = tf.placeholder("float", [None, n_classes], name='y-input') self.finetune_learning_rate = finetune_learning_rate next_layer_input = self.x assert len(layer_sizes) == len(layer_names) self.encoding_matrices = [] self.encoding_biases = [] for i in range(len(layer_sizes)): dim = layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) # Initialize W using xavier initialization W = tf.Variable(xavier_init(input_dim, dim, transfer_function), name=layer_names[i][0]) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1]) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) # the input into the next layer is the output of this layer next_layer_input = output # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input self.last_W = tf.Variable(tf.truncated_normal( [self.encoded_x.get_shape()[1].value, n_classes], stddev=0.1), name='sm-weigths') self.last_b = tf.Variable(tf.constant(0.1, shape=[n_classes]), name='sm-biases') self.last_out = tf.add(tf.matmul(self.encoded_x, self.last_W), self.last_b) #self.layer_nodes.append(last_out) #self.last_out = last_out # build the reconstruction layers by reversing the reductions # compute cost vars = [] vars.extend(self.encoding_matrices) vars.extend(self.encoding_biases) regterm = self.compute_regularization(vars) if self.loss_func == 'cross_entropy': clip_inf = tf.clip_by_value(self.last_out, 1e-10, float('inf')) clip_sup = tf.clip_by_value(1 - self.last_out, 1e-10, float('inf')) cost = -tf.reduce_mean(self.y * tf.log(clip_inf + 1e-50) + (1 - self.y) * tf.log(clip_sup + 1e-50)) elif self.loss_func == 'softmax_cross_entropy': #softmax = tf.add(tf.nn.softmax(self.last_out),1e-50) #sparse_ cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(self.last_out, self.y)) else: #mean_squared cost = tf.sqrt(tf.reduce_mean(tf.square(self.y - self.last_out))) with tf.name_scope('Loss-target'): self.cost = (cost + regterm) if regterm is not None else (cost) with tf.name_scope('OPT-target'): if self.opt == 'gradient_descent': self.optimizer = tf.train.GradientDescentOptimizer( self.finetune_learning_rate).minimize(self.cost) elif self.opt == 'ada_grad': self.optimizer = tf.train.AdagradOptimizer( self.finetune_learning_rate).minimize(self.cost) elif self.opt == 'momentum': self.optimizer = tf.train.MomentumOptimizer( self.finetune_learning_rate, self.momentum).minimize(self.cost) elif self.opt == 'adam': self.optimizer = tf.train.AdamOptimizer( self.finetune_learning_rate).minimize(self.cost) # initalize variables self.model_predictions = tf.argmax(self.last_out, 1) correct_prediction = tf.equal(self.model_predictions, tf.argmax(self.y, 1)) with tf.name_scope('Accuracy'): self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) self.loss_sup_summary = tf.scalar_summary("loss-sup", self.cost) self.accuracy_summary = tf.scalar_summary('accuracy', self.accuracy) init = tf.initialize_all_variables() self.sess = tf.Session() self.sess.run(init) logs_path = './tf-logs/' + dir_ self.summary_writer = tf.train.SummaryWriter( logs_path, graph=tf.get_default_graph())
def __init__(self, RANDOM_INIT, ALL_WEIGHTS_TRAINABLE, input_size, layer_sizes, layer_names, optimizer=tf.train.AdamOptimizer(), transfer_function=tf.nn.sigmoid): self.keep_prob = tf.placeholder(tf.float32) self.RANDOM_INIT = RANDOM_INIT self.ALL_WEIGHTS_TRAINABLE = ALL_WEIGHTS_TRAINABLE self.layer_names = layer_names # Build the encoding layers self.x = tf.placeholder(tf.float32, [None, input_size]) next_layer_input = self.x assert len(layer_sizes) == len(layer_names) self.encoding_matrices = [] self.encoding_biases = [] for i in range(len(layer_sizes)): dim = layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) # Initialize W using xavier initialization # W = tf.get_variable(name=layer_names[i][0],shape=(input_dim, dim),dtype=tf.float32,initializer=tf.contrib.layers.xavier_initializer()) W = tf.Variable(xavier_init(input_dim, dim, transfer_function, self.RANDOM_INIT), name=layer_names[i][0], trainable=self.ALL_WEIGHTS_TRAINABLE) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1], trainable=self.ALL_WEIGHTS_TRAINABLE) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) # the input into the next layer is the output of this layer next_layer_input = output # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input # Feed forward net self.ff_matrices = [] self.ff_biases = [] # W = tf.get_variable(name="ffw1", shape=(4, 50), dtype=tf.float32, # initializer=tf.contrib.layers.xavier_initializer()) W = tf.Variable(xavier_init(4, 50, transfer_function, self.RANDOM_INIT), name="ffw1") b = tf.Variable(tf.zeros([50]), name="ffb1") self.ff_matrices.append(W) self.ff_biases.append(b) output = transfer_function(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) next_layer_input = output # W = tf.get_variable(name="ffw2", shape=(50, 2), dtype=tf.float32, # initializer=tf.contrib.layers.xavier_initializer()) W = tf.Variable(xavier_init(50, 2, transfer_function, self.RANDOM_INIT), name="ffw2") b = tf.Variable(tf.zeros([2]), name="ffb2") self.ff_matrices.append(W) self.ff_biases.append(b) self.output = tf.nn.softmax(tf.matmul(next_layer_input, W) + b) self.y = tf.placeholder(tf.float32, shape=(None, 2)) # compute cost self.cost = tf.reduce_mean(-tf.reduce_sum(self.y * tf.log(self.output), reduction_indices=[1])) self.optimizer = optimizer.minimize(self.cost) # initalize variables init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def __init__(self, input_size, n_classes, layer_sizes, layer_names, tied_weights=False, keep_prob=1, momentum=0.5, opt_unsup='gradient_descent', opt_sup='ada_grad', finetune_learning_rate=0.3, transfer_function_enc=tf.nn.sigmoid, transfer_function_dec=tf.nn.sigmoid, l2reg=5e-4, regtype='none', loss_func_target='softmax_cross_entropy', loss_func_au='mean_squared', corr_frac=0, corr_type='none', dir_='mixDbn'): self.layer_names = layer_names self.tied_weights = tied_weights self.keep_prob = tf.placeholder(tf.float32) self.keep_prob_value = keep_prob self.l2reg = l2reg self.regtype = regtype self.loss_func_au = loss_func_au self.loss_func_target = loss_func_target self.finetune_learning_rate = finetune_learning_rate self.opt_unsup = opt_unsup self.opt_sup = opt_sup self.momentum = momentum self.corr_frac = corr_frac self.corr_type = corr_type # Build the encoding layers self.x = tf.placeholder(tf.float32, [None, input_size], name='x-input') self.y = tf.placeholder(tf.float32, [None, n_classes], name='y-input') self.x_corr = tf.placeholder(tf.float32, [None, input_size], name='x-corr-input') #next_layer_input = self.x next_layer_input = self.x_corr assert len(layer_sizes) == len(layer_names) self.encoding_matrices = [] self.encoding_biases = [] self.enconding_vars = [] for i in range(len(layer_sizes)): dim = layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) # Initialize W using xavier initialization W = tf.Variable(xavier_init(input_dim, dim, transfer_function_enc), name=layer_names[i][0]) # Initialize b to zero b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1]) # We are going to use tied-weights so store the W matrix for later reference. self.encoding_matrices.append(W) self.encoding_biases.append(b) self.enconding_vars.append(W) self.enconding_vars.append(b) output = transfer_function_enc(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) # the input into the next layer is the output of this layer next_layer_input = output with tf.name_scope('Model'): # The fully encoded x value is now stored in the next_layer_input self.encoded_x = next_layer_input # build the reconstruction layers by reversing the reductions layer_sizes.reverse() layer_names.reverse() self.encoding_matrices.reverse() self.decoding_matrices = [] self.decoding_biases = [] self.decoding_vars = [] for i, dim in enumerate(layer_sizes[1:] + [int(self.x.get_shape()[1])]): W = None # if we are using tied weights, so just lookup the encoding matrix for this step and transpose it if tied_weights: W = tf.transpose(self.encoding_matrices[i]) b = tf.Variable(tf.zeros([dim])) else: #W = tf.Variable(tf.transpose(self.encoding_matrices[i].initialized_value())) W = tf.Variable(xavier_init( self.encoding_matrices[i].get_shape()[1].value, self.encoding_matrices[i].get_shape()[0].value, transfer_function_dec), name=layer_names[i][0] + 'd') b = tf.Variable(tf.zeros([dim]), name=layer_names[i][1] + 'd') self.decoding_matrices.append(W) self.decoding_biases.append(b) self.decoding_vars.append(W) self.decoding_vars.append(b) output = transfer_function_dec(tf.matmul(next_layer_input, W) + b) output = tf.nn.dropout(output, self.keep_prob) next_layer_input = output # i need to reverse the encoding matrices back for loading weights self.encoding_matrices.reverse() self.decoding_matrices.reverse() # the fully encoded and reconstructed value of x is here: self.reconstructed_x = next_layer_input # The fully encoded x value is now stored in the next_layer_input self.last_W = tf.Variable(tf.truncated_normal( [self.encoded_x.get_shape()[1].value, n_classes], stddev=0.1), name='sm-weigths') self.last_b = tf.Variable(tf.constant(0.1, shape=[n_classes]), name='sm-biases') with tf.name_scope('Model'): self.last_out = tf.add(tf.matmul(self.encoded_x, self.last_W), self.last_b) # compute cost vars = [] vars.extend(self.encoding_matrices) vars.extend(self.encoding_biases) regterm = self.compute_regularization(vars) if self.loss_func_au == 'cross_entropy': clip_inf = tf.clip_by_value(self.reconstructed_x, 1e-10, float('inf')) clip_sup = tf.clip_by_value(1 - self.reconstructed_x, 1e-10, float('inf')) cost_au = -tf.reduce_mean(self.x * tf.log(clip_inf) + (1 - self.x) * tf.log(clip_sup)) elif self.loss_func_au == 'softmax_cross_entropy': cost_au = tf.contrib.losses.softmax_cross_entropy( self.reconstructed_x, self.x) else: #mean_squared cost_au = tf.sqrt( tf.reduce_mean(tf.square(self.x - self.reconstructed_x))) if self.loss_func_target == 'cross_entropy': clip_inf = tf.clip_by_value(self.last_out, 1e-10, float('inf')) clip_sup = tf.clip_by_value(1 - self.last_out, 1e-10, float('inf')) cost_target = -tf.reduce_mean(self.y * tf.log(clip_inf) + (1 - self.y) * tf.log(clip_sup)) elif self.loss_func_target == 'softmax_cross_entropy': cost_target = tf.contrib.losses.softmax_cross_entropy( self.last_out, self.y) else: #mean_squared cost_target = tf.sqrt( tf.reduce_mean(tf.square(self.y - self.last_out))) # cost = cost_target# tf.add(cost_au,cost_target) with tf.name_scope('Loss-au'): self.cost_target = (cost_target + regterm) if regterm is not None else ( cost_target) with tf.name_scope('Loss-target'): self.cost_au = (cost_au + regterm) if regterm is not None else (cost_au) #_ = tf.scalar_summary(self.loss_func, self.cost) with tf.name_scope('OPT-au'): if self.opt_sup == 'gradient_descent': all_var = self.enconding_vars + [self.last_W, self.last_b] self.train_step_sup = tf.train.GradientDescentOptimizer( self.finetune_learning_rate).minimize(self.cost_target, var_list=all_var) elif self.opt_sup == 'ada_grad': all_var = self.enconding_vars + [self.last_W, self.last_b] self.train_step_sup = tf.train.AdagradOptimizer( self.finetune_learning_rate).minimize(self.cost_target, var_list=all_var) elif self.opt_sup == 'momentum': all_var = self.enconding_vars + [self.last_W, self.last_b] self.train_step_sup = tf.train.MomentumOptimizer( self.finetune_learning_rate, self.momentum).minimize(self.cost_target, var_list=all_var) elif self.opt_sup == 'adam': all_var = self.enconding_vars + [self.last_W, self.last_b] self.train_step_sup = tf.train.AdamOptimizer( self.finetune_learning_rate).minimize(self.cost_target, var_list=all_var) else: self.train_step_sup = None with tf.name_scope('OPT-target'): if self.opt_unsup == 'gradient_descent': all_var = self.enconding_vars + self.decoding_vars self.train_step_unsup = tf.train.GradientDescentOptimizer( self.finetune_learning_rate).minimize(self.cost_au, var_list=all_var) elif self.opt_unsup == 'ada_grad': all_var = self.enconding_vars + self.decoding_vars self.train_step_unsup = tf.train.AdagradOptimizer( self.finetune_learning_rate).minimize(self.cost_au, var_list=all_var) elif self.opt_unsup == 'momentum': all_var = self.enconding_vars + self.decoding_vars self.train_step_unsup = tf.train.MomentumOptimizer( self.finetune_learning_rate, self.momentum).minimize(self.cost_au, var_list=all_var) elif self.opt_unsup == 'adam': all_var = self.enconding_vars + self.decoding_vars self.train_step_unsup = tf.train.AdamOptimizer( self.finetune_learning_rate).minimize(self.cost_au, var_list=all_var) else: self.train_step_unsup = None self.model_predictions = tf.argmax(self.last_out, 1) correct_prediction = tf.equal(self.model_predictions, tf.argmax(self.y, 1)) with tf.name_scope('Accuracy'): self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) self.loss_unsup_summary = tf.scalar_summary("loss-au", self.cost_au) self.loss_sup_summary = tf.scalar_summary("loss-sup", self.cost_target) # self.merged_summary_op = tf.merge_all_summaries() self.accuracy_summary = tf.scalar_summary('accuracy', self.accuracy) # scaledImageRec_x = tf.image.convert_image_dtype(self.reconstructed_x, dtype=tf.uint8) # reconstructed_x_transposed = tf.transpose(scaledImageRec_x, [None, 28, 28, 1]) # self.reconstruct_summary = tf.image_summary('reconstruct', reconstructed_x_transposed ,max_image=30) logs_path = './tf-logs/' + dir_ # initalize variables init = tf.initialize_all_variables() self.sess = tf.Session() self.sess.run(init) self.summary_writer = tf.train.SummaryWriter( logs_path, graph=tf.get_default_graph())