def _initialize_weights(self): self.x = tf.placeholder(tf.float32, [None, self.n_visible]) self.y = tf.placeholder(tf.float32, [None, self.n_hidden]) self.w = tf.Variable(tf_xavier_init(self.n_visible, self.n_hidden, const=1.0), dtype=tf.float32) self.vbias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32) self.hbias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32) self.delta_w = tf.Variable(tf.zeros([self.n_visible, self.n_hidden]), dtype=tf.float32) self.delta_vbias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32) self.delta_hbias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32)
def def_network(self, lr): next_layer_input = self.x for i in range(len(self.layer_sizes)): dim = self.layer_sizes[i] input_dim = int(next_layer_input.get_shape()[1]) W = tf.Variable(tf_xavier_init(input_dim, dim, const=1.0), dtype=tf.float32) b = tf.Variable(tf.zeros([dim]), dtype=tf.float32) self.encoding_matrices.append(W) self.encoding_biases.append(b) output = tf.nn.sigmoid(tf.matmul(next_layer_input, W) + b) next_layer_input = output self.encoded_x = next_layer_input # FFNN self.W = tf.Variable(tf.truncated_normal( [self.layer_sizes[-1], self.FFNN_layer]), name='Weight_FFNN', dtype=tf.float32) self.b = tf.Variable(tf.zeros([self.FFNN_layer], np.float32), name='bias_FFNN', dtype=tf.float32) self.W_out = tf.Variable(tf.truncated_normal([self.FFNN_layer, 1]), name='Weight_output', dtype=tf.float32) self.b_out = tf.Variable(tf.zeros([1], np.float32), name='bias_output', dtype=tf.float32) # compute cost # self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.x - self.reconstructed_x))) # self.cost = tf.sqrt(tf.reduce_mean(tf.square((tf.matmul(self.x, self.W) + self.b) - self.y))) self.y_ = tf.matmul(self.encoded_x, self.W) + self.b self.y_logits = tf.matmul(self.y_, self.W_out) + self.b_out tf.add_to_collection('pred_network', self.y_logits) self.cost = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=self.y_logits, labels=self.y, name='cross_entropy')) self.optimizer = tf.train.GradientDescentOptimizer(lr).minimize( self.cost) self.predict_op = tf.nn.sigmoid(self.y_logits) init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def __init__( self, n_visible, n_hidden, learning_rate=0.01, momentum=0.95, xavier_const=1.0, err_function='mse', use_tqdm=False, # DEPRECATED: tqdm=None): if not 0.0 <= momentum <= 1.0: raise ValueError('momentum should be in range [0, 1]') if err_function not in {'mse', 'cosine'}: raise ValueError( 'err_function should be either \'mse\' or \'cosine\'') self._use_tqdm = use_tqdm self._tqdm = None if use_tqdm or tqdm is not None: from tqdm import tqdm self._tqdm = tqdm self.n_visible = n_visible self.n_hidden = n_hidden self.learning_rate = learning_rate self.momentum = momentum self.x = tf.placeholder(tf.float32, [None, self.n_visible]) self.y = tf.placeholder(tf.float32, [None, self.n_hidden]) self.w = tf.Variable(tf_xavier_init(self.n_visible, self.n_hidden, const=xavier_const), dtype=tf.float32) self.visible_bias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32) self.hidden_bias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32) self.delta_w = tf.Variable(tf.zeros([self.n_visible, self.n_hidden]), dtype=tf.float32) self.delta_visible_bias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32) self.delta_hidden_bias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32) self.update_weights = None self.update_deltas = None self.compute_hidden = None self.compute_visible = None self.compute_visible_from_hidden = None self._initialize_vars() assert self.update_weights is not None assert self.update_deltas is not None assert self.compute_hidden is not None assert self.compute_visible is not None assert self.compute_visible_from_hidden is not None if err_function == 'cosine': x1_norm = tf.nn.l2_normalize(self.x, 1) x2_norm = tf.nn.l2_normalize(self.compute_visible, 1) cos_val = tf.reduce_mean(tf.reduce_sum(tf.mul(x1_norm, x2_norm), 1)) self.compute_err = tf.acos(cos_val) / tf.constant(np.pi) else: self.compute_err = tf.reduce_mean( tf.square(self.x - self.compute_visible)) init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def __init__(self, n_visible, n_hidden, momentum=0.95, xavier_const=1.0, err_function='mse', use_tqdm=False, logsdir = None, # DEPRECATED: tqdm=None): if not 0.0 <= momentum <= 1.0: raise ValueError('momentum should be in range [0, 1]') if err_function not in {'mse', 'cosine', 'rmse'}: raise ValueError('err_function should be either \'mse\', \'rmse\' or \'cosine\'') self._use_tqdm = use_tqdm self._tqdm = None if use_tqdm or tqdm is not None: from tqdm import tqdm self._tqdm = tqdm self.n_visible = n_visible self.n_hidden = n_hidden self.learning_rate_ph = tf.placeholder(tf.float32, ())#learning_rate self.momentum = momentum self.x = tf.placeholder(tf.float32, [None, self.n_visible]) self.y = tf.placeholder(tf.float32, [None, self.n_hidden]) # self.w = tf.Variable(tf.random_normal(mean=0.0, stddev=(1/self.n_visible), dtype = tf.float32, shape = [self.n_visible, self.n_hidden])) # self.w = tf.Variable(tf.random_normal(mean=0.0, stddev=(0.1), dtype = tf.float32, shape = [self.n_visible, self.n_hidden])) self.w = tf.Variable(tf_xavier_init(self.n_visible, self.n_hidden, const=xavier_const), dtype=tf.float32) self.visible_bias = tf.Variable(1*tf.ones([self.n_visible]), dtype=tf.float32) # self.hidden_bias = tf.Variable(tf.random_uniform(minval = 0, maxval = 1, dtype = tf.float32, shape = [self.n_hidden])) self.hidden_bias = tf.Variable(1*tf.ones([self.n_hidden]), dtype=tf.float32) self.delta_w = tf.Variable(tf.zeros([self.n_visible, self.n_hidden]), dtype=tf.float32) self.delta_visible_bias = tf.Variable(tf.zeros([self.n_visible]), dtype=tf.float32) self.delta_hidden_bias = tf.Variable(tf.zeros([self.n_hidden]), dtype=tf.float32) self.update_weights = None self.update_deltas = None self.compute_hidden = None self.compute_visible = None self.compute_visible_from_hidden = None self._initialize_vars() assert self.update_weights is not None assert self.update_deltas is not None assert self.compute_hidden is not None assert self.compute_visible is not None assert self.compute_visible_from_hidden is not None if err_function == 'cosine': x1_norm = tf.nn.l2_normalize(self.x, 1) x2_norm = tf.nn.l2_normalize(self.compute_visible, 1) cos_val = tf.reduce_mean(tf.reduce_sum(tf.mul(x1_norm, x2_norm), 1)) self.compute_err = tf.acos(cos_val) / tf.constant(np.pi) elif err_function == 'mse': self.compute_err = tf.reduce_mean(tf.square(self.x - self.compute_visible)) else: #rmse self.compute_err = tf.sqrt(tf.reduce_mean(tf.square(self.x - self.compute_visible))) hist_bv = tf.summary.histogram('visible_bias', self.visible_bias) hist_b = tf.summary.histogram('hidden_bias', self.hidden_bias) hist_w = tf.summary.histogram("w", self.w) self.hists_vars = tf.summary.merge([hist_w] + [hist_b] + [hist_bv]) self.loss_ph = tf.placeholder(tf.float32, shape = (), name="loss_placeholder") self.summary_loss = tf.summary.scalar("loss", self.loss_ph) self.hist_metrics = tf.summary.merge([self.summary_loss]) self.logs_dir = logsdir if self.logs_dir is not None: self.train_writer = tf.summary.FileWriter(self.logs_dir+"/logs/train") # self.test_writer = tf.summary.FileWriter(self.logs_dir+"/logs/test") self.hidden_bias_below_zero = tf.cast(tf.math.greater(tf.zeros_like(self.hidden_bias), self.hidden_bias), tf.float32) self.bias_below_zero = tf.cast(self.hidden_bias.assign(tf.abs(self.hidden_bias)), tf.float32) self.hidden_bias_correction = self.hidden_bias.assign(self.hidden_bias*self.hidden_bias_below_zero) # self.hidden_bias_below_zero = tf.cast(tf.math.greater(tf.zeros_like(self.hidden_bias), self.hidden_bias), tf.float32) # self.hidden_bias_correction = self.hidden_bias.assign_add(tf.abs(tf.reduce_mean(self.hidden_bias))*self.hidden_bias_below_zero) self.visible_bias_below_zero = tf.cast(tf.math.greater(tf.zeros_like(self.visible_bias), self.visible_bias), tf.float32) self.visible_bias_correction = self.visible_bias.assign_add(tf.abs(tf.reduce_mean(self.visible_bias))*self.visible_bias_below_zero) init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)