def cdl_estimate(self, data_x, num_iter): for i in range(num_iter): b_x, ids = utils.get_batch(data_x, self.params.batch_size) _, l, gen_loss, v_loss = self.sess.run( (self.optimizer, self.loss, self.gen_loss, self.v_loss), feed_dict={ self.x: b_x, self.v: self.m_V[ids, :] }) # Display logs per epoch step if i % self.print_step == 0 and self.verbose: print ("Iter:", '%04d' % (i+1), \ "loss=", "{:.5f}".format(l), \ "genloss=", "{:.5f}".format(gen_loss), \ "vloss=", "{:.5f}".format(v_loss)) return gen_loss
def run(self, data_x, hidden_dim, activation, loss, lr, print_step, epoch, batch_size=100): tf.reset_default_graph() input_dim = len(data_x[0]) n = data_x.shape[0] num_iter = int(n / batch_size) sess = tf.Session() x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x') x_ = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x_') encode = { 'weights': tf.Variable(xavier_init(input_dim, hidden_dim, dtype=tf.float32)), 'biases': tf.Variable(tf.zeros([hidden_dim], dtype=tf.float32)) } decode = { 'biases': tf.Variable(tf.zeros([input_dim], dtype=tf.float32)), 'weights': tf.transpose(encode['weights']) } encoded = self.activate( tf.matmul(x, encode['weights']) + encode['biases'], activation) decoded = tf.matmul(encoded, decode['weights']) + decode['biases'] # reconstruction loss if loss == 'rmse': # loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(x_, decoded)))) loss = tf.reduce_mean(tf.reduce_sum(tf.square(x_ - decoded), 1)) elif loss == 'cross-entropy': decoded = tf.nn.sigmoid(decoded, name='decoded') # loss = -tf.reduce_mean(x_ * tf.log(decoded)) loss = -tf.reduce_mean( tf.reduce_sum( x_ * tf.log(tf.maximum(decoded, 1e-16)) + (1 - x_) * tf.log(tf.maximum(1 - decoded, 1e-16)), 1)) train_op = tf.train.AdamOptimizer(lr).minimize(loss) sess.run(tf.global_variables_initializer()) for i in range(epoch): for it in range(num_iter): b_x_, ids = utils.get_batch(data_x, batch_size) b_x = self.add_noise(b_x_) _, l = sess.run((train_op, loss), feed_dict={x: b_x, x_: b_x_}) if (i + 1) % print_step == 0: l = sess.run(loss, feed_dict={x: b_x_, x_: b_x_}) logging.info('epoch {0}: batch loss = {1}'.format(i, l)) # debug # print('Decoded', sess.run(decoded, feed_dict={x: self.data_x_})[0]) self.weights.append(sess.run(encode['weights'])) self.biases.append(sess.run(encode['biases'])) self.de_weights.append(sess.run(decode['weights'])) self.de_biases.append(sess.run(decode['biases'])) return sess.run(encoded, feed_dict={x: data_x})
def run_latent(self, data_x, hidden_dim, batch_size, lr, epoch, print_step=100): tf.reset_default_graph() n = data_x.shape[0] input_dim = len(data_x[0]) num_iter = int(n / batch_size) sess = tf.Session() rec = { 'W_z_mean': tf.get_variable("W_z_mean", [self.dims[1], self.n_z], initializer=tf.contrib.layers.xavier_initializer(), dtype=tf.float32), 'b_z_mean': tf.get_variable("b_z_mean", [self.n_z], initializer=tf.constant_initializer(0.0), dtype=tf.float32), 'W_z_log_sigma': tf.get_variable("W_z_log_sigma", [self.dims[1], self.n_z], initializer=tf.contrib.layers.xavier_initializer(), dtype=tf.float32), 'b_z_log_sigma': tf.get_variable("b_z_log_sigma", [self.n_z], initializer=tf.constant_initializer(0.0), dtype=tf.float32) } gen = { 'W2': tf.get_variable("W2", [self.n_z, self.dims[1]], initializer=tf.contrib.layers.xavier_initializer(), dtype=tf.float32), 'b2': tf.get_variable("b2", [self.dims[1]], initializer=tf.constant_initializer(0.0), dtype=tf.float32) } x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x') z_mean = tf.matmul(x, rec['W_z_mean']) + rec['b_z_mean'] z_log_sigma_sq = tf.matmul(x, rec['W_z_log_sigma']) + rec['b_z_log_sigma'] eps = tf.random_normal((batch_size, hidden_dim), 0, 1, dtype=tf.float32) z = z_mean + tf.sqrt(tf.maximum(tf.exp(z_log_sigma_sq), 1e-10)) * eps x_recon = tf.matmul(z, gen['W2']) + gen['b2'] x_recon = tf.nn.sigmoid(x_recon, name='x_recon') gen_loss = -tf.reduce_mean( tf.reduce_sum( x * tf.log(tf.maximum(x_recon, 1e-10)) + (1 - x) * tf.log(tf.maximum(1 - x_recon, 1e-10)), 1)) latent_loss = 0.5 * tf.reduce_mean( tf.reduce_sum( tf.square(z_mean) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq - 1, 1)) loss = gen_loss + latent_loss train_op = tf.train.AdamOptimizer(lr).minimize(loss) sess.run(tf.global_variables_initializer()) for i in range(epoch): for it in range(num_iter): b_x, ids = utils.get_batch(data_x, batch_size) _, l, gl, ll = sess.run( (train_op, loss, gen_loss, latent_loss), feed_dict={x: b_x}) if (i + 1) % print_step == 0: logging.info( 'epoch {0}: batch loss = {1}, gen_loss={2}, latent_loss={3}' .format(i, l, gl, ll)) self.weights.append(sess.run(rec['W_z_mean'])) self.weights.append(sess.run(rec['W_z_log_sigma'])) self.biases.append(sess.run(rec['b_z_mean'])) self.biases.append(sess.run(rec['b_z_log_sigma'])) self.de_weights.append(sess.run(gen['W2'])) self.de_biases.append(sess.run(gen['b2']))
def run_all(self, data_x, lr, batch_size, epoch, print_step=100, x_valid=None): tf.reset_default_graph() n = data_x.shape[0] input_dim = len(data_x[0]) num_iter = int(n / batch_size) with tf.variable_scope("inference"): rec = { 'W1': tf.get_variable("W1", initializer=tf.constant(self.weights[0]), dtype=tf.float32), 'b1': tf.get_variable("b1", initializer=tf.constant(self.biases[0]), dtype=tf.float32), 'W2': tf.get_variable("W2", initializer=tf.constant(self.weights[1]), dtype=tf.float32), 'b2': tf.get_variable("b2", initializer=tf.constant(self.biases[1]), dtype=tf.float32), 'W_z_mean': tf.get_variable("W_z_mean", initializer=tf.constant(self.weights[2]), dtype=tf.float32), 'b_z_mean': tf.get_variable("b_z_mean", initializer=tf.constant(self.biases[2]), dtype=tf.float32), 'W_z_log_sigma': tf.get_variable("W_z_log_sigma", initializer=tf.constant(self.weights[3]), dtype=tf.float32), 'b_z_log_sigma': tf.get_variable("b_z_log_sigma", initializer=tf.constant(self.biases[3]), dtype=tf.float32) } with tf.variable_scope("generation"): gen = { 'W2': tf.get_variable("W2", initializer=tf.constant(self.de_weights[2]), dtype=tf.float32), 'b2': tf.get_variable("b2", initializer=tf.constant(self.de_biases[2]), dtype=tf.float32), 'W1': tf.transpose(rec['W2']), 'b1': rec['b1'], 'W_x': tf.transpose(rec['W1']), 'b_x': tf.get_variable("b_x", [input_dim], initializer=tf.constant_initializer(0.0), dtype=tf.float32) } weights = [] weights += [ rec['W1'], rec['b1'], rec['W2'], rec['b2'], rec['W_z_mean'], rec['b_z_mean'], rec['W_z_log_sigma'], rec['b_z_log_sigma'] ] weights += [gen['W2'], gen['b2'], gen['b_x']] saver = tf.train.Saver(weights) x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x') h1 = self.activate( tf.matmul(x, rec['W1']) + rec['b1'], self.activations[0]) h2 = self.activate( tf.matmul(h1, rec['W2']) + rec['b2'], self.activations[1]) z_mean = tf.matmul(h2, rec['W_z_mean']) + rec['b_z_mean'] z_log_sigma_sq = tf.matmul(h2, rec['W_z_log_sigma']) + rec['b_z_log_sigma'] eps = tf.random_normal((batch_size, self.n_z), 0, 1, dtype=tf.float32) z = z_mean + tf.sqrt(tf.maximum(tf.exp(z_log_sigma_sq), 1e-10)) * eps h2 = self.activate( tf.matmul(z, gen['W2']) + gen['b2'], self.activations[1]) h1 = self.activate( tf.matmul(h2, gen['W1']) + gen['b1'], self.activations[0]) x_recon = tf.matmul(h1, gen['W_x']) + gen['b_x'] x_recon = tf.nn.sigmoid(x_recon, name='x_recon') gen_loss = -tf.reduce_mean( tf.reduce_sum( x * tf.log(tf.maximum(x_recon, 1e-10)) + (1 - x) * tf.log(tf.maximum(1 - x_recon, 1e-10)), 1)) latent_loss = 0.5 * tf.reduce_mean( tf.reduce_sum( tf.square(z_mean) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq - 1, 1)) loss = gen_loss + latent_loss train_op = tf.train.AdamOptimizer(lr).minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) for i in range(epoch): for it in range(num_iter): b_x, ids = utils.get_batch(data_x, batch_size) _, l, gl, ll = sess.run( (train_op, loss, gen_loss, latent_loss), feed_dict={x: b_x}) if (i + 1) % print_step == 0: if x_valid is None: logging.info( 'epoch {0}: batch loss = {1}, gen_loss={2}, latent_loss={3}' .format(i, l, gl, ll)) else: valid_loss = self.validation(x_valid, sess, gen_loss, x, batch_size) logging.info( 'epoch {0}: batch loss = {1}, gen_loss={2}, latent_loss={3}, valid_loss={4}' .format(i, l, gl, ll, valid_loss)) from pathlib import Path path_o = Path() weight_path = path_o.cwd().joinpath("weights/pretrain") weight_folder = path_o.cwd().joinpath("weights") local_path = str(weight_path.absolute()) saver.save(sess, local_path) logging.info("Weights saved at {}".format(local_path)) self.s3_client.upload_folder_to_s3((weight_folder.absolute()), "weights/pretrain")