def cdl_estimate(self, data_x, num_iter):
for i in range(num_iter):
b_x, ids = utils.get_batch(data_x, self.params.batch_size)
_, l, gen_loss, v_loss = self.sess.run(
(self.optimizer, self.loss, self.gen_loss, self.v_loss),
feed_dict={
self.x: b_x,
self.v: self.m_V[ids, :]
})
# Display logs per epoch step
if i % self.print_step == 0 and self.verbose:
print ("Iter:", '%04d' % (i+1), \
"loss=", "{:.5f}".format(l), \
"genloss=", "{:.5f}".format(gen_loss), \
"vloss=", "{:.5f}".format(v_loss))
return gen_loss
def run(self,
data_x,
hidden_dim,
activation,
loss,
lr,
print_step,
epoch,
batch_size=100):
tf.reset_default_graph()
input_dim = len(data_x[0])
n = data_x.shape[0]
num_iter = int(n / batch_size)
sess = tf.Session()
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')
x_ = tf.placeholder(dtype=tf.float32,
shape=[None, input_dim],
name='x_')
encode = {
'weights':
tf.Variable(xavier_init(input_dim, hidden_dim, dtype=tf.float32)),
'biases':
tf.Variable(tf.zeros([hidden_dim], dtype=tf.float32))
}
decode = {
'biases': tf.Variable(tf.zeros([input_dim], dtype=tf.float32)),
'weights': tf.transpose(encode['weights'])
}
encoded = self.activate(
tf.matmul(x, encode['weights']) + encode['biases'], activation)
decoded = tf.matmul(encoded, decode['weights']) + decode['biases']
# reconstruction loss
if loss == 'rmse':
# loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(x_, decoded))))
loss = tf.reduce_mean(tf.reduce_sum(tf.square(x_ - decoded), 1))
elif loss == 'cross-entropy':
decoded = tf.nn.sigmoid(decoded, name='decoded')
# loss = -tf.reduce_mean(x_ * tf.log(decoded))
loss = -tf.reduce_mean(
tf.reduce_sum(
x_ * tf.log(tf.maximum(decoded, 1e-16)) +
(1 - x_) * tf.log(tf.maximum(1 - decoded, 1e-16)), 1))
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
sess.run(tf.global_variables_initializer())
for i in range(epoch):
for it in range(num_iter):
b_x_, ids = utils.get_batch(data_x, batch_size)
b_x = self.add_noise(b_x_)
_, l = sess.run((train_op, loss), feed_dict={x: b_x, x_: b_x_})
if (i + 1) % print_step == 0:
l = sess.run(loss, feed_dict={x: b_x_, x_: b_x_})
logging.info('epoch {0}: batch loss = {1}'.format(i, l))
# debug
# print('Decoded', sess.run(decoded, feed_dict={x: self.data_x_})[0])
self.weights.append(sess.run(encode['weights']))
self.biases.append(sess.run(encode['biases']))
self.de_weights.append(sess.run(decode['weights']))
self.de_biases.append(sess.run(decode['biases']))
return sess.run(encoded, feed_dict={x: data_x})
def run_latent(self,
data_x,
hidden_dim,
batch_size,
lr,
epoch,
print_step=100):
tf.reset_default_graph()
n = data_x.shape[0]
input_dim = len(data_x[0])
num_iter = int(n / batch_size)
sess = tf.Session()
rec = {
'W_z_mean':
tf.get_variable("W_z_mean", [self.dims[1], self.n_z],
initializer=tf.contrib.layers.xavier_initializer(),
dtype=tf.float32),
'b_z_mean':
tf.get_variable("b_z_mean", [self.n_z],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32),
'W_z_log_sigma':
tf.get_variable("W_z_log_sigma", [self.dims[1], self.n_z],
initializer=tf.contrib.layers.xavier_initializer(),
dtype=tf.float32),
'b_z_log_sigma':
tf.get_variable("b_z_log_sigma", [self.n_z],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
}
gen = {
'W2':
tf.get_variable("W2", [self.n_z, self.dims[1]],
initializer=tf.contrib.layers.xavier_initializer(),
dtype=tf.float32),
'b2':
tf.get_variable("b2", [self.dims[1]],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
}
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')
z_mean = tf.matmul(x, rec['W_z_mean']) + rec['b_z_mean']
z_log_sigma_sq = tf.matmul(x,
rec['W_z_log_sigma']) + rec['b_z_log_sigma']
eps = tf.random_normal((batch_size, hidden_dim),
0,
1,
dtype=tf.float32)
z = z_mean + tf.sqrt(tf.maximum(tf.exp(z_log_sigma_sq), 1e-10)) * eps
x_recon = tf.matmul(z, gen['W2']) + gen['b2']
x_recon = tf.nn.sigmoid(x_recon, name='x_recon')
gen_loss = -tf.reduce_mean(
tf.reduce_sum(
x * tf.log(tf.maximum(x_recon, 1e-10)) +
(1 - x) * tf.log(tf.maximum(1 - x_recon, 1e-10)), 1))
latent_loss = 0.5 * tf.reduce_mean(
tf.reduce_sum(
tf.square(z_mean) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq -
1, 1))
loss = gen_loss + latent_loss
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
sess.run(tf.global_variables_initializer())
for i in range(epoch):
for it in range(num_iter):
b_x, ids = utils.get_batch(data_x, batch_size)
_, l, gl, ll = sess.run(
(train_op, loss, gen_loss, latent_loss),
feed_dict={x: b_x})
if (i + 1) % print_step == 0:
logging.info(
'epoch {0}: batch loss = {1}, gen_loss={2}, latent_loss={3}'
.format(i, l, gl, ll))
self.weights.append(sess.run(rec['W_z_mean']))
self.weights.append(sess.run(rec['W_z_log_sigma']))
self.biases.append(sess.run(rec['b_z_mean']))
self.biases.append(sess.run(rec['b_z_log_sigma']))
self.de_weights.append(sess.run(gen['W2']))
self.de_biases.append(sess.run(gen['b2']))
def run_all(self,
data_x,
lr,
batch_size,
epoch,
print_step=100,
x_valid=None):
tf.reset_default_graph()
n = data_x.shape[0]
input_dim = len(data_x[0])
num_iter = int(n / batch_size)
with tf.variable_scope("inference"):
rec = {
'W1':
tf.get_variable("W1",
initializer=tf.constant(self.weights[0]),
dtype=tf.float32),
'b1':
tf.get_variable("b1",
initializer=tf.constant(self.biases[0]),
dtype=tf.float32),
'W2':
tf.get_variable("W2",
initializer=tf.constant(self.weights[1]),
dtype=tf.float32),
'b2':
tf.get_variable("b2",
initializer=tf.constant(self.biases[1]),
dtype=tf.float32),
'W_z_mean':
tf.get_variable("W_z_mean",
initializer=tf.constant(self.weights[2]),
dtype=tf.float32),
'b_z_mean':
tf.get_variable("b_z_mean",
initializer=tf.constant(self.biases[2]),
dtype=tf.float32),
'W_z_log_sigma':
tf.get_variable("W_z_log_sigma",
initializer=tf.constant(self.weights[3]),
dtype=tf.float32),
'b_z_log_sigma':
tf.get_variable("b_z_log_sigma",
initializer=tf.constant(self.biases[3]),
dtype=tf.float32)
}
with tf.variable_scope("generation"):
gen = {
'W2':
tf.get_variable("W2",
initializer=tf.constant(self.de_weights[2]),
dtype=tf.float32),
'b2':
tf.get_variable("b2",
initializer=tf.constant(self.de_biases[2]),
dtype=tf.float32),
'W1':
tf.transpose(rec['W2']),
'b1':
rec['b1'],
'W_x':
tf.transpose(rec['W1']),
'b_x':
tf.get_variable("b_x", [input_dim],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
}
weights = []
weights += [
rec['W1'], rec['b1'], rec['W2'], rec['b2'], rec['W_z_mean'],
rec['b_z_mean'], rec['W_z_log_sigma'], rec['b_z_log_sigma']
]
weights += [gen['W2'], gen['b2'], gen['b_x']]
saver = tf.train.Saver(weights)
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')
h1 = self.activate(
tf.matmul(x, rec['W1']) + rec['b1'], self.activations[0])
h2 = self.activate(
tf.matmul(h1, rec['W2']) + rec['b2'], self.activations[1])
z_mean = tf.matmul(h2, rec['W_z_mean']) + rec['b_z_mean']
z_log_sigma_sq = tf.matmul(h2,
rec['W_z_log_sigma']) + rec['b_z_log_sigma']
eps = tf.random_normal((batch_size, self.n_z), 0, 1, dtype=tf.float32)
z = z_mean + tf.sqrt(tf.maximum(tf.exp(z_log_sigma_sq), 1e-10)) * eps
h2 = self.activate(
tf.matmul(z, gen['W2']) + gen['b2'], self.activations[1])
h1 = self.activate(
tf.matmul(h2, gen['W1']) + gen['b1'], self.activations[0])
x_recon = tf.matmul(h1, gen['W_x']) + gen['b_x']
x_recon = tf.nn.sigmoid(x_recon, name='x_recon')
gen_loss = -tf.reduce_mean(
tf.reduce_sum(
x * tf.log(tf.maximum(x_recon, 1e-10)) +
(1 - x) * tf.log(tf.maximum(1 - x_recon, 1e-10)), 1))
latent_loss = 0.5 * tf.reduce_mean(
tf.reduce_sum(
tf.square(z_mean) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq -
1, 1))
loss = gen_loss + latent_loss
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for i in range(epoch):
for it in range(num_iter):
b_x, ids = utils.get_batch(data_x, batch_size)
_, l, gl, ll = sess.run(
(train_op, loss, gen_loss, latent_loss),
feed_dict={x: b_x})
if (i + 1) % print_step == 0:
if x_valid is None:
logging.info(
'epoch {0}: batch loss = {1}, gen_loss={2}, latent_loss={3}'
.format(i, l, gl, ll))
else:
valid_loss = self.validation(x_valid, sess, gen_loss, x,
batch_size)
logging.info(
'epoch {0}: batch loss = {1}, gen_loss={2}, latent_loss={3}, valid_loss={4}'
.format(i, l, gl, ll, valid_loss))
from pathlib import Path
path_o = Path()
weight_path = path_o.cwd().joinpath("weights/pretrain")
weight_folder = path_o.cwd().joinpath("weights")
local_path = str(weight_path.absolute())
saver.save(sess, local_path)
logging.info("Weights saved at {}".format(local_path))
self.s3_client.upload_folder_to_s3((weight_folder.absolute()),
"weights/pretrain")