def run(self,
data_x,
data_x_,
hidden_dim,
activation,
loss,
lr,
print_step,
epoch,
batch_size=100):
tf.reset_default_graph()
input_dim = len(data_x[0])
sess = tf.Session()
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')
x_ = tf.placeholder(dtype=tf.float32,
shape=[None, input_dim],
name='x_')
encode = {
'weights':
tf.Variable(
tf.truncated_normal([input_dim, hidden_dim],
dtype=tf.float32)),
'biases':
tf.Variable(tf.truncated_normal([hidden_dim], dtype=tf.float32))
}
decode = {
'biases':
tf.Variable(tf.truncated_normal([input_dim], dtype=tf.float32)),
'weights':
tf.transpose(encode['weights'])
}
encoded = self.activate(
tf.matmul(x, encode['weights']) + encode['biases'], activation)
decoded = tf.matmul(encoded, decode['weights']) + decode['biases']
# reconstruction loss
if loss == 'rmse':
loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(x_, decoded))))
elif loss == 'cross-entropy':
loss = -tf.reduce_mean(x_ * tf.log(decoded))
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
sess.run(tf.global_variables_initializer())
f = open('dA_loss.txt', 'a+')
for i in range(epoch):
b_x, b_x_ = utils.get_batch(data_x, data_x_, batch_size)
sess.run(train_op, feed_dict={x: b_x, x_: b_x_})
if (i + 1) % print_step == 0:
l = sess.run(loss, feed_dict={x: data_x, x_: data_x_})
f.write('epoch {0}: global loss = {1}\n'.format(i, l))
f.close()
# debug
# print('Decoded', sess.run(decoded, feed_dict={x: self.data_x_})[0])
self.weights.append(sess.run(encode['weights']))
self.biases.append(sess.run(encode['biases']))
return sess.run(encoded, feed_dict={x: data_x_})
def finetune(self,
data_x,
data_x_,
loss,
lr,
print_step,
epoch,
batch_size=100):
"""
Finetunes the entire network after each layer is trained.
"""
tf.reset_default_graph()
input_dim = len(data_x[0])
sess = tf.Session()
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')
x_ = tf.placeholder(dtype=tf.float32,
shape=[None, input_dim],
name='x_')
decoded = x
for w, b, a in zip(self.weights, self.biases, self.activations):
weight = tf.constant(w, dtype=tf.float32)
bias = tf.constant(b, dtype=tf.float32)
layer = tf.matmul(decoded, weight) + bias
decoded = self.activate(layer, a)
print decoded
if loss == 'rmse':
loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(x_, decoded))))
elif loss == 'cross-entropy':
loss = -tf.reduce_mean(x_ * tf.log(decoded))
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
for i in range(epoch):
b_x, b_x_ = utils.get_batch(data_x, data_x_, batch_size)
sess.run(train_op, feed_dict={x: b_x, x_: b_x_})
if (i + 1) % print_step == 0:
l = sess.run(loss, feed_dict={x: data_x, x_: data_x_})
print('epoch {0}: global loss = {1}'.format(i, l))
def run(self, data_x, data_x_, hidden_dim, activation, loss, lr,
print_step, epoch, batch_size=100):
tf.reset_default_graph()
input_dim = len(data_x[0])
sess = tf.Session()
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')
x_ = tf.placeholder(dtype=tf.float32, shape=[
None, input_dim], name='x_')
encode = {'weights': tf.Variable(tf.truncated_normal(
[input_dim, hidden_dim], dtype=tf.float32)),
'biases': tf.Variable(tf.truncated_normal([hidden_dim],
dtype=tf.float32))}
decode = {'biases': tf.Variable(tf.truncated_normal([input_dim],
dtype=tf.float32)),
'weights': tf.transpose(encode['weights'])}
encoded = self.activate(
tf.matmul(x, encode['weights']) + encode['biases'], activation)
decoded = tf.matmul(encoded, decode['weights']) + decode['biases']
# reconstruction loss
if loss == 'rmse':
loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(x_, decoded))))
elif loss == 'cross-entropy':
loss = -tf.reduce_mean(x_ * tf.log(decoded))
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
sess.run(tf.global_variables_initializer())
for i in range(epoch):
b_x, b_x_ = utils.get_batch(
data_x, data_x_, batch_size)
sess.run(train_op, feed_dict={x: b_x, x_: b_x_})
if (i + 1) % print_step == 0:
l = sess.run(loss, feed_dict={x: data_x, x_: data_x_})
print('epoch {0}: global loss = {1}'.format(i, l))
# self.loss_val = l
# debug
# print('Decoded', sess.run(decoded, feed_dict={x: self.data_x_})[0])
self.weights.append(sess.run(encode['weights']))
self.biases.append(sess.run(encode['biases']))
return sess.run(encoded, feed_dict={x: data_x_})
def _run(self,
data_x,
data_x_,
val_x,
val_x_,
hidden_dim,
activation,
loss,
lr,
print_step,
epoch,
batch_size=100,
depth=0,
load_model=False,
save_model=False):
tf.reset_default_graph()
with tf.Session() as sess:
if load_model:
saver = tf.train.import_meta_graph(
'./SDA_model/Layer{}/SDA_model.meta'.format(depth))
saver.restore(
sess,
tf.train.latest_checkpoint(
'./SDA_model/Layer{}/'.format(depth)))
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
x_ = graph.get_tensor_by_name("x_:0")
decoded = graph.get_tensor_by_name("decoded:0")
encoded = graph.get_tensor_by_name("encoded:0")
encode = {
'weights': graph.get_tensor_by_name("enc_weights:0"),
'biases': graph.get_tensor_by_name("enc_biases:0")
}
decode = {
'biases': graph.get_tensor_by_name("dec_biases:0"),
'weights': graph.get_tensor_by_name("dec_weights:0")
}
train_op = tf.get_collection("train_op")[0]
loss = graph.get_tensor_by_name("loss:0")
else:
input_dim = len(data_x[0])
x = tf.placeholder(dtype=tf.float32,
shape=[None, input_dim],
name='x')
x_ = tf.placeholder(dtype=tf.float32,
shape=[None, input_dim],
name='x_')
encode = {
'weights':
tf.Variable(tf.truncated_normal([input_dim, hidden_dim],
dtype=tf.float32),
name='enc_weights'),
'biases':
tf.Variable(tf.truncated_normal([hidden_dim],
dtype=tf.float32),
name='enc_biases')
}
decode = {
'biases':
tf.Variable(tf.truncated_normal([input_dim],
dtype=tf.float32),
name='dec_biases'),
'weights':
tf.transpose(encode['weights'], name='dec_weights')
}
encoded = self._activate(
tf.matmul(x, encode['weights']) + encode['biases'],
activation)
decoded = tf.add(tf.matmul(encoded, decode['weights']),
decode['biases'],
name='decoded')
# reconstruction loss
if loss == 'rmse':
loss = tf.sqrt(tf.reduce_mean(
tf.square(tf.subtract(x_, decoded))),
name='loss')
elif loss == 'cross-entropy':
loss = tf.negative(tf.reduce_mean(x_ * tf.log(decoded),
name='loss'),
name='loss')
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
# TODO: Epoch is wrong! Training is only with ONE random batch!
for i in range(epoch):
b_x, b_x_ = utils.get_batch(data_x, data_x_, batch_size)
sess.run(train_op, feed_dict={x: b_x, x_: b_x_})
if (i + 1) % print_step == 0:
loss_train = sess.run(loss,
feed_dict={
x: data_x,
x_: data_x_
})
dec_train = sess.run(decoded, feed_dict={x: data_x})
if val_x is None:
print('epoch {0}: train loss = {1:.5f}'.format(
i, loss_train))
else:
loss_val = sess.run(loss,
feed_dict={
x: val_x,
x_: val_x_
})
dec_val = sess.run(decoded, feed_dict={x: val_x})
print(
'epoch {0}: train loss = {1:.5f}, validation loss = {2:.5f}'
.format(i, loss_train, loss_val))
self.loss_val = loss_train
# debug
# print('Decoded', sess.run(decoded, feed_dict={x: self.data_x_})[0])
self.weights.append(sess.run(encode['weights']))
self.biases.append(sess.run(encode['biases']))
self.dec_biases.append(sess.run(decode['biases']))
if val_x is not None:
val_x = sess.run(encoded, feed_dict={x: val_x_})
train_x = sess.run(encoded, feed_dict={x: data_x_})
# save train_op and all layers
if save_model:
tf.add_to_collection("train_op", train_op)
layerPath = './SDA_model/Layer{}/SDA_model'.format(depth)
if not os.path.exists(layerPath):
os.makedirs(layerPath)
save_path = saver.save(sess, layerPath)
print("SDA-Model saved in path: %s" % save_path)
return train_x, val_x