train_writer = tf.summary.FileWriter('./boardlogs/train', sess.graph)
test_writer = tf.summary.FileWriter('./boardlogs/test')
tf.global_variables_initializer().run()
n_epoch = 100
batch_size = 512
steps = 0
max_acc = 0
delta = 0.00001
last_acc = 0
start_time = time.time()
for i in range(n_epoch):
n_batch = train_dt.num_examples / batch_size
for j in range(int(n_batch)):
batch_xs, batch_ys = train_dt.next_batch(batch_size)
sess.run(train_step,
feed_dict={
xs: batch_xs,
ys: batch_ys,
keep_prob: 0.7
})
steps += 1
summary, acc = sess.run([merged, accuracy],
feed_dict={
xs: df_te,
ys: y_te,
keep_prob: 1.0
})
train_writer.add_summary(summary, steps)
test_writer.add_summary(summary, steps)
def train(self, N, rows, cols, miss_rows=None, miss_cols=None,
n_factors=20, d_pairwise=1, hidden_layer_sizes=[], n_iterations=1000,
batch_size=None, holdout_ratio=None, learning_rate=0.001,
reg_param=0.01, l2_param=None,
root_savedir='saved', root_logdir=None,
seed=None):
"""
Training routine.
:param N: Number of nodes
:param rows: Rows for "on" entries
:param cols: Corresponding columns for "on" entries
:param n_factors: Number of non-bilinear terms
:param d_pairwise: Number of bilinear terms
:param hidden_layer_sizes:
:param n_iterations:
:param batch_size:
:param holdout_ratio:
:param learning_rate:
:param reg_param: Frobenius norm regularization terms for the features
:param l2_param: L2 regularization parameter for the nnet weights
:param root_savedir:
:param root_logdir:
\ :param seed:
:return:
"""
self.N = N
self.n_factors = n_factors
self.d_pairwise = d_pairwise
self.hidden_layer_sizes = hidden_layer_sizes
self.reg_param = reg_param
self.l2_param = l2_param
if not os.path.exists(root_savedir):
os.makedirs(root_savedir)
root_logdir = os.path.join(root_savedir, 'tf_logs') if root_logdir is None else root_logdir
### Data handling ###
dataset = Dataset(N, rows, cols, miss_rows=miss_rows, miss_cols=miss_cols,
batch_size=batch_size, holdout_ratio=holdout_ratio, seed=seed)
### Construct the TF graph ###
self.build()
all_vars = tf.trainable_variables()
latent_vars = [self.U, self.Up] # the inputs to the nnets
nnet_vars = [x for x in all_vars if x not in latent_vars] # the nnet variables
print("\nlatent vars:", latent_vars)
print("\nnnet vars:", nnet_vars)
train_lvars = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss, var_list=latent_vars)
train_nnet = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss, var_list=nnet_vars)
### Training ###
train_loss = tf.placeholder(dtype=tf.float32, shape=[], name='train_loss')
train_loss_summary = tf.summary.scalar('train_loss', train_loss)
if holdout_ratio is not None:
test_xent = tf.placeholder(dtype=tf.float32, shape=[], name='test_xent')
test_xent_summary = tf.summary.scalar('test_xent', test_xent)
# create tensorboard summary objects
scalar_summaries = [tf.summary.scalar(var_.name, var_) for var_ in all_vars if len(var_.shape) == 0]
array_summaries = [tf.summary.histogram(var_.name, var_) for var_ in all_vars if len(var_.shape) > 0]
writer = tf.summary.FileWriter(root_logdir)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
for iteration in range(n_iterations):
batch = dataset.next_batch()
batch_dict = {self.row: batch[:, 0],
self.col: batch[:, 1],
self.val: batch[:, 2]}
# alternate between optimizing inputs and nnet vars
sess.run(train_lvars, feed_dict=batch_dict)
sess.run(train_nnet, feed_dict=batch_dict)
if iteration % 20 == 0:
print(iteration, end="")
train_entropy_ = self.evaluate(dataset.train, sess)
train_loss_summary_str = sess.run(train_loss_summary, feed_dict={train_loss: train_entropy_})
writer.add_summary(train_loss_summary_str, iteration)
print("\ttrain xent: %.4f" % train_entropy_, end="")
if holdout_ratio is not None:
test_xent_ = self.evaluate(dataset.test, sess)
test_xent_summary_str = sess.run(test_xent_summary, feed_dict={test_xent: test_xent_})
writer.add_summary(test_xent_summary_str, iteration)
print("\ttest xent: %.4f" % test_xent_)
scalar_summaries_str = sess.run(scalar_summaries)
array_summaries_str = sess.run(array_summaries)
for summary_ in scalar_summaries_str + array_summaries_str:
writer.add_summary(summary_, iteration)
# save the model
saver.save(sess, os.path.join(root_savedir, "model.ckpt"))
# close the file writer
writer.close()
def train(self,
N,
rows,
cols,
miss_rows=None,
miss_cols=None,
n_factors=20,
d_pairwise=1,
hidden_layer_sizes=[],
n_iterations=1000,
batch_size=None,
holdout_ratio=None,
learning_rate=0.001,
n_samples=10,
root_savedir='saved',
root_logdir=None,
no_train_metric=False,
seed=None):
"""
Training routine.
:param N: Number of nodes
:param rows: Rows for "on" entries
:param cols: Corresponding columns for "on" entries
:param n_factors: Number of non-bilinear terms
:param d_pairwise: Number of bilinear terms
:param hidden_layer_sizes:
:param n_iterations:
:param batch_size:
:param holdout_ratio:
:param learning_rate:
:param n_samples:
:param root_savedir:
:param root_logdir:
:param no_train_metric:
:param seed:
:return:
"""
self.N = N
self.n_factors = n_factors
self.d_pairwise = d_pairwise
self.hidden_layer_sizes = hidden_layer_sizes
if not os.path.exists(root_savedir):
os.makedirs(root_savedir)
root_logdir = os.path.join(
root_savedir, 'tf_logs') if root_logdir is None else root_logdir
### Data handling ###
dataset = Dataset(N,
rows,
cols,
miss_rows=miss_rows,
miss_cols=miss_cols,
batch_size=batch_size,
holdout_ratio=holdout_ratio,
seed=seed)
### Construct the TF graph ###
self.build()
train_op = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(-self.elbo)
### Training ###
if not no_train_metric:
train_elbo = tf.placeholder(dtype=tf.float32,
shape=[],
name='train_elbo')
train_elbo_summary = tf.summary.scalar('train_elbo', train_elbo)
train_ll = tf.placeholder(dtype=tf.float32,
shape=[],
name='train_ll')
train_ll_summary = tf.summary.scalar('train_ll', train_ll)
if holdout_ratio is not None:
test_ll = tf.placeholder(dtype=tf.float32,
shape=[],
name='test_ll')
test_ll_summary = tf.summary.scalar('test_ll', test_ll)
# create tensorboard summary objects
all_vars = tf.trainable_variables()
scalar_summaries = [
tf.summary.scalar(var_.name, var_) for var_ in all_vars
if len(var_.shape) == 0
]
array_summaries = [
tf.summary.histogram(var_.name, var_) for var_ in all_vars
if len(var_.shape) > 0
]
writer = tf.summary.FileWriter(root_logdir)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
if not no_train_metric:
train_dict = {
self.row: dataset.train[:, 0],
self.col: dataset.train[:, 1],
self.val: dataset.train[:, 2],
self.n_samples: 100,
self.batch_scale: 1.0
}
if holdout_ratio is not None:
test_dict = {
self.row: dataset.test[:, 0],
self.col: dataset.test[:, 1],
self.val: dataset.test[:, 2],
self.n_samples: 100,
self.batch_scale: 1.0
}
for iteration in range(n_iterations):
batch = dataset.next_batch()
sess.run(train_op,
feed_dict={
self.row: batch[:, 0],
self.col: batch[:, 1],
self.val: batch[:, 2],
self.n_samples: n_samples,
self.batch_scale: len(dataset.train) / len(batch)
})
if iteration % 20 == 0:
print(iteration, end="")
if not no_train_metric:
train_ll_, train_elbo_ = sess.run(
[self.data_loglikel, self.elbo],
feed_dict=train_dict)
train_ll_summary_str, train_elbo_summary_str = sess.run(
[train_ll_summary, train_elbo_summary],
feed_dict={
train_ll: train_ll_,
train_elbo: train_elbo_
})
writer.add_summary(train_ll_summary_str, iteration)
writer.add_summary(train_elbo_summary_str, iteration)
print("\tTrain ELBO: %.4f" % train_elbo_, end="")
print("\tTrain LL: %.4f" % train_ll_, end="")
if holdout_ratio is not None:
test_ll_ = sess.run(self.data_loglikel,
feed_dict=test_dict)
test_ll_summary_str = sess.run(
test_ll_summary, feed_dict={test_ll: test_ll_})
writer.add_summary(test_ll_summary_str, iteration)
print("\tTest LL: %.4f" % test_ll_)
scalar_summaries_str = sess.run(scalar_summaries)
array_summaries_str = sess.run(array_summaries)
for summary_ in scalar_summaries_str + array_summaries_str:
writer.add_summary(summary_, iteration)
# save the model
saver.save(sess, os.path.join(root_savedir, "model.ckpt"))
# close the file writer
writer.close()
for i in range(i_save,n_epoch):
i_save = i
worksum = int(len(train_y)/batch_size)
pb = ProgressBar(worksum=worksum)
pb.startjob()
train_loss_list = []
train_acc_list = []
for j in range(worksum):
if restore == True and j < j_save:
pb.finishsum += 1
continue
restore = False
j_save = j
batch_x,batch_y = train_set.next_batch(batch_size)
lx = [seq_max_len] * batch_size
ly = [seq_max_len] * batch_size
bx = [np.sum(m > 0) for m in batch_x]
by = [np.sum(m > 0) for m in batch_y]
by =[m + 2 if m < seq_max_len - 1 else m for m in by ]
_, loss = session.run([optimizer,train_loss],feed_dict={x:batch_x,y:batch_y,x_len:lx,y_len:ly,learning_rate:lr,y_in:
np.concatenate((
np.ones((batch_y.shape[0],1),dtype=np.int) * ch2ind['<go>'],batch_y[:,:-1]) ,axis=1)
,y_real_len:by,
x_real_len:bx
})
train_loss_list.append(loss)
#tmp_train_acc = cal_acc(tran,batch_y)
#train_acc_list.append(tmp_train_acc)
pb.info = "iter {} loss:{} lr:{}".format(i + 1,loss,lr)
