train_writer = tf.summary.FileWriter('./boardlogs/train', sess.graph) test_writer = tf.summary.FileWriter('./boardlogs/test') tf.global_variables_initializer().run() n_epoch = 100 batch_size = 512 steps = 0 max_acc = 0 delta = 0.00001 last_acc = 0 start_time = time.time() for i in range(n_epoch): n_batch = train_dt.num_examples / batch_size for j in range(int(n_batch)): batch_xs, batch_ys = train_dt.next_batch(batch_size) sess.run(train_step, feed_dict={ xs: batch_xs, ys: batch_ys, keep_prob: 0.7 }) steps += 1 summary, acc = sess.run([merged, accuracy], feed_dict={ xs: df_te, ys: y_te, keep_prob: 1.0 }) train_writer.add_summary(summary, steps) test_writer.add_summary(summary, steps)
def train(self, N, rows, cols, miss_rows=None, miss_cols=None, n_factors=20, d_pairwise=1, hidden_layer_sizes=[], n_iterations=1000, batch_size=None, holdout_ratio=None, learning_rate=0.001, reg_param=0.01, l2_param=None, root_savedir='saved', root_logdir=None, seed=None): """ Training routine. :param N: Number of nodes :param rows: Rows for "on" entries :param cols: Corresponding columns for "on" entries :param n_factors: Number of non-bilinear terms :param d_pairwise: Number of bilinear terms :param hidden_layer_sizes: :param n_iterations: :param batch_size: :param holdout_ratio: :param learning_rate: :param reg_param: Frobenius norm regularization terms for the features :param l2_param: L2 regularization parameter for the nnet weights :param root_savedir: :param root_logdir: \ :param seed: :return: """ self.N = N self.n_factors = n_factors self.d_pairwise = d_pairwise self.hidden_layer_sizes = hidden_layer_sizes self.reg_param = reg_param self.l2_param = l2_param if not os.path.exists(root_savedir): os.makedirs(root_savedir) root_logdir = os.path.join(root_savedir, 'tf_logs') if root_logdir is None else root_logdir ### Data handling ### dataset = Dataset(N, rows, cols, miss_rows=miss_rows, miss_cols=miss_cols, batch_size=batch_size, holdout_ratio=holdout_ratio, seed=seed) ### Construct the TF graph ### self.build() all_vars = tf.trainable_variables() latent_vars = [self.U, self.Up] # the inputs to the nnets nnet_vars = [x for x in all_vars if x not in latent_vars] # the nnet variables print("\nlatent vars:", latent_vars) print("\nnnet vars:", nnet_vars) train_lvars = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss, var_list=latent_vars) train_nnet = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss, var_list=nnet_vars) ### Training ### train_loss = tf.placeholder(dtype=tf.float32, shape=[], name='train_loss') train_loss_summary = tf.summary.scalar('train_loss', train_loss) if holdout_ratio is not None: test_xent = tf.placeholder(dtype=tf.float32, shape=[], name='test_xent') test_xent_summary = tf.summary.scalar('test_xent', test_xent) # create tensorboard summary objects scalar_summaries = [tf.summary.scalar(var_.name, var_) for var_ in all_vars if len(var_.shape) == 0] array_summaries = [tf.summary.histogram(var_.name, var_) for var_ in all_vars if len(var_.shape) > 0] writer = tf.summary.FileWriter(root_logdir) saver = tf.train.Saver() init = tf.global_variables_initializer() with tf.Session() as sess: init.run() for iteration in range(n_iterations): batch = dataset.next_batch() batch_dict = {self.row: batch[:, 0], self.col: batch[:, 1], self.val: batch[:, 2]} # alternate between optimizing inputs and nnet vars sess.run(train_lvars, feed_dict=batch_dict) sess.run(train_nnet, feed_dict=batch_dict) if iteration % 20 == 0: print(iteration, end="") train_entropy_ = self.evaluate(dataset.train, sess) train_loss_summary_str = sess.run(train_loss_summary, feed_dict={train_loss: train_entropy_}) writer.add_summary(train_loss_summary_str, iteration) print("\ttrain xent: %.4f" % train_entropy_, end="") if holdout_ratio is not None: test_xent_ = self.evaluate(dataset.test, sess) test_xent_summary_str = sess.run(test_xent_summary, feed_dict={test_xent: test_xent_}) writer.add_summary(test_xent_summary_str, iteration) print("\ttest xent: %.4f" % test_xent_) scalar_summaries_str = sess.run(scalar_summaries) array_summaries_str = sess.run(array_summaries) for summary_ in scalar_summaries_str + array_summaries_str: writer.add_summary(summary_, iteration) # save the model saver.save(sess, os.path.join(root_savedir, "model.ckpt")) # close the file writer writer.close()
def train(self, N, rows, cols, miss_rows=None, miss_cols=None, n_factors=20, d_pairwise=1, hidden_layer_sizes=[], n_iterations=1000, batch_size=None, holdout_ratio=None, learning_rate=0.001, n_samples=10, root_savedir='saved', root_logdir=None, no_train_metric=False, seed=None): """ Training routine. :param N: Number of nodes :param rows: Rows for "on" entries :param cols: Corresponding columns for "on" entries :param n_factors: Number of non-bilinear terms :param d_pairwise: Number of bilinear terms :param hidden_layer_sizes: :param n_iterations: :param batch_size: :param holdout_ratio: :param learning_rate: :param n_samples: :param root_savedir: :param root_logdir: :param no_train_metric: :param seed: :return: """ self.N = N self.n_factors = n_factors self.d_pairwise = d_pairwise self.hidden_layer_sizes = hidden_layer_sizes if not os.path.exists(root_savedir): os.makedirs(root_savedir) root_logdir = os.path.join( root_savedir, 'tf_logs') if root_logdir is None else root_logdir ### Data handling ### dataset = Dataset(N, rows, cols, miss_rows=miss_rows, miss_cols=miss_cols, batch_size=batch_size, holdout_ratio=holdout_ratio, seed=seed) ### Construct the TF graph ### self.build() train_op = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(-self.elbo) ### Training ### if not no_train_metric: train_elbo = tf.placeholder(dtype=tf.float32, shape=[], name='train_elbo') train_elbo_summary = tf.summary.scalar('train_elbo', train_elbo) train_ll = tf.placeholder(dtype=tf.float32, shape=[], name='train_ll') train_ll_summary = tf.summary.scalar('train_ll', train_ll) if holdout_ratio is not None: test_ll = tf.placeholder(dtype=tf.float32, shape=[], name='test_ll') test_ll_summary = tf.summary.scalar('test_ll', test_ll) # create tensorboard summary objects all_vars = tf.trainable_variables() scalar_summaries = [ tf.summary.scalar(var_.name, var_) for var_ in all_vars if len(var_.shape) == 0 ] array_summaries = [ tf.summary.histogram(var_.name, var_) for var_ in all_vars if len(var_.shape) > 0 ] writer = tf.summary.FileWriter(root_logdir) saver = tf.train.Saver() init = tf.global_variables_initializer() with tf.Session() as sess: init.run() if not no_train_metric: train_dict = { self.row: dataset.train[:, 0], self.col: dataset.train[:, 1], self.val: dataset.train[:, 2], self.n_samples: 100, self.batch_scale: 1.0 } if holdout_ratio is not None: test_dict = { self.row: dataset.test[:, 0], self.col: dataset.test[:, 1], self.val: dataset.test[:, 2], self.n_samples: 100, self.batch_scale: 1.0 } for iteration in range(n_iterations): batch = dataset.next_batch() sess.run(train_op, feed_dict={ self.row: batch[:, 0], self.col: batch[:, 1], self.val: batch[:, 2], self.n_samples: n_samples, self.batch_scale: len(dataset.train) / len(batch) }) if iteration % 20 == 0: print(iteration, end="") if not no_train_metric: train_ll_, train_elbo_ = sess.run( [self.data_loglikel, self.elbo], feed_dict=train_dict) train_ll_summary_str, train_elbo_summary_str = sess.run( [train_ll_summary, train_elbo_summary], feed_dict={ train_ll: train_ll_, train_elbo: train_elbo_ }) writer.add_summary(train_ll_summary_str, iteration) writer.add_summary(train_elbo_summary_str, iteration) print("\tTrain ELBO: %.4f" % train_elbo_, end="") print("\tTrain LL: %.4f" % train_ll_, end="") if holdout_ratio is not None: test_ll_ = sess.run(self.data_loglikel, feed_dict=test_dict) test_ll_summary_str = sess.run( test_ll_summary, feed_dict={test_ll: test_ll_}) writer.add_summary(test_ll_summary_str, iteration) print("\tTest LL: %.4f" % test_ll_) scalar_summaries_str = sess.run(scalar_summaries) array_summaries_str = sess.run(array_summaries) for summary_ in scalar_summaries_str + array_summaries_str: writer.add_summary(summary_, iteration) # save the model saver.save(sess, os.path.join(root_savedir, "model.ckpt")) # close the file writer writer.close()
for i in range(i_save,n_epoch): i_save = i worksum = int(len(train_y)/batch_size) pb = ProgressBar(worksum=worksum) pb.startjob() train_loss_list = [] train_acc_list = [] for j in range(worksum): if restore == True and j < j_save: pb.finishsum += 1 continue restore = False j_save = j batch_x,batch_y = train_set.next_batch(batch_size) lx = [seq_max_len] * batch_size ly = [seq_max_len] * batch_size bx = [np.sum(m > 0) for m in batch_x] by = [np.sum(m > 0) for m in batch_y] by =[m + 2 if m < seq_max_len - 1 else m for m in by ] _, loss = session.run([optimizer,train_loss],feed_dict={x:batch_x,y:batch_y,x_len:lx,y_len:ly,learning_rate:lr,y_in: np.concatenate(( np.ones((batch_y.shape[0],1),dtype=np.int) * ch2ind['<go>'],batch_y[:,:-1]) ,axis=1) ,y_real_len:by, x_real_len:bx }) train_loss_list.append(loss) #tmp_train_acc = cal_acc(tran,batch_y) #train_acc_list.append(tmp_train_acc) pb.info = "iter {} loss:{} lr:{}".format(i + 1,loss,lr)