def trainIters(self, n_iters, model_file_path=None): iter, running_avg_loss = self.setup_train(model_file_path) start = time.time() min_val_loss = np.inf while iter < n_iters: batch = self.batcher.next_batch() loss = self.train_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % config.print_interval == 0: tf.logging.info( 'steps %d, seconds for %d batch: %.2f , loss: %f, min_val_loss: %f' % (iter, config.print_interval, time.time() - start, loss, min_val_loss)) start = time.time() if iter % config.model_save_iters == 0: self.summary_writer.flush() model_save_path = self.save_model(running_avg_loss, iter, mode='train') tf.logging.info('Evaluate the model %s at validation set....' % model_save_path) evl_model = Evaluate(model_save_path) val_avg_loss = evl_model.run_eval() if val_avg_loss < min_val_loss: min_val_loss = val_avg_loss best_model_save_path = self.save_model(running_avg_loss, iter, mode='eval') tf.logging.info('Save best model at %s' % best_model_save_path)
def trainIters(self, n_iters, model_file_path=None): # 训练设置,包括 iter_step, running_avg_loss = self.setup_train(model_file_path) start = time.time() while iter_step < n_iters: # 获取下一个batch数据 batch = self.batcher.next_batch() loss = self.train_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter_step) iter_step += 1 if iter_step % 100 == 0: self.summary_writer.flush() # print_interval = 1000 if iter_step % 100 == 0: # lr = self.optimizer.state_dict()['param_groups'][0]['lr'] logging.info('steps %d, seconds for %d steps: %.2f, loss: %f' % (iter_step, 10, time.time() - start, loss)) start = time.time() # 50000次迭代就保存一下模型 if iter_step % 50000 == 0: logging.info("model saved = {}/{}".format(int(iter_step / 50000) + 1, int(config.max_iterations/50000) + 1)) self.save_model(running_avg_loss, iter_step)
def run(self): start = time.time() running_avg_loss, iter = 0, 0 batch = self.batcher.next_batch() print_interval = 100 while batch is not None: loss = self.eval_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % print_interval == 0: self.summary_writer.flush() print('step: %d, second: %.2f , loss: %f' % (iter, time.time() - start, running_avg_loss)) start = time.time() batch = self.batcher.next_batch() return running_avg_loss
def trainIters(self, n_iters, model_file_path=None): iter, running_avg_loss = self.setup_train(model_file_path) start = time.time() while iter < n_iters: batch = self.batcher.next_batch() loss = self.train_one_batch(batch, iter) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, iter) iter += 1 print( 'global step %d/%d, step loss: %.8f, running avg loss: %.8f, speed: %.2f step/s' % (iter, n_iters, loss, running_avg_loss, 1.0 / (time.time() - start))) start = time.time() if iter % 5000 == 0 or iter == 1000: model_save_dir = self.save_model(running_avg_loss, iter) print( 'Saved model for iter %d with running avg loss %.8f to directory: %s' % (iter, running_avg_loss, model_save_dir))
def run_eval(self): running_avg_loss, iter = 0, 0 start = time.time() batch = self.batcher.next_batch() while batch is not None: loss = self.eval_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 # if iter % 500 == 0: # self.summary_writer.flush() # if iter % config.print_interval == 0: # print('steps %d, seconds for %d batch: %.2f , validation_loss: %f' % ( # iter, config.print_interval, time.time() - start, running_avg_loss)) # start = time.time() batch = self.batcher.next_batch() return running_avg_loss
def run_eval(self): running_avg_loss, iter = 0, 0 start = time.time() batch = self.batcher.next_batch() while batch is not None: loss = self.eval_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % 100 == 0: self.summary_writer.flush() print_interval = 1000 if iter % print_interval == 0: print('steps %d, seconds for %d batch: %.2f , loss: %f' % (iter, print_interval, time.time() - start, running_avg_loss)) start = time.time() batch = self.batcher.next_batch()
def run(self, n_iters, model_path=None): iter, running_avg_loss = self.setup_train(model_path) start = time.time() interval = 100 while iter < n_iters: batch = self.batcher.next_batch() loss, cove_loss = self.train_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % interval == 0: self.summary_writer.flush() print('step: %d, second: %.2f , loss: %f, cover_loss: %f' % (iter, time.time() - start, loss, cove_loss)) start = time.time() if iter % 5000 == 0: self.save_model(running_avg_loss, iter)
def trainIters(self, n_iters, model_file_path=None): iter, running_avg_loss = self.setup_train(model_file_path) start = time.time() while iter < n_iters: batch = self.batcher.next_batch() loss = self.train_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % 100 == 0: self.summary_writer.flush() print_interval = 1000 if iter % print_interval == 0: print('steps %d, seconds for %d batch: %.2f , loss: %f' % (iter, print_interval, time.time() - start, loss)) start = time.time() if iter % 5000 == 0: self.save_model(running_avg_loss, iter)
def trainIters(self, n_iters, model_file_path=None): # 训练设置,包括 iter_step, running_avg_loss = self.setup_train(model_file_path) start = time.time() while iter_step < n_iters: # 获取下一个batch数据 batch = self.batcher.next_batch() loss = self.train_one_batch(batch) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, iter_step) iter_step += 1 if running_avg_loss < 0.01: break if iter_step % 100 == 0: print('steps %d, seconds for %d ,' % (iter_step, time.time() - start)) print('loss:', loss) start = time.time() if iter_step % 500 == 0 and running_avg_loss > 0.001: self.save_model(running_avg_loss, iter_step)
def trainIters(self, n_iters, model_file_path=None): iter, running_avg_loss = self.setup_train(model_file_path) min_val_loss = np.inf alpha = config.alpha beta = config.beta k1 = config.k1 k2 = config.k2 delay = 0 while iter < n_iters: if config.mode == 'RL': alpha = 0. beta = 0. elif config.mode == 'GTI': alpha = 1. beta = 0. elif config.mode == 'SO': alpha = 1. beta = k2 / (k2 + np.exp((iter - delay) / k2)) elif config.mode == 'SIO': alpha *= k1 if alpha < 0.01: beta = k2 / (k2 + np.exp((iter - delay) / k2)) else: beta = 1. delay += 1 elif config.mode == 'DAGGER': alpha *= k1 beta = 1. elif config.mode == 'DAGGER*': alpha = config.alpha beta = 1. else: alpha = 1. beta = 1. batch = self.batcher.next_batch() loss, avg_reward = self.train_one_batch(batch, alpha, beta) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % config.print_interval == 0: print('steps %d, current_loss: %f, avg_reward: %f' % (iter, loss, avg_reward)) if iter % config.save_model_iter == 0: model_file_path = self.save_model(running_avg_loss, iter, mode='train') evl_model = Evaluate(model_file_path) val_avg_loss = evl_model.run_eval() if val_avg_loss < min_val_loss: min_val_loss = val_avg_loss best_model_file_path = self.save_model(running_avg_loss, iter, mode='eval') print('Save best model at %s' % best_model_file_path) print('steps %d, train_loss: %f, val_loss: %f' % (iter, loss, val_avg_loss)) # write val_loss into tensorboard loss_sum = tf.compat.v1.Summary() loss_sum.value.add(tag='val_avg_loss', simple_value=val_avg_loss) self.summary_writer.add_summary(loss_sum, global_step=iter) self.summary_writer.flush()
def trainIters(self, n_iters, model_file_path=None): if config.mode not in [ "MLE", "RL", "GTI", "SO", "SIO", "DAGGER", "DAGGER*" ]: print("\nTRAINING MODE ERROR\n") raise ValueError # log file path log_path = os.path.join(config.log_root, 'log') log = open(log_path, 'w') print_log("==============================", file=log) iter, running_avg_loss = self.setup_train( model_file_path, emb_v_path=config.emb_v_path, emb_list_path=config.vocab_path, vocab=self.vocab, log=log) min_val_loss = np.inf alpha = config.alpha beta = config.beta k1 = config.k1 k2 = config.k2 delay = iter # set to 0 in the original code (wyu-du) print("\nLog root is %s" % config.log_root) print_log("Train mode is %s" % config.mode, file=log) print_log("k1: %s, k2: %s" % (config.k1, config.k2), file=log) print_log("==============================", file=log) cur_time = time.time() while iter < n_iters: if config.mode == 'RL': alpha = 0. beta = 0. elif config.mode == 'GTI': alpha = 1. beta = 0. elif config.mode == 'SO': alpha = 1. beta = k2 / (k2 + np.exp((iter - delay) / k2)) elif config.mode == 'SIO': alpha *= k1 if alpha < 0.01: beta = k2 / (k2 + np.exp((iter - delay) / k2)) else: beta = 1. delay += 1 elif config.mode == 'DAGGER': alpha *= k1 beta = 1. elif config.mode == 'DAGGER*': alpha = config.alpha beta = 1. else: alpha = 1. beta = 1. batch = self.batcher.next_batch() loss, avg_reward = self.train_one_batch(batch, alpha, beta) running_avg_loss = calc_running_avg_loss(loss, running_avg_loss, self.summary_writer, iter) iter += 1 if iter % config.print_interval == 0: print_log('steps %d, current_loss: %f, avg_reward: %f, alpha: %f, beta: %f, delay: %d' % \ (iter, loss, avg_reward, alpha, beta, delay), file=log) if iter % config.save_model_iter == 0: model_file_path = self.save_model(running_avg_loss, iter, mode='train') evl_model = Evaluate(model_file_path) val_avg_loss = evl_model.run_eval() if val_avg_loss < min_val_loss: min_val_loss = val_avg_loss best_model_file_path = self.save_model(running_avg_loss, iter, mode='eval') print_log('Save best model at %s' % best_model_file_path, file=log) print_log('steps %d, train_loss: %f, val_loss: %f, time: %ds' % \ (iter, loss, val_avg_loss, time.time()-cur_time), file=log) # write val_loss into tensorboard loss_sum = tf.compat.v1.Summary() loss_sum.value.add(tag='val_avg_loss', simple_value=val_avg_loss) self.summary_writer.add_summary(loss_sum, global_step=iter) self.summary_writer.flush() cur_time = time.time() log.close()