def evaluate_models_from(GPT_saved_models_folder, eval_file, enc, args): # Prepare eval data eval_dataloader_loss = DynamicBatchingLoader(eval_file, enc, args.normalize_data, args.eval_batch_size, args.max_seq_length) eval_dataloader_gen = get_eval_list_same_length(eval_file, enc, args.eval_batch_size, True) # read eval_loss log file eval_loss_log_file = os.path.join(GPT_saved_models_folder, "eval_log.txt") min_ckpt_old_perplexity = None min_ckpt_new_perplexity = None min_old_perplexity = 1000000.0 min_new_perplexity = 1000000.0 with open(eval_loss_log_file, "r") as reader: head_row = next(reader) for line in reader: line = line.strip() epoch, ckpt_no, _, loss, perplexity = line.split(",") epoch = int(epoch) ckpt_no = int(ckpt_no) - 1 loss = float(loss) perplexity = float(perplexity) print(ckpt_no, loss, perplexity, end="") if min_old_perplexity > perplexity: min_old_perplexity = perplexity min_ckpt_old_perplexity = ckpt_no # calculate new loss and perplexity model_filename = "GP2-pretrain-step-{}.pkl" model = load_model(GPT2LMHeadModel(config), os.path.join(GPT_saved_models_folder, model_filename.format(ckpt_no)), args, verbose=True) eval_loss, eval_ppl = eval_model_loss(model, enc, eval_dataloader_loss, epoch, args) if min_new_perplexity > eval_ppl: min_new_perplexity = eval_ppl min_ckpt_new_perplexity = ckpt_no print("Old best ckpt and perplexity:", min_ckpt_old_perplexity, min_old_perplexity) print("New best ckpt and perplexity:", min_ckpt_new_perplexity, min_new_perplexity) return min_ckpt_old_perplexity, min_old_perplexity, min_ckpt_new_perplexity, min_new_perplexity
n_token_real_all_proc, n_token_total_all_proc, epoch_time), file=train_logger) if global_step % args.valid_step == 0: if args.local_rank == -1: # only rank 0 process evaluate torch.save( { k: (v.cpu() if v is not None else None ) # save to cpu tensors for k, v in model.state_dict().items() }, join(output_dir, f'GP2-pretrain-step-{global_step}.pkl')) eval_loss, eval_ppl = eval_model_loss( model, enc, eval_dataloader_loss, epoch, args) # enable generation step evaluation for now # gen_response = eval_model_generation( # model, enc, eval_dataloader_gen, epoch, args) ''' # probably use beam search only for test set if False: gen_response_beam = eval_model_generation( model, enc, eval_dataloader_gen, epoch, args, use_beam_search=True, beam_width=3) ''' print('{},{},{},{},{}'.format(epoch + 1, global_step + 1, step + 1, eval_loss, eval_ppl), file=eval_logger) logger.info('current learning rate: ' +
########################################################################## model = load_model(GPT2LMHeadModel(config), args.init_checkpoint, args, verbose=True) if args.local_rank != -1: # when from scratch make sure initial models are the same params = [p.data for p in model.parameters()] all_reduce_and_rescale_tensors(params, float(torch.distributed.get_world_size())) # FIXME is averaging the best way? init variance will change model_parameters = filter(lambda p: p.requires_grad, model.parameters()) total_params = sum([np.prod(p.size()) for p in model_parameters]) logger.info('Number of parameter = {}'.format(total_params)) ######################################################################### # Training ! ########################################################################## epoch = 0 if not args.skip_eval: eval_loss, eval_ppl = eval_model_loss(model, eval_dataloader_loss, epoch, args, encoder=enc, return_loss=True, setting=args.input_setting, outfile=args.output_file)