def evaluate(model, tokenizer, epoch_id, args):
# use the same signature with eval_model_generation
logger.info(
'compute eval model loss, using eval mode, please change it back to train after calling this function'
)
model.eval()
eval_dataloader = DynamicBatchingLoader(args.eval_input_file, tokenizer,
args.normalize_data,
args.eval_batch_size,
args.max_seq_length)
tot_loss = []
tot_sample = []
with torch.no_grad():
for step, batch in enumerate(eval_dataloader):
batch = tuple(t.to(args.device) for t in batch)
input_ids, position_ids, token_ids, label_ids, src_len, _ = batch
if args.no_token_id: token_ids = None
n_sample = input_ids.shape[0]
loss = model(input_ids, position_ids, token_ids, label_ids)
tot_loss.append(loss.mean().item() * n_sample)
tot_sample.append(n_sample)
print(
f"\n Epoch {epoch_id}: Val loss {np.sum(tot_loss) / np.sum(tot_sample)} "
)
return np.sum(tot_loss) / np.sum(tot_sample)
def evaluate_models_from(GPT_saved_models_folder, eval_file, enc, args):
# Prepare eval data
eval_dataloader_loss = DynamicBatchingLoader(eval_file, enc,
args.normalize_data,
args.eval_batch_size,
args.max_seq_length)
eval_dataloader_gen = get_eval_list_same_length(eval_file, enc,
args.eval_batch_size, True)
# read eval_loss log file
eval_loss_log_file = os.path.join(GPT_saved_models_folder, "eval_log.txt")
min_ckpt_old_perplexity = None
min_ckpt_new_perplexity = None
min_old_perplexity = 1000000.0
min_new_perplexity = 1000000.0
with open(eval_loss_log_file, "r") as reader:
head_row = next(reader)
for line in reader:
line = line.strip()
epoch, ckpt_no, _, loss, perplexity = line.split(",")
epoch = int(epoch)
ckpt_no = int(ckpt_no) - 1
loss = float(loss)
perplexity = float(perplexity)
print(ckpt_no, loss, perplexity, end="")
if min_old_perplexity > perplexity:
min_old_perplexity = perplexity
min_ckpt_old_perplexity = ckpt_no
# calculate new loss and perplexity
model_filename = "GP2-pretrain-step-{}.pkl"
model = load_model(GPT2LMHeadModel(config),
os.path.join(GPT_saved_models_folder,
model_filename.format(ckpt_no)),
args,
verbose=True)
eval_loss, eval_ppl = eval_model_loss(model, enc,
eval_dataloader_loss, epoch,
args)
if min_new_perplexity > eval_ppl:
min_new_perplexity = eval_ppl
min_ckpt_new_perplexity = ckpt_no
print("Old best ckpt and perplexity:", min_ckpt_old_perplexity,
min_old_perplexity)
print("New best ckpt and perplexity:", min_ckpt_new_perplexity,
min_new_perplexity)
return min_ckpt_old_perplexity, min_old_perplexity, min_ckpt_new_perplexity, min_new_perplexity
config = GPT2Config.from_json_file(join(args.model_name_or_path,
'config.json'))
if args.local_rank == -1:
train_dataloader = BucketingDataLoader(args.train_input_file,
args.train_batch_size,
args.max_seq_length)
else:
pass
# train_dataloader = DistributedBucketingDataLoader(
# get_rank(), get_world_size(),
# args.train_input_file, args.train_batch_size,
# args.max_seq_length)
eval_dataloader_loss = DynamicBatchingLoader(args.eval_input_file, enc,
args.normalize_data,
args.eval_batch_size,
args.max_seq_length)
eval_dataloader_gen = get_eval_list_same_length(args.eval_input_file, enc,
args.eval_batch_size, True)
#########################################################################
# Prepare Model and Optimizer
##########################################################################
model = load_model(GPT2LMHeadModel(config),
args.init_checkpoint,
args,
verbose=True)
if args.local_rank != -1:
# when from scratch make sure initial models are the same
params = [p.data for p in model.parameters()]
logger.info('Input Argument Information')
args_dict = vars(args)
for a in args_dict:
logger.info('%-28s %s' % (a, args_dict[a]))
#########################################################################
# Prepare Data Set
##########################################################################
print("Prepare Data")
enc = GPT2Tokenizer.from_pretrained(args.model_name_or_path)
config = GPT2Config.from_json_file(
join(args.model_name_or_path, 'config.json'))
inference_dataloader_loss = DynamicBatchingLoader(
args.inference_input_file, enc, args.normalize_data,
args.inference_batch_size, args.max_seq_length, True)
inference_dataloader_gen = get_eval_list_same_length(
args.inference_input_file, enc, args.inference_batch_size, True)
# eval_dataloader_loss = DynamicBatchingLoader(
# args.eval_input_file, enc, args.normalize_data,
# args.eval_batch_size, args.max_seq_length)
#
# eval_dataloader_gen = get_eval_list_same_length(
# args.eval_input_file, enc, args.eval_batch_size, True)
#########################################################################
# Prepare Model
##########################################################################