def eval(tokenizer: Tokenizer, model: GPT2LMHeadModel, dataset: MyDataset,
args: TrainingArguments):
model.eval()
loss = 0
iterator = build_data_iterator(tokenizer, dataset, args.eval_batch_size,
args.block_size)
for ids, attention_mask in tqdm(iterator, desc='eval'):
ids = ids.to(args.device)
with torch.no_grad():
loss += model(ids,
attention_mask=attention_mask.to(args.device),
labels=ids)[0].item()
model.train()
return loss / len(iterator)
def _validate(
model: GPT2LMHeadModel,
dev_dataloader: DataLoader,
device: torch.device,
logger: logging.Logger,
global_step: int,
):
model.eval()
loss_list = []
for batch_data in tqdm(dev_dataloader, desc="[EVAL]"):
with torch.no_grad():
input_ids, attention_mask, labels = tuple(value.to(device) for value in batch_data)
model_outputs = model.forward(input_ids, attention_mask=attention_mask, labels=labels, return_dict=True)
loss_list.append(model_outputs.loss.item())
mean_loss = np.mean(loss_list)
logger.info(f"[EVAL] global_step:{global_step} loss:{mean_loss:.4f} perplexity:{math.exp(mean_loss):.4f}")
model.train()