def evaluate(self, eval_df, output_dir, multi_label=False, prefix="", **kwargs):
"""
Evaluates the model on eval_df.
Utility function to be used by the eval_model() method. Not intended to be used directly.
"""
tokenizer = self.tokenizer
device = self.device
model = self.model
args = self.args
eval_output_dir = output_dir
results = {}
if "text" in eval_df.columns and "labels" in eval_df.columns:
eval_examples = [
InputExample(i, text, None, label)
for i, (text, label) in enumerate(zip(eval_df["text"], eval_df["labels"]))
]
else:
eval_examples = [
InputExample(i, text, None, label)
for i, (text, label) in enumerate(zip(eval_df.iloc[:, 0], eval_df.iloc[:, 1]))
]
eval_dataset = self.load_and_cache_examples(eval_examples, evaluate=True)
os.makedirs(eval_output_dir, exist_ok=True)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args["eval_batch_size"])
eval_loss = 0.0
nb_eval_steps = 0
preds = None
out_label_ids = None
model.eval()
for batch in tqdm(eval_dataloader, disable=args["silent"]):
batch = tuple(t.to(device) for t in batch)
with torch.no_grad():
inputs = self._get_inputs_dict(batch)
if self.sliding_window:
outputs = model(inputs)
else:
outputs = model(**inputs)
tmp_eval_loss, logits = outputs[:2]
if multi_label:
logits = logits.sigmoid()
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if self.sliding_window:
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = inputs[0]["labels"].detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(out_label_ids, inputs[0]["labels"].detach().cpu().numpy(), axis=0)
else:
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = inputs["labels"].detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
eval_loss = eval_loss / nb_eval_steps
model_outputs = preds
if not multi_label:
preds = np.argmax(preds, axis=1)
result, wrong = self.compute_metrics(preds, out_label_ids, eval_examples, **kwargs)
result["eval_loss"] = eval_loss
results.update(result)
output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
for key in sorted(result.keys()):
writer.write("{} = {}\n".format(key, str(result[key])))
return results, model_outputs, wrong
def predict(self, to_predict, multi_label=False):
"""
Performs predictions on a list of text.
Args:
to_predict: A python list of text (str) to be sent to the model for prediction.
Returns:
preds: A python list of the predictions (0 or 1) for each text.
model_outputs: A python list of the raw model outputs for each text.
"""
tokenizer = self.tokenizer
device = self.device
model = self.model
args = self.args
self._move_model_to_device()
if multi_label:
eval_examples = [
InputExample(i, text, None, [0 for i in range(self.num_labels)]) for i, text in enumerate(to_predict)
]
else:
eval_examples = [InputExample(i, text, None, 0) for i, text in enumerate(to_predict)]
eval_dataset = self.load_and_cache_examples(
eval_examples, evaluate=True, multi_label=multi_label, no_cache=True
)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args["eval_batch_size"])
eval_loss = 0.0
nb_eval_steps = 0
preds = None
out_label_ids = None
for batch in tqdm(eval_dataloader, disable=args["silent"]):
model.eval()
batch = tuple(t.to(device) for t in batch)
with torch.no_grad():
inputs = self._get_inputs_dict(batch)
if self.sliding_window:
outputs = model(inputs)
else:
outputs = model(**inputs)
tmp_eval_loss, logits = outputs[:2]
if multi_label:
logits = logits.sigmoid()
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if self.sliding_window:
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = inputs[0]["labels"].detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(out_label_ids, inputs[0]["labels"].detach().cpu().numpy(), axis=0)
else:
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = inputs["labels"].detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
eval_loss = eval_loss / nb_eval_steps
model_outputs = preds
if multi_label:
if isinstance(args["threshold"], list):
threshold_values = args["threshold"]
preds = [
[self._threshold(pred, threshold_values[i]) for i, pred in enumerate(example)] for example in preds
]
else:
preds = [[self._threshold(pred, args["threshold"]) for pred in example] for example in preds]
else:
preds = np.argmax(preds, axis=1)
return preds, model_outputs
def train_model(
self, train_df, multi_label=False, output_dir=None, show_running_loss=True, args=None, eval_df=None
):
"""
Trains the model using 'train_df'
Args:
train_df: Pandas Dataframe containing at least two columns. If the Dataframe has a header, it should contain a 'text' and a 'labels' column. If no header is present,
the Dataframe should contain at least two columns, with the first column containing the text, and the second column containing the label. The model will be trained on this Dataframe.
output_dir: The directory where model files will be saved. If not given, self.args['output_dir'] will be used.
show_running_loss (optional): Set to False to prevent running loss from being printed to console. Defaults to True.
args (optional): Optional changes to the args dict of the model. Any changes made will persist for the model.
eval_df (optional): A DataFrame against which evaluation will be performed when evaluate_during_training is enabled. Is required if evaluate_during_training is enabled.
Returns:
None
"""
if args:
self.args.update(args)
if self.args["silent"]:
show_running_loss = False
if self.args["evaluate_during_training"] and eval_df is None:
raise ValueError(
"evaluate_during_training is enabled but eval_df is not specified. Pass eval_df to model.train_model() if using evaluate_during_training."
)
if not output_dir:
output_dir = self.args["output_dir"]
if os.path.exists(output_dir) and os.listdir(output_dir) and not self.args["overwrite_output_dir"]:
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
output_dir
)
)
self._move_model_to_device()
if "text" in train_df.columns and "labels" in train_df.columns:
train_examples = [
InputExample(i, text, None, label)
for i, (text, label) in enumerate(zip(train_df["text"], train_df["labels"]))
]
else:
train_examples = [
InputExample(i, text, None, label)
for i, (text, label) in enumerate(zip(train_df.iloc[:, 0], train_df.iloc[:, 1]))
]
train_dataset = self.load_and_cache_examples(train_examples)
global_step, tr_loss = self.train(
train_dataset, output_dir, show_running_loss=show_running_loss, eval_df=eval_df
)
os.makedirs(output_dir, exist_ok=True)
model_to_save = self.model.module if hasattr(self.model, "module") else self.model
model_to_save.save_pretrained(output_dir)
self.tokenizer.save_pretrained(output_dir)
torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
print("Training of {} model complete. Saved to {}.".format(self.args["model_type"], output_dir))
