def training_step(self, batch, batch_nb):
X, y, weights = batch
y_hat = self.forward(X["adj"], X["x"], X["idx"])
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
self.train_metrics.update_metrics(y_hat, y, weights=None)
loss = self.loss(y_hat, y)
return {'loss': loss}
def validation_step(self, batch, batch_nb):
X, y, weights = batch
y_hat = self.forward(X["adj"], X["x"], X["idx"])
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
self.valid_metrics.update_metrics(y_hat, y, weights=None)
loss = self.loss(y_hat, y)
return {"val_loss": loss}
def test_step(self, batch, batch_nb):
X, y, weights = batch
y_hat = self.forward(X)
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
self.test_metrics.update_metrics(y_hat, y, weights=None)
loss = self.loss(y_hat, y)
return {"test_loss": loss}
def validation_step(self, batch, batch_nb):
X, y, weights = batch
embeddings = self.forward(X)
nids = (X["node_type"] == int(self.dataset.node_types.index(self.dataset.head_node_type)))
y_hat = self.classifier.forward(embeddings[nids])
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
loss = self.criterion(y_hat, y)
self.valid_metrics.update_metrics(y_hat, y, weights=None)
return {"val_loss": loss}
def update_metrics(self, y_hat: torch.Tensor, y: torch.Tensor, weights=None):
"""
:param y_pred:
:param y_true:
:param weights:
"""
y_pred = y_hat.detach()
y_true = y.detach()
y_pred, y_true = filter_samples(y_pred, y_true, weights)
# Apply softmax/sigmoid activation if needed
if "LOGITS" in self.loss_type or "FOCAL" in self.loss_type:
if "SOFTMAX" in self.loss_type:
y_pred = torch.softmax(y_pred, dim=1)
else:
y_pred = torch.sigmoid(y_pred)
elif "NEGATIVE_LOG_LIKELIHOOD" == self.loss_type or "SOFTMAX_CROSS_ENTROPY" in self.loss_type:
y_pred = torch.softmax(y_pred, dim=1)
for metric in self.metrics:
# torchmetrics metrics
if isinstance(self.metrics[metric], torchmetrics.metric.Metric):
self.metrics[metric].update(y_pred, y_true)
# Torch ignite metrics
elif "precision" in metric or "recall" in metric or "accuracy" in metric:
if not self.multilabel and y_true.dim() == 1:
self.metrics[metric].update((self.hot_encode(y_pred.argmax(1, keepdim=False), type_as=y_true),
self.hot_encode(y_true, type_as=y_pred)))
else:
self.metrics[metric].update(((y_pred > self.threshold).type_as(y_true), y_true))
# Torch ignite metrics
elif metric == "top_k":
self.metrics[metric].update((y_pred, y_true))
# OGB metrics
elif "ogb" in metric:
if metric in ["ogbl-ddi", "ogbl-collab"]:
y_true = y_true[:, 0]
elif "ogbg-mol" in metric:
# print(tensor_sizes({"y_pred": y_pred, "y_true": y_true}))
pass
self.metrics[metric].update((y_pred, y_true))
else:
raise Exception(f"Metric {metric} has problem at .update()")
def test_step(self, batch, batch_nb):
X, y, weights = batch
y_hat, proximity_loss = self.forward(X, save_betas=True)
if isinstance(y, dict) and len(y) > 1:
y = y[self.head_node_type]
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
test_loss = self.criterion(y_hat, y)
if batch_nb == 0:
self.print_pred_class_counts(y_hat, y, multilabel=self.dataset.multilabel)
self.test_metrics.update_metrics(y_hat, y, weights=None)
if self.hparams.use_proximity:
test_loss = test_loss + proximity_loss
return {"test_loss": test_loss}
def validation_step(self, batch, batch_nb):
X, y, weights = batch
y_hat, proximity_loss = self.forward(X)
if isinstance(y, dict) and len(y) > 1:
y = y[self.head_node_type]
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
val_loss = self.criterion.forward(y_hat, y)
# if batch_nb == 0:
# self.print_pred_class_counts(y_hat, y, multilabel=self.dataset.multilabel)
self.valid_metrics.update_metrics(y_hat, y, weights=None)
if self.hparams.use_proximity:
val_loss = val_loss + proximity_loss
return {"val_loss": val_loss}
def training_step(self, batch, batch_nb):
X, y, weights = batch
y_hat, proximity_loss = self.forward(X)
if isinstance(y, dict) and len(y) > 1:
y = y[self.head_node_type]
y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights)
loss = self.criterion.forward(y_hat, y)
self.train_metrics.update_metrics(y_hat, y, weights=None)
logs = None
if self.hparams.use_proximity:
loss = loss + proximity_loss
logs = {"proximity_loss": proximity_loss}
outputs = {'loss': loss}
if logs is not None:
outputs.update({'progress_bar': logs, "logs": logs})
return outputs
def evaluate(parameters, logger=None):
reranker = utils.get_reranker(parameters)
if parameters["full_evaluation"]:
eval_datasets = [
"aida-A",
"aida-B",
"msnbc",
"aquaint",
"ace2004",
"clueweb",
"wikipedia",
]
else:
eval_datasets = ["aida-B"]
candidates_key = ("pregenerated_candidates"
if parameters["evaluate_with_pregenerated_candidates"]
else "candidates")
gold_key = ("pregenerated_gold_pos"
if parameters["evaluate_with_pregenerated_candidates"] else
"gold_pos")
number_of_samples_per_dataset = {}
total_time = 0
for eval_dataset_name in eval_datasets:
time_start = time.time()
logger.info("\nEvaluating on the {} dataset".format(eval_dataset_name))
eval_samples = utils.read_dataset(
eval_dataset_name, parameters["path_to_preprocessed_json_data"])
eval_samples_filtered = utils.filter_samples(eval_samples,
parameters["top_k"],
gold_key)
logger.info("Retained {} out of {} samples".format(
len(eval_samples_filtered), len(eval_samples)))
number_of_samples_per_dataset[eval_dataset_name] = len(eval_samples)
# if args.num_preprocessing_threads == -1:
# eval_data, eval_tensor_data = process_samples_for_model(args.context_key, eval_samples_filtered, tokenizer, args.max_seq_length, logger = logger, top_k = args.top_k, example = False, debug = args.debug, tagged = args.tag_mention, candidates_key = candidates_key, gold_key = gold_key)
# else:
# eval_data, eval_tensor_data = preprocessing_multithreaded(eval_samples_filtered, logger, args, output_dir=True)
eval_data, eval_tensor_data = reranker._process_mentions_for_model(
parameters["context_key"],
eval_samples_filtered,
reranker.tokenizer,
parameters["max_seq_length"],
parameters["top_k"],
parameters["silent"],
candidates_key=candidates_key,
gold_key=gold_key,
debug=parameters["debug"],
)
eval_sampler = SequentialSampler(eval_tensor_data)
eval_dataloader = DataLoader(
eval_tensor_data,
sampler=eval_sampler,
batch_size=parameters["evaluation_batch_size"],
)
if parameters["output_eval_file"] is None:
output_eval_file = os.path.join(parameters["path_to_model"],
"eval_results.txt")
else:
output_eval_file = parameters["output_eval_file"]
result = evaluate_model_on_dataset(
reranker.model,
eval_dataloader,
eval_dataset_name,
eval_bm45_acc=True,
device=reranker.device,
logger=logger,
path_to_file_to_write_results=output_eval_file,
number_of_samples=number_of_samples_per_dataset[eval_dataset_name],
)
execution_time = (time.time() - time_start) / 60
total_time += execution_time
if logger != None:
logger.info("The execution for dataset {} took {} minutes".format(
eval_dataset_name, execution_time))
else:
print("The execution for dataset {} took {} minutes".format(
eval_dataset_name, execution_time))
if logger != None:
logger.info("The execution for dataset {} took {} minutes".format(
eval_dataset_name, execution_time))
else:
print("The evaluation took:", total_time, " minutes")