def training_step(self, batch, batch_nb): X, y, weights = batch y_hat = self.forward(X["adj"], X["x"], X["idx"]) y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) self.train_metrics.update_metrics(y_hat, y, weights=None) loss = self.loss(y_hat, y) return {'loss': loss}
def validation_step(self, batch, batch_nb): X, y, weights = batch y_hat = self.forward(X["adj"], X["x"], X["idx"]) y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) self.valid_metrics.update_metrics(y_hat, y, weights=None) loss = self.loss(y_hat, y) return {"val_loss": loss}
def test_step(self, batch, batch_nb): X, y, weights = batch y_hat = self.forward(X) y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) self.test_metrics.update_metrics(y_hat, y, weights=None) loss = self.loss(y_hat, y) return {"test_loss": loss}
def validation_step(self, batch, batch_nb): X, y, weights = batch embeddings = self.forward(X) nids = (X["node_type"] == int(self.dataset.node_types.index(self.dataset.head_node_type))) y_hat = self.classifier.forward(embeddings[nids]) y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) loss = self.criterion(y_hat, y) self.valid_metrics.update_metrics(y_hat, y, weights=None) return {"val_loss": loss}
def update_metrics(self, y_hat: torch.Tensor, y: torch.Tensor, weights=None): """ :param y_pred: :param y_true: :param weights: """ y_pred = y_hat.detach() y_true = y.detach() y_pred, y_true = filter_samples(y_pred, y_true, weights) # Apply softmax/sigmoid activation if needed if "LOGITS" in self.loss_type or "FOCAL" in self.loss_type: if "SOFTMAX" in self.loss_type: y_pred = torch.softmax(y_pred, dim=1) else: y_pred = torch.sigmoid(y_pred) elif "NEGATIVE_LOG_LIKELIHOOD" == self.loss_type or "SOFTMAX_CROSS_ENTROPY" in self.loss_type: y_pred = torch.softmax(y_pred, dim=1) for metric in self.metrics: # torchmetrics metrics if isinstance(self.metrics[metric], torchmetrics.metric.Metric): self.metrics[metric].update(y_pred, y_true) # Torch ignite metrics elif "precision" in metric or "recall" in metric or "accuracy" in metric: if not self.multilabel and y_true.dim() == 1: self.metrics[metric].update((self.hot_encode(y_pred.argmax(1, keepdim=False), type_as=y_true), self.hot_encode(y_true, type_as=y_pred))) else: self.metrics[metric].update(((y_pred > self.threshold).type_as(y_true), y_true)) # Torch ignite metrics elif metric == "top_k": self.metrics[metric].update((y_pred, y_true)) # OGB metrics elif "ogb" in metric: if metric in ["ogbl-ddi", "ogbl-collab"]: y_true = y_true[:, 0] elif "ogbg-mol" in metric: # print(tensor_sizes({"y_pred": y_pred, "y_true": y_true})) pass self.metrics[metric].update((y_pred, y_true)) else: raise Exception(f"Metric {metric} has problem at .update()")
def test_step(self, batch, batch_nb): X, y, weights = batch y_hat, proximity_loss = self.forward(X, save_betas=True) if isinstance(y, dict) and len(y) > 1: y = y[self.head_node_type] y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) test_loss = self.criterion(y_hat, y) if batch_nb == 0: self.print_pred_class_counts(y_hat, y, multilabel=self.dataset.multilabel) self.test_metrics.update_metrics(y_hat, y, weights=None) if self.hparams.use_proximity: test_loss = test_loss + proximity_loss return {"test_loss": test_loss}
def validation_step(self, batch, batch_nb): X, y, weights = batch y_hat, proximity_loss = self.forward(X) if isinstance(y, dict) and len(y) > 1: y = y[self.head_node_type] y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) val_loss = self.criterion.forward(y_hat, y) # if batch_nb == 0: # self.print_pred_class_counts(y_hat, y, multilabel=self.dataset.multilabel) self.valid_metrics.update_metrics(y_hat, y, weights=None) if self.hparams.use_proximity: val_loss = val_loss + proximity_loss return {"val_loss": val_loss}
def training_step(self, batch, batch_nb): X, y, weights = batch y_hat, proximity_loss = self.forward(X) if isinstance(y, dict) and len(y) > 1: y = y[self.head_node_type] y_hat, y = filter_samples(Y_hat=y_hat, Y=y, weights=weights) loss = self.criterion.forward(y_hat, y) self.train_metrics.update_metrics(y_hat, y, weights=None) logs = None if self.hparams.use_proximity: loss = loss + proximity_loss logs = {"proximity_loss": proximity_loss} outputs = {'loss': loss} if logs is not None: outputs.update({'progress_bar': logs, "logs": logs}) return outputs
def evaluate(parameters, logger=None): reranker = utils.get_reranker(parameters) if parameters["full_evaluation"]: eval_datasets = [ "aida-A", "aida-B", "msnbc", "aquaint", "ace2004", "clueweb", "wikipedia", ] else: eval_datasets = ["aida-B"] candidates_key = ("pregenerated_candidates" if parameters["evaluate_with_pregenerated_candidates"] else "candidates") gold_key = ("pregenerated_gold_pos" if parameters["evaluate_with_pregenerated_candidates"] else "gold_pos") number_of_samples_per_dataset = {} total_time = 0 for eval_dataset_name in eval_datasets: time_start = time.time() logger.info("\nEvaluating on the {} dataset".format(eval_dataset_name)) eval_samples = utils.read_dataset( eval_dataset_name, parameters["path_to_preprocessed_json_data"]) eval_samples_filtered = utils.filter_samples(eval_samples, parameters["top_k"], gold_key) logger.info("Retained {} out of {} samples".format( len(eval_samples_filtered), len(eval_samples))) number_of_samples_per_dataset[eval_dataset_name] = len(eval_samples) # if args.num_preprocessing_threads == -1: # eval_data, eval_tensor_data = process_samples_for_model(args.context_key, eval_samples_filtered, tokenizer, args.max_seq_length, logger = logger, top_k = args.top_k, example = False, debug = args.debug, tagged = args.tag_mention, candidates_key = candidates_key, gold_key = gold_key) # else: # eval_data, eval_tensor_data = preprocessing_multithreaded(eval_samples_filtered, logger, args, output_dir=True) eval_data, eval_tensor_data = reranker._process_mentions_for_model( parameters["context_key"], eval_samples_filtered, reranker.tokenizer, parameters["max_seq_length"], parameters["top_k"], parameters["silent"], candidates_key=candidates_key, gold_key=gold_key, debug=parameters["debug"], ) eval_sampler = SequentialSampler(eval_tensor_data) eval_dataloader = DataLoader( eval_tensor_data, sampler=eval_sampler, batch_size=parameters["evaluation_batch_size"], ) if parameters["output_eval_file"] is None: output_eval_file = os.path.join(parameters["path_to_model"], "eval_results.txt") else: output_eval_file = parameters["output_eval_file"] result = evaluate_model_on_dataset( reranker.model, eval_dataloader, eval_dataset_name, eval_bm45_acc=True, device=reranker.device, logger=logger, path_to_file_to_write_results=output_eval_file, number_of_samples=number_of_samples_per_dataset[eval_dataset_name], ) execution_time = (time.time() - time_start) / 60 total_time += execution_time if logger != None: logger.info("The execution for dataset {} took {} minutes".format( eval_dataset_name, execution_time)) else: print("The execution for dataset {} took {} minutes".format( eval_dataset_name, execution_time)) if logger != None: logger.info("The execution for dataset {} took {} minutes".format( eval_dataset_name, execution_time)) else: print("The evaluation took:", total_time, " minutes")