def _eval(self, model: torch.nn.Module, dataset: Dataset, input_reader: BaseInputReader, epoch: int = 0, updates_epoch: int = 0, iteration: int = 0): self._logger.info("Evaluate: %s" % dataset.label) if isinstance(model, DataParallel): # currently no multi GPU support during evaluation model = model.module # create evaluator predictions_path = os.path.join(self._log_path, f'predictions_{dataset.label}_epoch_{epoch}.json') examples_path = os.path.join(self._log_path, f'examples_%s_{dataset.label}_epoch_{epoch}.html') evaluator = Evaluator(dataset, input_reader, self._tokenizer, self._args.rel_filter_threshold, self._args.no_overlapping, predictions_path, examples_path, self._args.example_count) # create data loader dataset.switch_mode(Dataset.EVAL_MODE) data_loader = DataLoader(dataset, batch_size=self._args.eval_batch_size, shuffle=False, drop_last=False, num_workers=self._args.sampling_processes, collate_fn=sampling.collate_fn_padding) with torch.no_grad(): model.eval() # iterate batches total = math.ceil(dataset.document_count / self._args.eval_batch_size) for batch in tqdm(data_loader, total=total, desc='Evaluate epoch %s' % epoch): # move batch to selected device batch = util.to_device(batch, self._device) # run model (forward pass) result = model(encodings=batch['encodings'], context_masks=batch['context_masks'], entity_masks=batch['entity_masks'], entity_sizes=batch['entity_sizes'], entity_spans=batch['entity_spans'], entity_sample_masks=batch['entity_sample_masks'], inference=True) entity_clf, rel_clf, rels = result # evaluate batch evaluator.eval_batch(entity_clf, rel_clf, rels, batch) global_iteration = epoch * updates_epoch + iteration ner_eval, rel_eval, rel_nec_eval = evaluator.compute_scores() self._log_eval(*ner_eval, *rel_eval, *rel_nec_eval, epoch, iteration, global_iteration, dataset.label) if self._args.store_predictions and not self._args.no_overlapping: evaluator.store_predictions() if self._args.store_examples: evaluator.store_examples()
def forward(self, tokens): self.input_reader.read_for_infer(tokens) dataset = self.input_reader.get_dataset("infer") dataset.switch_mode(Dataset.EVAL_MODE) data_loader = DataLoader(dataset, batch_size=self.args.eval_batch_size, shuffle=False, drop_last=False, num_workers=self.args.sampling_processes, collate_fn=sampling.collate_fn_padding) evaluator = Evaluator(dataset, self.input_reader, self._tokenizer, self.args.rel_filter_threshold, self.args.no_overlapping, None, "predictions_%s_epoch_%s.json", self.args.example_count, 0, dataset.label) with torch.no_grad(): self.model.eval() # iterate batches total = math.ceil(dataset.document_count / self.args.eval_batch_size) for batch in tqdm(data_loader, total=total, desc='Evaluate epoch %s' % 0): # move batch to selected device batch = util.to_device(batch, self._device) # run model (forward pass) result = self.model( encodings=batch['encodings'], context_masks=batch['context_masks'], entity_masks=batch['entity_masks'], entity_sizes=batch['entity_sizes'], entity_spans=batch['entity_spans'], entity_sample_masks=batch['entity_sample_masks'], evaluate=True) entity_clf, rel_clf, rels = result evaluator.eval_batch(entity_clf, rel_clf, rels, batch) return evaluator.get_preds()
def _predict(self, model: torch.nn.Module, dataset: Dataset, input_reader: JsonInputReader, epoch: int = 0, updates_epoch: int = 0, iteration: int = 0): if isinstance(model, DataParallel): model = model.modules() # create evaluator evaluator = Evaluator(dataset, input_reader, self._tokenizer, self.args.rel_filter_threshold, self.args.no_overlapping, self.args.output_path, self._examples_path, self.args.example_count, epoch, dataset.label) # create data loader dataset.switch_mode(Dataset.EVAL_MODE) data_loader = DataLoader(dataset, batch_size=self.args.eval_batch_size, shuffle=False, drop_last=False, num_workers=self.args.sampling_processes, collate_fn=sampling.collate_fn_padding) returned_predict= list() with torch.no_grad(): model.eval() # iteraate batches total = math.ceil(dataset.document_count / self.args.eval_batch_size) for batch in tqdm(data_loader, total=total, desc='Evaluate epoch %s' % epoch): # move batch to selected device batch = util.to_device(batch, self._device) # run model (forward pass) result = model(encodings=batch['encodings'], context_masks=batch['context_masks'], entity_masks=batch['entity_masks'], entity_sizes=batch['entity_sizes'], entity_spans=batch['entity_spans'], entity_sample_masks=batch['entity_sample_masks'], evaluate=True) entity_clf, rel_clf, rels = result evaluator.predict_batch(entity_clf, rel_clf, rels, batch) evaluator.store_predictions()
def _eval(self, model: torch.nn.Module, dataset: Dataset, input_reader: JsonInputReader, epoch: int = 0, updates_epoch: int = 0, iteration: int = 0): self._logger.info("Evaluate: %s" % dataset.label) if isinstance(model, DataParallel): # currently no multi GPU support during evaluation model = model.module # create evaluator evaluator = Evaluator(dataset, input_reader, self._tokenizer, self.args.rel_filter_threshold, self.args.example_count, self._examples_path, epoch, dataset.label) # create batch sampler sampler = self._sampler.create_eval_sampler(dataset, self.args.eval_batch_size, self.args.max_span_size, input_reader.context_size, truncate=False) with torch.no_grad(): model.eval() # iterate batches total = math.ceil(dataset.document_count / self.args.eval_batch_size) for batch in tqdm(sampler, total=total, desc='Evaluate epoch %s' % epoch): # move batch to selected device batch = batch.to(self._device) # run model (forward pass) entity_clf, rel_clf, rels = model(batch.encodings, batch.ctx_masks, batch.entity_masks, batch.entity_sizes, batch.entity_spans, batch.entity_sample_masks, evaluate=True) # evaluate batch evaluator.eval_batch(entity_clf, rel_clf, rels, batch) global_iteration = epoch * updates_epoch + iteration ner_eval, rel_eval, rel_ner_eval = evaluator.compute_scores() self._log_eval(*ner_eval, *rel_eval, *rel_ner_eval, epoch, iteration, global_iteration, dataset.label) if self.args.store_examples: evaluator.store_examples()