def __call__(self, outputs: torch.Tensor, targets: torch.LongTensor, masks: torch.LongTensor = None, relevant_ignored: torch.LongTensor = None, irrelevant_ignored: torch.LongTensor = None) -> None: if not len(outputs.shape) == len(targets.shape): targets = torch.unsqueeze(targets, 1) targets = torch.zeros_like(outputs).scatter_(1, targets, 1) if relevant_ignored is None: relevant_ignored = np.zeros(outputs.shape[0]) else: relevant_ignored = relevant_ignored.cpu().numpy() if irrelevant_ignored is None: irrelevant_ignored = np.zeros(outputs.shape[0]) else: irrelevant_ignored = irrelevant_ignored.cpu().numpy() if masks is None: masks = torch.ones_like(targets).int() else: # try converting to int masks = masks.int() for output, target, mask, rel_ignored, irrel_ignored in zip( outputs, targets, masks, relevant_ignored, irrelevant_ignored): valid_docs = torch.sum(mask) output, target = output[:valid_docs], target[:valid_docs] output, target = paired_sort(output, target) output = self._cutoff(output) confusion = self.confusion_matrix(output, target) if torch.sum(target).cpu().numpy() + rel_ignored == 0: if self.version == 'tuning': # ignore both miss and false alarm when tuning pass elif self.version == 'program': # ignore miss when calculating program target false_alarm = confusion[0, 1] / float( sum(confusion[0, :]) + irrel_ignored) self.false_alarm.append(false_alarm) else: miss = (confusion[1, 0] + rel_ignored ) / float(sum(confusion[1, :]) + rel_ignored) false_alarm = confusion[0, 1] / float( sum(confusion[0, :]) + irrel_ignored) self.false_alarm.append(false_alarm) self.miss.append(miss)
def _get_loss(self, log_probs: torch.FloatTensor, target_tokens: Dict[str, torch.LongTensor], source_lengths: torch.LongTensor) -> torch.FloatTensor: targets = target_tokens[self._target_namespace] mask = (targets != self._pad_index).bool() relevant_mask = mask[:, 1:] relevant_targets = targets[:, 1:] target_lengths = util.get_lengths_from_binary_sequence_mask( relevant_mask) loss = self._loss(log_probs, relevant_targets.int(), source_lengths.int(), target_lengths.int()) return loss