def get_criterion(version): if version == "focal_v1": return BinaryFocalLoss(alpha=0.04, gamma=2, reduction="mean", ignore_index=None, normalized=False, reduced_threshold=None) elif version == "focal_v2": return BinaryFocalLoss(alpha=0.04, gamma=1.5, reduction="mean", ignore_index=None, normalized=False, reduced_threshold=None) elif version == "bce_v1": return BCEWithLogitsLoss(pos_weight=torch.tensor(24.16), reduction='mean', weight=None, size_average=None, reduce=None) elif version == "bce_v2": return BCEWithLogitsLoss(reduction='mean', weight=None, size_average=None, reduce=None) elif version == "bce_v3": return BCEWithLogitsLoss(pos_weight=torch.tensor(15.463626008840938), reduction='mean', weight=None, size_average=None, reduce=None) elif version == "dice_v1": return DiceLoss(mode="binary", from_logits=True, classes=None, log_loss=False, smooth=0.0, ignore_index=None, eps=1e-7) else: raise Exception(f"Criterion version '{version}' is UNKNOWN!")
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, text_encoder: Seq2SeqEncoder, classifier_feedforward: FeedForward, verbose_metrics: False, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, loss: Optional[dict] = None, ) -> None: super(TextClassifier, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size("labels") self.text_encoder = text_encoder self.classifier_feedforward = classifier_feedforward self.prediction_layer = torch.nn.Linear(self.classifier_feedforward.get_output_dim(), self.num_classes) self.pool = lambda text, mask: util.get_final_encoder_states(text, mask, bidirectional=True) self.label_accuracy = CategoricalAccuracy() self.label_f1_metrics = {} for i in range(self.num_classes): self.label_f1_metrics[vocab.get_token_from_index(index=i, namespace="labels")] = F1Measure(positive_label=i) self.verbose_metrics = verbose_metrics if loss is None: self.loss = torch.nn.CrossEntropyLoss() else: alpha = loss.get('alpha') gamma = loss.get('gamma') weight = loss.get('weight') if alpha is not None: alpha = float(alpha) if gamma is not None: gamma = float(gamma) if weight is not None: weight = torch.tensor([1.0, float(weight)]) if loss.get('type') == 'CrossEntropyLoss': self.loss = torch.nn.CrossEntropyLoss(weight=weight) elif loss.get('type') == 'BinaryFocalLoss': self.loss = BinaryFocalLoss(alpha=alpha, gamma=gamma) elif loss.get('type') == 'FocalLoss': self.loss = FocalLoss(alpha=alpha, gamma=gamma) elif loss.get('type') == 'MultiLabelMarginLoss': self.loss = torch.nn.MultiLabelMarginLoss() elif loss.get('type') == 'MultiLabelSoftMarginLoss': self.loss = torch.nn.MultiLabelSoftMarginLoss(weight) else: raise ValueError(f'Unexpected loss "{loss}"') initializer(self)
def __init__(self, hparams): super().__init__() self.hparams = hparams self.model = object_from_dict(self.hparams["model"]) if "resume_from_checkpoint" in self.hparams: corrections: Dict[str, str] = {"model.": ""} state_dict = state_dict_from_disk( file_path=self.hparams["resume_from_checkpoint"], rename_in_layers=corrections, ) self.model.load_state_dict(state_dict) self.losses = [ ("jaccard", 0.1, JaccardLoss(mode="binary", from_logits=True)), ("focal", 0.9, BinaryFocalLoss()), ]
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, verbose_metrics: False, dropout: float = 0.2, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, loss: Optional[dict] = None, ) -> None: super(TextClassifier, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.dropout = torch.nn.Dropout(dropout) self.num_classes = self.vocab.get_vocab_size("labels") self.classifier_feedforward = torch.nn.Linear( self.text_field_embedder.get_output_dim(), self.num_classes) self.label_accuracy = CategoricalAccuracy() self.label_f1_metrics = {} self.verbose_metrics = verbose_metrics for i in range(self.num_classes): self.label_f1_metrics[vocab.get_token_from_index( index=i, namespace="labels")] = F1Measure(positive_label=i) if loss is None or loss.get('type') == 'CrossEntropyLoss': self.loss = torch.nn.CrossEntropyLoss() elif loss.get('type') == 'BinaryFocalLoss': self.loss = BinaryFocalLoss(alpha=loss.get('alpha'), gamma=loss.get('gamma')) elif loss.get('type') == 'FocalLoss': self.loss = FocalLoss(alpha=loss.get('alpha'), gamma=loss.get('gamma')) elif loss.get('type') == 'MultiLabelMarginLoss': self.loss = torch.nn.MultiLabelMarginLoss() elif loss.get('type') == 'MultiLabelSoftMarginLoss': self.loss = torch.nn.MultiLabelSoftMarginLoss( weight=torch.tensor(loss.get('weight')) if 'weight' in loss else None) else: raise ValueError(f'Unexpected loss "{loss}"') initializer(self)
def __init__(self, hparams): super().__init__() self.hparams = hparams self.model = object_from_dict(self.hparams["model"]) if "resume_from_checkpoint" in self.hparams: corrections: Dict[str, str] = {"model.": ""} checkpoint = load_checkpoint( file_path=self.hparams["resume_from_checkpoint"], rename_in_layers=corrections, ) self.model.load_state_dict(checkpoint["state_dict"]) if hparams["sync_bn"]: self.model = apex.parallel.convert_syncbn_model(self.model) self.losses = [ ("jaccard", 0.1, JaccardLoss(mode="binary", from_logits=True)), ("focal", 0.9, BinaryFocalLoss()), ]
def get_loss(loss_name: str, tsa=False): if loss_name.lower() == "dice": return DiceLoss(mode="binary") if loss_name.lower() == "mse": return nn.MSELoss() if loss_name.lower() == "msle": return MSLELoss() if loss_name.lower() == "smooth_l1": return nn.SmoothL1Loss() if loss_name.lower() == "mask_bce": return ResizeToTarget2d(SoftBCEWithLogitsLoss()) if loss_name.lower() == "rank": return PairwiseRankingLoss() if loss_name.lower() == "kl": return LogSoftmaxKLDivLoss() if loss_name.lower() == "rank2": return PairwiseRankingLossV2() if loss_name.lower() == "ccos": return ContrastiveCosineEmbeddingLoss() if loss_name.lower() == "cntr": return EmbeddingLoss() if loss_name.lower() == "roc_auc": return RocAucLoss() if loss_name.lower() == "roc_auc_ce": return RocAucLossCE() if loss_name.lower() == "bce": return nn.BCEWithLogitsLoss(reduction="none" if tsa else "mean") if loss_name.lower() == "wbce": return nn.BCEWithLogitsLoss(reduction="none" if tsa else "mean", pos_weight=torch.tensor(0.33).float()).cuda() if loss_name.lower() == "wbce2": return nn.BCEWithLogitsLoss(reduction="none" if tsa else "mean", pos_weight=torch.tensor(0.66).float()).cuda() if loss_name.lower() == "ce": return nn.CrossEntropyLoss(reduction="none" if tsa else "mean") if loss_name.lower() == "soft_ce": return SoftCrossEntropyLoss(reduction="none" if tsa else "mean", smooth_factor=0.1) if loss_name.lower() == "soft_bce": return SoftBCEWithLogitsLoss(reduction="none" if tsa else "mean", smooth_factor=0.1, ignore_index=None) if loss_name.lower() == "wce": return nn.CrossEntropyLoss( reduction="none" if tsa else "mean", weight=torch.tensor([2, 1, 2, 1]).float() ).cuda() if loss_name.lower() == "focal": return FocalLoss(alpha=None, gamma=2, reduction="none" if tsa else "mean") if loss_name.lower() == "binary_focal": return BinaryFocalLoss(alpha=None, gamma=2, reduction="none" if tsa else "mean") if loss_name.lower() == "nfl": return FocalLoss(alpha=None, gamma=2, normalized=True, reduction="sum" if tsa else "mean") if loss_name.lower() == "ohem_ce": return OHEMCrossEntropyLoss() # losses for embedding if loss_name.lower() == "cntrv2": return EmbeddingLossV2() if loss_name.lower() == "arc_face": return ArcFaceLoss() raise KeyError(loss_name)