def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder = None, encoder: Seq2VecEncoder = None, dropout: float = 0.3, highway: bool = False): super().__init__(vocab, embedder, encoder, dropout) self.embedder = embedder self.encoder = encoder or BertCLSPooler(self.embedder.get_output_dim()) self.dropout = nn.Dropout(dropout) num_classes = self.vocab.get_vocab_size("labels") assert num_classes > 0, "Wrong namespace for labels apparently" self.highway = highway if not highway: self.clf = nn.Linear(self.encoder.get_output_dim() * 2, num_classes) else: output_dim = self.encoder.get_output_dim() self.clf = nn.Sequential(nn.Linear(output_dim * 2, output_dim), nn.Sigmoid(), nn.Linear(output_dim, num_classes)) self.accuracy = CategoricalAccuracy() self.f1 = FBetaMeasure(average=None, labels=list( range(self.vocab.get_vocab_size("labels"))))
class VisualEntailmentHead(Head): def __init__(self, vocab: Vocabulary, embedding_dim: int, label_namespace: str = "labels"): super().__init__(vocab) num_labels = vocab.get_vocab_size(label_namespace) self.label_namespace = label_namespace self.classifier = torch.nn.Linear(embedding_dim, num_labels) from allennlp.training.metrics import CategoricalAccuracy from allennlp.training.metrics import FBetaMeasure self.accuracy = CategoricalAccuracy() self.fbeta = FBetaMeasure(beta=1.0, average="macro") @overrides def forward( self, # type: ignore encoded_boxes: torch.Tensor, encoded_boxes_mask: torch.Tensor, encoded_boxes_pooled: torch.Tensor, encoded_text: torch.Tensor, encoded_text_mask: torch.Tensor, encoded_text_pooled: torch.Tensor, pooled_boxes_and_text: torch.Tensor, labels: Optional[torch.Tensor] = None, label_weights: Optional[torch.Tensor] = None, ) -> Dict[str, torch.Tensor]: logits = self.classifier(pooled_boxes_and_text) probs = torch.softmax(logits, dim=-1) output = {"logits": logits, "probs": probs} assert label_weights is None if labels is not None: output["loss"] = torch.nn.functional.cross_entropy(logits, labels) / logits.size(0) self.accuracy(logits, labels) self.fbeta(probs, labels) return output @overrides def get_metrics(self, reset: bool = False) -> Dict[str, float]: result = self.fbeta.get_metric(reset) result["acc"] = self.accuracy.get_metric(reset) return result def make_output_human_readable( self, output_dict: Dict[str, torch.Tensor] ) -> Dict[str, torch.Tensor]: if len(output_dict) <= 0: return output_dict logits = output_dict["logits"] entailment_answer_index = logits.argmax(-1) entailment_answer = [ self.vocab.get_token_from_index(int(i), "labels") for i in entailment_answer_index ] output_dict["entailment_answer"] = entailment_answer return output_dict default_predictor = "vilbert_ve"
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder = None, encoder: Seq2VecEncoder = None, dropout: float = 0.3): super().__init__(vocab) self.embedder = embedder self.encoder = encoder or BertCLSPooler(self.embedder.get_output_dim()) self.dropout = nn.Dropout(dropout) num_classes = self.vocab.get_vocab_size("labels") assert num_classes > 0, "Wrong namespace for labels apparently" self.clf = nn.Linear(self.encoder.get_output_dim(), num_classes) self.accuracy = CategoricalAccuracy() assert num_classes == 2 or num_classes == 3 labels = list(range(num_classes)) self.f1 = FBetaMeasure(average=None, labels=labels)
def test_fbeta_handles_no_prediction_true_all_class(self, device: str): predictions = torch.tensor([[0.65, 0.35], [0.0, 0.0]], device=device) # preds = [0, NA] targets = torch.tensor([1, 1], device=device) fbeta = FBetaMeasure() fbeta(predictions, targets) metric = fbeta.get_metric() precisions = metric["precision"] recalls = metric["recall"] fscores = metric["fscore"] assert_allclose(precisions, [0.0, 0.0]) assert_allclose(recalls, [0.0, 0.0]) assert_allclose(fscores, [0.0, 0.0])
def __init__( self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary, metrics: Dict[str, Metric] = None, initializer: InitializerApplicator = InitializerApplicator(), ) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder hidden_dim = self.encoder.get_output_dim() self.span_starts = nn.Linear(hidden_dim, 1) self.span_ends = nn.Linear(hidden_dim, 1) self._target_namespace = 'answer_labels' self.label_head = nn.Sequential( nn.Linear(encoder.get_output_dim(), 32), nn.Dropout(0.3), nn.Linear(32, self.vocab.get_vocab_size(self._target_namespace))) self._label_loss_fn = CrossEntropyLoss() self._qa_loss_fn = CrossEntropyLoss() metrics = metrics or {} metrics.update({ "label_accuracy": CategoricalAccuracy(), "start_accuracy": CategoricalAccuracy(), "end_accuracy": CategoricalAccuracy(), "label_f1measure": FBetaMeasure(), }) self.metrics = metrics initializer(self)
def __init__( self, vocab: Vocabulary, input_unit: Seq2VecEncoder, text_field_embedder: TextFieldEmbedder, # embedding_projection_dim: int = None, classifier_feedforward: FeedForward = None, max_step: int = 12, n_memories: int = 3, self_attention: bool = False, memory_gate: bool = False, dropout: int = 0.15, loss_weights=None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.num_classes = max(self.vocab.get_vocab_size("labels"), 2) self.text_field_embedder = text_field_embedder self.proj = nn.Linear(text_field_embedder.get_output_dim(), input_unit.get_input_dim()) self.input_unit = input_unit self.mac = MACCell( text_field_embedder.get_output_dim( ), # input_unit.get_output_dim(), max_step=max_step, n_memories=n_memories, self_attention=self_attention, memory_gate=memory_gate, dropout=dropout, save_attns=False, ) hidden_size = 2 * input_unit.get_output_dim() n_layers = 3 self.classifier = classifier_feedforward or FeedForward( input_dim=hidden_size, num_layers=n_layers, hidden_dims=(n_layers - 1) * [hidden_size] + [self.num_classes], activations=[ Activation.by_name("relu")(), Activation.by_name("relu")(), Activation.by_name("linear")() ], dropout=[dropout, dropout, 0.0]) self.metrics = { "accuracy": CategoricalAccuracy(), "f1": F1Measure(positive_label=1), "weighted_f1": WeightedF1Measure(), "fbeta": FBetaMeasure(average='micro') } weights = loss_weights and torch.FloatTensor(loss_weights) self.loss = nn.CrossEntropyLoss(weight=weights) initializer(self)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.vocab = vocab self.label_vocab = vocab.get_index_to_token_vocabulary( namespace='labels') inf_vec = torch.Tensor([float('-inf')] * encoder.get_input_dim()) self.class_avgs = [ inf_vec.clone() for i in range(len(self.label_vocab)) ] self.accuracy = CategoricalAccuracy() self.f_beta = FBetaMeasure(1.0, None, [0, 1, 2])
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, hidden_dim, **kwargs): super().__init__(vocab, **kwargs) out_class = vocab.get_vocab_size('labels') T = 3 self._text_embed_dropout = nn.Dropout(0.3) self._text_field_embedder = text_field_embedder self._dual_transformers = nn.ModuleList([copy.deepcopy(DualTransformer(hidden_dim)) for _ in range(T)]) self._W_plum = nn.Linear(2 * hidden_dim, hidden_dim, bias=False) self._W3 = torch.nn.Linear(hidden_dim, hidden_dim, bias=False) self._final_classifier = nn.Linear(hidden_dim, out_class) self._loss_fn = nn.CrossEntropyLoss() self._positive_class_f1 = FBetaMeasure(average='macro') self._accuracy = CategoricalAccuracy()
def __init__(self, vocab: Vocabulary, lambd: float, embedder: TextFieldEmbedder = None, dropout: float = 0.3): super().__init__(vocab) self.lambd = lambd self.embedder = embedder self.dropout = nn.Dropout(dropout) num_classes = self.vocab.get_vocab_size("labels") assert num_classes > 0, "Wrong namespace for labels apparently" self.sic = SICModel(self.embedder.get_output_dim()) self.interp = InterpretationModel(self.embedder.get_output_dim()) self.clf = nn.Linear(self.embedder.get_output_dim(), num_classes) self.accuracy = CategoricalAccuracy() self.f1 = FBetaMeasure(average=None, labels=list( range(self.vocab.get_vocab_size("labels"))))
def test_fbeta_multiclass_with_explicit_labels(self): # same prediction but with and explicit label ordering fbeta = FBetaMeasure(labels=[4, 3, 2, 1, 0]) fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric['precision'] recalls = metric['recall'] fscores = metric['fscore'] desired_precisions = self.desired_precisions[::-1] desired_recalls = self.desired_recalls[::-1] desired_fscores = self.desired_fscores[::-1] # check value numpy.testing.assert_almost_equal(precisions, desired_precisions, decimal=2) numpy.testing.assert_almost_equal(recalls, desired_recalls, decimal=2) numpy.testing.assert_almost_equal(fscores, desired_fscores, decimal=2)
def __init__(self, vocab: Vocabulary, embedding_dim: int, label_namespace: str = "labels"): super().__init__(vocab) self.label_namespace = label_namespace self.layer1 = torch.nn.Linear(embedding_dim * 2, embedding_dim) self.layer2 = torch.nn.Linear(embedding_dim, 2) self.activation = torch.nn.ReLU() from allennlp.training.metrics import CategoricalAccuracy from allennlp.training.metrics import FBetaMeasure self.accuracy = CategoricalAccuracy() self.fbeta = FBetaMeasure(beta=1.0, average="macro")
class RationaleBaseModel(Model): def __init__( self, vocab: Vocabulary, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(RationaleBaseModel, self).__init__(vocab, regularizer) self._vocabulary = vocab self._f1_metric = FBetaMeasure() self._accuracy = CategoricalAccuracy() self.prediction_mode = False initializer(self) def forward(self, document, sentence_indices, query=None, labels=None, metadata=None): raise NotImplementedError def decode(self, output_dict): output_dict = self._decode(output_dict) return output_dict def _call_metrics(self, output_dict) : self._f1_metric(output_dict['logits'], output_dict['gold_labels']) self._accuracy(output_dict['logits'], output_dict['gold_labels']) def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics = self._f1_metric.get_metric(reset) output_labels = self._vocabulary.get_index_to_token_vocabulary("labels") output_labels = [output_labels[i] for i in range(len(output_labels))] class_metrics = {} for k, v in metrics.items(): assert len(v) == len(output_labels) class_nums = dict(zip(output_labels, v)) class_metrics.update({k + "_" + str(kc): x for kc, x in class_nums.items()}) class_metrics.update({"accuracy": self._accuracy.get_metric(reset)}) modified_class_metrics = {} for k, v in class_metrics.items() : if k.endswith('_1') or k == 'accuracy': modified_class_metrics[k] = v else : modified_class_metrics['_' + k] = v return modified_class_metrics def normalize_attentions(self, output_dict) : ''' In case, attention is over subtokens rather than at token level. Combine subtoken attention into token attention. ''' return output_dict
def test_fbeta_multiclass_with_macro_average(self): labels = [0, 1] fbeta = FBetaMeasure(average="macro", labels=labels) fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric["precision"] recalls = metric["recall"] fscores = metric["fscore"] macro_precision = numpy.array(self.desired_precisions)[labels].mean() macro_recall = numpy.array(self.desired_recalls)[labels].mean() macro_fscore = numpy.array(self.desired_fscores)[labels].mean() # check value numpy.testing.assert_almost_equal(precisions, macro_precision, decimal=2) numpy.testing.assert_almost_equal(recalls, macro_recall, decimal=2) numpy.testing.assert_almost_equal(fscores, macro_fscore, decimal=2)
def test_fbeta_multiclass_with_explicit_labels(self, device: str): self.predictions = self.predictions.to(device) self.targets = self.targets.to(device) # same prediction but with and explicit label ordering fbeta = FBetaMeasure(labels=[4, 3, 2, 1, 0]) fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric["precision"] recalls = metric["recall"] fscores = metric["fscore"] desired_precisions = self.desired_precisions[::-1] desired_recalls = self.desired_recalls[::-1] desired_fscores = self.desired_fscores[::-1] # check value assert_allclose(precisions, desired_precisions) assert_allclose(recalls, desired_recalls) assert_allclose(fscores, desired_fscores)
class NLIModel(Model): default_predictor = "NLIPredictor" def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder = None, encoder: Seq2VecEncoder = None, dropout: float = 0.3): super().__init__(vocab) self.embedder = embedder self.encoder = encoder or BertCLSPooler(self.embedder.get_output_dim()) self.dropout = nn.Dropout(dropout) num_classes = self.vocab.get_vocab_size("labels") assert num_classes > 0, "Wrong namespace for labels apparently" self.clf = nn.Linear(self.encoder.get_output_dim(), num_classes) self.accuracy = CategoricalAccuracy() assert num_classes == 2 or num_classes == 3 labels = list(range(num_classes)) self.f1 = FBetaMeasure(average=None, labels=labels) def forward(self, tokens: Dict[str, Dict[str, torch.LongTensor]], labels: torch.LongTensor = None, **kwargs) -> Dict[str, torch.Tensor]: mask = get_text_field_mask(tokens) embedded = self.embedder(tokens) embedded = self.dropout(embedded) encoded_cls = self.encoder(embedded, mask) logits = self.clf(encoded_cls) # logits - batch_size, num_classes output_dict = {"logits": logits} if labels is not None: # labels - batch_size labels = labels.view(-1) loss = cross_entropy(logits, labels) output_dict["loss"] = loss self.accuracy(logits, labels) self.f1(logits, labels) return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics = {} acc: float = self.accuracy.get_metric(reset) metrics["accuracy"] = acc f1 = self.f1.get_metric(reset) for name, idx in self.vocab.get_token_to_index_vocabulary( "labels").items(): for metric_name, value in f1.items(): metrics[name + "_" + metric_name] = value[idx] return metrics def make_output_human_readable( self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, Any]: return output_dict
def __init__(self, vocab: Vocabulary, embedding_dim: int, feature_size: int, max_span_width: int, lexical_dropout: float = 0.2, mlp_dropout: float = 0.4, sampling_rate: float = 1, regularizer: Optional[RegularizerApplicator] = None, embedder_type=None) -> None: super(SEPT, self).__init__(vocab, regularizer) self.sampling_rate = sampling_rate self.class_num = self.vocab.get_vocab_size('labels') word_embeddings = get_embeddings(embedder_type, self.vocab, embedding_dim, True) embedding_dim = word_embeddings.get_output_dim() self._text_field_embedder = word_embeddings self._span_extractor = PoolingSpanExtractor(embedding_dim) # self._endpoint_span_extractor = EndpointSpanExtractor(embedding_dim, # combination="x,y", # num_width_embeddings=max_span_width, # span_width_embedding_dim=feature_size, # bucket_widths=False) # self._attentive_span_extractor = SelfAttentiveSpanExtractor(input_dim=embedding_dim) # entity_feedforward = FeedForward(self._span_extractor.get_output_dim() # + self._endpoint_span_extractor.get_output_dim() # + self._attentive_span_extractor.get_output_dim(), 3, feature_size, # F.relu, mlp_dropout) entity_feedforward = FeedForward(self._span_extractor.get_output_dim(), 3, feature_size, F.relu, mlp_dropout) self._entity_scorer = torch.nn.Sequential( TimeDistributed(entity_feedforward), TimeDistributed(torch.nn.Linear(entity_feedforward.get_output_dim(), self.class_num))) self._max_span_width = max_span_width if lexical_dropout > 0: self._lexical_dropout = torch.nn.Dropout(p=lexical_dropout) else: self._lexical_dropout = lambda x: x self._metric_all = FBetaMeasure() self._metric_avg = NERF1Metric()
def test_fbeta_multiclass_metric(self, device: str): self.predictions = self.predictions.to(device) self.targets = self.targets.to(device) fbeta = FBetaMeasure() fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric["precision"] recalls = metric["recall"] fscores = metric["fscore"] # check value assert_allclose(precisions, self.desired_precisions) assert_allclose(recalls, self.desired_recalls) assert_allclose(fscores, self.desired_fscores) # check type assert isinstance(precisions, List) assert isinstance(recalls, List) assert isinstance(fscores, List)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, seq2vec_encoder: Seq2VecEncoder, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super().__init__(vocab, regularizer) self._vocab = vocab self._text_field_embedder = text_field_embedder self._pooler = seq2vec_encoder self._num_labels = self._vocab.get_vocab_size("labels") self._classifier = nn.Linear(self._pooler.get_output_dim(), self._num_labels, bias=False) self._f1 = FBetaMeasure() self._accuracy = CategoricalAccuracy() initializer(self)
def test_fbeta_multiclass_with_macro_average(self, device: str): self.predictions = self.predictions.to(device) self.targets = self.targets.to(device) labels = [0, 1] fbeta = FBetaMeasure(average="macro", labels=labels) fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric["precision"] recalls = metric["recall"] fscores = metric["fscore"] # We keep the expected values in CPU because FBetaMeasure returns them in CPU. macro_precision = torch.tensor(self.desired_precisions)[labels].mean() macro_recall = torch.tensor(self.desired_recalls)[labels].mean() macro_fscore = torch.tensor(self.desired_fscores)[labels].mean() # check value assert_allclose(precisions, macro_precision) assert_allclose(recalls, macro_recall) assert_allclose(fscores, macro_fscore)
def test_fbeta_multiclass_marco_average_metric(self): fbeta = FBetaMeasure(average='macro') fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric['precision'] recalls = metric['recall'] fscores = metric['fscore'] macro_precision = numpy.mean(self.desired_precisions) macro_recall = numpy.mean(self.desired_recalls) macro_fscore = numpy.mean(self.desired_fscores) # check value numpy.testing.assert_almost_equal(precisions, macro_precision, decimal=2) numpy.testing.assert_almost_equal(recalls, macro_recall, decimal=2) numpy.testing.assert_almost_equal(fscores, macro_fscore, decimal=2) # check type assert isinstance(precisions, float) assert isinstance(recalls, float) assert isinstance(fscores, float)
def test_fbeta_multiclass_state(self, device: str): self.predictions = self.predictions.to(device) self.targets = self.targets.to(device) fbeta = FBetaMeasure() fbeta(self.predictions, self.targets) # check state assert_allclose(fbeta._pred_sum.tolist(), self.pred_sum) assert_allclose(fbeta._true_sum.tolist(), self.true_sum) assert_allclose(fbeta._true_positive_sum.tolist(), self.true_positive_sum) assert_allclose(fbeta._true_negative_sum.tolist(), self.true_negative_sum) assert_allclose(fbeta._total_sum.tolist(), self.total_sum)
def test_fbeta_multiclass_with_weighted_average(self, device: str): self.predictions = self.predictions.to(device) self.targets = self.targets.to(device) labels = [0, 1] fbeta = FBetaMeasure(average="weighted", labels=labels) fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric["precision"] recalls = metric["recall"] fscores = metric["fscore"] weighted_precision, weighted_recall, weighted_fscore, _ = precision_recall_fscore_support( self.targets.cpu().numpy(), self.predictions.argmax(dim=1).cpu().numpy(), labels=labels, average="weighted", ) # check value assert_allclose(precisions, weighted_precision) assert_allclose(recalls, weighted_recall) assert_allclose(fscores, weighted_fscore)
def __init__(self, voc: Vocabulary, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, out_sz: int, multi=True): super(BaseModelWithoutKnowledge, self).__init__(voc) self.word_embeddings = word_embeddings self.encoder = encoder self.projection = nn.Linear(in_features=self.encoder.get_output_dim(), out_features=out_sz) self.loss = nn.BCEWithLogitsLoss() if multi else nn.CrossEntropyLoss() self.accuracy = AccuracyMultiLabel() if multi else FBetaMeasure( average='micro')
def test_fbeta_multiclass_metric(self): fbeta = FBetaMeasure() fbeta(self.predictions, self.targets) metric = fbeta.get_metric() precisions = metric['precision'] recalls = metric['recall'] fscores = metric['fscore'] # check value numpy.testing.assert_almost_equal(precisions, self.desired_precisions, decimal=2) numpy.testing.assert_almost_equal(recalls, self.desired_recalls, decimal=2) numpy.testing.assert_almost_equal(fscores, self.desired_fscores, decimal=2) # check type assert isinstance(precisions, List) assert isinstance(recalls, List) assert isinstance(fscores, List)
def test_fbeta_multiclass_state(self): fbeta = FBetaMeasure() fbeta(self.predictions, self.targets) # check state numpy.testing.assert_almost_equal(fbeta._pred_sum.tolist(), self.pred_sum) numpy.testing.assert_almost_equal(fbeta._true_sum.tolist(), self.true_sum) numpy.testing.assert_almost_equal(fbeta._true_positive_sum.tolist(), self.true_positive_sum) numpy.testing.assert_almost_equal(fbeta._true_negative_sum.tolist(), self.true_negative_sum) numpy.testing.assert_almost_equal(fbeta._total_sum.tolist(), self.total_sum)
def __init__( self, vocab: Vocabulary, transformer_model: str = "roberta-large", num_labels: Optional[int] = None, label_namespace: str = "labels", override_weights_file: Optional[str] = None, **kwargs, ) -> None: super().__init__(vocab, **kwargs) transformer_kwargs = { "model_name": transformer_model, "weights_path": override_weights_file, } self.embeddings = TransformerEmbeddings.from_pretrained_module( **transformer_kwargs) self.transformer_stack = TransformerStack.from_pretrained_module( **transformer_kwargs) self.pooler = TransformerPooler.from_pretrained_module( **transformer_kwargs) self.pooler_dropout = Dropout(p=0.1) self.label_tokens = vocab.get_index_to_token_vocabulary( label_namespace) if num_labels is None: num_labels = len(self.label_tokens) self.linear_layer = torch.nn.Linear(self.pooler.get_output_dim(), num_labels) self.linear_layer.weight.data.normal_(mean=0.0, std=0.02) self.linear_layer.bias.data.zero_() from allennlp.training.metrics import CategoricalAccuracy, FBetaMeasure self.loss = torch.nn.CrossEntropyLoss() self.acc = CategoricalAccuracy() self.f1 = FBetaMeasure()
def __init__( self, vocab: Vocabulary, text_embeddings: TransformerEmbeddings, image_embeddings: ImageFeatureEmbeddings, encoder: BiModalEncoder, pooled_output_dim: int, fusion_method: str = "mul", dropout: float = 0.1, label_namespace: str = "labels", *, ignore_text: bool = False, ignore_image: bool = False, ) -> None: super().__init__( vocab, text_embeddings, image_embeddings, encoder, pooled_output_dim, fusion_method, dropout, label_namespace, is_multilabel=False, ) self.pooled_output_dim = pooled_output_dim self.layer1 = torch.nn.Linear(pooled_output_dim * 2, pooled_output_dim) self.layer2 = torch.nn.Linear(pooled_output_dim, 2) self.activation = torch.nn.ReLU() self.accuracy = CategoricalAccuracy() self.fbeta = FBetaMeasure(beta=1.0, average="macro")
def __init__( self, vocab: Vocabulary, classification_type: str = 'multi-class', pos_label: str = None, threshold: float = 0.5, neg_weight: float = 1.0, label_namespace: str = 'labels', regularizer: Optional[RegularizerApplicator] = None, ): super().__init__(vocab, regularizer) self._classification_type = classification_type self._label_namespace = label_namespace self._threshold = threshold self._neg_weight = neg_weight self._pos_label_index = vocab.get_token_index( pos_label, namespace=label_namespace) if pos_label else None self._use_threshold = False if self._classification_type == "ce": self._loss = torch.nn.CrossEntropyLoss() self._accuracy = CategoricalAccuracy() if self._pos_label_index is not None: self._f1 = FBetaMeasure(average=None) else: self._f1 = FBetaMeasure(average='micro') elif self._classification_type == "bce": # BCE 是否可以指定全负样本 assert self._pos_label_index is None self._loss = torch.nn.BCEWithLogitsLoss() self._accuracy = BooleanAccuracy() self._f1 = BooleanF1() self._use_threshold = True elif self._classification_type == "as": # AS should given _pos_label_index assert self._pos_label_index is not None self._loss = AdaptiveScalingLossLayer( num_label=vocab.get_vocab_size(label_namespace), positive_idx=[self._pos_label_index]) self._accuracy = CategoricalAccuracy() self._f1 = FBetaMeasure(average=None) else: raise NotImplementedError( 'Classification Type Not Implemented: %s' % self._classification_type)
def test_multiple_distributed_runs(self): predictions = [ torch.tensor( [[0.35, 0.25, 0.1, 0.1, 0.2], [0.1, 0.6, 0.1, 0.2, 0.0], [0.1, 0.6, 0.1, 0.2, 0.0]] ), torch.tensor( [[0.1, 0.5, 0.1, 0.2, 0.0], [0.1, 0.2, 0.1, 0.7, 0.0], [0.1, 0.6, 0.1, 0.2, 0.0]] ), ] targets = [torch.tensor([0, 4, 1]), torch.tensor([0, 3, 0])] metric_kwargs = {"predictions": predictions, "gold_labels": targets} desired_metrics = { "precision": self.desired_precisions, "recall": self.desired_recalls, "fscore": self.desired_fscores, } run_distributed_test( [-1, -1], multiple_runs, FBetaMeasure(), metric_kwargs, desired_metrics, exact=False, )
def multiple_runs( global_rank: int, world_size: int, gpu_id: Union[int, torch.device], metric: FBetaMeasure, metric_kwargs: Dict[str, List[Any]], desired_values: Dict[str, Any], exact: Union[bool, Tuple[float, float]] = True, ): kwargs = {} # Use the arguments meant for the process with rank `global_rank`. for argname in metric_kwargs: kwargs[argname] = metric_kwargs[argname][global_rank] for i in range(200): metric(**kwargs) metric_values = metric.get_metric() for key in desired_values: assert_allclose(desired_values[key], metric_values[key])