def test_auc_computation(self, device: str): auc = Auc() all_predictions = [] all_labels = [] for _ in range(5): predictions = torch.randn(8, device=device) labels = torch.randint(0, 2, (8,), dtype=torch.long, device=device) auc(predictions, labels) all_predictions.append(predictions) all_labels.append(labels) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( torch.cat(all_labels, dim=0).cpu().numpy(), torch.cat(all_predictions, dim=0).cpu().numpy(), ) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_allclose(real_auc_value, computed_auc_value) # One more computation to assure reset works. predictions = torch.randn(8, device=device) labels = torch.randint(0, 2, (8,), dtype=torch.long, device=device) auc(predictions, labels) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( labels.cpu().numpy(), predictions.cpu().numpy() ) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_allclose(real_auc_value, computed_auc_value)
def test_auc_computation(self): auc = Auc() all_predictions = [] all_labels = [] for _ in range(5): predictions = torch.randn(8).float() labels = torch.randint(0, 2, (8, )).long() auc(predictions, labels) all_predictions.append(predictions) all_labels.append(labels) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( torch.cat(all_labels, dim=0).numpy(), torch.cat(all_predictions, dim=0).numpy()) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_almost_equal(real_auc_value, computed_auc_value) # One more computation to assure reset works. predictions = torch.randn(8).float() labels = torch.randint(0, 2, (8, )).long() auc(predictions, labels) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( labels.numpy(), predictions.numpy()) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_almost_equal(real_auc_value, computed_auc_value)
class Classifier(BasicClassifier): def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, seq2vec_encoder: Seq2VecEncoder, seq2seq_encoder: Seq2SeqEncoder = None, num_labels: int = None) -> None: super().__init__(vocab, text_field_embedder, seq2vec_encoder, seq2seq_encoder, num_labels=num_labels) if self._num_labels == 2: self._auc = Auc() self._f1 = F1Measure(1) def forward(self, tokens: Dict[str, torch.LongTensor], label: torch.IntTensor = None) -> Dict[str, torch.Tensor]: embedded_text = self._text_field_embedder(tokens) mask = get_text_field_mask(tokens).float() if self._seq2seq_encoder: embedded_text = self._seq2seq_encoder(embedded_text, mask=mask) embedded_text = self._seq2vec_encoder(embedded_text, mask=mask) if self._dropout: embedded_text = self._dropout(embedded_text) logits = self._classification_layer(embedded_text) probs = torch.nn.functional.softmax(logits, dim=-1) output_dict = {"logits": logits, "probs": probs} if label is not None: loss = self._loss(logits, label.long().view(-1)) output_dict["loss"] = loss self._accuracy(logits, label) if label is not None and self._num_labels == 2: self._auc(output_dict['probs'][:, 1], label.long().view(-1)) self._f1(output_dict['probs'], label.long().view(-1)) return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics = super().get_metrics(reset) if self._num_labels == 2: metrics.update({ 'auc': self._auc.get_metric(reset), 'f1': self._f1.get_metric(reset)[2] }) return metrics
class MortalityClassifier(Model): def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, regularizer_applicator: RegularizerApplicator = None): super().__init__(vocab, regularizer_applicator) self.embedder = embedder self.encoder = encoder num_labels = vocab.get_vocab_size( "labels" ) # the labels was constructed. i.e. even though we did not explicitly do anything to it, it knows how large it should be! logger.info("num labels is as follows: {}".format(num_labels) ) # why does it ned to know the labels converison however? self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels) self.accuracy = CategoricalAccuracy() self.auc = Auc() self.reg_app = regularizer_applicator def forward(self, text: Dict[str, torch.Tensor], label: torch.Tensor) -> Dict[str, torch.Tensor]: # Shape: (batch_size, num_tokens, embedding_dim) embedded_text = self.embedder(text) logger.info("Embedded text shape is as follows: {}".format( (embedded_text.shape))) # Shape: (batch_size, num_tokens) mask = util.get_text_field_mask(text) # Shape: (batch_size, encoding_dim) encoded_text = self.encoder(embedded_text, mask) print(encoded_text.shape) # Shape: (batch_size, num_labels) logits = self.classifier(encoded_text) print(logits.shape) # Shape: (batch_size, num_labels) probs = torch.nn.functional.softmax(logits, dim=-1) # reg_loss = self.get_regularization_penalty() # should not have to manually apply the regularization # Shape: (1,) loss = torch.nn.functional.cross_entropy(logits, label) self.accuracy(logits, label) preds = logits.argmax(-1) self.auc(preds, label) output = {'loss': loss, 'probs': probs} return output '''this is called''' def get_metrics(self, reset: bool = False) -> Dict[str, float]: return { "accuracy": self.accuracy.get_metric(reset), "auc": self.auc.get_metric(reset) }
def test_auc_with_mask(self): auc = Auc() predictions = torch.randn(8).float() labels = torch.randint(0, 2, (8, )).long() mask = torch.ByteTensor([1, 1, 1, 1, 0, 0, 0, 0]) auc(predictions, labels, mask) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( labels[:4].numpy(), predictions[:4].numpy()) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_almost_equal(real_auc_value, computed_auc_value)
class HatefulMemeModel(Model): def __init__(self, vocab: Vocabulary, text_model_name: str): super().__init__(vocab) self._text_model = BertForSequenceClassification.from_pretrained( text_model_name) self._num_labels = vocab.get_vocab_size() self._accuracy = Average() self._auc = Auc() self._softmax = torch.nn.Softmax(dim=1) def forward( self, source_tokens: TextFieldTensors, box_features: Optional[Tensor] = None, box_coordinates: Optional[Tensor] = None, box_mask: Optional[Tensor] = None, label: Optional[Tensor] = None, metadata: Optional[Dict] = None, ) -> Dict[str, torch.Tensor]: input_ids = source_tokens["tokens"]["token_ids"] input_mask = source_tokens["tokens"]["mask"] token_type_ids = source_tokens["tokens"]["type_ids"] outputs = self._text_model( input_ids=input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, return_dict=True, labels=label, ) if label is not None: predictions = torch.argmax(self._softmax(outputs.logits), dim=-1) for index in range(predictions.shape[0]): correct = float((predictions[index] == label[index])) self._accuracy(int(correct)) self._auc(predictions, label) return outputs @overrides def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics: Dict[str, float] = {} if not self.training: metrics["accuracy"] = self._accuracy.get_metric(reset=reset) metrics["auc"] = self._auc.get_metric(reset=reset) return metrics
def test_auc_with_mask(self, device: str): auc = Auc() predictions = torch.randn(8, device=device) labels = torch.randint(0, 2, (8,), dtype=torch.long, device=device) mask = torch.tensor([True, True, True, True, False, False, False, False], device=device) auc(predictions, labels, mask) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( labels[:4].cpu().numpy(), predictions[:4].cpu().numpy() ) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_allclose(real_auc_value, computed_auc_value)
class DecompensationClassifier(Model): def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, regularizer_applicator: RegularizerApplicator = None): super().__init__(vocab, regularizer_applicator) self.embedder = embedder self.encoder = encoder num_labels = vocab.get_vocab_size("labels") self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels) self.accuracy = CategoricalAccuracy() self.auc = Auc() self.reg_app = regularizer_applicator def forward(self, text: Dict[str, torch.Tensor], label: torch.Tensor) -> Dict[str, torch.Tensor]: # Shape: (batch_size, num_tokens, embedding_dim) embedded_text = self.embedder(text) # Shape: (batch_size, num_tokens) mask = util.get_text_field_mask(text) # Shape: (batch_size, encoding_dim) encoded_text = self.encoder( embedded_text, mask) #horizontal; vertical (partial depth) might be good # Shape: (batch_size, num_labels) logits = self.classifier(encoded_text) # Shape: (batch_size, num_labels) probs = torch.nn.functional.softmax(logits, dim=-1) # reg_loss = self.get_regularization_penalty() # should not have to manually apply the regularization # Shape: (1,) loss = torch.nn.functional.cross_entropy(logits, label) self.accuracy(logits, label) preds = logits.argmax(-1) self.auc(preds, label) output = {'loss': loss, 'probs': probs} return output '''this is called''' def get_metrics(self, reset: bool = False) -> Dict[str, float]: return { "accuracy": self.accuracy.get_metric(reset), "auc": self.auc.get_metric(reset) }
def test_auc_gold_labels_behaviour(self): # Check that it works with different pos_label auc = Auc(positive_label=4) predictions = torch.randn(8).float() labels = torch.randint(3, 5, (8, )).long() auc(predictions, labels) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( labels.numpy(), predictions.numpy(), pos_label=4) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_almost_equal(real_auc_value, computed_auc_value) # Check that it errs on getting more than 2 labels. with pytest.raises(ConfigurationError) as _: labels = torch.LongTensor([3, 4, 5, 6, 7, 8, 9, 10]) auc(predictions, labels)
def test_auc_gold_labels_behaviour(self, device: str): # Check that it works with different pos_label auc = Auc(positive_label=4) predictions = torch.randn(8, device=device) labels = torch.randint(3, 5, (8, ), dtype=torch.long, device=device) # We make sure that the positive label is always present. labels[0] = 4 auc(predictions, labels) computed_auc_value = auc.get_metric(reset=True) false_positive_rates, true_positive_rates, _ = metrics.roc_curve( labels.cpu().numpy(), predictions.cpu().numpy(), pos_label=4) real_auc_value = metrics.auc(false_positive_rates, true_positive_rates) assert_allclose(real_auc_value, computed_auc_value) # Check that it errs on getting more than 2 labels. with pytest.raises(ConfigurationError) as _: labels = torch.tensor([3, 4, 5, 6, 7, 8, 9, 10], device=device) auc(predictions, labels)
class L2AFBertBaseline(Model): """ BERT baseline model """ def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, attnpool: AttnPooling, output_ffl: FeedForward, initializer: InitializerApplicator, dropout: float = 0.3, ) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder self._variational_dropout = InputVariationalDropout(dropout) self._attn_pool = attnpool self._output_ffl = output_ffl self._num_labels = vocab.get_vocab_size(namespace="labels") self._auc = Auc() self._loss = torch.nn.BCELoss() initializer(self) def forward( self, # type: ignore combined_source: Dict[str, torch.LongTensor], label: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ :param combined_source: :param label: :param metadata: :return: """ embedded_source = self._text_field_embedder( combined_source) # B * T * H source_mask = get_text_field_mask(combined_source) # B * T embedded_source = self._variational_dropout(embedded_source) pooled = self._attn_pool(embedded_source, source_mask) # B * H choice_score = self._output_ffl(pooled) # B * 1 output = torch.sigmoid(choice_score).squeeze(-1) # B output_dict = { "label_logits": choice_score.squeeze(-1), "label_probs": output, "metadata": metadata } if label is not None: label = label.long().view(-1) loss = self._loss(output, label.float()) self._auc(output, label) output_dict["loss"] = loss return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: return { 'auc': self._auc.get_metric(reset), }
class DocumentRanker(Model): def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, relevance_matcher: RelevanceMatcher, dropout: float = None, num_labels: int = None, initializer: InitializerApplicator = InitializerApplicator(), **kwargs, ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder self._relevance_matcher = TimeDistributed(relevance_matcher) self._dropout = dropout and torch.nn.Dropout(dropout) self._auc = Auc() self._mrr = MRR(padding_value=-1) self._ndcg = NDCG(padding_value=-1) self._loss = torch.nn.MSELoss(reduction='none') initializer(self) # @torchsnooper.snoop() def forward( # type: ignore self, tokens: TextFieldTensors, # batch * words options: TextFieldTensors, # batch * num_options * words labels: torch.IntTensor = None # batch * num_options ) -> Dict[str, torch.Tensor]: embedded_text = self._text_field_embedder(tokens) mask = get_text_field_mask(tokens).long() embedded_options = self._text_field_embedder( options, num_wrapping_dims=1) # options_mask.dim() - 2 options_mask = get_text_field_mask(options).long() if self._dropout: embedded_text = self._dropout(embedded_text) embedded_options = self._dropout(embedded_options) """ This isn't exactly a 'hack', but it's definitely not the most efficient way to do it. Our matcher expects a single (query, document) pair, but we have (query, [d_0, ..., d_n]). To get around this, we expand the query embeddings to create these pairs, and then flatten both into the 3D tensor [batch*num_options, words, dim] expected by the matcher. The expansion does this: [ (q_0, [d_{0,0}, ..., d_{0,n}]), (q_1, [d_{1,0}, ..., d_{1,n}]) ] => [ [ (q_0, d_{0,0}), ..., (q_0, d_{0,n}) ], [ (q_1, d_{1,0}), ..., (q_1, d_{1,n}) ] ] Which we then flatten along the batch dimension. It would likely be more efficient to rewrite the matrix multiplications in the relevance matchers, but this is a more general solution. """ embedded_text = embedded_text.unsqueeze(1).expand( -1, embedded_options.size(1), -1, -1) # [batch, num_options, words, dim] mask = mask.unsqueeze(1).expand(-1, embedded_options.size(1), -1) scores = self._relevance_matcher(embedded_text, embedded_options, mask, options_mask).squeeze(-1) probs = torch.sigmoid(scores) output_dict = {"logits": scores, "probs": probs} output_dict["token_ids"] = util.get_token_ids_from_text_field_tensors( tokens) if labels is not None: label_mask = (labels != -1) self._mrr(probs, labels, label_mask) self._ndcg(probs, labels, label_mask) probs = probs.view(-1) labels = labels.view(-1) label_mask = label_mask.view(-1) self._auc(probs, labels.ge(0.5).long(), label_mask) loss = self._loss(probs, labels) output_dict["loss"] = loss.masked_fill(~label_mask, 0).sum() / label_mask.sum() return output_dict @overrides def make_output_human_readable( self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics = { "auc": self._auc.get_metric(reset), "mrr": self._mrr.get_metric(reset), "ndcg": self._ndcg.get_metric(reset), } return metrics default_predictor = "document_ranker"
def test_auc_works_without_calling_metric_at_all(self, device: str): auc = Auc() auc.get_metric()
class MultiStepParaRankModel(Model): def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, phrase_layer: Seq2SeqEncoder, pq_attention: MatrixAttention, p_selfattention: MatrixAttention, supports_pooling: Seq2VecEncoder, query_pooling: Seq2VecEncoder, candidates_pooling: Seq2VecEncoder, decoder: Decoder, dropout: float = 0.2, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(MultiStepParaRankModel, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.phrase_layer = phrase_layer self.pq_attention = pq_attention self.p_selfattention = p_selfattention self.supports_pooling = supports_pooling self.query_pooling = query_pooling self.candidates_pooling = candidates_pooling self.decoder = decoder self.dropout = InputVariationalDropout(p=dropout) self._support_accuracy = Auc() self._candidate_accuracy = CategoricalAccuracy() initializer(self) def forward( self, query: Dict[str, torch.LongTensor], supports: Dict[str, torch.LongTensor], candidates: Dict[str, torch.LongTensor], answer: Dict[str, torch.LongTensor] = None, answer_index: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None, supports_labels: torch.Tensor = None, ) -> Dict[str, torch.Tensor]: embedded_supports = self.text_field_embedder(supports) embedded_query = self.text_field_embedder(query) embedded_candidates = self.text_field_embedder(candidates) batch_size, support_num, support_length, embed_dim = embedded_supports.size( ) _, candidate_num, candidate_length, _ = embedded_candidates.size() _, query_length, _ = embedded_query.size() supports_mask_para = util.get_text_field_mask(supports) supports_mask_seq = util.get_text_field_mask(supports, num_wrapping_dims=1) supports_mask_seq_expand = supports_mask_seq.view( -1, supports_mask_seq.size(-1)) candidates_mask_para = util.get_text_field_mask(candidates) candidates_mask_seq = util.get_text_field_mask(candidates, num_wrapping_dims=1) candidates_mask_seq_expand = candidates_mask_seq.view( -1, candidates_mask_seq.size(-1)) query_mask = util.get_text_field_mask(query) query_mask_expand = query_mask.unsqueeze(1).expand( query_mask.size(0), support_num, query_mask.size(1)) query_mask_expand = query_mask_expand.contiguous().view( -1, query_mask_expand.size(-1)) embedded_supports_expand = embedded_supports.view( -1, support_length, embed_dim) embedded_candidates_expand = embedded_candidates.view( -1, candidate_length, embed_dim) encoded_query = self.phrase_layer(self.dropout(embedded_query), query_mask) encoded_supports = self.phrase_layer( self.dropout(embedded_supports_expand), supports_mask_seq_expand) encoded_candidates = self.phrase_layer( self.dropout(embedded_candidates_expand), candidates_mask_seq_expand) encoded_query_expand = encoded_query.unsqueeze(1).expand( batch_size, support_num, encoded_query.size(1), encoded_query.size(2)) encoded_query_expand = encoded_query_expand.contiguous().view( -1, encoded_query.size(1), encoded_query.size(2)) # Co-attention # shape: (batch_size*passage_num, passage_length, question_length ) supports_query_similarity = self.pq_attention(encoded_supports, encoded_query_expand) # shape: (batch_size*passage_num, passage_length, question_length ) supports_query_attention = util.masked_softmax( supports_query_similarity, query_mask_expand, memory_efficient=True) # shape: (batch_size*passage_num, passage_length, encoding_dim) supports_query_vectors = util.weighted_sum(encoded_query_expand, supports_query_attention) # shape: (batch_size*passage_num, query_length, passage_length) query_passage_attention = util.masked_softmax( supports_query_similarity.transpose(1, 2), supports_mask_seq_expand, memory_efficient=True) # shape: (batch_size*passage_num, query_length, encoding_dim) query_supports_vectors = util.weighted_sum(encoded_supports, query_passage_attention) # shape: (batch_size*passage_num, passage_length, encoding_dim) supports_query_vectors_2 = torch.bmm(supports_query_attention, query_supports_vectors) # shape: (batch_size*passage_num, passage_length, encoding_dim*2) supports_coattention_vectors = torch.cat( [supports_query_vectors, supports_query_vectors_2], dim=-1) # Fusion, 暂时用简单的fusion函数 #supports_coattention_vectors = co_attention_fusion(torch.cat([encoded_supports,supports_query_vectors_final], dim=-1)) suppports_self_similarity = self.p_selfattention( supports_coattention_vectors, supports_coattention_vectors) supports_selfattention = util.masked_softmax(suppports_self_similarity, supports_mask_seq_expand, memory_efficient=True) supports_selfatt_vectors = util.weighted_sum( supports_coattention_vectors, supports_selfattention) #support_selfatt_fusion = self_attention_fusion(util.combine_tensors('1,2,1-2,1*2',[supports_coattention_vectors, supports_selfatt_vectors])) supports_pooling_vectors = self.supports_pooling( supports_selfatt_vectors, supports_mask_seq_expand) supports_pooling_vectors = supports_pooling_vectors.view( batch_size, support_num, -1) question_pooling_vectors = self.query_pooling(encoded_query, query_mask) candidates_pooling_vectors = self.candidates_pooling( encoded_candidates, candidates_mask_seq_expand) candidates_pooling_vectors = candidates_pooling_vectors.view( batch_size, -1, candidates_pooling_vectors.size(-1)) # supports porb normalized, candidates_score: unnormalized supports_prob, candidates_score = self.decoder( supports_pooling_vectors, question_pooling_vectors, candidates_pooling_vectors, supports_mask_para) candidates_score = util.replace_masked_values(candidates_score, candidates_mask_para, -1e7) output_dict = { "supports_prob": supports_prob, "candidates_score": candidates_score } if supports_labels is not None: supports_prob = util.replace_masked_values(supports_prob, supports_mask_para, -1e32) s_loss = binary_cross_entropy_with_logits(supports_prob, supports_labels) c_loss = nll_loss( util.masked_log_softmax(candidates_score, candidates_mask_para), answer_index.squeeze(-1)) loss = s_loss + c_loss self._support_accuracy( supports_prob.view(-1, 1).squeeze(), supports_labels.view(-1, 1).squeeze(), supports_mask_para.view(-1).squeeze().detach().cpu()) self._candidate_accuracy(candidates_score, answer_index.squeeze(-1)) output_dict['loss'] = loss output_dict['s_loss'] = s_loss output_dict['c_loss'] = c_loss return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: return { 'support_auc': self._support_accuracy.get_metric(reset), 'candidate_acc': self._candidate_accuracy.get_metric(reset) }
class AnswerHelpfulPredictionModel(Model): """ This is the implementation of the RAPH model proposed in the paper. Given a question, its answer and relevant reviews, predict whether the answer is helpful or not. """ def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, context_encoder: Seq2SeqEncoder, qa_attention_module: MatrixAttention, text_encoder_qa_matching: Seq2VecEncoder, qa_matching_layer: FeedForward, qr_attention_module: Attention, text_encoder_ra_entailment: Seq2VecEncoder, ra_matching_layer: FeedForward, predict_layer: FeedForward, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None): super(AnswerHelpfulPredictionModel, self).__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.context_encoder = context_encoder self.qa_attention_module = qa_attention_module self.text_encoder_qa_matching = text_encoder_qa_matching self.qa_matching_layer = qa_matching_layer self.qr_attention_module = qr_attention_module self.text_encoder_ra_entailment = text_encoder_ra_entailment self.ra_matching_layer = ra_matching_layer self.predict_layer = predict_layer # performance scores are running values, reset the values every epoch self.f1_measure = F1Measure(positive_label=1) self.auc_score = Auc(positive_label=1) self.accuracy = CategoricalAccuracy() self.criterion = torch.nn.CrossEntropyLoss() initializer(self) @overrides def forward(self, question, answer, reviews, helpful=None): # ---------------------------------------------------------- # layer-1: Embed q/a/reviews and Encode context infomation # shape = (batch_size, seq_len) q_mask = get_text_field_mask(question) a_mask = get_text_field_mask(answer) # shape = (batch_size, 5, seq_len) r_mask = get_text_field_mask(reviews, num_wrapping_dims=1) # shape = (batch_size, seq_len, embed_dim) embedded_q = self.text_field_embedder(question) embedded_a = self.text_field_embedder(answer) # shape = (batch_size, 5, seq_len, embed_dim) embedded_r = self.text_field_embedder(reviews, num_wrapping_dims=1) review_ls = [(embedded_r[:, i, :, :], r_mask[:, i, :]) for i in range(5)] context_q = self.context_encoder(embedded_q, q_mask) context_a = self.context_encoder(embedded_a, a_mask) # shape of context_r[i]: (bs, seq_len, encoding_dim) context_r = [self.context_encoder(r[0], r[1]) for r in review_ls] # ---------------------------------------------------------- # layer-2: QA Matching # shape = (bs, len_q, len_a) sim_matrix = self.qa_attention_module(context_q, context_a) # masked attention to remove those paddings in Q/A a2q_attention = masked_softmax(sim_matrix, a_mask) q2a_attention = masked_softmax( sim_matrix.transpose(1, 2).contiguous(), q_mask) # shape = (batch_size, len_q, encoding_dim) attended_q_from_a = weighted_sum(context_a, a2q_attention) attended_a_from_q = weighted_sum(context_q, q2a_attention) v_q = torch.cat([context_q, attended_q_from_a], dim=-1) v_a = torch.cat([context_a, attended_a_from_q], dim=-1) # encoding the sequence info to a fixed vector o_q = self.text_encoder_qa_matching(v_q, q_mask) o_a_q = self.text_encoder_qa_matching(v_a, a_mask) qa_matching_score = self.qa_matching_layer( torch.cat([o_q, o_a_q], dim=-1)) # ---------------------------------------------------------- # layer-3: RA coherence modeling # use question text to highlight relevant review info q_enhanced_r = [] qr_attention_weights = [] for i in range(5): # Q2R attention (vec-matrix attention) # shape = (bs, len_r_i) beta = self.qr_attention_module(o_q, context_r[i], review_ls[i][1]) qr_attention_weights.append(beta) # for visulazation enhanced_r_i = torch.matmul(beta.unsqueeze(1), context_r[i]).squeeze(1) q_enhanced_r.append(enhanced_r_i) # for visulazation attention weights qr_attention_weights_op = torch.cat(qr_attention_weights, dim=-1) # v_a/v_r shape = (bs, encoding_shape) o_a_r = self.text_encoder_ra_entailment(context_a, a_mask) o_r = [ self.text_encoder_ra_entailment(context_r[i], review_ls[i][1]) + q_enhanced_r[i] for i in range(5) ] # "K entailment checking" entailment_result = [ self.ra_matching_layer(torch.cat([o_a_r, o_r[i]], dim=-1)) for i in range(5) ] entailment_score = torch.cat(entailment_result, dim=1) # ---------------------------------------------------------- # layer-4: Final prediction layer logits = self.predict_layer( torch.cat([qa_matching_score, entailment_score], dim=-1)) probs = F.softmax(logits, dim=-1) output_dict = { 'probs': probs, 'entailment_score': entailment_score, 'qr_attention': qr_attention_weights_op, "qa_attention": sim_matrix } if helpful is not None: loss = self.criterion(logits, helpful) self.f1_measure(logits, helpful) self.auc_score(probs[:, 1], helpful) # self.accuracy(logits, helpful) output_dict['loss'] = loss return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics = {} precision, recall, f1_measure = self.f1_measure.get_metric(reset) metrics["f1"] = f1_measure metrics["auc"] = self.auc_score.get_metric(reset) # metrics["accuracy"] = self.accuracy.get_metric(reset) return metrics
class ContextualAtt(Model): def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, sec_name_encoder: Seq2SeqEncoder, sent_encoder: Seq2SeqEncoder, classifier_feedforward: FeedForward, encoder_attention: Attention = DotProductAttention( normalize=True), label_namespace: str = "labels", class_weight=[1.0, 1.0], dropout: Optional[float] = None, calculate_f1: bool = None, calculate_auc: bool = None, calculate_auc_pr: bool = None, positive_label: int = 1, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(ContextualAtt, self).__init__(vocab, regularizer) self.label_namespace = label_namespace self.num_tags = self.vocab.get_vocab_size() self.text_field_embedder = text_field_embedder self.num_classes = self.vocab.get_vocab_size(label_namespace) self.sec_name_encoder = sec_name_encoder self.sent_encoder = sent_encoder self.attention = encoder_attention # self.attention_scale = math.sqrt(encoder.get_output_dim()) if dropout: self.dropout = torch.nn.Dropout(dropout) else: self.dropout = None self.classifier_feedforward = classifier_feedforward if classifier_feedforward is not None: output_dim = classifier_feedforward.get_output_dim() self.metrics = {"accuracy": CategoricalAccuracy()} # if isinstance(class_weight, list) and len(class_weight)>0: # # [0.2419097587861097, 1.0] # self.loss = torch.nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weight)) # else: # self.loss = torch.nn.CrossEntropyLoss() self.loss = torch.nn.CrossEntropyLoss() self.positive_label = positive_label self.calculate_f1 = calculate_f1 self.calculate_auc = calculate_auc self.calculate_auc_pr = calculate_auc_pr if calculate_f1: self._f1_metric = F1Measure(positive_label) if calculate_auc: self._auc = Auc(positive_label) if calculate_auc_pr: self._auc_pr = AucPR(positive_label) # check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(), # "text field embedding dim", "encoder input dim") if classifier_feedforward is not None: check_dimensions_match( sent_encoder.get_output_dim() * 3 + sec_name_encoder.get_output_dim() + 8, classifier_feedforward.get_input_dim(), "encoder output dim", "feedforward input dim") initializer(self) @overrides def forward( self, # type: ignore section_name: Dict[str, torch.LongTensor], prev_sent: Dict[str, torch.LongTensor], cur_sent: Dict[str, torch.LongTensor], next_sent: Dict[str, torch.LongTensor], additional_features: torch.Tensor = None, label: torch.LongTensor = None, metadata: List[str] = None, **kwargs) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """ Parameters ---------- section_name : Dict[str, torch.LongTensor], required The output of ``TextField.as_tensor()``, which should typically be passed directly to a ``TextFieldEmbedder``. This output is a dictionary mapping keys to ``TokenIndexer`` tensors. At its most basic, using a ``SingleIdTokenIndexer`` this is: ``{"tokens": Tensor(batch_size, num_tokens)}``. This dictionary will have the same keys as were used for the ``TokenIndexers`` when you created the ``TextField`` representing your sequence. The dictionary is designed to be passed directly to a ``TextFieldEmbedder``, which knows how to combine different word representations into a single vector per token in your input. prev_sent : Dict[str, Variable], required The output of ``TextField.as_array()``. cur_sent : Dict[str, Variable], required The output of ``TextField.as_array()``. next_sent : Dict[str, Variable], required The output of ``TextField.as_array()``. additional_features : Variable A variable representing the additional features for each instance in the batch. label : Variable A variable representing the label for each instance in the batch. metadata : ``List[Dict[str, Any]]``, optional (default = None) Metadata containing the original "cur_sent_id" field. Returns ------- An output dictionary consisting of: class_probabilities : torch.FloatTensor A tensor of shape ``(batch_size, num_classes)`` representing a distribution over the label classes for each instance. loss : torch.FloatTensor, optional A scalar loss to be optimised. """ def build_encoder_layer(field, encoder, need_attention=True): mask = util.get_text_field_mask(field) embedded_text_input = self.text_field_embedder(field) if self.dropout: embedded_text_input = self.dropout(embedded_text_input) encoded_text = encoder(embedded_text_input, mask) if self.dropout: encoded_text = self.dropout(encoded_text) # (batch_size, num_directions * hidden_size) try: final_hidden_states = util.get_final_encoder_states( encoded_text, mask, encoder.is_bidirectional()) except: print(field) if not need_attention: return final_hidden_states # Add attention here attention_weights = self.attention(final_hidden_states, encoded_text, mask).unsqueeze(-1) # rnn_output (batch_size, seq_len, num_directions * hidden_size) # ==> (batch_size, num_directions * hidden_size, seq_len) rnn_output_re_order = encoded_text.permute(0, 2, 1) attention_output = torch.bmm(rnn_output_re_order, attention_weights).squeeze(-1) return attention_output final_states_sec_name = build_encoder_layer(section_name, self.sec_name_encoder, need_attention=True) final_states_prev_sent = build_encoder_layer(prev_sent, self.sent_encoder, need_attention=True) final_states_cur_sent = build_encoder_layer(cur_sent, self.sent_encoder, need_attention=True) final_states_next_sent = build_encoder_layer(next_sent, self.sent_encoder, need_attention=True) embeded_features = torch.cat( (final_states_sec_name, final_states_prev_sent, final_states_cur_sent, final_states_next_sent, additional_features), dim=-1) logits = self.classifier_feedforward(embeded_features) output_dict = { "cur_sent_id": metadata, "logits": logits, "golden_label": label } if label is not None: loss = self.loss(logits, label) output_dict["loss"] = loss for metric in self.metrics.values(): metric(logits, label) if self.calculate_f1: self._f1_metric(logits, label) class_probabilities = F.softmax(logits, dim=-1) output_dict['class_probabilities'] = class_probabilities positive_class_prob = class_probabilities[:, 1].detach() # label_prob, label_index = torch.max(class_probabilities, -1) # argmax_indices = numpy.argmax(class_probabilities, axis=-1) output_dict['positive_class_prob'] = positive_class_prob if self.calculate_auc: self._auc(positive_class_prob, label) if self.calculate_auc_pr: self._auc_pr(positive_class_prob, label) return output_dict @overrides def decode( self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: return output_dict @overrides def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics_to_return = { metric_name: metric.get_metric(reset) for metric_name, metric in self.metrics.items() } if self.calculate_f1: p_r_f1 = self._f1_metric.get_metric(reset=reset) precision, recall, f1_measure = p_r_f1 f1_dict = { 'precision': precision, 'recall': recall, "f1": f1_measure } metrics_to_return.update(f1_dict) if self.calculate_auc: metrics_to_return["auc"] = self._auc.get_metric(reset=reset) if self.calculate_auc_pr: metrics_to_return["auc_pr"] = self._auc_pr.get_metric(reset=reset) return metrics_to_return
class MortalityClassifier(Model): def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, regularizer_applicator: RegularizerApplicator = None, num_classes=1): super().__init__(vocab, regularizer_applicator) self.embedder = embedder self.encoder = encoder self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_classes) self.accuracy = CategoricalAccuracy() self.auc = Auc() self.reg_app = regularizer_applicator def forward(self, text: Dict[str, torch.Tensor], label: torch.Tensor, metadata) -> Dict[str, torch.Tensor]: # assert that metadata has the same length as the other ones. Then, they are parallel # Shape: (batch_size, num_tokens, embedding_dim) embedded_text = self.embedder(text) # Shape: (batch_size, num_tokens) mask = util.get_text_field_mask(text) # Shape: (batch_size, encoding_dim) encoded_text = self.encoder( embedded_text, mask) #horizontal; vertical (partial depth) might be good # Shape: (batch_size, num_labels) logits = self.classifier(encoded_text) # Shape: (batch_size, num_labels) # probs = torch.nn.functional.softmax(logits, dim=-1) probs = torch.sigmoid(logits) # reg_loss = self.get_regularization_penalty() # should not have to manually apply the regularization # Shape: (1,) # loss = torch.nn.functional.cross_entropy(logits, label) loss = torch.nn.functional.binary_cross_entropy_with_logits( logits, label.float()) # self.accuracy(logits, label.squeeze()) # preds = logits.argmax(-1) # probs_1 = logits[:,-1] # AUC can no longer be computed directly now. instead, we will need to supply a user function # self.auc(probs_1, label) # bleed everything through into the output output = { 'loss': loss, 'probs': probs, "metadata": metadata, "label": label } #no need to yield the label here return output '''this is called. it both gets, and resets, if reset=True''' def get_metrics(self, reset: bool = False) -> Dict[str, float]: return { "accuracy": self.accuracy.get_metric(reset), "auc": self.auc.get_metric(reset) }
class L2AF3WayAPM(Model): """ 3-way Attentive Pooling Network """ def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, pseqlevelenc: Seq2SeqEncoder, qaseqlevelenc: Seq2SeqEncoder, choicelevelenc: Seq2SeqEncoder, cartesian_attn: SeqAttnMat, pcattnmat: SeqAttnMat, gate_qdep_penc: GatedEncoding, qdep_penc_rnn: Seq2SeqEncoder, mfa_enc: GatedMultifactorSelfAttnEnc, mfa_rnn: Seq2SeqEncoder, pqaattnmat: SeqAttnMat, cqaattnmat: SeqAttnMat, initializer: InitializerApplicator, dropout: float = 0.3, is_qdep_penc: bool = True, is_mfa_enc: bool = True, with_knowledge: bool = True, is_qac_ap: bool = True, shared_rnn: bool = True) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder self._pseqlevel_enc = pseqlevelenc self._qaseqlevel_enc = qaseqlevelenc self._cseqlevel_enc = choicelevelenc self._cart_attn = cartesian_attn self._pqaattnmat = pqaattnmat self._pcattnmat = pcattnmat self._cqaattnmat = cqaattnmat self._gate_qdep_penc = gate_qdep_penc self._qdep_penc_rnn = qdep_penc_rnn self._multifactor_attn = mfa_enc self._mfarnn = mfa_rnn self._with_knowledge = with_knowledge self._qac_ap = is_qac_ap if not self._with_knowledge: if not self._qac_ap: raise AssertionError self._is_qdep_penc = is_qdep_penc self._is_mfa_enc = is_mfa_enc self._shared_rnn = shared_rnn self._variational_dropout = InputVariationalDropout(dropout) self._num_labels = vocab.get_vocab_size(namespace="labels") self._auc = Auc() self._loss = torch.nn.BCELoss() initializer(self) def forward( self, # type: ignore passage: Dict[str, torch.LongTensor], all_qa: Dict[str, torch.LongTensor], candidate: Dict[str, torch.LongTensor], combined_source: Dict[str, torch.LongTensor], label: torch.IntTensor = None, metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ """""" if self._with_knowledge: embedded_passage = self._text_field_embedder(passage) # B * T * d passage_len = embedded_passage.size(1) embedded_all_qa = self._text_field_embedder(all_qa) # B * U * d embedded_choice = self._text_field_embedder(candidate) # B * V * d if self._with_knowledge: embedded_passage = self._variational_dropout( embedded_passage) # B * T * d embedded_all_qa = self._variational_dropout(embedded_all_qa) embedded_choice = self._variational_dropout( embedded_choice) # B * V * d all_qa_mask = util.get_text_field_mask(all_qa) # B * U choice_mask = util.get_text_field_mask(candidate) # B * V # Encoding if self._with_knowledge: # B * T * H passage_mask = util.get_text_field_mask(passage) # B * T encoded_passage = self._variational_dropout( self._pseqlevel_enc(embedded_passage, passage_mask)) # B * U * H if self._shared_rnn: encoded_allqa = self._variational_dropout( self._pseqlevel_enc(embedded_all_qa, all_qa_mask)) else: encoded_allqa = self._variational_dropout( self._qaseqlevel_enc(embedded_all_qa, all_qa_mask)) if self._with_knowledge and self._is_qdep_penc: # similarity matrix _, normalized_attn_mat = self._cart_attn(encoded_passage, encoded_allqa, all_qa_mask) # B * T * U # question dependent passage encoding q_aware_passage_rep = sequential_weighted_avg( encoded_allqa, normalized_attn_mat) # B * T * H q_dep_passage_enc_rnn_input = torch.cat( [encoded_passage, q_aware_passage_rep], 2) # B * T * 2H # gated question dependent passage encoding gated_qaware_passage_rep = self._gate_qdep_penc( q_dep_passage_enc_rnn_input) # B * T * 2H encoded_qdep_penc = self._qdep_penc_rnn(gated_qaware_passage_rep, passage_mask) # B * T * H # multi factor attentive encoding if self._with_knowledge and self._is_mfa_enc: if self._is_qdep_penc: mfa_enc = self._multifactor_attn(encoded_qdep_penc, passage_mask) # B * T * 2H else: mfa_enc = self._multifactor_attn(encoded_passage, passage_mask) # B * T * 2H encoded_passage = self._mfarnn(mfa_enc, passage_mask) # B * T * H # B * V * H if self._shared_rnn: encoded_choice = self._variational_dropout( self._pseqlevel_enc(embedded_choice, choice_mask)) # B * V * H else: encoded_choice = self._variational_dropout( self._cseqlevel_enc(embedded_choice, choice_mask)) # B * V * H if self._with_knowledge: attn_pq, _ = self._pqaattnmat(encoded_passage, encoded_allqa, all_qa_mask) # B * T * U combined_pqa_mask = passage_mask.unsqueeze(-1) * \ all_qa_mask.unsqueeze(1) # B * T * U max_attn_pqa = masked_max(attn_pq, combined_pqa_mask, dim=1) # B * U norm_attn_pqa = masked_softmax(max_attn_pqa, all_qa_mask, dim=-1) # B * U agg_prev_qa = norm_attn_pqa.unsqueeze(1).bmm( encoded_allqa).squeeze(1) # B * H attn_pc, _ = self._pcattnmat(encoded_passage, encoded_choice, choice_mask) # B * T * V combined_pc_mask = passage_mask.unsqueeze(-1) * \ choice_mask.unsqueeze(1) # B * T * V max_attn_pc = masked_max(attn_pc, combined_pc_mask, dim=1) # B * V norm_attn_pc = masked_softmax(max_attn_pc, choice_mask, dim=-1) # B * V agg_c = norm_attn_pc.unsqueeze(1).bmm(encoded_choice) # B * 1 * H choice_scores_wk = agg_c.bmm(agg_prev_qa.unsqueeze(-1)).squeeze( -1) # B * 1 if self._qac_ap: attn_qac, _ = self._cqaattnmat(encoded_allqa, encoded_choice, choice_mask) # B * U * V combined_qac_mask = all_qa_mask.unsqueeze(-1) * \ choice_mask.unsqueeze(1) # B * U * V max_attn_c = masked_max(attn_qac, combined_qac_mask, dim=1) # B * V max_attn_qa = masked_max(attn_qac, combined_qac_mask, dim=2) # B * U norm_attn_c = masked_softmax(max_attn_c, choice_mask, dim=-1) # B * V norm_attn_qa = masked_softmax(max_attn_qa, all_qa_mask, dim=-1) # B * U agg_c_qa = norm_attn_c.unsqueeze(1).bmm(encoded_choice).squeeze( 1) # B * H agg_qa_c = norm_attn_qa.unsqueeze(1).bmm(encoded_allqa).squeeze( 1) # B * H choice_scores_nk = agg_c_qa.unsqueeze(1).bmm( agg_qa_c.unsqueeze(-1)).squeeze(-1) # B * 1 if self._with_knowledge and self._qac_ap: choice_score = choice_scores_wk + choice_scores_nk elif self._qac_ap: choice_score = choice_scores_nk elif self._with_knowledge: choice_score = choice_scores_wk else: raise NotImplementedError output = torch.sigmoid(choice_score).squeeze(-1) # B output_dict = { "label_logits": choice_score.squeeze(-1), "label_probs": output, "metadata": metadata } if label is not None: label = label.long().view(-1) loss = self._loss(output, label.float()) self._auc(output, label) output_dict["loss"] = loss return output_dict def get_metrics(self, reset: bool = False) -> Dict[str, float]: return { 'auc': self._auc.get_metric(reset), }