def __init__(self, word_embeddings: TextFieldEmbedder, text_encoder: Seq2SeqEncoder, relation_encoder: Seq2VecEncoder, vocab: Vocabulary, encoder_dropout: float = 0.5) -> None: # We have to pass the vocabulary to the constructor. super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder_dropout = torch.nn.Dropout(p=encoder_dropout) self.text_encoder = text_encoder self.text_attn = LinearAttention( input_dim=text_encoder.get_output_dim()) self.relation_encoder = relation_encoder self.relation_attn = BilinearAttention( vector_dim=text_encoder.get_output_dim(), matrix_dim=relation_encoder.get_output_dim()) hidden_dim = (text_encoder.get_output_dim() + relation_encoder.get_output_dim()) self.output = torch.nn.Linear(in_features=hidden_dim, out_features=1)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary, positive_label: int = 1) -> None: super().__init__(vocab) # We need the embeddings to convert word IDs to their vector representations self.word_embeddings = word_embeddings self.encoder = encoder # After converting a sequence of vectors to a single vector, we feed it into # a fully-connected linear layer to reduce the dimension to the total number of labels. self.linear = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) # Monitor the metrics - we use accuracy, as well as prec, rec, f1 for 4 (very positive) self.accuracy = CategoricalAccuracy() self.f1_measure = F1Measure(positive_label) # We use the cross entropy loss because this is a classification task. # Note that PyTorch's CrossEntropyLoss combines softmax and log likelihood loss, # which makes it unnecessary to add a separate softmax layer. self.loss_function = torch.nn.CrossEntropyLoss() self.W = nn.Parameter( torch.zeros(size=(2 * encoder.get_output_dim(), 1))) nn.init.xavier_uniform_(self.W.data) self.LeakyReLU = torch.nn.LeakyReLU(0.1)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder = None, dropout: float = 0.0, num_samples: int = None, sparse_embeddings: bool = False, bidirectional: bool = False, initializer=InitializerApplicator(), **kwargs) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder self._contextualizer = contextualizer self._bidirectional = bidirectional if self._bidirectional: self._forward_dim = contextualizer.get_output_dim() // 2 else: self._forward_dim = contextualizer.get_output_dim() self._softmax_loss = SoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=self._forward_dim) self._perplexity = Perplexity() if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x if initializer is not None: initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, dropout: float = None, num_samples: int = None, sparse_embeddings: bool = False, bidirectional: bool = False, initializer: InitializerApplicator = None, **kwargs, ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = text_field_embedder if contextualizer.is_bidirectional() is not bidirectional: raise ConfigurationError( "Bidirectionality of contextualizer must match bidirectionality of " "language model. " f"Contextualizer bidirectional: {contextualizer.is_bidirectional()}, " f"language model bidirectional: {bidirectional}") self._contextualizer = contextualizer self._bidirectional = bidirectional # The dimension for making predictions just in the forward # (or backward) direction. if self._bidirectional: self._forward_dim = contextualizer.get_output_dim() // 2 else: self._forward_dim = contextualizer.get_output_dim() if num_samples is not None: self._softmax_loss = SampledSoftmaxLoss( num_words=vocab.get_vocab_size("transactions"), embedding_dim=self._forward_dim, num_samples=num_samples, sparse=sparse_embeddings, ) else: self._softmax_loss = SoftmaxLoss( num_words=vocab.get_vocab_size("transactions"), embedding_dim=self._forward_dim, ) # This buffer is now unused and exists only for backwards compatibility reasons. self.register_buffer("_last_average_loss", torch.zeros(1)) self._perplexity = Perplexity() if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x if initializer is not None: initializer(self)
def __init__( self, highway_encoder: Seq2SeqEncoder, transform_gate_encoder: Seq2SeqEncoder, carry_gate_encoder: Optional[Seq2SeqEncoder] = None, projection: bool = True, ) -> None: stateful = highway_encoder.stateful or transform_gate_encoder.stateful check_dimensions_match( highway_encoder.get_input_dim(), transform_gate_encoder.get_input_dim(), "highway_encoder input dim", "transform_gate_encoder input dim", ) if carry_gate_encoder is not None: stateful = stateful or carry_gate_encoder.stateful check_dimensions_match( highway_encoder.get_input_dim(), carry_gate_encoder.get_input_dim(), "highway_encoder input dim", "carry_gate_encoder input dim", ) super().__init__(stateful=stateful) self._input_dim = highway_encoder.get_input_dim() self._highway_encoder = highway_encoder self._transform_gate_encoder = transform_gate_encoder self._carry_gate_encoder = carry_gate_encoder self._highway_projection: Optional[torch.nn.Module] = None self._transform_gate_projection: Optional[torch.nn.Module] = None self._carry_gate_projection: Optional[torch.nn.Module] = None if projection: self._highway_projection = TimeDistributed( # type: ignore torch.nn.Linear( highway_encoder.get_output_dim(), highway_encoder.get_input_dim(), )) self._transform_gate_projection = TimeDistributed( # type: ignore torch.nn.Linear( transform_gate_encoder.get_output_dim(), transform_gate_encoder.get_input_dim(), ), ) if carry_gate_encoder is not None: self._carry_gate_projection = TimeDistributed( # type: ignore torch.nn.Linear( carry_gate_encoder.get_output_dim(), carry_gate_encoder.get_input_dim(), ), ) else: assert highway_encoder.get_output_dim() in (self._input_dim, 1) assert transform_gate_encoder.get_output_dim() in (self._input_dim, 1) if carry_gate_encoder is not None: assert carry_gate_encoder.get_output_dim() in (self._input_dim, 1)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, dropout: float = None, loss_scale: Union[float, str] = 1.0, num_samples: int = None, sparse_embeddings: bool = False, bidirectional: bool = False, initializer: InitializerApplicator = None) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder if contextualizer.is_bidirectional() is not bidirectional: raise ConfigurationError( "Bidirectionality of contextualizer must match bidirectionality of " "language model. " f"Contextualizer bidirectional: {contextualizer.is_bidirectional()}, " f"language model bidirectional: {bidirectional}") self._contextualizer = contextualizer self._bidirectional = bidirectional # The dimension for making predictions just in the forward # (or backward) direction. if self._bidirectional: self._forward_dim = contextualizer.get_output_dim() // 2 else: self._forward_dim = contextualizer.get_output_dim() # TODO(joelgrus): more sampled softmax configuration options, as needed. if num_samples is not None: self._softmax_loss = SampledSoftmaxLoss( num_words=vocab.get_vocab_size(), embedding_dim=self._forward_dim, num_samples=num_samples, sparse=sparse_embeddings) else: self._softmax_loss = _SoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=self._forward_dim) # TODO(brendanr): Output perplexity here. e^loss self.register_buffer('_last_average_loss', torch.zeros(1)) if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x self._loss_scale = loss_scale if initializer is not None: initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, dropout: float = None, num_samples: int = None, sparse_embeddings: bool = False, bidirectional: bool = False, initializer: InitializerApplicator = None) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder if contextualizer.is_bidirectional() is not bidirectional: raise ConfigurationError( "Bidirectionality of contextualizer must match bidirectionality of " "language model. " f"Contextualizer bidirectional: {contextualizer.is_bidirectional()}, " f"language model bidirectional: {bidirectional}") self._contextualizer = contextualizer self._bidirectional = bidirectional # The dimension for making predictions just in the forward # (or backward) direction. if self._bidirectional: self._forward_dim = contextualizer.get_output_dim() // 2 else: self._forward_dim = contextualizer.get_output_dim() # TODO(joelgrus): more sampled softmax configuration options, as needed. if num_samples is not None: self._softmax_loss = SampledSoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=self._forward_dim, num_samples=num_samples, sparse=sparse_embeddings) else: self._softmax_loss = _SoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=self._forward_dim) # TODO(brendanr): Output perplexity here. e^loss self.register_buffer('_last_average_loss', torch.zeros(1)) if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x if initializer is not None: initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, layer_norm: Optional[MaskedLayerNorm] = None, dropout: float = None, loss_scale: Union[float, str] = 1.0, remove_bos_eos: bool = True) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder self._layer_norm = layer_norm or (lambda x: x) if not contextualizer.is_bidirectional(): raise ConfigurationError("contextualizer must be bidirectional") self._contextualizer = contextualizer # The dimension for making predictions just in the forward # (or backward) direction. self._forward_dim = contextualizer.get_output_dim() // 2 # TODO(joelgrus): Allow SampledSoftmaxLoss here by configuration self._softmax_loss = _SoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=self._forward_dim) self.register_buffer('_last_average_loss', torch.zeros(1)) if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x self._loss_scale = loss_scale self._remove_bos_eos = remove_bos_eos
def __init__( self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, dropout: float = 0.2, decoder_hidden_dim: int = 128, decoder_ff_dim: int = 128, decoder_num_layers: int = 1, decoder_num_heads: int = 4, teacher_forcing: float = 1.0, num_teacher_forcing_steps: int = None, num_tags: int = 2, label_smoothing: float = None, ): super().__init__(vocab) # teacher forcing is how often we choose to force the correct answer. self.embedder = embedder self.encoder = encoder self.laserdecoder = LaserDecoder( hidden_dim=decoder_hidden_dim, encoder_dim=encoder.get_output_dim(), num_layers=decoder_num_layers, ff_dim=decoder_ff_dim, num_heads=decoder_num_heads, num_classes=num_tags, ) self.dropout = torch.nn.Dropout(dropout) self.accuracy = CategoricalAccuracy() self.f1 = F1Measure(1) self.teacher_forcing = teacher_forcing self.num_tf_steps = num_teacher_forcing_steps self.cur_tf_steps = 0 self.label_smoothing = label_smoothing
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, dropout: float = 0.1, ff_dim: int = 100): super().__init__(vocab) self.embedder = embedder self.encoder = encoder assert self.embedder.get_output_dim() == self.encoder.get_input_dim() self.feedforward = FeedForward( encoder.get_output_dim(), 1, hidden_dims=ff_dim, activations=Activation.by_name('relu')(), dropout=dropout) self.out = torch.nn.Linear( in_features=self.feedforward.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.crf = ConditionalRandomField(vocab.get_vocab_size('labels')) self.f1 = FBetaMeasure(average='micro') self.accuracy = CategoricalAccuracy() self.idx_to_label = vocab.get_index_to_token_vocabulary('labels')
def __init__( self, #### The embedding layer is specified as an AllenNLP <code>TextFieldEmbedder</code> #### which represents a general way of turning tokens into tensors. #### (Here we know that we want to represent each unique word with a learned tensor, #### but using the general class allows us to easily experiment with different types #### of embeddings, for example <a href = "https://allennlp.org/elmo">ELMo</a>.) word_embeddings: TextFieldEmbedder, #### Similarly, the encoder is specified as a general <code>Seq2SeqEncoder</code> #### even though we know we want to use an LSTM. Again, this makes it easy to #### experiment with other sequence encoders, for example a Transformer. encoder: Seq2SeqEncoder, #### Every AllenNLP model also expects a <code>Vocabulary</code>, #### which contains the namespaced mappings of tokens to indices and labels to indices. vocab: Vocabulary ) -> None: #### Notice that we have to pass the vocab to the base class constructor. super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder #### The feed forward layer is not passed in as a parameter, but is constructed by us. #### Notice that it looks at the encoder to find the correct input dimension and looks #### at the vocabulary (and, in particular, at the label -> index mapping) to find the correct output dimension. self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) #### The last thing to notice is that we also instantiate a #### <code>CategoricalAccuracy</code> metric, which we'll use to track accuracy #### during each training and validation epoch. self.accuracy = CategoricalAccuracy()
def __init__(self, word_embeddings: TextFieldEmbedder, sentence_encoder: Seq2SeqEncoder, document_encoder: Seq2SeqEncoder, vocab: Vocabulary, encoder_dropout: float = 0.0) -> None: # We have to pass the vocabulary to the constructor. super().__init__(vocab) self.word_embeddings = word_embeddings if encoder_dropout > 0: self.encoder_dropout = torch.nn.Dropout(p=encoder_dropout) else: self.encoder_dropout = lambda x: x self.sentence_encoder = sentence_encoder self.sentence_attn = LinearSelfAttention( input_dim=self.sentence_encoder.get_output_dim(), bias=True) self.document_encoder = document_encoder self.document_attn = LinearSelfAttention( input_dim=self.document_encoder.get_output_dim(), bias=True) self.output = torch.nn.Linear( in_features=document_encoder.get_output_dim(), out_features=1)
def __init__(self, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.embedder = embedder self.encoder = encoder self.linear = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('pos')) self.accuracy = CategoricalAccuracy()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self._f1_metric = SpanBasedF1Measure( vocab, 'labels') # SpanBasedF1Measure: NER の評価
def __init__(self, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.embedder = embedder self.encoder = encoder self.hidden2labels = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() self.f1 = SpanBasedF1Measure(vocab, tag_namespace='labels')
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, hparams: Dict, ) -> None: super().__init__(vocab) self.text_field_embedder = text_field_embedder self.contextualizer = contextualizer self.bidirectional = contextualizer.is_bidirectional() if self.bidirectional: self.forward_dim = contextualizer.get_output_dim() // 2 else: self.forward_dim = contextualizer.get_output_dim() dropout = hparams["dropout"] if dropout: self.dropout = torch.nn.Dropout(dropout) else: self.dropout = lambda x: x self.hidden2chord = torch.nn.Sequential( torch.nn.Linear(self.forward_dim, hparams["fc_hidden_dim"]), torch.nn.ReLU(True), torch.nn.Linear(hparams["fc_hidden_dim"], vocab.get_vocab_size()), ) self.perplexity = PerplexityCustom() self.accuracy = CategoricalAccuracy() self.real_loss = Average() self.similarity_matrix = hparams["similarity_matrix"] self.training_mode = hparams["training_mode"] self.T_initial = hparams["T_initial"] self.T = self.T_initial self.decay_rate = hparams["decay_rate"] self.batches_per_epoch = hparams["batches_per_epoch"] self.epoch = 0 self.batch_counter = 0
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder) -> None: super().__init__(vocab) self._embedder = embedder self._encoder = encoder self._classifier = nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.f1 = SpanBasedF1Measure(vocab, 'labels')
def __init__( self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary, dropout: float = 0.5, n_linear_layers=1, ) -> None: """ :param word_embeddings: the embeddings to start with :param encoder: the seq2seq transformer of embeddings can be LSTM for example :param vocab: dataset input and output vocabulary """ super(BaseTextClassifier, self).__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder # Representations this is the layer that is just above the last layer and the non linearity (hidden[-1]) # is is used to calculate FID score, and similar metrics that's why we expose it into self.representations # class attribute self.representations = self.encoder if n_linear_layers > 0: extra_hiddens = [] for k in range(n_linear_layers): extra_hiddens += [ nn.Linear(self.encoder.get_output_dim(), self.encoder.get_output_dim()), nn.ReLU(True) ] self.extra_hiddens = nn.Sequential(*extra_hiddens) else: self.extra_hiddens = None self.hidden2label = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) # self.accuracy = CategoricalAccuracy() self.criterion = CrossEntropyLoss() self.metrics = { "accuracy": CategoricalAccuracy(), "hinge-loss": Loss(HingeEmbeddingLoss()), "huber-loss": Loss(SmoothL1Loss()), "cross-entropy-loss": Loss(CrossEntropyLoss()), "confidence": Confidence() } self.dropout = nn.Dropout(dropout)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.dbg_ctr = 0 self.word_embeddings = word_embeddings self.encoder = encoder.cuda(CUDA_DEVICE) self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')).cuda(CUDA_DEVICE) self.accuracy = CategoricalAccuracy() self.blank_index = vocab.get_token_index("_")
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, forward_segmental_contextualizer: Seq2SeqEncoder, backward_segmental_contextualizer: Seq2SeqEncoder, label_feature_dim: int, softmax_projection_dim: int, label_namespace: str = "labels", dropout: float = None, num_samples: int = None, sparse_embeddings: bool = False, bidirectional: bool = True, initializer: InitializerApplicator = None) -> None: super().__init__(vocab=vocab, text_field_embedder=text_field_embedder, contextualizer=contextualizer, dropout=dropout, num_samples=num_samples, sparse_embeddings=sparse_embeddings, bidirectional=bidirectional, initializer=initializer) self._forward_segmental_contextualizer = forward_segmental_contextualizer self._backward_segmental_contextualizer = backward_segmental_contextualizer if num_samples is not None: self._softmax_loss = SampledSoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=softmax_projection_dim, num_samples=num_samples, sparse=sparse_embeddings) else: self._softmax_loss = _SoftmaxLoss(num_words=vocab.get_vocab_size(), embedding_dim=softmax_projection_dim) self.num_classes = self.vocab.get_vocab_size(label_namespace) self.label_feature_embedding = Embedding(self.num_classes, label_feature_dim) self._forward_dim = contextualizer.get_output_dim() // 2 + \ forward_segmental_contextualizer.get_output_dim() // 2 + \ label_feature_dim self.projection_layer = TimeDistributed(Linear(self._forward_dim, softmax_projection_dim))
def __init__(self, word_embeddings: TextFieldEmbedder, sequence_encoder: Seq2SeqEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.sequence_encoder = sequence_encoder # Fully connected layer from sequence encoding to tags self.fc = torch.nn.Linear( in_features=sequence_encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary, num_categories: int) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() self.num_categories = num_categories self.fms = [F1Measure(i) for i in range(1, self.num_categories + 1)]
def __init__( self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, # you pass in the model with layers here. LSTM, etc. ): super().__init__(vocab) self.embedder = embedder self.encoder = encoder num_labels = vocab.get_vocab_size( "tokens") #get from the tokens namespace self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels)
def hierarchical_seq_over_seq(encoder: Seq2SeqEncoder, sentences: torch.Tensor, masks: torch.Tensor) -> torch.Tensor: num_batch, num_sent, num_tokens = masks.shape enc_size = encoder.get_output_dim() sentence_hiddens = sentences.new_empty( num_batch, num_sent, num_tokens, enc_size) for i in range(num_sent): sentence_emb = sentences[:, i, :, :] sentence_mask = masks[:, i, :] sentence_hiddens[:, i, :, :] = encoder(sentence_emb, sentence_mask) return sentence_hiddens
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, dropout_p: int, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.embedding2input = FeedForward( input_dim=word_embeddings.get_output_dim(), num_layers=1, hidden_dims=encoder.get_input_dim(), activations=Activation.by_name('relu')(), dropout=dropout_p) self.encoder = encoder self.hidden2intermediate = FeedForward( input_dim=encoder.get_output_dim(), num_layers=1, hidden_dims=int(encoder.get_output_dim() / 2), activations=Activation.by_name('relu')(), dropout=dropout_p) self.intermediate2tag = nn.Linear( in_features=int(encoder.get_output_dim() / 2), out_features=vocab.get_vocab_size('labels')) # self.accuracy = CategoricalAccuracy() label_vocab = vocab.get_token_to_index_vocabulary('labels').copy() # print("label_vocab: ", label_vocab) [label_vocab.pop(x) for x in ['O', 'OR']] labels_for_metric = list(label_vocab.values()) # print("labels_for_metric: ", labels_for_metric) self.accuracy = CustomFBetaMeasure(beta=1.0, average='micro', labels=labels_for_metric)
def __init__(self, embedders: nn.ModuleList, encoder: Seq2SeqEncoder, vocabulary: Vocabulary, config: argparse.Namespace) -> None: super().__init__(vocabulary) self.config = config self.embedders = embedders self.encoder = encoder self.linear_layer = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocabulary.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() self.F1 = SpanBasedF1Measure(vocabulary, tag_namespace='labels')
def __init__( self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, emb_to_enc_proj: FeedForward = None, feedforward: FeedForward = None, dropout: float = 0.0, num_tags: int = 2, use_crf: bool = False, ): super().__init__(vocab) self.embedder = embedder self.emb_to_enc_proj = None if emb_to_enc_proj is not None: self.emb_to_enc_proj = emb_to_enc_proj self.encoder = encoder assert (embedder.get_output_dim() == encoder.get_input_dim() or emb_to_enc_proj is not None and emb_to_enc_proj.get_output_dim() == encoder.get_input_dim()) self.feedforward = None pre_output_dim = encoder.get_output_dim() if feedforward is not None: assert feedforward.get_input_dim() == encoder.get_output_dim() self.feedforward = feedforward pre_output_dim = self.feedforward.get_output_dim() self.hidden2tag = torch.nn.Linear(in_features=pre_output_dim, out_features=num_tags) self.dropout = torch.nn.Dropout(dropout) self.accuracy = CategoricalAccuracy() self.f1 = F1Measure(1) self.use_crf = use_crf if use_crf: self.crf = ConditionalRandomField( num_tags, include_start_end_transitions=True)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary, output_feature_key: str = 'pos', **kwargs) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.output_size = vocab.get_vocab_size(output_feature_key) logger.info( f"Key {output_feature_key} : feature size {self.output_size}") self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=self.output_size) self.accuracy = CategoricalAccuracy()
def __init__(self, char_embeddings: TextFieldEmbedder, encoder: Seq2SeqEncoder, vocab: Vocabulary, padding: int) -> None: super().__init__(vocab) self.padding = padding self.char_embeddings = char_embeddings self.encoder = encoder self.linear_u = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=1) self.linear_db = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=1) self.bilinear = torch.nn.Bilinear(in1_features=self.padding, in2_features=self.padding, out_features=1) self.sigmoid = torch.nn.Sigmoid() self.accuracy = BooleanAccuracy() self.loss = torch.nn.BCELoss()
def __init__(self, word_embeddings: TextFieldEmbedder, hidden_size: int, max_len: int, vocab: Vocabulary, encoder: Seq2SeqEncoder = None, **kwargs) -> None: super().__init__(vocab) self.embeddings = word_embeddings self.encoder = encoder self.hidden2out = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('tokens')) self.hidden_size = hidden_size self.max_len = max_len self.vocab = vocab
def __init__(self, #### The embedding layer is specified as an AllenNLP <code>TextFieldEmbedder</code> which represents a general way of turning tokens into tensors. (Here we know that we want to represent each unique word with a learned tensor, but using the general class allows us to easily experiment with different types of embeddings, for example <a href = "https://allennlp.org/elmo">ELMo</a>.) word_embeddings: TextFieldEmbedder, #### Similarly, the encoder is specified as a general <code>Seq2SeqEncoder</code> even though we know we want to use an LSTM. Again, this makes it easy to experiment with other sequence encoders, for example a Transformer. encoder: Seq2SeqEncoder, #### Every AllenNLP model also expects a <code>Vocabulary</code>, which contains the namespaced mappings of tokens to indices and labels to indices. vocab: Vocabulary) -> None: #### Notice that we have to pass the vocab to the base class constructor. super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder #### The feed forward layer is not passed in as a parameter, but is constructed by us. Notice that it looks at the encoder to find the correct input dimension and looks at the vocabulary (and, in particular, at the label -> index mapping) to find the correct output dimension. self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) #### The last thing to notice is that we also instantiate a <code>CategoricalAccuracy</code> metric, which we'll use to track accuracy during each training and validation epoch. self.accuracy = CategoricalAccuracy()