def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary, positive_label: int = 4) -> None: super().__init__(vocab) # We need the embeddings to convert word IDs to their vector representations self.word_embeddings = word_embeddings # bottle-neck self.linear_bn = torch.nn.Linear( in_features=word_embeddings.get_output_dim(), out_features=encoder.get_input_dim()) self.encoder = encoder # After converting a sequence of vectors to a single vector, we feed it into # a fully-connected linear layer to reduce the dimension to the total number of labels. self.linear = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) # Monitor the metrics - we use accuracy, as well as prec, rec, f1 for 4 (very positive) self.accuracy = CategoricalAccuracy() self.f1_measure = F1Measure(positive_label) # We use the cross entropy loss because this is a classification task. # Note that PyTorch's CrossEntropyLoss combines softmax and log likelihood loss, # which makes it unnecessary to add a separate softmax layer. self.loss_function = torch.nn.CrossEntropyLoss()
def __init__(self, word_embeddings: TextFieldEmbedder, text_encoder: Seq2SeqEncoder, relation_encoder: Seq2VecEncoder, vocab: Vocabulary, encoder_dropout: float = 0.5) -> None: # We have to pass the vocabulary to the constructor. super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder_dropout = torch.nn.Dropout(p=encoder_dropout) self.text_encoder = text_encoder self.text_attn = LinearAttention( input_dim=text_encoder.get_output_dim()) self.relation_encoder = relation_encoder self.relation_attn = BilinearAttention( vector_dim=text_encoder.get_output_dim(), matrix_dim=relation_encoder.get_output_dim()) hidden_dim = (text_encoder.get_output_dim() + relation_encoder.get_output_dim()) self.output = torch.nn.Linear(in_features=hidden_dim, out_features=1)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, dropout_p: int, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.embedding2input = FeedForward( input_dim=word_embeddings.get_output_dim(), num_layers=1, hidden_dims=encoder.get_input_dim(), activations=Activation.by_name('relu')(), dropout=dropout_p) self.encoder = encoder self.hidden2intermediate = FeedForward( input_dim=encoder.get_output_dim(), num_layers=1, hidden_dims=int(encoder.get_output_dim() / 2), activations=Activation.by_name('relu')(), dropout=dropout_p) self.intermediate2tag = nn.Linear( in_features=int(encoder.get_output_dim() / 2), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() self.loss_function = torch.nn.CrossEntropyLoss()
def __init__(self, args, kgemb_dim, encoder: Seq2VecEncoder, vocab): super().__init__(vocab) self.args = args self.encoder = encoder self.first_and_last_emb = encoder.get_output_dim() self.loss = nn.MSELoss() self.projector = nn.Linear(kgemb_dim, encoder.get_output_dim()) self.projector2 = nn.Linear(kgemb_dim, encoder.get_output_dim()) self.projector3 = nn.Linear(kgemb_dim, encoder.get_output_dim()) self.nonlinear = nn.Tanhshrink()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) # We need the embeddings to convert word IDs to their vector representations self.word_embeddings = word_embeddings # Seq2VecEncoder is a neural network abstraction that takes a sequence of something # (usually a sequence of embedded word vectors), processes it, and returns it as a single # vector. Oftentimes, this is an RNN-based architecture (e.g., LSTM or GRU), but # AllenNLP also supports CNNs and other simple architectures (for example, # just averaging over the input vectors). self.encoder = encoder # After converting a sequence of vectors to a single vector, we feed it into # a fully-connected linear layer to reduce the dimension to the total number of labels. self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() # We use the cross-entropy loss because this is a classification task. # Note that PyTorch's CrossEntropyLoss combines softmax and log likelihood loss, # which makes it unnecessary to add a separate softmax layer. self.loss_function = torch.nn.CrossEntropyLoss()
def __init__(self, encoder: Seq2VecEncoder, first_n_states: int = 1, weight: float = None, disc_grad_norm: float = None, gen_grad_norm: float = None, steps_per_update: int = 5, target_distribution: str = 'uniform', num_tasks: int = 1): super().__init__() self.first_n_states = first_n_states self.weight = weight self.disc_grad_norm = disc_grad_norm self.gen_grad_norm = gen_grad_norm self._encoder = encoder self.steps_per_update = steps_per_update self._projection = FeedForward( input_dim=encoder.get_output_dim(), hidden_dims=num_tasks, num_layers=1, activations=Activation.by_name("relu")(), ) self.log_softmax = torch.nn.LogSoftmax() self._d_loss = torch.nn.NLLLoss(reduction='mean') self._g_loss = None if target_distribution == 'uniform': kl_loss = torch.nn.KLDivLoss(reduction='batchmean') self._g_loss = lambda inputs: kl_loss( inputs, inputs.new_ones(inputs.size()) / inputs.size(1) )
def from_params(cls, vocab, params: Params) -> 'WikiTablesSemanticParser': question_embedder = TextFieldEmbedder.from_params(vocab, params.pop("question_embedder")) action_embedding_dim = params.pop_int("action_embedding_dim") encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) entity_encoder = Seq2VecEncoder.from_params(params.pop('entity_encoder')) max_decoding_steps = params.pop_int("max_decoding_steps") mixture_feedforward_type = params.pop('mixture_feedforward', None) if mixture_feedforward_type is not None: mixture_feedforward = FeedForward.from_params(mixture_feedforward_type) else: mixture_feedforward = None decoder_beam_search = BeamSearch.from_params(params.pop("decoder_beam_search")) # If no attention function is specified, we should not use attention, not attention with # default similarity function. attention_function_type = params.pop("attention_function", None) if attention_function_type is not None: attention_function = SimilarityFunction.from_params(attention_function_type) else: attention_function = None dropout = params.pop_float('dropout', 0.0) num_linking_features = params.pop_int('num_linking_features', 8) rule_namespace = params.pop('rule_namespace', 'rule_labels') params.assert_empty(cls.__name__) return cls(vocab, question_embedder=question_embedder, action_embedding_dim=action_embedding_dim, encoder=encoder, entity_encoder=entity_encoder, mixture_feedforward=mixture_feedforward, decoder_beam_search=decoder_beam_search, max_decoding_steps=max_decoding_steps, attention_function=attention_function, dropout=dropout, num_linking_features=num_linking_features, rule_namespace=rule_namespace)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, dropout_proba: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), regularizer: RegularizerApplicator = None, ) -> None: super().__init__(vocab, regularizer) self.text_field_embedder = text_field_embedder self.encoder = encoder num_dense = encoder.get_output_dim() * 2 self.projection = nn.Sequential( nn.BatchNorm1d(num_dense, num_dense), nn.Dropout(p=dropout_proba), nn.Linear(num_dense, num_dense), nn.ReLU(inplace=True), nn.BatchNorm1d(num_dense, num_dense), nn.Dropout(p=dropout_proba), nn.Linear(num_dense, 1), nn.Sigmoid(), ) self.lossfun = nn.BCEWithLogitsLoss() self.metrics = {"accuracy": CategoricalAccuracy()} initializer(self)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embedding = word_embeddings self.encoder = encoder self.hidden2out = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size("labels")) self.accuracy = MicroMetrics(vocab) self.lstm = nn.LSTM(input_size=word_embeddings.get_output_dim(), hidden_size=128, num_layers=1, batch_first=True) self.label_index_to_label = self.vocab.get_index_to_token_vocabulary('labels')
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=2) self.accuracy = CategoricalAccuracy()
def test_stacked_bidirectional_lstm_can_build_from_params_seq2vec(self): params = Params({"type": "stacked_bidirectional_lstm", "input_size": 5, "hidden_size": 9, "num_layers": 3}) encoder = Seq2VecEncoder.from_params(params) assert encoder.get_input_dim() == 5 assert encoder.get_output_dim() == 18
def from_params(cls, vocab, params: Params) -> 'WikiTablesErmSemanticParser': question_embedder = TextFieldEmbedder.from_params( vocab, params.pop("question_embedder")) action_embedding_dim = params.pop_int("action_embedding_dim") encoder = Seq2SeqEncoder.from_params(params.pop("encoder")) entity_encoder = Seq2VecEncoder.from_params( params.pop('entity_encoder')) mixture_feedforward_type = params.pop('mixture_feedforward', None) if mixture_feedforward_type is not None: mixture_feedforward = FeedForward.from_params( mixture_feedforward_type) else: mixture_feedforward = None # If no attention function is specified, we should not use attention, not attention with # default similarity function. attention_function_type = params.pop("attention_function", None) if attention_function_type is not None: attention_function = SimilarityFunction.from_params( attention_function_type) else: attention_function = None decoder_beam_size = params.pop_int("decoder_beam_size") decoder_num_finished_states = params.pop_int( "decoder_num_finished_states", None) max_decoding_steps = params.pop_int("max_decoding_steps") normalize_beam_score_by_length = params.pop( "normalize_beam_score_by_length", False) use_neighbor_similarity_for_linking = params.pop_bool( "use_neighbor_similarity_for_linking", False) dropout = params.pop_float('dropout', 0.0) num_linking_features = params.pop_int('num_linking_features', 10) tables_directory = params.pop('tables_directory', '/wikitables/') rule_namespace = params.pop('rule_namespace', 'rule_labels') checklist_cost_weight = params.pop_float("checklist_cost_weight", 0.6) mml_model_file = params.pop('mml_model_file', None) params.assert_empty(cls.__name__) return cls( vocab, question_embedder=question_embedder, action_embedding_dim=action_embedding_dim, encoder=encoder, entity_encoder=entity_encoder, mixture_feedforward=mixture_feedforward, attention_function=attention_function, decoder_beam_size=decoder_beam_size, decoder_num_finished_states=decoder_num_finished_states, max_decoding_steps=max_decoding_steps, normalize_beam_score_by_length=normalize_beam_score_by_length, checklist_cost_weight=checklist_cost_weight, use_neighbor_similarity_for_linking= use_neighbor_similarity_for_linking, dropout=dropout, num_linking_features=num_linking_features, tables_directory=tables_directory, rule_namespace=rule_namespace, initial_mml_model_file=mml_model_file)
def test_stacked_bidirectional_lstm_can_build_from_params_seq2vec(self): params = Params({"type": "stacked_bidirectional_lstm", "input_size": 5, "hidden_size": 9, "num_layers": 3}) encoder = Seq2VecEncoder.from_params(params) assert encoder.get_input_dim() == 5 assert encoder.get_output_dim() == 18
def __init__(self, text_field_embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super(ClaimRank, self).__init__(vocab) self.text_field_embedder = text_field_embedder self.encoder = encoder self.hidden2score = torch.nn.Linear( in_features=3 * encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.f1 = F1Measure(positive_label=1)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.out = nn.Linear(in_features=encoder.get_output_dim(), out_features=1) self.loss_function = nn.L1Loss()
def test_stacked_bidirectional_lstm_can_complete_forward_pass_seq2vec(self): params = Params({"type": "stacked_bidirectional_lstm", "input_size": 3, "hidden_size": 9, "num_layers": 3}) encoder = Seq2VecEncoder.from_params(params) input_tensor = torch.rand(4, 5, 3) mask = torch.ones(4, 5) output = encoder(input_tensor, mask) assert output.detach().numpy().shape == (4, 18)
def test_stacked_bidirectional_lstm_can_complete_forward_pass_seq2vec(self): params = Params({"type": "stacked_bidirectional_lstm", "input_size": 3, "hidden_size": 9, "num_layers": 3}) encoder = Seq2VecEncoder.from_params(params) input_tensor = torch.rand(4, 5, 3) mask = torch.ones(4, 5) output = encoder(input_tensor, mask) assert output.detach().numpy().shape == (4, 18)
def __init__(self, args, kgemb_dim, encoder: Seq2VecEncoder, vocab): super().__init__(vocab) self.args = args self.encoder = encoder self.first_and_last_emb = encoder.get_output_dim() self.fc1 = nn.Linear(self.first_and_last_emb, kgemb_dim) self.fc21 = nn.Linear(kgemb_dim, 300) self.fc22 = nn.Linear(kgemb_dim, 300) self.fc3 = nn.Linear(300, kgemb_dim) self.fc4 = nn.Linear(kgemb_dim, self.first_and_last_emb)
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.classification_layer = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self._loss = torch.nn.CrossEntropyLoss() self._accuracy = CategoricalAccuracy()
def __init__(self, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.encoder = encoder self.h2o = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() self.loss = torch.nn.NLLLoss() self.softmax = torch.nn.LogSoftmax(dim=1)
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder): super().__init__(vocab) self.embedder = embedder self.encoder = encoder num_labels = vocab.get_vocab_size("labels") self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels) self.accuracy = CategoricalAccuracy() self.macrof1 = FBetaMeasure(average='macro') self.microf1 = FBetaMeasure(average='micro') self.weightedf1 = FBetaMeasure(average='weighted')
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary, n_classes: int) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.linear = torch.nn.Linear(in_features = encoder.get_output_dim(), out_features = n_classes) self.accuracy = CategoricalAccuracy() self.loss = torch.nn.CrossEntropyLoss()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.out = nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() self.f1_measure = F1Measure(4) self.loss_function = nn.CrossEntropyLoss()
def __init__(self, char_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.char_embeddings = char_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear( in_features = encoder.get_output_dim(), out_features = vocab.get_vocab_size("labels")) self.accuracy = CategoricalAccuracy() self.loss_function = torch.nn.CrossEntropyLoss()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2decision = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size("grammaticality_labels")) self.loss_function = nn.CrossEntropyLoss() self.accuracy = CategoricalAccuracy() self.vocab = vocab self.specificAccuracies = {} for ind in range(vocab.get_vocab_size(namespace="ugtype_labels")): self.specificAccuracies[ind] = CategoricalAccuracy()
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) # We need the embeddings to convert word IDs to their vector representations self.word_embeddings = word_embeddings self.encoder = encoder self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=2) self.accuracy = CategoricalAccuracy() #self.projection = nn.Linear(self.encoder.get_output_dim(), out_sz) self.out_act = torch.nn.Sigmoid() #self.loss_function = torch.nn.BCELoss() #BCEWithLogitsLoss() #CrossEntropyLoss() self.loss = torch.nn.CrossEntropyLoss() #torch.nn.BCEWithLogitsLoss()
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, initializer: InitializerApplicator = None, **kwargs ): super().__init__(vocab, **kwargs) self.embedder = embedder self.encoder = encoder self.vocab = vocab num_labels = vocab.get_vocab_size("labels") self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels) self.accuracy = CategoricalAccuracy() if initializer: initializer(self)
def __init__(self, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary, positive_label: str = '4') -> None: super().__init__(vocab) self.embedder = embedder self.encoder = encoder self.linear = torch.nn.Linear(in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) positive_index = vocab.get_token_index(positive_label, namespace='labels') self.accuracy = CategoricalAccuracy() self.f1_measure = F1Measure(positive_index) self.loss_function = torch.nn.CrossEntropyLoss()
def from_params(cls, vocab: Vocabulary, params: Params) -> 'SyllableEmbedder': # type: ignore # pylint: disable=arguments-differ embedding_params: Params = params.pop("syllable_embedding") # Embedding.from_params() uses "tokens" as the default namespace, but we need to change # that to be "token_characters" by default. If num_embeddings is present, set default namespace # to None so that extend_vocab call doesn't misinterpret that some namespace was originally used. default_namespace = None if embedding_params.get( "num_embeddings", None) else "token_characters" embedding_params.setdefault("vocab_namespace", default_namespace) embedding = Embedding.from_params(vocab, embedding_params) encoder_params: Params = params.pop("syllable_encoder") encoder = Seq2VecEncoder.from_params(encoder_params) dropout = params.pop_float("dropout", 0.0) params.assert_empty(cls.__name__) return cls(embedding, encoder, dropout)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, sentence_encoder: Seq2VecEncoder, claim_encoder: Seq2SeqEncoder, attention: Attention, max_steps: int = 100, beam_size: int = 5, beta: float = 1.0) -> None: super(Seq2SeqClaimRank, self).__init__(vocab) self.text_field_embedder = text_field_embedder self.sentence_encoder = sentence_encoder self.claim_encoder = TimeDistributed(claim_encoder) # Handles additional sequence dim self.claim_encoder_dim = claim_encoder.get_output_dim() self.attention = attention self.decoder_embedding_dim = text_field_embedder.get_output_dim() self.max_steps = max_steps self.beam_size = beam_size self.beta = beta # self.target_embedder = torch.nn.Embedding(vocab.get_vocab_size(), decoder_embedding_dim) # Since we are using the sentence encoding as the initial hidden state to the decoder, the # decoder hidden dim must match the sentence encoder hidden dim. self.decoder_output_dim = sentence_encoder.get_output_dim() self.decoder_0_cell = torch.nn.LSTMCell(self.decoder_embedding_dim + self.claim_encoder_dim, self.decoder_output_dim) self.decoder_1_cell = torch.nn.LSTMCell(self.decoder_output_dim, self.decoder_output_dim) # When projecting out we will use attention to combine claim embeddings into a single # context embedding, this will be concatenated with the decoder cell output before being # fed to the projection layer. Hence the expected input size is: # decoder output dim + claim encoder output dim projection_input_dim = self.decoder_output_dim + self.claim_encoder_dim self.output_projection_layer = torch.nn.Linear(projection_input_dim, vocab.get_vocab_size()) self._start_index = self.vocab.get_token_index('<s>') self._end_index = self.vocab.get_token_index('</s>') self.beam_search = BeamSearch(self._end_index, max_steps=max_steps, beam_size=beam_size) pad_index = vocab.get_token_index(vocab._padding_token) self.bleu = BLEU(exclude_indices={pad_index, self._start_index, self._end_index}) self.avg_reconstruction_loss = Average() self.avg_claim_scoring_loss = Average()
def seq_over_seq(encoder: Seq2VecEncoder, sentences: torch.Tensor, masks: Optional[torch.Tensor] = None ) -> torch.Tensor: num_batch, num_sent, _, _ = sentences.shape enc_size = encoder.get_output_dim() sentence_encodings = sentences.new_empty(num_batch, num_sent, enc_size) for i in range(num_sent): sentence_emb = sentences[:, i, :, :] sentence_mask: Optional[torch.Tensor] if masks is not None: sentence_mask = masks[:, i, :] else: sentence_mask = None sentence_encodings[:, i, :] = encoder(sentence_emb, sentence_mask) return sentence_encodings
def __init__(self, word_embeddings: TextFieldEmbedder, encoder: Seq2VecEncoder, vocab: Vocabulary) -> None: super().__init__(vocab) # We need the embeddings to convert word IDs to their vector representations self.word_embeddings = word_embeddings self.encoder = encoder # After converting a sequence of vectors to a single vector, we feed it into # a fully-connected linear layer to reduce the dimension to the total number of labels. self.hidden2tag = torch.nn.Linear( in_features=encoder.get_output_dim(), out_features=vocab.get_vocab_size('labels')) self.accuracy = CategoricalAccuracy() # We use the cross-entropy loss because this is a classification task. # Note that PyTorch's CrossEntropyLoss combines softmax and log likelihood loss, # which makes it unnecessary to add a separate softmax layer. self.loss_function = torch.nn.CrossEntropyLoss()