def __init__( self, input_size: int, hidden_size: int, num_layers: int, recurrent_dropout_probability: float = 0.0, layer_dropout_probability: float = 0.0, use_highway: bool = True, ) -> None: super().__init__() self.input_size = input_size self.hidden_size = hidden_size self.num_layers = num_layers self.bidirectional = True self.layer_dropout_probability = layer_dropout_probability layers = [] lstm_input_size = input_size for layer_index in range(num_layers): forward_layer = AugmentedLstm( lstm_input_size, hidden_size, go_forward=True, recurrent_dropout_probability=recurrent_dropout_probability, use_highway=use_highway, use_input_projection_bias=False, ) lstm_input_size = hidden_size self.add_module("forward_layer_{}".format(layer_index), forward_layer) layers.append(forward_layer) self.lstm_layers = layers self.layer_dropout = InputVariationalDropout(layer_dropout_probability)
def __init__(self, hidden_size, drop_prob=0.): super(Co_Attention, self).__init__() self.drop_prob = drop_prob # For get_similarity_matrix_slqa self.shared_weight = nn.Parameter(torch.zeros(hidden_size, hidden_size)) nn.init.xavier_uniform_(self.shared_weight) # dropout mar 13 YL self._variational_dropout = InputVariationalDropout(drop_prob) # fuse self.linear_fuse_pq = nn.Linear(hidden_size * 4, hidden_size, bias=True) self.linear_fuse_qp = nn.Linear(hidden_size * 4, hidden_size, bias=True) self.tanh = nn.Tanh() # gate self.linear_gate_pq = nn.Linear(hidden_size * 4, 1, bias=True) self.linear_gate_qp = nn.Linear(hidden_size * 4, 1, bias=True) self.sigmoid = nn.Sigmoid()
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, qdep_henc_rnn: Seq2SeqEncoder, senc_self_attn: GatedMultifactorSelfAttnEnc, attnpool: AttnPooling, output_ffl: FeedForward, initializer: InitializerApplicator, dropout: float = 0.3, ) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder self._qdep_henc_rnn = qdep_henc_rnn self._senc_self_attn = senc_self_attn self._variational_dropout = InputVariationalDropout(dropout) self._attn_pool = attnpool self._output_ffl = output_ffl self._num_labels = vocab.get_vocab_size(namespace="labels") self._accuracy = CategoricalAccuracy() self._mae = MeanAbsoluteError() self._loss = torch.nn.MSELoss() self._softmax = torch.nn.Softmax(dim=1) initializer(self)
def __init__(self, inp_dim: int, hidden_dim: int, dropout: float = 0.0): super(ProjectedEncoder, self).__init__() self.encoder = torch.nn.Linear(inp_dim, hidden_dim, bias=False) self.dropout = InputVariationalDropout(dropout) self.hidden_dim = hidden_dim self.reset_parameters()
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, phrase_layer: Seq2SeqEncoder, residual_encoder: Seq2SeqEncoder, span_start_encoder: Seq2SeqEncoder, span_end_encoder: Seq2SeqEncoder, initializer: InitializerApplicator, dropout: float = 0.2, num_context_answers: int = 0, marker_embedding_dim: int = 10, max_span_length: int = 30) -> None: super().__init__(vocab) self._num_context_answers = num_context_answers self._max_span_length = max_span_length self._text_field_embedder = text_field_embedder self._phrase_layer = phrase_layer self._marker_embedding_dim = marker_embedding_dim self._encoding_dim = phrase_layer.get_output_dim() max_turn_length = 12 self._matrix_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y') self._merge_atten = TimeDistributed(torch.nn.Linear(self._encoding_dim * 4, self._encoding_dim)) self._residual_encoder = residual_encoder if num_context_answers > 0: self._question_num_marker = torch.nn.Embedding(max_turn_length, marker_embedding_dim * num_context_answers) self._prev_ans_marker = torch.nn.Embedding((num_context_answers * 4) + 1, marker_embedding_dim) self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y') self._followup_lin = torch.nn.Linear(self._encoding_dim, 3) self._merge_self_attention = TimeDistributed(torch.nn.Linear(self._encoding_dim * 3, self._encoding_dim)) self._span_start_encoder = span_start_encoder self._span_end_encoder = span_end_encoder self._span_start_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1)) self._span_end_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 1)) self._span_yesno_predictor = TimeDistributed(torch.nn.Linear(self._encoding_dim, 3)) self._span_followup_predictor = TimeDistributed(self._followup_lin) initializer(self) self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_yesno_accuracy = CategoricalAccuracy() self._span_followup_accuracy = CategoricalAccuracy() self._span_gt_yesno_accuracy = CategoricalAccuracy() self._span_gt_followup_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._official_f1 = Average() self._variational_dropout = InputVariationalDropout(dropout)
def __init__( self, vocab: Vocabulary, elmo_embedder: TextFieldEmbedder, tokens_embedder: TextFieldEmbedder, features_embedder: TextFieldEmbedder, phrase_layer: Seq2SeqEncoder, projected_layer: Seq2SeqEncoder, contextual_passage: Seq2SeqEncoder, contextual_question: Seq2SeqEncoder, dropout: float = 0.2, regularizer: Optional[RegularizerApplicator] = None, initializer: InitializerApplicator = InitializerApplicator(), ): super(MultiGranularityHierarchicalAttentionFusionNetworks, self).__init__(vocab, regularizer) self.elmo_embedder = elmo_embedder self.tokens_embedder = tokens_embedder self.features_embedder = features_embedder self._phrase_layer = phrase_layer self._encoding_dim = self._phrase_layer.get_output_dim() self.projected_layer = torch.nn.Linear(self._encoding_dim + 1024, self._encoding_dim) self.fuse_p = FusionLayer(self._encoding_dim) self.fuse_q = FusionLayer(self._encoding_dim) self.fuse_s = FusionLayer(self._encoding_dim) self.projected_lstm = projected_layer self.contextual_layer_p = contextual_passage self.contextual_layer_q = contextual_question self.linear_self_align = torch.nn.Linear(self._encoding_dim, 1) # self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y') self._self_attention = BilinearMatrixAttention(self._encoding_dim, self._encoding_dim) self.bilinear_layer_s = BilinearSeqAtt(self._encoding_dim, self._encoding_dim) self.bilinear_layer_e = BilinearSeqAtt(self._encoding_dim, self._encoding_dim) self.yesno_predictor = FeedForward(self._encoding_dim, self._encoding_dim, 3) self.relu = torch.nn.ReLU() self._max_span_length = 30 self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._squad_metrics = SquadEmAndF1() self._span_yesno_accuracy = CategoricalAccuracy() self._official_f1 = Average() self._variational_dropout = InputVariationalDropout(dropout) self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__( self, vocab: Vocabulary, embedder: PretrainedBertEmbedder, passage_BiLSTM: Seq2SeqEncoder, columns_BiLSTM: Seq2SeqEncoder, sentence_BiLSTM: Seq2SeqEncoder, passage_contextual: Seq2SeqEncoder, columns_contextual: Seq2SeqEncoder, sentence_contextual: Seq2SeqEncoder, dropout: float = 0.2, regularizer: Optional[RegularizerApplicator] = None, initializer: InitializerApplicator = InitializerApplicator(), ): super(NumSeq2SQL, self).__init__(vocab, regularizer) # BERT Embedding self._embedder = embedder self._baidu_embedder = self.load_word_emb() self._passage_BiLSTM = passage_BiLSTM self._columns_BiLSTM = columns_BiLSTM self._sentence_BiLSTM = sentence_BiLSTM self._passage_contextual = passage_contextual self._columns_contextual = columns_contextual self._sentence_contextual = sentence_contextual self._encoding_dim = self._passage_BiLSTM.get_output_dim() self.projected_layer = torch.nn.Linear(self._encoding_dim, self._encoding_dim) self.fuse_p = FusionLayer(self._encoding_dim) self.fuse_c = FusionLayer(self._encoding_dim) self.fuse_s = FusionLayer(self._encoding_dim) self.linear_self_align = torch.nn.Linear(self._encoding_dim, 1) self.bilinear_layer_s = BilinearSeqAtt(self._encoding_dim, self._encoding_dim) self.bilinear_layer_e = BilinearSeqAtt(self._encoding_dim, self._encoding_dim) self.yesno_predictor = torch.nn.Linear(self._encoding_dim, 3) self.relu = torch.nn.ReLU() self._max_span_length = 30 self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._variational_dropout = InputVariationalDropout(dropout) self._span_yesno_accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() initializer(self)
def __init__(self, char_to_index, char_embed_size, hidden_size, output_size, dropout, cuda_flag, batch_first=True): """ Args: char_to_index: char_embed_size: char embeddings dim hidden_size: lstm reccurent dim dropout: dropout probability batch_first: batch first option """ super(Char_RNN, self).__init__() self.char_to_index = char_to_index self.char_embed_size = char_embed_size self.hidden_size = hidden_size self.dropout = dropout self.output_size = output_size self.batch_first = batch_first self.padding_index = self.char_to_index['__PADDING__'] self.cuda_flag = cuda_flag self.char_encoder = nn.Embedding(len(self.char_to_index), self.char_embed_size, sparse=True, padding_idx=self.padding_index) torch.nn.init.xavier_uniform_(self.char_encoder.weight.data) self.char_rnn = AugmentedLstm( input_size=self.char_embed_size, hidden_size=self.hidden_size, go_forward=True, recurrent_dropout_probability=self.dropout, use_highway=False, use_input_projection_bias=False) self.char_rnn.state_linearity.bias.data.fill_(0.0) self.var_drop = InputVariationalDropout(self.dropout) self.w_atten = nn.Linear(self.hidden_size, 1, bias=False) self.w_atten.weight.data.fill_(0.0) self.char_projection = nn.Linear(self.hidden_size * 2, self.output_size, bias=True) self.char_projection.weight.data.fill_(0.0) self.char_projection.bias.data.fill_(0.0) self.drp = nn.Dropout(self.dropout)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, pseqlevelenc: Seq2SeqEncoder, qaseqlevelenc: Seq2SeqEncoder, choicelevelenc: Seq2SeqEncoder, cartesian_attn: SeqAttnMat, pcattnmat: SeqAttnMat, gate_qdep_penc: GatedEncoding, qdep_penc_rnn: Seq2SeqEncoder, mfa_enc: GatedMultifactorSelfAttnEnc, mfa_rnn: Seq2SeqEncoder, pqaattnmat: SeqAttnMat, cqaattnmat: SeqAttnMat, initializer: InitializerApplicator, dropout: float = 0.3, is_qdep_penc: bool = True, is_mfa_enc: bool = True, with_knowledge: bool = True, is_qac_ap: bool = True, shared_rnn: bool = True) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder self._pseqlevel_enc = pseqlevelenc self._qaseqlevel_enc = qaseqlevelenc self._cseqlevel_enc = choicelevelenc self._cart_attn = cartesian_attn self._pqaattnmat = pqaattnmat self._pcattnmat = pcattnmat self._cqaattnmat = cqaattnmat self._gate_qdep_penc = gate_qdep_penc self._qdep_penc_rnn = qdep_penc_rnn self._multifactor_attn = mfa_enc self._mfarnn = mfa_rnn self._with_knowledge = with_knowledge self._qac_ap = is_qac_ap if not self._with_knowledge: if not self._qac_ap: raise AssertionError self._is_qdep_penc = is_qdep_penc self._is_mfa_enc = is_mfa_enc self._shared_rnn = shared_rnn self._variational_dropout = InputVariationalDropout(dropout) self._num_labels = vocab.get_vocab_size(namespace="labels") self._auc = Auc() self._loss = torch.nn.BCELoss() initializer(self)
def __init__( self, input_size: int, hidden_size: int, num_layers: int, recurrent_dropout_probability: float = 0.0, layer_dropout_probability: float = 0.0, use_highway: bool = True, ) -> None: super().__init__() # Required to be wrapped with a :class:`PytorchSeq2SeqWrapper`. self.input_size = input_size self.hidden_size = hidden_size self.num_layers = num_layers self.bidirectional = True layers = [] lstm_input_size = input_size for layer_index in range(num_layers): forward_layer = AugmentedLstm( lstm_input_size, hidden_size, go_forward=True, recurrent_dropout_probability=recurrent_dropout_probability, use_highway=use_highway, use_input_projection_bias=False, ) backward_layer = AugmentedLstm( lstm_input_size, hidden_size, go_forward=False, recurrent_dropout_probability=recurrent_dropout_probability, use_highway=use_highway, use_input_projection_bias=False, ) lstm_input_size = hidden_size * 2 self.add_module("forward_layer_{}".format(layer_index), forward_layer) self.add_module("backward_layer_{}".format(layer_index), backward_layer) layers.append([forward_layer, backward_layer]) self.lstm_layers = layers self.layer_dropout = InputVariationalDropout(layer_dropout_probability)
def __init__(self, input_size: int, hidden_size: int, num_layers: int, layer_dropout_probability: float = 0.0, use_residual: bool = True, use_residual_projection: bool = False) -> None: super(ResidualBidirectionalLstm, self).__init__() # Required to be wrapped with a :class:`PytorchSeq2SeqWrapper`. self.input_size = input_size self.hidden_size = hidden_size self.num_layers = num_layers self.bidirectional = True self.use_residual = use_residual self.use_residual_projection = use_residual_projection layers = [] lstm_input_size = input_size for layer_index in range(num_layers): layer = nn.LSTM(lstm_input_size, hidden_size, num_layers=1, batch_first=True, bidirectional=True) if use_residual and layer_index < (self.num_layers - 1): if use_residual_projection or lstm_input_size != hidden_size * 2: residual_projection = nn.Linear(lstm_input_size, hidden_size * 2, bias=False) else: residual_projection = nn.Identity() self.add_module('res_proj_{}'.format(layer_index), residual_projection) lstm_input_size = hidden_size * 2 self.add_module('layer_{}'.format(layer_index), layer) layers.append(layer) self.lstm_layers = layers self.layer_dropout = InputVariationalDropout(layer_dropout_probability)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, embedding_dim: int, max_mention_length: int, max_embeddings: int, variational_dropout_rate: float = 0.0, dropout_rate: float = 0.0, initializer: InitializerApplicator = InitializerApplicator() ) -> None: super(EntityNLMDiscriminator, self).__init__(vocab) self._text_field_embedder = text_field_embedder self._encoder = encoder self._embedding_dim = embedding_dim self._max_mention_length = max_mention_length self._max_embeddings = max_embeddings self._state: Optional[StateDict] = None # Input variational dropout self._variational_dropout = InputVariationalDropout( variational_dropout_rate) self._dropout = torch.nn.Dropout(dropout_rate) # For entity type prediction self._entity_type_projection = torch.nn.Linear( in_features=embedding_dim, out_features=2, bias=False) self._dynamic_embeddings = DynamicEmbedding( embedding_dim=embedding_dim, max_embeddings=max_embeddings) # For mention length prediction self._mention_length_projection = torch.nn.Linear( in_features=2 * embedding_dim, out_features=max_mention_length) self._entity_type_accuracy = CategoricalAccuracy() self._entity_id_accuracy = CategoricalAccuracy() self._mention_length_accuracy = CategoricalAccuracy() initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, attnpool: AttnPooling, output_ffl: FeedForward, initializer: InitializerApplicator, dropout: float = 0.3, ) -> None: super().__init__(vocab) self._text_field_embedder = text_field_embedder self._variational_dropout = InputVariationalDropout(dropout) self._attn_pool = attnpool self._output_ffl = output_ffl self._num_labels = vocab.get_vocab_size(namespace="labels") self._auc = Auc() self._loss = torch.nn.BCELoss() initializer(self)
def __init__(self, word_vectors, hidden_size, drop_prob=0.): super(Encoder, self).__init__() self.input_size_query = 300 self.input_size_doc = 300 self.embed_size = 300 self.drop_prob = 0.5 # Word embedding: glove self.glove_embedding = nn.Embedding.from_pretrained(word_vectors) # two Bi-LSTM self.bilstm1 = RNNEncoder(input_size=self.input_size_doc, hidden_size=hidden_size, drop_prob=drop_prob, num_layers=1) self.bilstm2 = RNNEncoder(input_size=self.input_size_query, hidden_size=hidden_size, drop_prob=drop_prob, num_layers=1) # dropout mar 13 YL self._variational_dropout = InputVariationalDropout(drop_prob)
def __init__(self, vocab: Vocabulary, embedder: TextFieldEmbedder, contextualizer: Seq2SeqEncoder, dropout: float = None, tie_embeddings: bool = True, num_samples: int = None, use_variational_dropout: bool = False): super().__init__(vocab) self._embedder = embedder self._contextualizer = contextualizer self._context_dim = contextualizer.get_output_dim() if use_variational_dropout: self._dropout = InputVariationalDropout( dropout) if dropout else lambda x: x else: self._dropout = Dropout(dropout) if dropout else lambda x: x vocab_size = self.vocab.get_vocab_size() padding_index = self.vocab.get_token_index(DEFAULT_PADDING_TOKEN) if num_samples: self._softmax_loss = SampledSoftmaxLoss(vocab_size, self._context_dim, num_samples) else: self._softmax_loss = SoftmaxLoss(vocab_size, self._context_dim, padding_index) self._tie_embeddings = tie_embeddings if self._tie_embeddings: embedder_children = dict(self._embedder.named_children()) word_embedder = embedder_children["token_embedder_tokens"] assert self._softmax_loss.softmax_w.size( ) == word_embedder.weight.size() self._softmax_loss.softmax_w = word_embedder.weight
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, encoder: Seq2SeqEncoder, embedding_dim: int, max_mention_length: int, max_embeddings: int, tie_weights: bool, variational_dropout_rate: float = 0.0, dropout_rate: float = 0.0, initializer: InitializerApplicator = InitializerApplicator() ) -> None: super(EntityNLM, self).__init__(vocab) self._text_field_embedder = text_field_embedder self._encoder = encoder self._embedding_dim = embedding_dim self._max_mention_length = max_mention_length self._max_embeddings = max_embeddings self._tie_weights = tie_weights self._variational_dropout_rate = variational_dropout_rate self._dropout_rate = dropout_rate self._state: Optional[StateDict] = None # Input variational dropout self._variational_dropout = InputVariationalDropout( variational_dropout_rate) self._dropout = torch.nn.Dropout(dropout_rate) # For entity type prediction self._entity_type_projection = torch.nn.Linear( in_features=embedding_dim, out_features=2, bias=False) self._dynamic_embeddings = DynamicEmbedding( embedding_dim=embedding_dim, max_embeddings=max_embeddings) # For mention length prediction self._mention_length_projection = torch.nn.Linear( in_features=2 * embedding_dim, out_features=max_mention_length) # For next word prediction self._dummy_context_embedding = Parameter( F.normalize(torch.randn(1, embedding_dim))) # TODO: Maybe squeeze self._entity_output_projection = torch.nn.Linear( in_features=embedding_dim, out_features=embedding_dim, bias=False) self._context_output_projection = torch.nn.Linear( in_features=embedding_dim, out_features=embedding_dim, bias=False) self._vocab_projection = torch.nn.Linear( in_features=embedding_dim, out_features=vocab.get_vocab_size('tokens')) if tie_weights: self._vocab_projection.weight = self._text_field_embedder._token_embedders[ 'tokens'].weight # pylint: disable=W0212 # self._perplexity = Perplexity() # self._unknown_penalized_perplexity = UnknownPenalizedPerplexity(self.vocab) self._entity_type_accuracy = CategoricalAccuracy() self._entity_id_accuracy = CategoricalAccuracy() self._mention_length_accuracy = CategoricalAccuracy() if tie_weights: self._vocab_projection.weight = self._text_field_embedder._token_embedders[ 'tokens'].weight # pylint: disable=W0212 initializer(self)
def __init__( self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, phrase_layer: Seq2SeqEncoder, residual_encoder: Seq2SeqEncoder, ctx_q_encoder: ContextualizedQuestionEncoder, # this is used to get some m_t span_start_encoder: Seq2SeqEncoder, span_end_encoder: Seq2SeqEncoder, initializer: InitializerApplicator, dropout: float = 0.2, num_context_answers: int = 0, marker_embedding_dim: int = 10, max_span_length: int = 30) -> None: super().__init__(vocab) print('INIT MODEL') self._num_context_answers = num_context_answers self._max_span_length = max_span_length self._text_field_embedder = text_field_embedder self._phrase_layer = phrase_layer self._marker_embedding_dim = marker_embedding_dim self._encoding_dim = phrase_layer.get_output_dim() self._train_coref_module = True # combine memory with question max_turn_length = 12 self._ctx_q_encoder = ctx_q_encoder self._matrix_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y') self._merge_atten = TimeDistributed( torch.nn.Linear(self._encoding_dim * 4, self._encoding_dim)) self._residual_encoder = residual_encoder if num_context_answers > 0: self._question_num_marker = torch.nn.Embedding( max_turn_length, marker_embedding_dim) self._prev_ans_marker = torch.nn.Embedding( (num_context_answers * 4) + 1, marker_embedding_dim) if self._ctx_q_encoder.use_ling: pos_tags = self.vocab.get_vocab_size('pos_tags') self._pos_emb = torch.nn.Embedding(pos_tags, marker_embedding_dim) self._self_attention = LinearMatrixAttention(self._encoding_dim, self._encoding_dim, 'x,y,x*y') self._followup_lin = torch.nn.Linear(self._encoding_dim, 3) self._merge_self_attention = TimeDistributed( torch.nn.Linear(self._encoding_dim * 3, self._encoding_dim)) self._span_start_encoder = span_start_encoder self._span_end_encoder = span_end_encoder self._span_start_predictor = TimeDistributed( torch.nn.Linear(self._encoding_dim, 1)) self._span_end_predictor = TimeDistributed( torch.nn.Linear(self._encoding_dim, 1)) self._span_yesno_predictor = TimeDistributed( torch.nn.Linear(self._encoding_dim, 3)) self._span_followup_predictor = TimeDistributed(self._followup_lin) initializer(self) self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_yesno_accuracy = CategoricalAccuracy() self._span_followup_accuracy = CategoricalAccuracy() self._span_gt_yesno_accuracy = CategoricalAccuracy() self._span_gt_followup_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._official_f1 = Average() self._variational_dropout = InputVariationalDropout(dropout)
def __init__(self, vocab: Vocabulary, text_encoder: Seq2SeqEncoder, word_embedder: TextFieldEmbedder, enable_training_log: bool = False, inp_drop_rate: float = 0.2, out_drop_rate: float = 0.2, loss_weights: List = (0.2, 0.4, 0.4), super_mode: str = 'before', backbone: str = 'unet', unet_down_channel: int = 256, feature_sel: int = 127): super(UnifiedFollowUp, self).__init__(vocab) self.text_encoder = text_encoder self.word_embedder = word_embedder """ Define model arch choices """ self.backbone = backbone # input dropout if inp_drop_rate > 0: self.var_inp_dropout = InputVariationalDropout(p=inp_drop_rate) else: self.var_inp_dropout = lambda x: x # output dropout if out_drop_rate > 0: self.var_out_dropout = InputVariationalDropout(p=out_drop_rate) else: self.var_out_dropout = lambda x: x self.hidden_size = text_encoder.get_output_dim() // 2 if text_encoder.is_bidirectional() \ else text_encoder.get_output_dim() self.output_size = text_encoder.get_output_dim() # ele -> element wise multiply # dot -> dot product # cos -> cosine similarity # emb_dot -> embedding dot product # emb_cos -> embedding cosine similarity # linear -> linear similarity # bilinear -> bilinear similarity feature_sel = feature_sel sel_arr = "{0:07b}".format(int(feature_sel)) nni_choices = ['ele', 'dot', 'cos', 'emb_dot', 'emb_cos', 'linear', 'bilinear'] self.segment_choices = [nni_choices[i] for i in range(7) if sel_arr[i] == '1'] # if expand bi-direction, we will regard forward/backward as two channels self.expand_bidir = False self.similar_function = ModuleDict({ 'ele': ElementWiseMatrixAttention(), 'dot': DotProductMatrixAttention(), 'cos': CosineMatrixAttention(), 'emb_dot': DotProductMatrixAttention(), 'emb_cos': CosineMatrixAttention(), 'bilinear': BilinearMatrixAttention(matrix_1_dim=self.output_size, matrix_2_dim=self.output_size), 'linear': LinearMatrixAttention(tensor_1_dim=self.output_size, tensor_2_dim=self.output_size) }) self.attn_channel = 0 for choice in self.segment_choices: if choice == 'ele': self.attn_channel += self.output_size elif choice in ['dot', 'cos', 'emb_dot', 'emb_cos', 'bilinear', 'linear']: if self.expand_bidir: self.attn_channel += 2 else: self.attn_channel += 1 self.class_mapping: Dict[str, int] = get_class_mapping(super_mode=super_mode) # Here we have two choices now, one is MLP, and another is UNet if self.backbone == 'unet': self.segmentation_net = AttentionUNet(input_channels=self.attn_channel, class_number=len(self.class_mapping.keys()), down_channel=unet_down_channel) else: raise Exception("Currently we do not support for other arches.") class_zero_weight = loss_weights[0] class_one_weight = loss_weights[1] self.register_buffer('weight_tensor', torch.tensor([class_zero_weight, class_one_weight, 1 - class_zero_weight - class_one_weight])) self.loss = nn.CrossEntropyLoss(ignore_index=-1, weight=self.weight_tensor) # initialize metrics measurement self.metrics = {'ROUGE': BatchAverage(), '_ROUGE1': BatchAverage(), '_ROUGE2': BatchAverage(), # TODO: You can speed up the code by disable BLEU since # the corpus-based BLEU metric is much time-consuming. 'BLEU': CorpusBLEUMetric(), 'EM': BatchAverage(), 'F1': FScoreMetric(prefix="1"), 'F2': FScoreMetric(prefix="2"), 'F3': FScoreMetric(prefix="3")} parameter_num = count_parameters(self) print(parameter_num) self.min_width = 8 self.min_height = 8 self.enable_training_log = enable_training_log
def __init__(self, n_relations: int, conf: Dict, input_batchers: Dict[str, InputBatch], use_cuda: bool): super(BiaffineParser, self).__init__() self.n_relations = n_relations self.conf = conf self.use_cuda = use_cuda self.use_mst_decoding_for_validation = conf[ 'use_mst_decoding_for_validation'] input_layers = {} for i, c in enumerate(conf['input']): if c['type'] == 'embeddings': if 'pretrained' in c: embs = load_embedding_txt(c['pretrained'], c['has_header']) logger.info('loaded {0} embedding entries.'.format( len(embs[0]))) else: embs = None name = c['name'] mapping = input_batchers[name].mapping layer = Embeddings(name, c['dim'], mapping, fix_emb=c['fixed'], embs=embs, normalize=c.get('normalize', False)) logger.info('embedding for field {0} ' 'created with {1} x {2}.'.format( c['field'], layer.n_V, layer.n_d)) input_layers[name] = layer elif c['type'] == 'cnn_encoder' or c['type'] == 'lstm_encoder': name = c['name'] mapping = input_batchers[name].mapping embeddings = Embeddings('{0}_ch_emb', c['dim'], mapping, fix_emb=False, embs=None, normalize=False) logger.info('character embedding for field {0} ' 'created with {1} x {2}.'.format( c['field'], embeddings.n_V, embeddings.n_d)) if c['type'] == 'lstm_encoder': layer = LstmTokenEmbedder(name, c['dim'], embeddings, conf['dropout'], use_cuda) elif c['type'] == 'cnn_encoder': layer = ConvTokenEmbedder(name, c['dim'], embeddings, c['filters'], c.get('n_highway', 1), c.get('activation', 'relu'), use_cuda) else: raise ValueError('Unknown type: {}'.format(c['type'])) input_layers[name] = layer elif c['type'] == 'elmo': name = c['name'] layer = ContextualizedWordEmbeddings(name, c['path'], use_cuda) input_layers[name] = layer else: raise ValueError('{} unknown input layer'.format(c['type'])) self.input_layers = torch.nn.ModuleDict(input_layers) input_encoders = [] input_dim = 0 for i, c in enumerate(conf['input_encoder']): input_info = { name: [ entry['dim'] for entry in conf['input'] if entry['name'] == name ][0] for name in c['input'] } if c['type'] == 'affine': input_encoder = AffineTransformInputEncoder( input_info, c['dim'], use_cuda) elif c['type'] == 'sum': input_encoder = SummationInputEncoder(input_info, use_cuda) elif c['type'] == 'concat': input_encoder = ConcatenateInputEncoder(input_info, use_cuda) else: raise ValueError('{} unknown input encoder'.format(c['type'])) input_dim += input_encoder.get_output_dim() input_encoders.append(input_encoder) self.input_encoders = torch.nn.ModuleList(input_encoders) c = conf['context_encoder'] if c['type'] == 'stacked_bidirectional_lstm_dozat': self.encoder = PytorchSeq2SeqWrapper( InputDropoutedStackedBidirectionalLstm( DozatLstmCell, num_layers=c['num_layers'], input_size=input_dim, hidden_size=c['hidden_dim'], recurrent_dropout_probability=c[ 'recurrent_dropout_probability'], layer_dropout_probability=c['layer_dropout_probability'], activation=Activation.by_name("leaky_relu")()), stateful=False) elif c['type'] == 'stacked_bidirectional_lstm_ma': self.encoder = PytorchSeq2SeqWrapper( InputDropoutedStackedBidirectionalLstm( MaLstmCell, num_layers=c['num_layers'], input_size=input_dim, hidden_size=c['hidden_dim'], recurrent_dropout_probability=c[ 'recurrent_dropout_probability'], layer_dropout_probability=c['layer_dropout_probability'], activation=Activation.by_name("tanh")()), stateful=False) elif c['type'] == 'stacked_bidirectional_lstm': self.encoder = PytorchSeq2SeqWrapper(StackedBidirectionalLstm( num_layers=c['num_layers'], input_size=input_dim, hidden_size=c['hidden_dim'], recurrent_dropout_probability=c[ 'recurrent_dropout_probability'], layer_dropout_probability=c['layer_dropout_probability']), stateful=False) else: self.encoder = DummyContextEncoder() encoder_dim = self.encoder.get_output_dim() c = conf['biaffine_parser'] self.arc_representation_dim = arc_representation_dim = c[ 'arc_representation_dim'] self.tag_representation_dim = tag_representation_dim = c[ 'tag_representation_dim'] self.head_sentinel_ = torch.nn.Parameter( torch.randn([1, 1, encoder_dim])) self.head_arc_feedforward = FeedForward(encoder_dim, 1, arc_representation_dim, Activation.by_name("elu")()) self.child_arc_feedforward = FeedForward(encoder_dim, 1, arc_representation_dim, Activation.by_name("elu")()) self.head_tag_feedforward = FeedForward(encoder_dim, 1, tag_representation_dim, Activation.by_name("elu")()) self.child_tag_feedforward = FeedForward(encoder_dim, 1, tag_representation_dim, Activation.by_name("elu")()) arc_attention_version = c.get('arc_attention_version', 'v1') if arc_attention_version == 'v2': self.arc_attention = BilinearMatrixAttentionV2( arc_representation_dim, arc_representation_dim, use_input_biases=True) else: self.arc_attention = BilinearMatrixAttention( arc_representation_dim, arc_representation_dim, use_input_biases=True) self.tag_bilinear = BilinearWithBias(tag_representation_dim, tag_representation_dim, n_relations) self.input_dropout_ = torch.nn.Dropout2d(p=conf['dropout']) self.dropout_ = InputVariationalDropout(p=conf['dropout']) self.input_encoding_timer = TimeRecoder() self.context_encoding_timer = TimeRecoder() self.classification_timer = TimeRecoder()
def __init__(self, conf: Dict, input_batchers: Dict[str, Union[WordBatch, CharacterBatch]], n_class: int, use_cuda: bool): super(SeqLabelModel, self).__init__() self.n_class = n_class self.use_cuda = use_cuda self.input_dropout = torch.nn.Dropout2d(p=conf["dropout"]) self.dropout = InputVariationalDropout(p=conf['dropout']) input_layers = {} for i, c in enumerate(conf['input']): if c['type'] == 'embeddings': if 'pretrained' in c: embs = load_embedding_txt(c['pretrained'], c['has_header']) logger.info('loaded {0} embedding entries.'.format( len(embs[0]))) else: embs = None name = c['name'] mapping = input_batchers[name].mapping layer = Embeddings(c['dim'], mapping, fix_emb=c['fixed'], embs=embs, normalize=c.get('normalize', False), input_field_name=name) logger.info('embedding for field {0} ' 'created with {1} x {2}.'.format( c['field'], layer.n_V, layer.n_d)) input_layers[name] = layer elif c['type'] == 'cnn_encoder' or c['type'] == 'lstm_encoder': name = c['name'] mapping = input_batchers[name].mapping embeddings = Embeddings( c['dim'], mapping, fix_emb=False, embs=None, normalize=False, input_field_name='{0}_ch_emb'.format(name)) logger.info('character embedding for field {0} ' 'created with {1} x {2}.'.format( c['field'], embeddings.n_V, embeddings.n_d)) if c['type'] == 'lstm_encoder': layer = LstmTokenEmbedder(c['dim'], embeddings, conf['dropout'], use_cuda, input_field_name=name) elif c['type'] == 'cnn_encoder': layer = ConvTokenEmbedder(c['dim'], embeddings, c['filters'], c.get('n_highway', 1), c.get('activation', 'relu'), use_cuda, input_field_name=name) else: raise ValueError('Unknown type: {}'.format(c['type'])) input_layers[name] = layer elif c['type'] == 'elmo': name = c['name'] layer = ContextualizedWordEmbeddings(name, c['path'], use_cuda) input_layers[name] = layer else: raise ValueError('{} unknown input layer'.format(c['type'])) self.input_layers = torch.nn.ModuleDict(input_layers) input_encoders = [] input_dim = 0 for i, c in enumerate(conf['input_encoder']): input_info = { name: self.input_layers[name].get_output_dim() for name in c['input'] } if c['type'] == 'affine': input_encoder = AffineTransformInputEncoder( input_info, c['dim'], use_cuda) elif c['type'] == 'sum': input_encoder = SummationInputEncoder(input_info, use_cuda) elif c['type'] == 'concat': input_encoder = ConcatenateInputEncoder(input_info, use_cuda) else: raise ValueError('{} unknown input encoder'.format(c['type'])) input_dim += input_encoder.get_output_dim() input_encoders.append(input_encoder) self.input_encoders = torch.nn.ModuleList(input_encoders) encoder_name = conf['encoder']['type'].lower() if encoder_name == 'stacked_bidirectional_lstm': lstm = StackedBidirectionalLstm( input_size=input_dim, hidden_size=conf['encoder']['hidden_dim'], num_layers=conf['encoder']['n_layers'], recurrent_dropout_probability=conf['dropout'], layer_dropout_probability=conf['dropout'], use_highway=conf['encoder'].get('use_highway', True)) self.encoder = PytorchSeq2SeqWrapper(lstm, stateful=False) encoded_input_dim = self.encoder.get_output_dim() elif encoder_name == 'project': self.encoder = ProjectedEncoder(input_dim, conf['encoder']['hidden_dim'], dropout=conf['dropout']) encoded_input_dim = self.encoder.get_output_dim() elif encoder_name == 'dummy': self.encoder = DummyEncoder() encoded_input_dim = input_dim else: raise ValueError('Unknown input encoder: {}'.format(encoder_name)) if conf["classifier"]["type"].lower() == 'crf': self.classify_layer = CRFLayer(encoded_input_dim, n_class, use_cuda) else: self.classify_layer = ClassifyLayer(encoded_input_dim, n_class, use_cuda) self.encode_time = 0 self.emb_time = 0 self.classify_time = 0
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, label_namespace: str = "labels", encoder: Optional[Seq2SeqEncoder] = None, feedforward: Optional[FeedForward] = None, label_encoding: Optional[str] = None, calculate_span_f1: bool = None, dropout: Optional[float] = None, verbose_metrics: bool = False, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.num_tags = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self._verbose_metrics = verbose_metrics embedding_output_dim = self.text_field_embedder.get_output_dim() if dropout is not None: self.dropout = torch.nn.Dropout(dropout) self.variational_dropout = InputVariationalDropout(dropout) else: self.dropout = None self._feedforward = feedforward if feedforward is not None: output_dim = feedforward.get_output_dim() elif encoder is not None: output_dim = self.encoder.get_output_dim() else: output_dim = embedding_output_dim self.tag_projection_layer = TimeDistributed( Linear(output_dim, self.num_tags)) if calculate_span_f1 is None: calculate_span_f1 = label_encoding is not None self.label_encoding = label_encoding self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3) } self.calculate_span_f1 = calculate_span_f1 if calculate_span_f1 is not None: if not label_encoding: raise ConfigurationError("calculate_span_f1 is True, but " "no label_encoding was specified.") self._f1_metric = SpanBasedF1Measure(vocab, tag_namespace=label_namespace, label_encoding=label_encoding) if encoder is not None: check_dimensions_match(embedding_output_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") if feedforward is not None and encoder is not None: check_dimensions_match(encoder.get_output_dim(), feedforward.get_input_dim(), "encoder output dim", "feedforward input dim") elif feedforward is not None and encoder is None: check_dimensions_match(embedding_output_dim, feedforward.get_input_dim(), "text field output dim", "feedforward input dim") initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, label_namespace: str = "labels", encoder: Optional[Seq2VecEncoder] = None, seq_encoder: Optional[Seq2SeqEncoder] = None, feedforward: Optional[FeedForward] = None, dropout: Optional[float] = None, incl_neutral: Optional[bool] = False, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.num_labels = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self.seq_encoder = seq_encoder if self.seq_encoder is not None: self.attention_vector = Parameter(torch.Tensor(self.seq_encoder.get_output_dim())) self.attention_layer = DotProductAttention(normalize=True) embedding_output_dim = self.text_field_embedder.get_output_dim() if dropout is not None: self.dropout = torch.nn.Dropout(dropout) self.variational_dropout = InputVariationalDropout(dropout) else: self.dropout = None self._feedforward = feedforward if feedforward is not None: output_dim = feedforward.get_output_dim() elif encoder is not None: output_dim = self.encoder.get_output_dim() elif seq_encoder is not None: output_dim = self.seq_encoder.get_output_dim() else: output_dim = embedding_output_dim # Have to create a tag projection layer for each label in the # multi label classifier self._tag_projection_layers: Any = [] for k in range(self.num_labels): tag_projection_layer = Linear(output_dim, 1) self.add_module(f'tag_projection_layer_{k}', tag_projection_layer) self._tag_projection_layers.append(tag_projection_layer) self.output_activation = torch.nn.Sigmoid() self.loss_criterion = torch.nn.BCEWithLogitsLoss(reduction='mean') self.incl_neutral = incl_neutral self.metrics = {"jaccard_index": JaccardIndex(self.incl_neutral)} if encoder is not None: check_dimensions_match(embedding_output_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") if feedforward is not None and encoder is not None: check_dimensions_match(encoder.get_output_dim(), feedforward.get_input_dim(), "encoder output dim", "feedforward input dim") elif feedforward is not None and encoder is None: check_dimensions_match(embedding_output_dim, feedforward.get_input_dim(), "text field output dim", "feedforward input dim") if self.seq_encoder is not None: self.reset_parameters() initializer(self)
def __init__(self, vocab: Vocabulary, text_field_embedder: TextFieldEmbedder, pos_tag_embedding: Embedding = None, pos_tag_loss: Optional[float] = None, label_namespace: str = "labels", encoder: Optional[Seq2SeqEncoder] = None, feedforward: Optional[FeedForward] = None, label_encoding: Optional[str] = None, crf: bool = True, include_start_end_transitions: bool = True, constrain_crf_decoding: bool = None, calculate_span_f1: bool = None, dropout: Optional[float] = None, verbose_metrics: bool = False, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) if pos_tag_loss is not None or pos_tag_embedding is not None: pos_tag_err = (f"Model uses POS tags but the Vocabulary {vocab} " "does not contain `pos_tags` namespace") if 'pos_tags' not in vocab._token_to_index: raise ConfigurationError(pos_tag_err) elif not len(vocab._token_to_index['pos_tags']): raise ConfigurationError(pos_tag_err) self.label_namespace = label_namespace self.text_field_embedder = text_field_embedder self.pos_tag_embedding = pos_tag_embedding self.num_tags = self.vocab.get_vocab_size(label_namespace) self.encoder = encoder self._verbose_metrics = verbose_metrics embedding_output_dim = self.text_field_embedder.get_output_dim() if self.pos_tag_embedding is not None: embedding_output_dim += self.pos_tag_embedding.get_output_dim() if dropout is not None: self.dropout = torch.nn.Dropout(dropout) self.variational_dropout = InputVariationalDropout(dropout) else: self.dropout = None self._feedforward = feedforward if feedforward is not None: output_dim = feedforward.get_output_dim() elif encoder is not None: output_dim = self.encoder.get_output_dim() else: output_dim = embedding_output_dim self.tag_projection_layer = TimeDistributed( Linear(output_dim, self.num_tags)) self.pos_tag_loss = pos_tag_loss if self.pos_tag_loss: self.num_pos_tags = self.vocab.get_vocab_size("pos_tags") self.pos_tag_projection_layer = TimeDistributed( Linear(output_dim, self.num_pos_tags)) self.pos_crf = None if crf: self.pos_crf = ConditionalRandomField(self.num_pos_tags, None, False) # if constrain_crf_decoding and calculate_span_f1 are not # provided, (i.e., they're None), set them to True # if label_encoding is provided and False if it isn't. if crf: if constrain_crf_decoding is None: constrain_crf_decoding = label_encoding is not None if calculate_span_f1 is None: calculate_span_f1 = label_encoding is not None self.label_encoding = label_encoding if constrain_crf_decoding and crf: if not label_encoding: raise ConfigurationError("constrain_crf_decoding is True, but " "no label_encoding was specified.") labels = self.vocab.get_index_to_token_vocabulary(label_namespace) constraints = allowed_transitions(label_encoding, labels) else: constraints = None if crf: self.include_start_end_transitions = include_start_end_transitions self.crf = ConditionalRandomField( self.num_tags, constraints, include_start_end_transitions=include_start_end_transitions) else: self.crf = None self.metrics = { "accuracy": CategoricalAccuracy(), "accuracy3": CategoricalAccuracy(top_k=3) } self.calculate_span_f1 = calculate_span_f1 if calculate_span_f1 is not None: if not label_encoding: raise ConfigurationError("calculate_span_f1 is True, but " "no label_encoding was specified.") self._f1_metric = SpanBasedF1Measure(vocab, tag_namespace=label_namespace, label_encoding=label_encoding) # If performing POS tagging would be good to keep updated on POS # accuracy if self.pos_tag_loss: self.metrics['POS_accuracy'] = CategoricalAccuracy() if encoder is not None: check_dimensions_match(embedding_output_dim, encoder.get_input_dim(), "text field embedding dim", "encoder input dim") if feedforward is not None and encoder is not None: check_dimensions_match(encoder.get_output_dim(), feedforward.get_input_dim(), "encoder output dim", "feedforward input dim") elif feedforward is not None and encoder is None: check_dimensions_match(embedding_output_dim, feedforward.get_input_dim(), "text field output dim", "feedforward input dim") initializer(self)
def __init__(self, **kwargs): super().__init__() content_id_size = kwargs["content_id_size"] content_id_dim = kwargs["content_id_dim"] self.content_id_emb = nn.Embedding(num_embeddings=content_id_size, embedding_dim=content_id_dim, padding_idx=0) if kwargs["emb_dropout"] > 0: self.emb_dropout = InputVariationalDropout(p=kwargs["emb_dropout"]) feature_dim = kwargs["feature_dim"] + kwargs["content_id_dim"] if not kwargs.get("no_prev_ans", False): feature_dim += 1 if "lstm_in_dim" in kwargs and kwargs["lstm_in_dim"] != feature_dim: lstm_in_dim = kwargs["lstm_in_dim"] self.lstm_in_proj = nn.Linear(in_features=feature_dim, out_features=lstm_in_dim, bias=True) else: lstm_in_dim = feature_dim lstm_hidden_dim = kwargs["lstm_hidden_dim"] lstm_num_layers = kwargs["lstm_num_layers"] lstm_dropout = kwargs["lstm_dropout"] self.encoder_type = kwargs.get("encoder_type", "vanilla_lstm") if self.encoder_type == "vanilla_lstm": self.encoder = nn.LSTM(input_size=lstm_in_dim, hidden_size=lstm_hidden_dim, bidirectional=False, batch_first=True, num_layers=lstm_num_layers, dropout=lstm_dropout) elif self.encoder_type == "GRU": self.encoder = nn.GRU(input_size=lstm_in_dim, hidden_size=lstm_hidden_dim, bidirectional=False, batch_first=True, num_layers=lstm_num_layers, dropout=lstm_dropout) elif self.encoder_type == "attention": self.encoder = nn.MultiheadAttention(embed_dim=lstm_in_dim, num_heads=lstm_num_layers, dropout=lstm_dropout) elif self.encoder_type == "augmented_lstm": self.encoder = StackedAugmentedLSTM( input_size=lstm_in_dim, hidden_size=lstm_hidden_dim, num_layers=lstm_num_layers, recurrent_dropout_probability=lstm_dropout, use_highway=kwargs.get("lstm_use_highway", True)) if kwargs.get("layer_norm", False): self.layer_norm = nn.LayerNorm(lstm_hidden_dim) if kwargs["output_dropout"] > 0: self.output_dropout = InputVariationalDropout( p=kwargs["output_dropout"]) if kwargs.get("highway_connection", False): self.highway_H = nn.Linear(in_features=lstm_in_dim, out_features=lstm_hidden_dim) self.highway_C = nn.Linear(in_features=lstm_in_dim, out_features=lstm_hidden_dim) hidden2logit_num_layers = kwargs.get("hidden2logit_num_layers", 1) self.hidden2logit = [] in_features = lstm_hidden_dim for i in range(hidden2logit_num_layers): if i == hidden2logit_num_layers - 1: self.hidden2logit.append( nn.Linear(in_features=in_features, out_features=1)) else: self.hidden2logit.append( nn.Linear(in_features=in_features, out_features=in_features // 2)) self.hidden2logit.append(nn.ReLU()) self.hidden2logit.append( nn.Dropout(p=kwargs["output_dropout"])) in_features = in_features // 2 self.hidden2logit = nn.Sequential(*self.hidden2logit) if "content_adj_mat" in kwargs and "smoothness_alpha" in kwargs: # normalize adjacency weight with w_ij/(sqrt(d_i)*sqrt(d_j)) adjacency_mat = kwargs["content_adj_mat"] del kwargs["content_adj_mat"] degree_mat = torch.clamp(adjacency_mat.sum(dim=1), 1e-8) inv_degree_mat = torch.diag(torch.pow(degree_mat, -0.5)) self.content_id_adj = inv_degree_mat @ adjacency_mat @ inv_degree_mat self.smoothness_alpha = kwargs["smoothness_alpha"] self.criterion = self.BCE_logit_emb_smooth_loss else: self.criterion = self.__class__.binary_cross_entropy_with_logits self.hparams = kwargs
def __init__(self, vocab: Vocabulary, char_embedder: TextFieldEmbedder, word_embedder: TextFieldEmbedder, tokens_encoder: Seq2SeqEncoder, model_args, inp_drop_rate: float = 0.5, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: """ :param vocab: vocabulary from train and dev dataset :param char_embedder: character embedding + cnn encoder :param word_embedder: word embedding :param tokens_encoder: Bi-LSTM backbone for split :param model_args: model arguments :param inp_drop_rate: input dropout rate """ super(FollowUpSnippetModel, self).__init__(vocab, regularizer) self.tokens_encoder = tokens_encoder self.projection_layer = torch.nn.Linear( in_features=word_embedder.get_output_dim() + 1 + char_embedder.get_output_dim(), out_features=self.tokens_encoder.get_input_dim(), bias=False) # integer to mark field, 0 or 1 self.num_classes = 2 self.num_conflicts = 2 self._non_linear = torch.nn.PReLU() self.hidden_size = int(self.tokens_encoder.get_output_dim() / 2) self.policy_net = PolicyNet(self.tokens_encoder.get_output_dim() * 3, self.num_classes) self.token_field_embedding = word_embedder self.char_field_embedding = char_embedder self._scaled_value = 1.0 self._self_attention = CosineMatrixAttention() self.margin_loss = MarginRankingLoss(margin=model_args.margin) # calculate span similarity self.cosine_similar = CosineSimilarity(dim=0) if inp_drop_rate > 0: self._variational_dropout = InputVariationalDropout(p=inp_drop_rate) else: self._variational_dropout = lambda x: x self.metrics = { "bleu": BLEUScore(), "reward": RewardScore(), "symbol": SymbolScore(), "reward_var": RewardScore(), "overall": RewardScore() } initializer(self)
def __init__(self, word_to_index, ext_word_to_index, char_to_index, char_embed_size, pos_embed_size, char_lstm_hidden_size, word_embed_size, ext_embed_tensor, num_layers, hidden_size, dropout, r_dropout, mlp_arc_size, mlp_label_size, pos_to_index, xpos_to_index, rel_to_index, cuda_flag, batch_first=True): """ Args: word_to_index: char_to_index: char_embed_size: word embeddings dim pos_embed_size: pos/xpos embeddings dim char_lstm_hidden_size: char LSTM reccurent dim word_embed_size: word embeddings dim num_layers: Bi-LSTM of layers hidden_size: Bi-lstm reccurent dim dropout: dropout probability r_dropout: dropout probability for reccurent units (Gal & Grahami) mlp_arc_size: arc mlp hidden dim mlp_label_size: label mlp hidden dim pos_to_index: xpos_to_index: rel_to_index: cuda_flag: batch_first: batch first option """ super(RNN, self).__init__() self.word_to_index = word_to_index self.ext_word_to_index = ext_word_to_index self.char_to_index = char_to_index self.char_embed_size = char_embed_size self.pos_embed_size = pos_embed_size self.char_lstm_hidden_size = char_lstm_hidden_size self.word_embed_size = word_embed_size self.num_layers = num_layers self.hidden_size = hidden_size self.dropout = dropout self.r_dropout = r_dropout self.mlp_arc_size = mlp_arc_size self.mlp_label_size = mlp_label_size self.pos_to_index = pos_to_index self.xpos_to_index = xpos_to_index self.rel_to_index = rel_to_index self.rels = len(self.rel_to_index) self.cuda_flag = cuda_flag self.batch_first = batch_first self.word_encoder = nn.Embedding( len(self.word_to_index), self.word_embed_size, sparse=True, padding_idx=self.word_to_index['__PADDING__']) torch.nn.init.xavier_uniform_(self.word_encoder.weight.data) self.pos_encoder = nn.Embedding( len(self.pos_to_index), self.pos_embed_size, sparse=True, padding_idx=self.pos_to_index['__PADDING__']) torch.nn.init.xavier_uniform_(self.pos_encoder.weight.data) self.xpos_encoder = nn.Embedding( len(self.xpos_to_index), self.pos_embed_size, sparse=True, padding_idx=self.xpos_to_index['__PADDING__']) torch.nn.init.xavier_uniform_(self.xpos_encoder.weight.data) self.ROOT = nn.Parameter( torch.zeros(self.word_embed_size + self.pos_embed_size)) torch.nn.init.normal_(self.ROOT, 0, 0.001) self.rnn = StackedBidirectionalLstm( input_size=self.word_embed_size + self.pos_embed_size, hidden_size=self.hidden_size, num_layers=self.num_layers, recurrent_dropout_probability=self.r_dropout, use_highway=False) self.char_embeds = Char_RNN(self.char_to_index, self.char_embed_size, self.char_lstm_hidden_size, self.word_embed_size, self.dropout, self.cuda_flag) self.fasttext_embs = Fasttext(ext_embed_tensor, self.word_embed_size, self.ext_word_to_index['__PADDING__'], self.cuda_flag) #Set forget bias to zero for layer_index in range(self.num_layers): eval( "self.rnn.forward_layer_{}.state_linearity.bias.data.fill_(0.0)" .format(layer_index)) eval( "self.rnn.backward_layer_{}.state_linearity.bias.data.fill_(0.0)" .format(layer_index)) self.var_drop = InputVariationalDropout(self.dropout) self.arc_head = nn.Linear(2 * self.hidden_size, self.mlp_arc_size, bias=True) self.arc_dep = nn.Linear(2 * self.hidden_size, self.mlp_arc_size, bias=True) torch.nn.init.orthogonal_(self.arc_head.weight.data, gain=np.sqrt(2 / (1 + 0.1**2))) torch.nn.init.orthogonal_(self.arc_dep.weight.data, gain=np.sqrt(2 / (1 + 0.1**2))) self.arc_head.bias.data.fill_(0.0) self.arc_head.bias.data.fill_(0.0) self.label_head = nn.Linear(2 * self.hidden_size, self.mlp_label_size, bias=True) self.label_dep = nn.Linear(2 * self.hidden_size, self.mlp_label_size, bias=True) torch.nn.init.orthogonal_(self.label_head.weight.data, gain=np.sqrt(2 / (1 + 0.1**2))) torch.nn.init.orthogonal_(self.label_dep.weight.data, gain=np.sqrt(2 / (1 + 0.1**2))) self.label_head.bias.data.fill_(0.0) self.label_head.bias.data.fill_(0.0) #Add biaffine layers #self.arc_biaffine = Biaffine(self.mlp_arc_size, self.mlp_arc_size, 1, bias=(True, False, False)) #self.label_biaffine = Biaffine(self.mlp_label_size, self.mlp_label_size,self.rels,bias=(True, True, True)) self.arc_biaf = nn.Parameter( torch.zeros(self.mlp_arc_size, self.mlp_arc_size)) self.arc_head_aff = nn.Linear(self.mlp_arc_size, 1, bias=False) self.arc_head_aff.weight.data.fill_(0.0) self.label_biaf = nn.Parameter( torch.zeros(self.mlp_label_size, self.mlp_label_size, self.rels)) self.label_head_aff = nn.Linear(self.mlp_label_size, self.rels, bias=False) self.label_head_aff.weight.data.fill_(0.0) self.label_dep_aff = nn.Linear(self.mlp_label_size, self.rels, bias=False) self.label_dep_aff.weight.data.fill_(0.0) self.label_bias = nn.Parameter(torch.zeros(self.rels)) self.Relu = nn.LeakyReLU(0.1) self.param_group_sparse = [] self.param_group_dense = [] for name, param in self.named_parameters(): if ((name == "word_encoder.weight") or (name == "char_embeds.char_encoder.weight") or (name == "pos_encoder.weight") or (name == "xpos_encoder.weight")): print("Sparse:", name) self.param_group_sparse.append(param) else: self.param_group_dense.append(param) self.param_group_sparse = iter(self.param_group_sparse) self.param_group_dense = iter(self.param_group_dense)