def __init__(self, word_vectors, char_vectors, hidden_size, drop_prob=0.): super(BiDAF_charCNN, self).__init__() self.emb = layers.Embedding(word_vectors=word_vectors, hidden_size=hidden_size, drop_prob=drop_prob) self.char_emb = layers.CharEmbedding(char_vectors=char_vectors, hidden_size=hidden_size, drop_prob=drop_prob) self.hwy = layers.HighwayEncoder(2, 2*hidden_size) self.enc = layers.RNNEncoder(input_size=2*hidden_size, hidden_size=2*hidden_size, num_layers=1, drop_prob=drop_prob) self.att = layers.BiDAFAttention(hidden_size=2 * 2*hidden_size, drop_prob=drop_prob) self.mod = layers.RNNEncoder(input_size=8 * 2*hidden_size, hidden_size=hidden_size, num_layers=2, drop_prob=drop_prob) self.out = layers.BiDAFOutput(hidden_size=hidden_size, drop_prob=drop_prob)
def __init__(self, word_vectors, char_vectors, hidden_size, drop_prob=0.,twist_embeddings=False): super(BiDAF_charCNN_BERTEnc_BERTMod, self).__init__() ### self.twist_embeddings = twist_embeddings idx_list = [] for i in range(hidden_size): idx_list.append(i) idx_list.append(hidden_size+i) self.register_buffer('idx_twist',torch.tensor(idx_list)) ### self.emb = layers.Embedding(word_vectors=word_vectors, hidden_size=hidden_size, drop_prob=drop_prob) self.char_emb = layers.CharEmbedding(char_vectors=char_vectors, hidden_size=hidden_size, drop_prob=drop_prob) self.hwy = layers.HighwayEncoder(2, 2*hidden_size) self.enc = bert_layers.BertEncoder(n_layers=3, #n_layers=4, d_feature=2*hidden_size, n_heads=8, out_size=2*hidden_size, #d_ff=2048, d_ff = 2*hidden_size, dropout_prob=0.1, #dropout_prob=drop_prob, ff_activation=F.relu) self.att = layers.BiDAFAttention(hidden_size=2 * hidden_size, drop_prob=drop_prob) self.mod = bert_layers.BertEncoder(n_layers=3, #n_layers=3, d_feature=8*hidden_size, n_heads=8, out_size=2*hidden_size, #d_ff=2048, d_ff = 2*hidden_size, dropout_prob=0.1, #dropout_prob=drop_prob, ff_activation=F.relu) # self.mod = layers.RNNEncoder(input_size=8 * hidden_size, # hidden_size=hidden_size, # num_layers=2, # drop_prob=drop_prob) self.out = layers.BiDAFOutput(hidden_size=hidden_size, drop_prob=drop_prob)
def __init__(self, word_vectors, char_vectors, hidden_size, drop_prob=0.): super(BiDAF, self).__init__() self.hidden_size = hidden_size self.word_emb = layers.WordEmbedding(word_vectors, hidden_size) self.char_emb = layers.CharEmbedding(char_vectors, hidden_size) # assert hidden_size * 2 == (char_channel_size + word_dim) # highway network self.hwy = layers.HighwayEncoder(2, hidden_size * 2) # highway network # for i in range(2): # setattr(self, f'highway_linear{i}', nn.Sequential( # nn.Linear(hidden_size * 2, hidden_size * 2), nn.ReLU())) # setattr(self, f'hightway_gate{i}', nn.Sequential( # nn.Linear(hidden_size * 2, hidden_size * 2), nn.Sigmoid())) # self.emb = layers.Embedding(word_vectors=word_vectors, # hidden_size=hidden_size, # drop_prob=drop_prob) self.enc = layers.RNNEncoder(input_size=hidden_size * 2, hidden_size=hidden_size, num_layers=1, drop_prob=drop_prob) self.att = layers.BiDAFAttention(hidden_size=2 * hidden_size, drop_prob=drop_prob) self.mod = layers.RNNEncoder(input_size=8 * hidden_size, hidden_size=hidden_size, num_layers=2, drop_prob=drop_prob) self.out = layers.BiDAFOutput(hidden_size=hidden_size, drop_prob=drop_prob)
def __init__(self, word_vectors, char_vectors, device, hidden_size, drop_prob=0.): super(RNet, self).__init__() self.device = device self.word_emb = nn.Embedding.from_pretrained(word_vectors) self.char_emb = layers.CharEmbedding(char_vectors=char_vectors, e_char=char_vectors.size(1), e_word=word_vectors.size(1), drop_prob=drop_prob, freeze=False) self.proj = nn.Linear(word_vectors.size(1) * 2, hidden_size, bias=False) self.hwy = layers.HighwayEncoder(2, hidden_size) self.encoder = Encoder(input_size=hidden_size, h_size=hidden_size, device=device, drop_prob=drop_prob) self.gatedAttn = GatedAttn(input_size=hidden_size, h_size=hidden_size, device=device, drop_prob=drop_prob) self.selfAttn = SelfAttn(self.gatedAttn.out_size, device=device, drop_prob=drop_prob) self.pointer = Pointer(self.selfAttn.out_size, self.encoder.out_size, device=device)
def __init__(self, word_vectors, hidden_size, use_char=False, char_vectors=None, use_syll=False, syll_vectors=None, drop_prob=0.): super(BiDAF, self).__init__() self.word_emb_size = word_vectors.size(1) self.emb = layers.WordEmbedding(word_vectors=word_vectors, hidden_size=hidden_size, drop_prob=drop_prob) self.use_char = use_char self.use_syll = use_syll if use_char and use_syll: self.char_emb = layers.CharEmbedding(char_vectors, e_char=char_vectors.size(1), e_word=hidden_size, drop_prob=drop_prob, freeze=False) self.syll_emb = layers.SyllEmbedding(syll_vectors, e_syll=syll_vectors.size(1), e_word=hidden_size, drop_prob=drop_prob, freeze=False) self.input_size = self.word_emb_size + 2 * hidden_size elif use_char: self.char_emb = layers.CharEmbedding(char_vectors, e_char=char_vectors.size(1), e_word=hidden_size, drop_prob=drop_prob, freeze=False) self.input_size = self.word_emb_size + hidden_size elif use_syll: self.syll_emb = layers.SyllEmbedding(syll_vectors, e_syll=syll_vectors.size(1), e_word=hidden_size, drop_prob=drop_prob, freeze=False) self.input_size = self.word_emb_size + hidden_size else: self.input_size = self.word_emb_size self.proj = nn.Linear(self.input_size, hidden_size, bias=False) self.hwy = layers.HighwayEncoder(2, hidden_size) self.enc = layers.RNNEncoder(input_size=hidden_size, hidden_size=hidden_size, num_layers=1, drop_prob=drop_prob) self.att = layers.BiDAFAttention(hidden_size=2 * hidden_size, drop_prob=drop_prob) self.mod = layers.RNNEncoder(input_size=8 * hidden_size, hidden_size=hidden_size, num_layers=2, drop_prob=drop_prob) self.out = layers.BiDAFOutput(hidden_size=hidden_size, drop_prob=drop_prob)