def test_case3(self): # 测试crf的loss不会出现负数 import torch from fastNLP.modules.decoder.crf import ConditionalRandomField from fastNLP.core.utils import seq_len_to_mask from torch import optim from torch import nn num_tags, include_start_end_trans = 4, True num_samples = 4 lengths = torch.randint(3, 50, size=(num_samples, )).long() max_len = lengths.max() tags = torch.randint(num_tags, size=(num_samples, max_len)) masks = seq_len_to_mask(lengths) feats = nn.Parameter(torch.randn(num_samples, max_len, num_tags)) crf = ConditionalRandomField(num_tags, include_start_end_trans) optimizer = optim.SGD( [param for param in crf.parameters() if param.requires_grad] + [feats], lr=0.1) for _ in range(10): loss = crf(feats, tags, masks).mean() optimizer.zero_grad() loss.backward() optimizer.step() if _ % 1000 == 0: print(loss) self.assertGreater(loss.item(), 0, "CRF loss cannot be less than 0.")
def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, hidden_size=200, bidirectional=True, embed_drop_p=0.2, num_layers=1, tag_size=4): """ 默认使用BMES的标注方式 :param vocab_num: :param embed_dim: :param bigram_vocab_num: :param bigram_embed_dim: :param num_bigram_per_char: :param hidden_size: :param bidirectional: :param embed_drop_p: :param num_layers: :param tag_size: """ super(CWSBiLSTMCRF, self).__init__() self.tag_size = tag_size self.encoder_model = CWSBiLSTMEncoder(vocab_num, embed_dim, bigram_vocab_num, bigram_embed_dim, num_bigram_per_char, hidden_size, bidirectional, embed_drop_p, num_layers) size_layer = [hidden_size, 200, tag_size] self.decoder_model = MLP(size_layer) allowed_trans = allowed_transitions({0:'b', 1:'m', 2:'e', 3:'s'}, encoding_type='bmes') self.crf = ConditionalRandomField(num_tags=tag_size, include_start_end_trans=False, allowed_transitions=allowed_trans)
def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, embed_drop_p=0.3, hidden_size=200, kernel_size=3, dilate='none', num_layers=1, num_heads=8, tag_size=4, relative_pos_embed_dim=0): super().__init__() self.embedding = nn.Embedding(vocab_num, embed_dim) input_size = embed_dim if bigram_vocab_num: self.bigram_embedding = nn.Embedding(bigram_vocab_num, bigram_embed_dim) input_size += num_bigram_per_char * bigram_embed_dim self.drop = nn.Dropout(embed_drop_p, inplace=True) self.fc1 = nn.Linear(input_size, hidden_size) # value_size = hidden_size//num_heads # self.transformer = TransformerEncoder(num_layers, model_size=hidden_size, inner_size=hidden_size, # key_size=value_size, # value_size=value_size, num_head=num_heads) self.transformer = TransformerDilateEncoder( num_layers=num_layers, model_size=hidden_size, num_heads=num_heads, hidden_size=hidden_size, kernel_size=kernel_size, dilate=dilate, relative_pos_embed_dim=relative_pos_embed_dim) self.fc2 = nn.Linear(hidden_size, tag_size) allowed_trans = allowed_transitions({ 0: 'b', 1: 'm', 2: 'e', 3: 's' }, encoding_type='bmes') self.crf = ConditionalRandomField(num_tags=tag_size, include_start_end_trans=False, allowed_transitions=allowed_trans)
def __init__(self, encoder, src_embed, position, d_model, tag_size, crf=None): super(CWSModel, self).__init__() self.encoder = encoder self.src_embed = src_embed self.pos = copy.deepcopy(position) self.proj = nn.Linear(d_model, tag_size) self.tag_size = tag_size if crf is None: self.crf = None self.loss_f = nn.CrossEntropyLoss(reduction="mean", ignore_index=-100) else: print("crf") trans = fastNLP.modules.decoder.crf.allowed_transitions( crf, encoding_type="bmes" ) self.crf = ConditionalRandomField(tag_size, allowed_transitions=trans)
def test_masking(self): # 测试crf的pad masking正常运行 import torch from fastNLP.modules.decoder.crf import ConditionalRandomField max_len = 5 n_tags = 5 pad_len = 5 torch.manual_seed(4) logit = torch.rand(1, max_len+pad_len, n_tags) # logit[0, -1, :] = 0.0 mask = torch.ones(1, max_len+pad_len) mask[0,-pad_len] = 0 model = ConditionalRandomField(n_tags) pred, score = model.viterbi_decode(logit[:,:-pad_len], mask[:,:-pad_len]) mask_pred, mask_score = model.viterbi_decode(logit, mask) self.assertEqual(pred[0].tolist(), mask_pred[0,:-pad_len].tolist())
class CWSModel(nn.Module): def __init__(self, encoder, src_embed, position, d_model, tag_size, crf=None): super(CWSModel, self).__init__() self.encoder = encoder self.src_embed = src_embed self.pos = copy.deepcopy(position) self.proj = nn.Linear(d_model, tag_size) self.tag_size = tag_size if crf is None: self.crf = None self.loss_f = nn.CrossEntropyLoss(size_average=False) else: print("crf") trans = fastNLP.modules.decoder.crf.allowed_transitions( crf, encoding_type='bmes') self.crf = ConditionalRandomField(tag_size, allowed_transitions=trans) #self.norm=nn.LayerNorm(d_model) def forward(self, task, uni, seq_len, bi1=None, bi2=None, tags=None): #mask=fastNLP.core.utils.seq_len_to_mask(seq_len) mask = seq_len_to_mask(seq_len, uni.size(1)) out = self.src_embed(task, uni, bi1, bi2) out = self.pos(out) #out=self.norm(out) #print(uni.size(),out.size(),mask.size(),seq_len) out = self.proj(self.encoder(out, mask.float())) if self.crf is not None: if tags is not None: out = self.crf(out, tags, mask) return {"loss": out} else: out, _ = self.crf.viterbi_decode(out, mask) return {"pred": out} else: if tags is not None: num = out.size(0) loss = self.loss_f( torch.masked_select( out, mask.unsqueeze(-1).expand_as(out)).contiguous().view( -1, self.tag_size), torch.masked_select(tags, mask)) return {"loss": loss / num} else: out = torch.argmax(out, dim=-1) return {"pred": out}
def test_case2(self): # 测试CRF是否正常work。 import json import torch from fastNLP import seq_len_to_mask with open('tests/data_for_tests/modules/decoder/crf.json', 'r') as f: data = json.load(f) bio_logits = torch.FloatTensor(data['bio_logits']) bio_scores = data['bio_scores'] bio_path = data['bio_path'] bio_trans_m = torch.FloatTensor(data['bio_trans_m']) bio_seq_lens = torch.LongTensor(data['bio_seq_lens']) bmes_logits = torch.FloatTensor(data['bmes_logits']) bmes_scores = data['bmes_scores'] bmes_path = data['bmes_path'] bmes_trans_m = torch.FloatTensor(data['bmes_trans_m']) bmes_seq_lens = torch.LongTensor(data['bmes_seq_lens']) labels = ['O'] for label in ['X', 'Y']: for tag in 'BI': labels.append('{}-{}'.format(tag, label)) id2label = {idx: label for idx, label in enumerate(labels)} num_tags = len(id2label) mask = seq_len_to_mask(bio_seq_lens) from fastNLP.modules.decoder.crf import ConditionalRandomField, allowed_transitions fast_CRF = ConditionalRandomField(num_tags=num_tags, allowed_transitions=allowed_transitions(id2label, include_start_end=True)) fast_CRF.trans_m.data = bio_trans_m fast_res = fast_CRF.viterbi_decode(bio_logits, mask, unpad=True) # score equal self.assertListEqual(bio_scores, [round(s, 4) for s in fast_res[1].tolist()]) # seq equal self.assertListEqual(bio_path, fast_res[0]) labels = [] for label in ['X', 'Y']: for tag in 'BMES': labels.append('{}-{}'.format(tag, label)) id2label = {idx: label for idx, label in enumerate(labels)} num_tags = len(id2label) mask = seq_len_to_mask(bmes_seq_lens) from fastNLP.modules.decoder.crf import ConditionalRandomField, allowed_transitions fast_CRF = ConditionalRandomField(num_tags=num_tags, allowed_transitions=allowed_transitions(id2label, encoding_type='BMES', include_start_end=True)) fast_CRF.trans_m.data = bmes_trans_m fast_res = fast_CRF.viterbi_decode(bmes_logits, mask, unpad=True) # score equal self.assertListEqual(bmes_scores, [round(s, 4) for s in fast_res[1].tolist()]) # seq equal self.assertListEqual(bmes_path, fast_res[0])
class CWSModel(nn.Module): def __init__(self, encoder, src_embed, position, d_model, tag_size, crf=None): super(CWSModel, self).__init__() self.encoder = encoder self.src_embed = src_embed self.pos = copy.deepcopy(position) self.proj = nn.Linear(d_model, tag_size) self.tag_size = tag_size if crf is None: self.crf = None self.loss_f = nn.CrossEntropyLoss(reduction="mean", ignore_index=-100) else: print("crf") trans = fastNLP.modules.decoder.crf.allowed_transitions( crf, encoding_type="bmes" ) self.crf = ConditionalRandomField(tag_size, allowed_transitions=trans) # self.norm=nn.LayerNorm(d_model) def forward(self, task, uni, seq_len, bi1=None, bi2=None, tags=None): # mask=fastNLP.core.utils.seq_len_to_mask(seq_len,uni.size(1)) # for dev 0.5.1 mask = seq_len_to_mask(seq_len, uni.size(1)) out = self.src_embed(task, uni, bi1, bi2) out = self.pos(out) # out=self.norm(out) out = self.proj(self.encoder(out, mask.float())) if self.crf is not None: if tags is not None: out = self.crf(out, tags, mask) return {"loss": out} else: out, _ = self.crf.viterbi_decode(out, mask) return {"pred": out} else: if tags is not None: out = out.contiguous().view(-1, self.tag_size) tags = tags.data.masked_fill_(mask.eq(False), -100).view(-1) loss = self.loss_f(out, tags) return {"loss": loss} else: out = torch.argmax(out, dim=-1) return {"pred": out}
class TransformerCWS(nn.Module): def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, hidden_size=200, embed_drop_p=0.3, num_layers=1, num_heads=8, tag_size=4): super().__init__() self.embedding = nn.Embedding(vocab_num, embed_dim) input_size = embed_dim if bigram_vocab_num: self.bigram_embedding = nn.Embedding(bigram_vocab_num, bigram_embed_dim) input_size += num_bigram_per_char * bigram_embed_dim self.drop = nn.Dropout(embed_drop_p, inplace=True) self.fc1 = nn.Linear(input_size, hidden_size) # value_size = hidden_size//num_heads # self.transformer = TransformerEncoder(num_layers, model_size=hidden_size, inner_size=hidden_size, # key_size=value_size, # value_size=value_size, num_head=num_heads) self.transformer = TransformerEncoder(num_layers=num_layers, model_size=hidden_size, num_heads=num_heads, hidden_size=hidden_size) self.fc2 = nn.Linear(hidden_size, tag_size) allowed_trans = allowed_transitions({ 0: 'b', 1: 'm', 2: 'e', 3: 's' }, encoding_type='bmes') self.crf = ConditionalRandomField(num_tags=tag_size, include_start_end_trans=False, allowed_transitions=allowed_trans) def forward(self, chars, target, seq_lens, bigrams=None): masks = seq_len_to_byte_mask(seq_lens) x = self.embedding(chars) batch_size = x.size(0) length = x.size(1) if hasattr(self, 'bigram_embedding'): bigrams = self.bigram_embedding( bigrams) # batch_size x seq_lens x per_char x embed_size x = torch.cat([x, bigrams.view(batch_size, length, -1)], dim=-1) self.drop(x) x = self.fc1(x) feats = self.transformer(x, masks) feats = self.fc2(feats) losses = self.crf(feats, target, masks.float()) pred_dict = {} pred_dict['seq_lens'] = seq_lens pred_dict['loss'] = torch.mean(losses) return pred_dict def predict(self, chars, seq_lens, bigrams=None): masks = seq_len_to_byte_mask(seq_lens) x = self.embedding(chars) batch_size = x.size(0) length = x.size(1) if hasattr(self, 'bigram_embedding'): bigrams = self.bigram_embedding( bigrams) # batch_size x seq_lens x per_char x embed_size x = torch.cat([x, bigrams.view(batch_size, length, -1)], dim=-1) self.drop(x) x = self.fc1(x) feats = self.transformer(x, masks) feats = self.fc2(feats) probs = self.crf.viterbi_decode(feats, masks, get_score=False) return {'pred': probs, 'seq_lens': seq_lens}
class CWSBiLSTMCRF(BaseModel): def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, hidden_size=200, bidirectional=True, embed_drop_p=0.2, num_layers=1, tag_size=4): """ 默认使用BMES的标注方式 :param vocab_num: :param embed_dim: :param bigram_vocab_num: :param bigram_embed_dim: :param num_bigram_per_char: :param hidden_size: :param bidirectional: :param embed_drop_p: :param num_layers: :param tag_size: """ super(CWSBiLSTMCRF, self).__init__() self.tag_size = tag_size self.encoder_model = CWSBiLSTMEncoder(vocab_num, embed_dim, bigram_vocab_num, bigram_embed_dim, num_bigram_per_char, hidden_size, bidirectional, embed_drop_p, num_layers) size_layer = [hidden_size, 200, tag_size] self.decoder_model = MLP(size_layer) allowed_trans = allowed_transitions({ 0: 'b', 1: 'm', 2: 'e', 3: 's' }, encoding_type='bmes') self.crf = ConditionalRandomField(num_tags=tag_size, include_start_end_trans=False, allowed_transitions=allowed_trans) def forward(self, chars, target, seq_lens, bigrams=None): device = self.parameters().__next__().device chars = chars.to(device).long() if not bigrams is None: bigrams = bigrams.to(device).long() else: bigrams = None seq_lens = seq_lens.to(device).long() masks = seq_lens_to_mask(seq_lens) feats = self.encoder_model(chars, bigrams, seq_lens) feats = self.decoder_model(feats) losses = self.crf(feats, target, masks) pred_dict = {} pred_dict['seq_lens'] = seq_lens pred_dict['loss'] = torch.mean(losses) return pred_dict def predict(self, chars, seq_lens, bigrams=None): device = self.parameters().__next__().device chars = chars.to(device).long() if not bigrams is None: bigrams = bigrams.to(device).long() else: bigrams = None seq_lens = seq_lens.to(device).long() masks = seq_lens_to_mask(seq_lens) feats = self.encoder_model(chars, bigrams, seq_lens) feats = self.decoder_model(feats) paths, _ = self.crf.viterbi_decode(feats, masks) return {'pred': paths, 'seq_lens': seq_lens}