예제 #1
0
 def __init__(self, model_name_or_path, dropout=0.1):
     super(MyModel, self).__init__()
     self.bert = BertModel.from_pretrained(model_name_or_path)
     for param in self.bert.parameters():
         param.requires_grad = True
     self.linear1 = nn.Linear(768, 768)
     self.linear2 = nn.Linear(768, 1)
     self.dropout = dropout
예제 #2
0
    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels

        self.bert = BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        self.init_weights()
예제 #3
0
    def __init__(self, config: Config, train_config: TrainerConfig):
        super(BertForIntentDetection, self).__init__()
        self.config = config
        self.train_config = train_config

        # 1. init the bert model
        bert_config = BertConfig.from_pretrained(config.pretrain_model)
        self.encoder = BertModel(bert_config)

        self.loss_function = nn.LogSoftmax(dim=-1)
예제 #4
0
    def __init__(self, model_name_or_path, dropout=0.1):
        super().__init__()
        # self.bert = AutoModel.from_pretrained(model_name_or_path)
        self.bert = BertModel.from_pretrained(model_name_or_path)

        # self.bert.requires_grad_(True)
        for param in self.bert.parameters():
            param.requires_grad = True
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, 1)
        self.n_class = 4
예제 #5
0
class BertComputation(RequestHandler):
    """
    Request handler that computes embeddings on english sentences
    """

    enable_cuda = True if torch.cuda.is_available() else False
    CLS = '[CLS]'
    SEP = '[SEP]'
    MSK = '[MASK]'
    pretrained_weights = 'bert-base-uncased'
    tokenizer = BertTokenizer.from_pretrained(pretrained_weights)
    model = BertModel(config=BertConfig())
    model.eval()
    if enable_cuda:
        model.cuda(torch.device('cuda'))

    def process_request(self, data):
        tokens = []
        mask_ids = []
        seg_ids = []
        seq_len = 0
        for txt in data:
            utt = txt.strip().lower()
            toks = self.tokenizer.tokenize(utt)
            if len(toks) > max_seq_len - 2:
                toks = toks[:max_seq_len - 2]
            toks.insert(0, self.CLS)
            toks.insert(-1, self.SEP)
            seq_len = max(len(toks), seq_len)
            mask_ids.append([1] * len(toks) + [0] * (seq_len - len(toks)))
            seg_ids.append([0] * seq_len)
            tokens.append(toks + [self.MSK] * (seq_len - len(toks)))
        input_ids = []
        for i in range(len(tokens)):
            for _ in range(seq_len - len(tokens[i])):
                tokens[i].append(self.MSK)
                mask_ids[i].append(0)
                seg_ids[i].append(0)
            input_ids.append(self.tokenizer.convert_tokens_to_ids(tokens[i]))
        return (torch.tensor(input_ids), torch.tensor(mask_ids),
                torch.tensor(seg_ids))

    def run_inference(self, model_input):
        if self.enable_cuda:
            input_ids, mask_ids, seg_ids = [x.to('cuda') for x in model_input]
        else:
            input_ids, mask_ids, seg_ids = model_input
        cls_emb = self.model.forward(input_ids=input_ids,
                                     attention_mask=mask_ids,
                                     token_type_ids=seg_ids)[1]
        return cls_emb

    def process_response(self, model_output_item):
        return encode_pickle(model_output_item.cpu().detach().numpy())
예제 #6
0
    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels

        self.bert = BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
        self.loss_type = config.loss_type
        self.focalloss_gamma = config.focalloss_gamma
        self.focalloss_alpha = config.focalloss_alpha
        self.diceloss_weight = config.diceloss_weight

        self.init_weights()
예제 #7
0
 def __init__(self,
              model_name_or_path,
              hidden_size=768,
              dropout=0.1,
              num_choices=4):
     super(BertForMultipleChoiceWithMatch, self).__init__()
     self.num_choices = num_choices
     self.bert = BertModel.from_pretrained(model_name_or_path)
     self.dropout = nn.Dropout(dropout)
     # self.classifier = nn.Linear(hidden_size, 1)
     # self.classifier2 = nn.Linear(2 * hidden_size, 1)
     self.classifier3 = nn.Linear(3 * hidden_size, 1)
     # self.classifier4 = nn.Linear(4 * hidden_size, 1)
     # self.classifier6 = nn.Linear(6 * hidden_size, 1)
     self.ssmatch = SSingleMatchNet(hidden_size, dropout)
     self.fuse = FuseNet(hidden_size)
예제 #8
0
 def __init__(self, config):
     super(BertCRFForAttr, self).__init__(config)
     self.bert = BertModel(config)
     self.dropout = nn.Dropout(config.hidden_dropout_prob)
     self.t_lstm = nn.LSTM(input_size=config.hidden_size,
                           hidden_size=config.hidden_size // 2,
                           batch_first=True,
                           bidirectional=True)
     self.a_lstm = nn.LSTM(input_size=config.hidden_size,
                           hidden_size=config.hidden_size // 2,
                           batch_first=True,
                           bidirectional=True)
     self.attention = CosAttention()
     self.ln = LayerNorm(config.hidden_size * 2)
     self.classifier = nn.Linear(config.hidden_size * 2, config.num_label)
     self.crf = CRF(num_tags=config.num_labels, batch_first=True)
     self.init_weights()
예제 #9
0
파일: trainer.py 프로젝트: kelvincjr/myRepo
    def __init__(self, config):
        super(BertSpanForNer, self).__init__(config)
        self.soft_label = config.soft_label
        self.num_labels = config.num_labels
        self.loss_type = config.loss_type
        self.focalloss_gamma = config.focalloss_gamma
        self.focalloss_alpha = config.focalloss_alpha
        self.diceloss_weight = config.diceloss_weight

        self.bert = BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.start_fc = PoolerStartLogits(config.hidden_size, self.num_labels)
        if self.soft_label:
            self.end_fc = PoolerEndLogits(config.hidden_size + self.num_labels,
                                          self.num_labels)
        else:
            self.end_fc = PoolerEndLogits(config.hidden_size + 1,
                                          self.num_labels)
        self.init_weights()
    def __init__(self, config: JointBertConfig):
        """ init JointBert model with configuration
        """
        super(JointBERT, self).__init__(config.bert_config)
        self.config: JointBertConfig = config

        self.bert: BertModel = BertModel(
            config=config.bert_config)  # Load pretrained bert

        self.intent_classifier = nn.Sequential(
            nn.Dropout(self.config.intent_dropout),
            nn.Linear(768, self.config.intent_size))

        self.slot_classifier = nn.Sequential(
            nn.Dropout(self.config.intent_dropout),
            nn.Linear(768, self.config.slot_label_size))

        if self.config.use_crf:
            self.crf = CRF(num_tags=self.config.slot_label_size,
                           batch_first=True)
예제 #11
0
파일: trainer.py 프로젝트: kelvincjr/myRepo
    def __init__(self, config):
        super(BertCrfForNer, self).__init__(config)
        #  self.num_labels = len(label2id)

        self.num_labels = config.num_labels
        self.crf_type = config.crf_type
        self.no_crf_loss = config.no_crf_loss
        self.loss_type = config.loss_type
        self.focalloss_gamma = config.focalloss_gamma
        self.focalloss_alpha = config.focalloss_alpha
        self.diceloss_weight = config.diceloss_weight

        self.bert = BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.classifier = nn.Linear(config.hidden_size, self.num_labels)

        label2id = config.label2id
        start_tag_idx = label2id["[CLS]"]
        stop_tag_idx = label2id["[SEP]"]

        tagset_size = len(label2id)

        # CRFPP
        if self.crf_type == 'ncrfpp':
            from ..models.ncrfpp_crf import CRF as CRFPP
            self.crf = CRFPP(tagset_size, start_tag_idx, stop_tag_idx)
        elif self.crf_type == 'pytorch-crf':
            from torchcrf import CRF
            # https://github.com/kmkurn/pytorch-crf/blob/master/torchcrf/__init__.py
            self.crf = CRF(len(label2id), batch_first=torch.BoolTensor([True]))
            #  self.crf = PyTorchCRF(tagset_size, start_tag_idx, stop_tag_idx)
        elif self.crf_type == 'old_crf':
            from ..models.crf_0 import CRF
            #  self.crf_0 = CRF0(tagset_size=len(label2id),
            #                    tag_dictionary=label2id,
            #                    device=device)
            self.crf_0 = CRF(tagset_size, start_tag_idx, stop_tag_idx)
            #  self.crf = OldCRF(tagset_size, start_tag_idx, stop_tag_idx)
        elif self.crf_type == 'new_crf':
            from ..models.crf import CRF
            self.crf = CRF(len(label2id), batch_first=True)
            # self.crf = NewCRF(tagset_size, start_tag_idx, stop_tag_idx)
        elif self.crf_type == 'lstm_crf':
            from ..models.lstm_crf import CRF
            self.crf = CRF(tagset_size, start_tag_idx, stop_tag_idx)
            #  self.crf = LSTMCRF(tagset_size, start_tag_idx, stop_tag_idx)

        # lstm_crf
        #  self.crf = CRF(tagset_size, start_tag_idx, stop_tag_idx)
        #  self.crf = LSTMCRF(tagset_size, start_tag_idx, stop_tag_idx)

        # TorchCRF
        #  self.crf = CRF(tagset_size, start_tag_idx, stop_tag_idx)
        #  self.crf = TorchCRF(tagset_size, start_tag_idx, stop_tag_idx)

        # torchcrf
        #  self.crf = CRF(len(label2id), batch_first=torch.BoolTensor([True]))
        #  self.crf = PyTorchCRF(tagset_size, start_tag_idx, stop_tag_idx)

        # NewCRF
        #  self.crf = CRF(len(label2id), batch_first=True)
        # self.crf = NewCRF(tagset_size, start_tag_idx, stop_tag_idx)

        #  self.crf_0 = CRF0(tagset_size=len(label2id),
        #                    tag_dictionary=label2id,
        #                    device=device)
        #  self.crf_0 = CRF0(tagset_size, start_tag_idx, stop_tag_idx)
        #  self.crf = OldCRF(tagset_size, start_tag_idx, stop_tag_idx)

        # CRFPP
        #  self.crf = CRFPP(tagset_size, start_tag_idx, stop_tag_idx)
        #  self.crf = NCRFPP(tagset_size, start_tag_idx, stop_tag_idx)

        # CRFSLTK
        #  self.crf = CRFSLTK(target_size, start_tag_idx, stop_tag_idx)
        #  self.crf = CRFSLTK(tagset_size, start_tag_idx, stop_tag_idx)

        self.init_weights()
예제 #12
0
import torch
from transformers.models.bert import BertModel, BertTokenizer

model_name = '/data/project/learn_code/data/chinese-bert-wwm-ext/'
# 读取模型对应的tokenizer
tokenizer = BertTokenizer.from_pretrained(model_name)
# 载入模型
model = BertModel.from_pretrained(model_name)
# 输入文本
# input_text = "Here is some text to encode"
input_text = "今天天气很好啊,你好吗"
# 通过tokenizer把文本变成 token_id
input_ids = tokenizer.encode(input_text, add_special_tokens=True)
print(len(input_ids))
# input_ids: [101, 2182, 2003, 2070, 3793, 2000, 4372, 16044, 102]
input_ids = torch.tensor([input_ids])
# 获得BERT模型最后一个隐层结果
print(input_ids.shape)
with torch.no_grad():
    last_hidden_states = model(input_ids)[0]
    print(last_hidden_states)
    print(last_hidden_states.shape)
예제 #13
0
def bert_model(model_name, num_labels) -> BertModel:
    bert_config = BertConfig.from_pretrained(model_name)
    bert_config.num_labels = num_labels
    return BertModel(bert_config)
예제 #14
0
 def __init__(self, config):
     super(BojoneModelWithPooler, self).__init__(config)
     self.bert = BertModel(config)
     self.cls = modeling_bert.BertPreTrainingHeads(config)
     self.init_weights()
예제 #15
0
 def from_tinybert(cls):
     config = TinyBertConfig()
     tokenizer = BertTokenizerFast.from_pretrained(config.name)
     q_encoder = BertModel.from_pretrained(config.name)
     c_encoder = BertModel.from_pretrained(config.name)
     return cls(config, tokenizer, q_encoder, c_encoder)