def __init__(self, model_name_or_path, dropout=0.1):
super(MyModel, self).__init__()
self.bert = BertModel.from_pretrained(model_name_or_path)
for param in self.bert.parameters():
param.requires_grad = True
self.linear1 = nn.Linear(768, 768)
self.linear2 = nn.Linear(768, 1)
self.dropout = dropout
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
def __init__(self, config: Config, train_config: TrainerConfig):
super(BertForIntentDetection, self).__init__()
self.config = config
self.train_config = train_config
# 1. init the bert model
bert_config = BertConfig.from_pretrained(config.pretrain_model)
self.encoder = BertModel(bert_config)
self.loss_function = nn.LogSoftmax(dim=-1)
def __init__(self, model_name_or_path, dropout=0.1):
super().__init__()
# self.bert = AutoModel.from_pretrained(model_name_or_path)
self.bert = BertModel.from_pretrained(model_name_or_path)
# self.bert.requires_grad_(True)
for param in self.bert.parameters():
param.requires_grad = True
self.dropout = nn.Dropout(dropout)
self.linear = nn.Linear(768, 1)
self.n_class = 4
class BertComputation(RequestHandler):
"""
Request handler that computes embeddings on english sentences
"""
enable_cuda = True if torch.cuda.is_available() else False
CLS = '[CLS]'
SEP = '[SEP]'
MSK = '[MASK]'
pretrained_weights = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(pretrained_weights)
model = BertModel(config=BertConfig())
model.eval()
if enable_cuda:
model.cuda(torch.device('cuda'))
def process_request(self, data):
tokens = []
mask_ids = []
seg_ids = []
seq_len = 0
for txt in data:
utt = txt.strip().lower()
toks = self.tokenizer.tokenize(utt)
if len(toks) > max_seq_len - 2:
toks = toks[:max_seq_len - 2]
toks.insert(0, self.CLS)
toks.insert(-1, self.SEP)
seq_len = max(len(toks), seq_len)
mask_ids.append([1] * len(toks) + [0] * (seq_len - len(toks)))
seg_ids.append([0] * seq_len)
tokens.append(toks + [self.MSK] * (seq_len - len(toks)))
input_ids = []
for i in range(len(tokens)):
for _ in range(seq_len - len(tokens[i])):
tokens[i].append(self.MSK)
mask_ids[i].append(0)
seg_ids[i].append(0)
input_ids.append(self.tokenizer.convert_tokens_to_ids(tokens[i]))
return (torch.tensor(input_ids), torch.tensor(mask_ids),
torch.tensor(seg_ids))
def run_inference(self, model_input):
if self.enable_cuda:
input_ids, mask_ids, seg_ids = [x.to('cuda') for x in model_input]
else:
input_ids, mask_ids, seg_ids = model_input
cls_emb = self.model.forward(input_ids=input_ids,
attention_mask=mask_ids,
token_type_ids=seg_ids)[1]
return cls_emb
def process_response(self, model_output_item):
return encode_pickle(model_output_item.cpu().detach().numpy())
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.loss_type = config.loss_type
self.focalloss_gamma = config.focalloss_gamma
self.focalloss_alpha = config.focalloss_alpha
self.diceloss_weight = config.diceloss_weight
self.init_weights()
def __init__(self,
model_name_or_path,
hidden_size=768,
dropout=0.1,
num_choices=4):
super(BertForMultipleChoiceWithMatch, self).__init__()
self.num_choices = num_choices
self.bert = BertModel.from_pretrained(model_name_or_path)
self.dropout = nn.Dropout(dropout)
# self.classifier = nn.Linear(hidden_size, 1)
# self.classifier2 = nn.Linear(2 * hidden_size, 1)
self.classifier3 = nn.Linear(3 * hidden_size, 1)
# self.classifier4 = nn.Linear(4 * hidden_size, 1)
# self.classifier6 = nn.Linear(6 * hidden_size, 1)
self.ssmatch = SSingleMatchNet(hidden_size, dropout)
self.fuse = FuseNet(hidden_size)
def __init__(self, config):
super(BertCRFForAttr, self).__init__(config)
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.t_lstm = nn.LSTM(input_size=config.hidden_size,
hidden_size=config.hidden_size // 2,
batch_first=True,
bidirectional=True)
self.a_lstm = nn.LSTM(input_size=config.hidden_size,
hidden_size=config.hidden_size // 2,
batch_first=True,
bidirectional=True)
self.attention = CosAttention()
self.ln = LayerNorm(config.hidden_size * 2)
self.classifier = nn.Linear(config.hidden_size * 2, config.num_label)
self.crf = CRF(num_tags=config.num_labels, batch_first=True)
self.init_weights()
def __init__(self, config):
super(BertSpanForNer, self).__init__(config)
self.soft_label = config.soft_label
self.num_labels = config.num_labels
self.loss_type = config.loss_type
self.focalloss_gamma = config.focalloss_gamma
self.focalloss_alpha = config.focalloss_alpha
self.diceloss_weight = config.diceloss_weight
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.start_fc = PoolerStartLogits(config.hidden_size, self.num_labels)
if self.soft_label:
self.end_fc = PoolerEndLogits(config.hidden_size + self.num_labels,
self.num_labels)
else:
self.end_fc = PoolerEndLogits(config.hidden_size + 1,
self.num_labels)
self.init_weights()
def __init__(self, config: JointBertConfig):
""" init JointBert model with configuration
"""
super(JointBERT, self).__init__(config.bert_config)
self.config: JointBertConfig = config
self.bert: BertModel = BertModel(
config=config.bert_config) # Load pretrained bert
self.intent_classifier = nn.Sequential(
nn.Dropout(self.config.intent_dropout),
nn.Linear(768, self.config.intent_size))
self.slot_classifier = nn.Sequential(
nn.Dropout(self.config.intent_dropout),
nn.Linear(768, self.config.slot_label_size))
if self.config.use_crf:
self.crf = CRF(num_tags=self.config.slot_label_size,
batch_first=True)
def __init__(self, config):
super(BertCrfForNer, self).__init__(config)
# self.num_labels = len(label2id)
self.num_labels = config.num_labels
self.crf_type = config.crf_type
self.no_crf_loss = config.no_crf_loss
self.loss_type = config.loss_type
self.focalloss_gamma = config.focalloss_gamma
self.focalloss_alpha = config.focalloss_alpha
self.diceloss_weight = config.diceloss_weight
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.num_labels)
label2id = config.label2id
start_tag_idx = label2id["[CLS]"]
stop_tag_idx = label2id["[SEP]"]
tagset_size = len(label2id)
# CRFPP
if self.crf_type == 'ncrfpp':
from ..models.ncrfpp_crf import CRF as CRFPP
self.crf = CRFPP(tagset_size, start_tag_idx, stop_tag_idx)
elif self.crf_type == 'pytorch-crf':
from torchcrf import CRF
# https://github.com/kmkurn/pytorch-crf/blob/master/torchcrf/__init__.py
self.crf = CRF(len(label2id), batch_first=torch.BoolTensor([True]))
# self.crf = PyTorchCRF(tagset_size, start_tag_idx, stop_tag_idx)
elif self.crf_type == 'old_crf':
from ..models.crf_0 import CRF
# self.crf_0 = CRF0(tagset_size=len(label2id),
# tag_dictionary=label2id,
# device=device)
self.crf_0 = CRF(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf = OldCRF(tagset_size, start_tag_idx, stop_tag_idx)
elif self.crf_type == 'new_crf':
from ..models.crf import CRF
self.crf = CRF(len(label2id), batch_first=True)
# self.crf = NewCRF(tagset_size, start_tag_idx, stop_tag_idx)
elif self.crf_type == 'lstm_crf':
from ..models.lstm_crf import CRF
self.crf = CRF(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf = LSTMCRF(tagset_size, start_tag_idx, stop_tag_idx)
# lstm_crf
# self.crf = CRF(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf = LSTMCRF(tagset_size, start_tag_idx, stop_tag_idx)
# TorchCRF
# self.crf = CRF(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf = TorchCRF(tagset_size, start_tag_idx, stop_tag_idx)
# torchcrf
# self.crf = CRF(len(label2id), batch_first=torch.BoolTensor([True]))
# self.crf = PyTorchCRF(tagset_size, start_tag_idx, stop_tag_idx)
# NewCRF
# self.crf = CRF(len(label2id), batch_first=True)
# self.crf = NewCRF(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf_0 = CRF0(tagset_size=len(label2id),
# tag_dictionary=label2id,
# device=device)
# self.crf_0 = CRF0(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf = OldCRF(tagset_size, start_tag_idx, stop_tag_idx)
# CRFPP
# self.crf = CRFPP(tagset_size, start_tag_idx, stop_tag_idx)
# self.crf = NCRFPP(tagset_size, start_tag_idx, stop_tag_idx)
# CRFSLTK
# self.crf = CRFSLTK(target_size, start_tag_idx, stop_tag_idx)
# self.crf = CRFSLTK(tagset_size, start_tag_idx, stop_tag_idx)
self.init_weights()
import torch
from transformers.models.bert import BertModel, BertTokenizer
model_name = '/data/project/learn_code/data/chinese-bert-wwm-ext/'
# 读取模型对应的tokenizer
tokenizer = BertTokenizer.from_pretrained(model_name)
# 载入模型
model = BertModel.from_pretrained(model_name)
# 输入文本
# input_text = "Here is some text to encode"
input_text = "今天天气很好啊,你好吗"
# 通过tokenizer把文本变成 token_id
input_ids = tokenizer.encode(input_text, add_special_tokens=True)
print(len(input_ids))
# input_ids: [101, 2182, 2003, 2070, 3793, 2000, 4372, 16044, 102]
input_ids = torch.tensor([input_ids])
# 获得BERT模型最后一个隐层结果
print(input_ids.shape)
with torch.no_grad():
last_hidden_states = model(input_ids)[0]
print(last_hidden_states)
print(last_hidden_states.shape)
def bert_model(model_name, num_labels) -> BertModel:
bert_config = BertConfig.from_pretrained(model_name)
bert_config.num_labels = num_labels
return BertModel(bert_config)
def __init__(self, config):
super(BojoneModelWithPooler, self).__init__(config)
self.bert = BertModel(config)
self.cls = modeling_bert.BertPreTrainingHeads(config)
self.init_weights()
def from_tinybert(cls):
config = TinyBertConfig()
tokenizer = BertTokenizerFast.from_pretrained(config.name)
q_encoder = BertModel.from_pretrained(config.name)
c_encoder = BertModel.from_pretrained(config.name)
return cls(config, tokenizer, q_encoder, c_encoder)