def initialize(self, ctx):
properties = ctx.system_properties
MODEL_DIR = properties.get("model_dir")
self.device = torch.device("cuda:" +
str(properties.get("gpu_id")) if torch.cuda.
is_available() else "cpu")
self.labelencoder = preprocessing.LabelEncoder()
self.labelencoder.classes_ = np.load(
os.path.join(MODEL_DIR, 'classes.npy'))
config = BertConfig(os.path.join(MODEL_DIR, 'bert_config.json'))
self.model = BertForSequenceClassification(
config, num_labels=len(self.labelencoder.classes_))
self.model.load_state_dict(
torch.load(os.path.join(MODEL_DIR, 'pytorch_model.bin'),
map_location="cpu"))
self.model.to(self.device)
self.model.eval()
self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
self.softmax = torch.nn.Softmax(dim=-1)
# self.batch_size = batch_size
logger.debug(
'Transformer model from path {0} loaded successfully'.format(
MODEL_DIR))
self.manifest = ctx.manifest
self.initialized = True
def get_bert_inference_model(bert_model, batch_size=16, max_seq_length=100):
tokenizer = BertTokenizer.from_pretrained(os.path.join(
bert_model, "vocab.txt"),
do_lower_case=True)
transform = BertClassificationTransform(tokenizer=tokenizer,
is_test=True,
max_len=max_seq_length)
state_save_path = os.path.join(bert_model, 'model.state')
if os.path.exists(state_save_path):
state = torch.load(state_save_path, map_location="cpu")
model = BertForSequenceClassification.from_pretrained(
bert_model, num_labels=2, state_dict=state['model_state'])
else:
previous_model_file = os.path.join(bert_model, "pytorch_model.bin")
model_state_dict = torch.load(previous_model_file, map_location="cpu")
model = BertForSequenceClassification.from_pretrained(
bert_model, state_dict=model_state_dict, num_labels=2)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
inference_model = BertInferenceModel(model,
transform,
device,
batch_size=batch_size)
return inference_model
def create_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
model = BertForSequenceClassification(config=config, num_labels=self.num_labels)
model.eval()
loss = model(input_ids, token_type_ids, input_mask, sequence_labels)
logits = model(input_ids, token_type_ids, input_mask)
outputs = {
"loss": loss,
"logits": logits,
}
return outputs
def load_pretrained_model_tokenizer(model_type="BertForSequenceClassification",
device="cuda",
config=None):
bert_model = config['bert_model']
# Load pre-trained model (weights)
if model_type == "BertForSequenceClassification":
model = BertForSequenceClassification.from_pretrained(bert_model,
num_labels=2)
# Load pre-trained model tokenizer (vocabulary)
elif model_type == "BertForNextSentencePrediction":
model = BertForNextSentencePrediction.from_pretrained(bert_model)
elif model_type == "specific_shared":
model = SpecificShared(config)
elif model_type == "siamese_bert":
model = SiameseBert(config)
elif model_type == "n_bert":
model = nBert(config)
elif model_type == "bert_sts":
model = BertSts(config)
elif model_type == "bert_fine_tune":
model = BertFineTune(config)
else:
print("[Error]: unsupported model type")
return None, None
tokenizer = BertTokenizer.from_pretrained(bert_model)
model.to(device)
print("Initialized model and tokenizer")
return model, tokenizer
def load_pretrained_model_tokenizer(model_type="BertForSequenceClassification",
base_model=None,
base_tokenizer=None,
device="cuda",
chinese=False,
num_labels=2):
# Load pre-trained model (weights)
if base_model is None:
if chinese:
base_model = "bert-base-chinese"
else:
base_model = "bert-base-uncased"
if model_type == "BertForSequenceClassification":
model = BertForSequenceClassification.from_pretrained(
base_model, num_labels=num_labels)
# Load pre-trained model tokenizer (vocabulary)
elif model_type == "BertForNextSentencePrediction":
model = BertForNextSentencePrediction.from_pretrained(base_model)
elif model_type == "BertForTokenClassification":
model = BertForTokenClassification.from_pretrained(
base_model, num_labels=num_labels)
elif model_type == "BertMSE":
model = BertMSE()
else:
print("[Error]: unsupported model type")
return None, None
if base_tokenizer is None:
# Download from huggingface
tokenizer = BertTokenizer.from_pretrained(base_model)
else:
# Load local file
tokenizer = BertTokenizer.from_pretrained(base_tokenizer)
model.to(device)
return model, tokenizer
def __init__(self):
credentialFilePath = 'twitter_credentials.json'
# TODO add that file to .gitignore
with open(credentialFilePath, "r") as file:
creds = json.load(file)
self.twitter = Twython(creds['CONSUMER_KEY'], creds['CONSUMER_SECRET'],
creds['ACCESS_TOKEN'], creds['ACCESS_SECRET'])
verification = self.twitter.verify_credentials()
print('Login Verification:' + verification['name'])
model_state_dict = torch.load(output_model_file)
self.loaded_model = BertForSequenceClassification.from_pretrained(
modelfilePath, state_dict=model_state_dict, num_labels=2)
self.loaded_model.cuda()
device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
torch.cuda.get_device_name(0)
if self.loaded_model is None:
raise ModelError(
"Sequence Classification Model was not properly loaded!!")
print("Model loaded")
if TwitterAnalyzer.__instance != None:
raise Exception("This class is a singleton!")
else:
TwitterAnalyzer.__instance = self
def __init__(self, model_type, db, freeze_bert=False):
super(CodahClassifier, self).__init__()
# instantiating BERT model object
size = model_type.split('-')[1]
if size == 'large':
output_dim = 1024
else:
output_dim = 768
self.bert_layer = BertForSequenceClassification.from_pretrained(
model_type, num_labels=1)
# freeze bert layers
if freeze_bert:
for p in self.bert_layer.parameters():
p.requires_grad = False
self.asn = DrawModel(db)
for p in self.asn.parameters():
p.requires_grad = False
self.obj_cls = nn.Linear(db.cfg.output_cls_size, 1)
self.att_layer = nn.Linear(db.cfg.num_scales * 2, 1)
self.att_cls = nn.Linear(db.cfg.grid_size[0]**2, 1)
self.pos_cls = nn.Linear(db.cfg.grid_size[0]**2, 1)
self.relu = nn.ReLU()
def test():
# 配置文件
cf = Config('./config.yaml')
# 有GPU用GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# 测试数据
test_data = NewsDataset("./data/cnews_final_test.txt", cf.max_seq_len)
test_dataloader = DataLoader(test_data,
batch_size=cf.batch_size,
shuffle=True)
# 模型
config = BertConfig("./output/pytorch_bert_config.json")
model = BertForSequenceClassification(config, num_labels=cf.num_labels)
model.load_state_dict(torch.load("./output/pytorch_model.bin"))
# 把模型放到指定设备
model.to(device)
# 让模型并行化运算
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# 训练
start_time = time.time()
data_len = len(test_dataloader)
model.eval()
y_pred = np.array([])
y_test = np.array([])
# for step,batch in enumerate(tqdm(test_dataloader,"batch",total=len(test_dataloader))):
for step, batch in enumerate(test_dataloader):
label_id = batch['label_id'].squeeze(1).to(device)
word_ids = batch['word_ids'].to(device)
segment_ids = batch['segment_ids'].to(device)
word_mask = batch['word_mask'].to(device)
loss = model(word_ids, segment_ids, word_mask, label_id)
with torch.no_grad():
pred = get_model_labels(model, word_ids, segment_ids, word_mask)
y_pred = np.hstack((y_pred, pred))
y_test = np.hstack((y_test, label_id.to("cpu").numpy()))
# 评估
print("Precision, Recall and F1-Score...")
print(
metrics.classification_report(y_test,
y_pred,
target_names=get_labels('./data/label')))
# 混淆矩阵
print("Confusion Matrix...")
cm = metrics.confusion_matrix(y_test, y_pred)
print(cm)
def __init__(self, vocab: Vocabulary, pretrained_bert_model_file: str,
num_labels: int) -> None:
super().__init__(vocab)
self.bert_sc_model = HuggingFaceBertSC.from_pretrained(
pretrained_bert_model_file, num_labels=num_labels)
self._accuracy = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
def __init__(self):
log.info('Instantiating model')
model_type = 'bert-base-cased'
self.model = BertForSequenceClassification.from_pretrained(
model_type, cache_dir=None, num_labels=1)
log.info('Instantiating tokenizer')
self.tokenizer = BertTokenizer.from_pretrained(model_type)
def __init__(self,model="",num_labels =2):
print('kaishi')
self.num_labels =num_labels
self.tokenizer = BertTokenizer.from_pretrained(model)
self.model = BertForSequenceClassification.from_pretrained(model,num_labels = num_labels)
# model = BertForSequenceClassification.from_pretrained('bert-base-chinese')
# model = BertForNextSentencePrediction.from_pretrained('bert-base-chinese')
self.model.eval()
def __init__(self, pretrain_path, dropout=0.1):
super(PairModel, self).__init__()
self.bert = BertForSequenceClassification.from_pretrained(
pretrain_path, cache_dir=None, num_labels=1)
self.head = nn.Sequential(
OrderedDict([
('dropout', nn.Dropout(dropout)),
('clf', nn.Linear(self.bert.config.hidden_size, 1)),
]))
def from_scratch(cls, bert_model, verbose=True):
tokenizer = BertTokenizer.from_pretrained(bert_model)
bert = BertForSequenceClassification.from_pretrained(bert_model,
num_labels=2)
model = cls(tokenizer, bert)
if verbose:
print('Intialized the model and the tokenizer from scratch')
return model
def finetune_bert(args, model_save_directory, bert_file_path):
train_data, valid_data = data_utils.make_datasets(args)
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
num_labels=4)
args.save_path = model_save_directory + "bert_model.pt"
ACC, _ = train_utils.train_model(train_data, valid_data, model, args, 0,
"bert")
def __init__(self, n_epochs=4, batch_size=10, lr=2e-5):
self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True)
self.MAX_LEN = 128
self.batch_size = batch_size
self.n_epochs = n_epochs
self.device = torch.device('cuda:0')
self.model = BertForSequenceClassification.from_pretrained(
"bert-base-uncased", num_labels=2)
self.lr = lr
def from_model_path(cls, output_model_path, verbose=True):
tokenizer = BertTokenizer.from_pretrained(output_model_path)
bert = BertForSequenceClassification.from_pretrained(output_model_path,
num_labels=2)
model = cls(tokenizer, bert)
if verbose:
print('Restored the model and the tokenizer from {}'.format(
output_model_path))
return model
def load_pytorch_model(path, *args, **kwargs):
state_dict = torch.load(path)
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k[7:] # remove `module.`
new_state_dict[name] = v
#model = SimpleLSTM(*args, **kwargs)
model = BertForSequenceClassification.from_pretrained(BERT_MODEL_PATH,
num_labels=1)
model.load_state_dict(new_state_dict)
return model
def __init__(self, device, n_ctx_embs, ctx_emb_dim, pretrain):
"""
The value of the parameters should also be specified in the BCN model config.
"""
self.device = device
self.n_ctx_embs = n_ctx_embs
self.ctx_emb_dim = ctx_emb_dim
self.tokenizer = BertTokenizer.from_pretrained(pretrain)
self.model = BertForSequenceClassification.from_pretrained(pretrain)
self.model.eval()
self.model.to(self.device)
def _load_model(self) -> None:
self.device = torch.device("cuda" if torch.cuda.is_available()
and not self.no_cuda else "cpu")
self.n_gpu = torch.cuda.device_count()
# Load a trained model and vocabulary that you have fine-tuned
self.model = BertForSequenceClassification.from_pretrained(
self.model_dir, num_labels=self.num_labels)
self.tokenizer = BertTokenizer.from_pretrained(
self.model_dir, do_lower_case=self.do_lower_case)
self.model.to(self.device)
def __init__(self):
self.args = GlobalVariable.get_value('BERT_ARGS')
self.model = BertForSequenceClassification.from_pretrained(self.args.get('bert_model'),
cache_dir=PYTORCH_PRETRAINED_BERT_CACHE / 'distributed_{}'.format(
self.args.get('local_rank')))
self.processors = {'SentencePro': sentencePro}
self.processor = self.processors['SentencePro']()
self.label_list = self.processor.get_labels()
self.device = torch.device("cuda" if torch.cuda.is_available() and not self.args.get('no_cuda') else "cpu")
self.tokenizer = BertTokenizer.from_pretrained(self.args.get('bert_model'))
self.model.load_state_dict(GlobalVariable.get_value('NEW_STATE_DICT'))
def __init__(self, output_dir="./models/"):
# get device
self.__device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
# get model from weight
self.__model = BertForSequenceClassification.from_pretrained(
output_dir, num_labels=28)
# create tokenizer
self.__tokenizer = BertTokenizer.from_pretrained(output_dir)
self.__model.eval()
def load_info():
# Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top.
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
num_labels=2)
model.cuda()
model.load_state_dict(torch.load("model/model.pt"))
model.eval()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True)
return model, tokenizer
def __init__(self, dir_path, max_seq_length=30):
self.max_seq_length = max_seq_length
self.processor = PredicateClassificationProcessor.load(os.path.join(dir_path, PROCESSOR_NAME))
self.tokenizer = BertTokenizer.from_pretrained(dir_path)
self.classifier = BertForSequenceClassification.from_pretrained(dir_path,
len(self.processor.labels)).to(self.device)
self.classifier.eval()
self.id2label = {i: label for i, label in enumerate(self.processor.labels)}
global debug_message
debug_message = False
def init_bert(pre_trained='bert-base-cased', num_labels=2):
# 分词器
tokenizer = BertTokenizer.from_pretrained(pre_trained)
# 获得hidden states
model = BertForSequenceClassification.from_pretrained(
pre_trained, num_labels=num_labels)
model.eval()
# print('成功加载bert模型')
return tokenizer, model
def _create_net_and_optim(self, net_cfg, optim_cfg, num_train_optimization_steps):
net = BertForSequenceClassification.from_pretrained(net_cfg.bert_pretrain, net_cfg.num_labels)
net.to(device=self._device)
param_optimizer = filter(lambda p: p.requires_grad, net.parameters())
if num_train_optimization_steps != None:
optim = BertAdam(param_optimizer,
t_total=num_train_optimization_steps,
**optim_cfg.kwargs)
else:
optim = None
return net, optim
def __init__(self,
bert_model_dir,
class_size,
feature_lens,
seed=None,
hidden_size=75,
dropout_rate_fc=0.2,
num_layers=2,
input_dropout=.3,
lstm_dropout=0.5,
fine_tune_embeddings=True):
self.fine_tune_embeddings = fine_tune_embeddings
if seed is None:
seed = torch.initial_seed() & ((1 << 63) - 1)
self.logger.info("Using seed {}".format(seed))
torch.manual_seed(seed)
super(RelationExtractorBertBiLstmNetworkNoPos, self).__init__()
# Use random weights if vocab size if passed in else load pretrained weights
self.text_column_index = feature_lens.argmax(axis=0)
self.max_sequence_len = feature_lens[self.text_column_index]
self.logger.info(
"The text feature is index {}, the feature lengths are {}".format(
self.text_column_index, feature_lens))
bidirectional = True
num_directions = 2 if bidirectional else 1
self.embeddings = BertForSequenceClassification.from_pretrained(
bert_model_dir,
num_labels=class_size).bert.embeddings.word_embeddings
self.lstm = nn.Sequential(
nn.Dropout(p=input_dropout),
nn.LSTM(self.embeddings.embedding_dim,
hidden_size=hidden_size,
num_layers=num_layers,
batch_first=True,
bidirectional=bidirectional,
dropout=lstm_dropout))
#
self.fc_input_size = (hidden_size *
num_directions) * self.max_sequence_len
self._class_size = class_size
self.fc = nn.Sequential(nn.Dropout(dropout_rate_fc),
nn.Linear(self.fc_input_size, class_size))
def create_bert_for_sequence_classification_attn(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels):
# Disable attention dropout
drop = config.attention_probs_dropout_prob
config.attention_probs_dropout_prob = 0.0
model = BertForSequenceClassification(config=config, num_labels=self.num_labels)
config.attention_probs_dropout_prob = drop
loss, _ = model(input_ids, token_type_ids, input_mask, sequence_labels, return_att=True)
logits, attn = model(input_ids, token_type_ids, input_mask, return_att=True)
outputs = {
"loss": loss,
"logits": logits,
"attn": attn,
}
return outputs
def main_bert(args, DEVICE):
train_dataloader, valid_dataloader, test_dataloader, num_train_examples = load_data_bert(
args, DEVICE)
model = BertForSequenceClassification.from_pretrained(
'bert-base-uncased', cache_dir='../result/', num_labels=2)
# Train if no pre-trained model is given
if args.new_train:
train_bert(args, [
train_dataloader, valid_dataloader, test_dataloader,
num_train_examples
], model, DEVICE)
interpret_bert(args, test_dataloader, DEVICE)
def build_model(self, param):
num_classes = param['num_classes']
bert_base = param['bert_base']
bert_model = param['bert_model']
model = BertForSequenceClassification.from_pretrained(
bert_model, num_labels=num_classes)
# model = BertForSequenceClassification.from_pretrained(bert_base, num_labels=num_classes)
if torch.cuda.is_available():
model.cuda()
# model.summary()
self.model = model
self.model_param = param
def configure_model():
model = BertForSequenceClassification.from_pretrained(BERT_MODEL_PATH,cache_dir=None,num_labels=1)
model.zero_grad()
model = model.to(device)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
lr = 3e-5
epsilon=1
lr_d = {}
weight_d = {}
for n, p in param_optimizer:
if any(nd in n for nd in no_decay):
weight_d[n] = 0.0
else:
weight_d[n] = 0.01
for n, p in param_optimizer[:5]:
lr_d[n] = lr*(epsilon**(11))
for n, p in param_optimizer:
if 'bert.encoder.layer.' in n:
for i in range(0, 12):
if 'bert.encoder.layer.'+str(i)+'.' in n:
lr_d[n] = lr*(epsilon**(11-i))
break
for n, p in param_optimizer[-4:]:
lr_d[n] = lr
comb_dict = {}
for n, p in param_optimizer:
para = (weight_d[n], lr_d[n])
if para in comb_dict:
comb_dict[para].append(p)
else:
comb_dict[para] = [p]
optimizer_grouped_parameters = []
for i, j in comb_dict.items():
optimizer_grouped_parameters.append({'params':j, 'weight_decay' : i[0], 'lr' : i[1]})
train = train_dataset
num_train_optimization_steps = int(EPOCHS*len(train)/batch_size/accumulation_steps)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=lr,
warmup=0.05,
t_total=num_train_optimization_steps)
model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)
model=model.train()
return model, optimizer, train
