def test_load_trained(self):
current_path = os.path.dirname(os.path.abspath(__file__))
config_path = os.path.join(current_path, 'test_checkpoint',
'bert_config.json')
model_path = os.path.join(current_path, 'test_checkpoint',
'bert_model.ckpt')
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False)
model.summary()
def test_load_adapter(self):
current_path = os.path.dirname(os.path.abspath(__file__))
config_path = os.path.join(current_path, 'test_checkpoint',
'bert_config.json')
model_path = os.path.join(current_path, 'test_checkpoint',
'bert_model.ckpt')
model = load_trained_model_from_checkpoint(
config_path,
model_path,
training=False,
use_adapter=True,
trainable=[
'Encoder-{}-MultiHeadSelfAttention-Adapter'.format(i + 1)
for i in range(2)
] +
['Encoder-{}-FeedForward-Adapter'.format(i + 1)
for i in range(2)] + [
'Encoder-{}-MultiHeadSelfAttention-Norm'.format(i + 1)
for i in range(2)
] +
['Encoder-{}-FeedForward-Norm'.format(i + 1) for i in range(2)],
)
model.summary()
def test_load_output_layer_num(self):
current_path = os.path.dirname(os.path.abspath(__file__))
config_path = os.path.join(current_path, 'test_checkpoint',
'bert_config.json')
model_path = os.path.join(current_path, 'test_checkpoint',
'bert_model.ckpt')
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False,
output_layer_num=4)
model.summary()
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False,
output_layer_num=[0])
model.summary()
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False,
output_layer_num=[1])
model.summary()
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False,
output_layer_num=[-1])
model.summary()
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False,
output_layer_num=[-2])
model.summary()
model = load_trained_model_from_checkpoint(config_path,
model_path,
training=False,
output_layer_num=[0, -1])
model.summary()
output_layer_num=4,
poolings=[POOL_NSP, POOL_MAX])
print("句子对:", np.array(embeddings).shape)
# 3. 可以使用adapter来对预训练模型进行微调,下面的代码只让adapter和layer normalization成为可训练的层
layer_num = 12
config_path = os.path.join(pretrained_path, 'bert_config.json')
model_path = os.path.join(pretrained_path, 'bert_model.ckpt')
model = load_trained_model_from_checkpoint(
config_path,
model_path,
training=False,
use_adapter=True,
trainable=[
'Encoder-{}-MultiHeadSelfAttention-Adapter'.format(i + 1)
for i in range(layer_num)
] +
['Encoder-{}-FeedForward-Adapter'.format(i + 1)
for i in range(layer_num)] + [
'Encoder-{}-MultiHeadSelfAttention-Norm'.format(i + 1)
for i in range(layer_num)
] +
['Encoder-{}-FeedForward-Norm'.format(i + 1) for i in range(layer_num)],
)
# 4. 使用任务嵌入:如果有多任务训练的需求，可以启用任务嵌入层,
# 针对不同任务将嵌入的结果加上不同的编码，注意要让Embedding-Task层可训练
from bertTAT.bert import load_trained_model_from_checkpoint
config_path = os.path.join(pretrained_path, 'bert_config.json')
model_path = os.path.join(pretrained_path, 'bert_model.ckpt')
model = load_trained_model_from_checkpoint(
加载预训练模型，并检测两个句子是否是连续的
"""
import os
import numpy as np
from bertTAT.bert import load_vocabulary, load_trained_model_from_checkpoint,\
Tokenizer, get_checkpoint_paths
# 1. 提取预训练模型文件的路径
now_path = os.path.dirname(__file__)
model_path = now_path + "/../pretrained_model/chinese_L-12_H-768_A-12"
paths = get_checkpoint_paths(model_path)
# 2. 加载模型
model = load_trained_model_from_checkpoint(paths.config,
paths.checkpoint,
training=True,
seq_len=None)
# model.summary(line_length=120)
# 3. 加载字典
token_dict = load_vocabulary(paths.vocab)
token_dict_inv = {v: k for k, v in token_dict.items()}
# 4. 分词
tokenizer = Tokenizer(token_dict)
text = '数学是利用符号语言研究数量、结构、变化以及空间等概念的一门学科'
tokens = tokenizer.tokenize(text)
tokens[1] = tokens[2] = '[MASK]'
print('Tokens:', tokens)
indices = np.array([[token_dict[token] for token in tokens]])
vocab_path = os.path.join(pretrained_path, 'vocab.txt')
# TPU使用的设置
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
resolver = tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)
tf.contrib.distribute.initialize_tpu_system(resolver)
strategy = tf.contrib.distribute.TPUStrategy(resolver)
# 加载基础模型
token_dict = {}
with codecs.open(vocab_path, 'r', 'utf8') as reader:
for line in reader:
token = line.strip()
token_dict[token] = len(token_dict)
with strategy.scope():
model = load_trained_model_from_checkpoint(config_path, checkpoint_path)
# 提取特征
tokenizer = Tokenizer(token_dict)
text = 'From that day forth... my arm changed... and a voice echoed'
tokens = tokenizer.tokenize(text)
indices, segments = tokenizer.encode(first=text, max_len=512)
print(tokens)
predicts = model.predict([np.array([indices] * 8),
np.array([segments] * 8)])[0]
for i, token in enumerate(tokens):
print(token, predicts[i].tolist()[:19])