def build_model():
model = load_trained_model_from_checkpoint(
config_path=paths.config,
checkpoint_path=paths.model,
batch_size=BATCH_SIZE,
memory_len=MEMORY_LEN,
target_len=TEXT_LEN,
in_train_phase=False,
attention_type=ATTENTION_TYPE_BI)
# 加载预训练权重
# Build classification model
last = model.output
extract = Extract(index=-1, name='Extract')(last)
dense = keras.layers.Dense(units=768, name='Dense')(extract)
norm = keras.layers.BatchNormalization(name='Normal')(dense)
output = keras.layers.Dense(units=2, activation='softmax',
name='Softmax')(norm)
model = keras.models.Model(inputs=model.inputs, outputs=output)
model.compile(
optimizer=Adam(learning_rate=LEARNING_RATE),
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'],
)
return model
def test_load_empty_memory(self):
current_path = os.path.dirname(os.path.abspath(__file__))
checkpoint_path = os.path.join(current_path, 'test_checkpoint_empty')
model = load_trained_model_from_checkpoint(
config_path=os.path.join(checkpoint_path, 'xlnet_config.json'),
checkpoint_path=os.path.join(checkpoint_path, 'xlnet_model.ckpt'),
batch_size=2,
memory_len=5,
target_len=5,
in_train_phase=False,
mask_index=0,
attention_type=ATTENTION_TYPE_UNI,
)
model_path = os.path.join(tempfile.gettempdir(), 'test_xlnet_%f.h5' % np.random.random())
model.save(model_path)
model = keras.models.load_model(model_path, custom_objects=get_custom_objects())
model.summary()
def _load_numpy(name):
return np.load(os.path.join(checkpoint_path, name + '.npy'))
input_ids = _load_numpy('input_ids')
seg_ids = _load_numpy('seg_ids')
tune_output = _load_numpy('empty_output')
inputs = [input_ids, seg_ids, np.zeros((2, 1))]
output = model.predict_on_batch(inputs)
self.assertTrue(np.allclose(tune_output, output, atol=1e-6))
def build_xlnet(args):
# Load pretrained model
model = load_trained_model_from_checkpoint(
config_path=args.config_path,
checkpoint_path=args.model_path,
batch_size=args.batch_size,
memory_len=0,
target_len=args.maxlen,
in_train_phase=False,
attention_type=ATTENTION_TYPE_BI,
)
# Build classification model
last = model.output
extract = Extract(index=-1, name='Extract')(last)
output = keras.layers.Dense(units=args.nclass,
activation='softmax',
name='Softmax')(extract)
model = keras.models.Model(inputs=model.inputs, outputs=output)
model.summary()
# Fit model
model.compile(
optimizer=RAdam(args.lr),
loss='categorical_crossentropy',
metrics=['accuracy'],
)
print(model.summary())
return model
def _test_pretrained_tune(self):
checkpoint_path = ''
model = load_trained_model_from_checkpoint(
config_path=os.path.join(checkpoint_path, 'xlnet_config.json'),
checkpoint_path=os.path.join(checkpoint_path, 'xlnet_model.ckpt'),
batch_size=8,
memory_len=512,
target_len=128,
in_train_phase=False,
attention_type=ATTENTION_TYPE_UNI,
)
current_path = os.path.dirname(os.path.abspath(__file__))
visual_path = os.path.join(current_path, 'xlnet_large_tune.jpg')
keras.utils.vis_utils.plot_model(model, visual_path, show_shapes=True)
def test_load_training(self):
current_path = os.path.dirname(os.path.abspath(__file__))
checkpoint_path = os.path.join(current_path, 'test_checkpoint_pre')
model = load_trained_model_from_checkpoint(
config_path=os.path.join(checkpoint_path, 'xlnet_config.json'),
checkpoint_path=os.path.join(checkpoint_path, 'xlnet_model.ckpt'),
batch_size=2,
memory_len=5,
target_len=5,
in_train_phase=True,
mask_index=0,
attention_type=ATTENTION_TYPE_UNI,
)
set_custom_objects()
model_path = os.path.join(tempfile.gettempdir(),
'test_xlnet_%f.h5' % np.random.random())
model.save(model_path)
model = keras.models.load_model(model_path)
model.summary()
def _load_numpy(name):
return np.load(os.path.join(checkpoint_path, name + '.npy'))
input_ids = _load_numpy('input_ids')
seg_ids = _load_numpy('seg_ids')
input_mask = _load_numpy('input_mask')
mems_0 = _load_numpy('mems_0')
mems_1 = _load_numpy('mems_1')
pre_output = _load_numpy('pre_output')
pre_new_mems_0 = _load_numpy('pre_new_mems_0')
pre_new_mems_1 = _load_numpy('pre_new_mems_1')
inputs = [input_ids, seg_ids, np.ones((2, 1)) * 5, input_mask]
self._update_memory(model, 'Memory-0', mems_0)
self._update_memory(model, 'Memory-1', mems_1)
output = model.predict_on_batch(inputs)
self.assertTrue(
np.allclose(pre_new_mems_0,
self._get_memory(model, 'Memory-0', 5),
atol=1e-6))
self.assertTrue(
np.allclose(pre_new_mems_1,
self._get_memory(model, 'Memory-1', 5),
atol=1e-6))
self.assertTrue(np.allclose(pre_output, output, atol=1e-6))
def build_xlnet_model(X):
'''
构建xlnet模型
:param X: encode后的结果
:return: np array形式做训练
'''
XLNetmodel = load_trained_model_from_checkpoint(
config_path=config_path,
checkpoint_path=checkpoint_path,
batch_size=16,
memory_len=0,
target_len=100,
in_train_phase=False,
attention_type=ATTENTION_TYPE_UNI,
)
memory_length_input = np.zeros((1, 1))
wordvec = []
for line in X:
token_input = np.expand_dims(np.array(line), axis=0)
segment_input = np.zeros_like(token_input)
XLNet_output = XLNetmodel.predict_on_batch(
[token_input, segment_input, memory_length_input])
wordvec.append(XLNet_output[0])
return np.array(wordvec)
# -*- coding: utf-8 -*-
import os
from keras_xlnet import Tokenizer, load_trained_model_from_checkpoint, ATTENTION_TYPE_BI
from keras.utils import plot_model
checkpoint_path = './xlnet_cased_L-12_H-768_A-12'
tokenizer = Tokenizer(os.path.join(checkpoint_path, 'spiece.model'))
model = load_trained_model_from_checkpoint(
config_path=os.path.join(checkpoint_path, 'xlnet_config.json'),
checkpoint_path=os.path.join(checkpoint_path, 'xlnet_model.ckpt'),
batch_size=16,
memory_len=512,
target_len=128,
in_train_phase=False,
attention_type=ATTENTION_TYPE_BI,
)
model.summary()
plot_model(model, to_file="xlnet.png", show_shapes=True)
segments = np.zeros_like(tokens)
segments[:, -1] = 1
lengths = np.zeros_like(tokens[:, :1])
return DataSequence([tokens, segments, lengths], classes)
current_path = os.path.dirname(os.path.abspath(__file__))
train_seq = generate_sequence(train_path)
dev_seq = generate_sequence(dev_path)
# Load pretrained model
model = load_trained_model_from_checkpoint(
config_path=paths.config,
checkpoint_path=paths.model,
batch_size=BATCH_SIZE,
memory_len=0,
target_len=SEQ_LEN,
in_train_phase=False,
attention_type=ATTENTION_TYPE_BI,
)
# Build classification model
last = Extract(index=-1, name='Extract')(model.output)
dense = keras.layers.Dense(units=768, activation='tanh', name='Dense')(last)
dropout = keras.layers.Dropout(rate=0.1, name='Dropout')(dense)
output = keras.layers.Dense(units=2, activation='softmax',
name='Softmax')(dropout)
model = keras.models.Model(inputs=model.inputs, outputs=output)
model.summary()
# Fit model
def build(self):
from keras_xlnet import load_trained_model_from_checkpoint, set_custom_objects
from keras_xlnet import Tokenizer, ATTENTION_TYPE_BI, ATTENTION_TYPE_UNI
self.embedding_type = 'xlnet'
self.checkpoint_path = os.path.join(self.corpus_path,
'xlnet_model.ckpt')
self.config_path = os.path.join(self.corpus_path, 'xlnet_config.json')
self.spiece_model = os.path.join(self.corpus_path, 'spiece.model')
self.attention_type = self.xlnet_embed.get('attention_type',
'bi') # or 'uni'
self.attention_type = ATTENTION_TYPE_BI if self.attention_type == 'bi' else ATTENTION_TYPE_UNI
self.memory_len = self.xlnet_embed.get('memory_len', 0)
self.target_len = self.xlnet_embed.get('target_len', 5)
print('load xlnet model start!')
# 模型加载
model = load_trained_model_from_checkpoint(
checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.trainable,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
#
set_custom_objects()
# 字典加载
self.tokenizer = Tokenizer(self.spiece_model)
# debug时候查看layers
self.model_layers = model.layers
len_layers = self.model_layers.__len__()
print(len_layers)
layer_real = [i for i in range(25)] + [-i for i in range(25)]
# 简要判别一下
self.layer_indexes = [
i if i in layer_real else -2 for i in self.layer_indexes
]
len_couche = int((len_layers - 6) / 10)
# 一共246个layer
# 每层10个layer（MultiHeadAttention,Dropout,Add,LayerNormalization）,第一是9个layer的输入和embedding层
# 一共24层
layer_dict = []
layer_0 = 7
for i in range(len_couche):
layer_0 = layer_0 + 10
layer_dict.append(layer_0)
layer_dict.append(247)
# 测试 get_output_at
# def get_number(index):
# try:
# model_node = model.get_output_at(node_index=index)
# gg = 0
# except:
# print('node index wrong!')
# print(index)
# list_index = [i for i in range(25)] + [-i for i in range(25)]
# for li in list_index:
# get_number(li)
# 输出它本身
if len(self.layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，取得不正确的话就取倒数第二层
elif len(self.layer_indexes) == 1:
if self.layer_indexes[0] in layer_real:
encoder_layer = model.get_layer(
index=layer_dict[self.layer_indexes[0]]).get_output_at(
node_index=0)
else:
encoder_layer = model.get_layer(
index=layer_dict[-1]).get_output_at(node_index=0)
# 否则遍历需要取的层，把所有层的weight取出来并加起来shape:768*层数
else:
# layer_indexes must be [0, 1, 2,3,......24]
all_layers = [
model.get_layer(index=layer_dict[lay]).get_output_at(
node_index=0) if lay in layer_real else model.get_layer(
index=layer_dict[-1]).get_output_at(
node_index=0) # 如果给出不正确，就默认输出倒数第一层
for lay in self.layer_indexes
]
print(self.layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
# def xlnet_concat(x):
# x_concat = K.concatenate(x, axis=1)
# return x_concat
# encoder_layer = Lambda(xlnet_concat, name='xlnet_concat')(all_layers)
self.output = NonMaskingLayer()(encoder_layer)
self.input = model.inputs
self.model = Model(self.input, self.output)
print("load KerasXlnetEmbedding end")
model.summary(132)
self.embedding_size = self.model.output_shape[-1]
self.vocab_size = len(self.tokenizer.sp)
text = "这个苹果很好吃"
tokens = tokenizer.encode(text)
print(np.array(tokens).shape)
token_input = np.expand_dims(np.array(tokens), axis=0)
print(token_input.shape)
segment_input = np.zeros_like(token_input)
print(segment_input.shape)
memory_length_input = np.zeros((1, 1))
# Load pre-trained model
model = load_trained_model_from_checkpoint(
config_path=config_path,
checkpoint_path=model_path,
batch_size=1,
memory_len=0,
target_len=14,
in_train_phase=False,
attention_type=ATTENTION_TYPE_UNI,
)
# Predict
results = model.predict_on_batch(
[token_input, segment_input, memory_length_input])
print('# Uni-directional')
for i in range(len(tokens)):
print(results[0, i, :5])
"""
Official outputs of [0, i, :5]:
'_All': [ 1.3914602 0.47409844 -0.18970338 -1.9293687 -0.97905093]
def build(self):
from keras_xlnet import load_trained_model_from_checkpoint, set_custom_objects
from keras_xlnet import Tokenizer, ATTENTION_TYPE_BI, ATTENTION_TYPE_UNI
self.embedding_type = 'xlnet'
self.checkpoint_path = os.path.join(self.corpus_path, 'xlnet_model.ckpt')
self.config_path = os.path.join(self.corpus_path, 'xlnet_config.json')
self.spiece_model = os.path.join(self.corpus_path, 'spiece.model')
self.attention_type = self.xlnet_embed.get('attention_type', 'bi') # or 'uni'
self.attention_type = ATTENTION_TYPE_BI if self.attention_type == 'bi' else ATTENTION_TYPE_UNI
self.memory_len = self.xlnet_embed.get('memory_len', 0)
self.target_len = self.xlnet_embed.get('target_len', 5)
print('load xlnet model start!')
# 模型加载
model = load_trained_model_from_checkpoint(checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.trainable,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
#
set_custom_objects()
self.build_config(self.config_path)
# 字典加载
self.tokenizer = Tokenizer(self.spiece_model)
# # debug时候查看layers
# self.model_layers = model.layers
# len_layers = self.model_layers.__len__()
# print(len_layers)
num_hidden_layers = self.configs.get("n_layer", 12)
layer_real = [i for i in range(num_hidden_layers)] + [-i for i in range(num_hidden_layers)]
# 简要判别一下
self.layer_indexes = [i if i in layer_real else -2 for i in self.layer_indexes]
output_layer = "FeedForward-Normal-{0}"
layer_dict = [model.get_layer(output_layer.format(i + 1)).get_output_at(node_index=0)
for i in range(num_hidden_layers)]
# 输出它本身
if len(self.layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，取得不正确的话就取倒数第二层
elif len(self.layer_indexes) == 1:
if self.layer_indexes[0] in layer_real:
encoder_layer = layer_dict[self.layer_indexes[0]]
else:
encoder_layer = layer_dict[-1]
# 否则遍历需要取的层，把所有层的weight取出来并加起来shape:768*层数
else:
# layer_indexes must be [0, 1, 2,3,......24]
all_layers = [layer_dict[lay] if lay in layer_real
else layer_dict[-1] # 如果给出不正确，就默认输出倒数第一层
for lay in self.layer_indexes]
print(self.layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
# def xlnet_concat(x):
# x_concat = K.concatenate(x, axis=1)
# return x_concat
# encoder_layer = Lambda(xlnet_concat, name='xlnet_concat')(all_layers)
self.output = NonMaskingLayer()(encoder_layer)
self.input = model.inputs
self.model = Model(self.input, self.output)
print("load KerasXlnetEmbedding end")
model.summary(132)
self.embedding_size = self.model.output_shape[-1]
self.vocab_size = len(self.tokenizer.sp)
def __init__(self):
self.attention_type = ATTENTION_TYPE_BI if args.attention_type[
0] == 'bi' else ATTENTION_TYPE_UNI
self.memory_len, self.target_len, self.batch_size = args.memory_len, args.target_len, args.batch_size
self.checkpoint_path, self.config_path = args.ckpt_name, args.config_name
self.layer_indexes, self.in_train_phase = args.layer_indexes, False
print("load KerasXlnetEmbedding start! ")
# 全局使用，使其可以django、flask、tornado等调用
global graph
graph = tf.get_default_graph()
global model
# 模型加载
model = load_trained_model_from_checkpoint(
checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.in_train_phase,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
# 字典加载
self.tokenizer = Tokenizer(args.spiece_model)
# debug时候查看layers
self.model_layers = model.layers
len_layers = self.model_layers.__len__()
print(len_layers)
len_couche = int((len_layers - 6) / 10)
# 一共246个layer
# 每层10个layer（MultiHeadAttention,Dropout,Add,LayerNormalization）,第一是9个layer的输入和embedding层
# 一共24层
layer_dict = [5]
layer_0 = 6
for i in range(len_couche):
layer_0 = layer_0 + 10
layer_dict.append(layer_0 - 2)
# 输出它本身
if len(self.layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，取得不正确的话就取倒数第二层
elif len(self.layer_indexes) == 1:
if self.layer_indexes[0] in [i + 1 for i in range(len_couche + 1)]:
encoder_layer = model.get_layer(
index=layer_dict[self.layer_indexes[0]]).output
else:
encoder_layer = model.get_layer(index=layer_dict[-2]).output
# 否则遍历需要取的层，把所有层的weight取出来并加起来shape:768*层数
else:
# layer_indexes must be [0, 1, 2,3,......24]
all_layers = [
model.get_layer(index=layer_dict[lay]).output if lay
in [i + 1 for i in range(len_couche + 1)] else model.get_layer(
index=layer_dict[-2]).output # 如果给出不正确，就默认输出倒数第二层
for lay in self.layer_indexes
]
print(self.layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(model.inputs, output_layer)
print("load KerasXlnetEmbedding end")
model.summary(132)
yield [
np.array(X1),
np.array(X2),
np.array(X3),
np.array(X4)
], np.array(Y)
X1, X2, X3, X4, Y = [], [], [], [], []
if sign_num == 2:
# 模型加载
xlnet_model = load_trained_model_from_checkpoint(
checkpoint_path=ckpt_name,
attention_type=ATTENTION_TYPE_BI,
in_train_phase=True,
config_path=config_name,
memory_len=0,
target_len=seq_max_len,
batch_size=batch_size,
mask_index=0)
# MODEL结构
x1_in = Input(shape=(None, ))
x2_in = Input(shape=(None, ))
x3_in = Input(shape=(None, ))
x4_in = Input(shape=(None, ))
x = xlnet_model([x1_in, x2_in, x3_in, x4_in])
x = Lambda(function=lambda x: x[:, 0])(x) # !!!!!!
p = Dense(1, activation='sigmoid')(x)
model = Model([x1_in, x2_in, x3_in, x4_in], p)
else:
def __init__(self, batch_size, gpu_name, gpu_num):
set_gpu_option(gpu_name, gpu_num)
self.attention_type = ATTENTION_TYPE_BI if args.attention_type[0] == 'bi' else ATTENTION_TYPE_UNI
self.memory_len, self.target_len, self.batch_size = args.memory_len, args.target_len, batch_size
self.checkpoint_path, self.config_path = args.ckpt_name, args.config_name
self.layer_indexes, self.in_train_phase = args.layer_indexes, False
print("##### load KerasXlnet start #####")
self.graph = tf.get_default_graph()
# 模型加载
self.model = load_trained_model_from_checkpoint(checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.in_train_phase,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
# 字典加载
self.tokenizer = Tokenizer(args.spiece_model)
# debug时候查看layers
self.model_layers = self.model.layers
len_layers = self.model_layers.__len__()
len_couche = int((len_layers - 6) / 10)
# 一共126个layer
# 每层10个layer,第一是7个layer的输入和embedding层
# 一共12层
layer_dict = [5]
layer_0 = 6
for i in range(len_couche):
layer_0 = layer_0 + 10
layer_dict.append(layer_0 - 2)
# 输出它本身
if len(self.layer_indexes) == 0:
encoder_layer = self.model.output
# 分类如果只有一层，取得不正确的话就取倒数第二层
elif len(self.layer_indexes) == 1:
if self.layer_indexes[0] in [i + 1 for i in range(len_couche + 1)]:
encoder_layer = self.model.get_layer(index=layer_dict[self.layer_indexes[0]]).output
else:
encoder_layer = self.model.get_layer(index=layer_dict[-2]).output
# 否则遍历需要取的层，把所有层的weight取出来并加起来shape:768*层数
else:
# layer_indexes must be [0, 1, 2,3,......12]
all_layers = [self.model.get_layer(index=layer_dict[lay]).output
if lay in [i + 1 for i in range(len_couche + 1)]
else self.model.get_layer(index=layer_dict[-3]).output # 如果给出不正确，就默认输出倒数第二层
for lay in self.layer_indexes]
all_layers = all_layers[1:]
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(self.model.inputs, output_layer)
if gpu_num >= 2:
self.par_model = multi_gpu_model(model, gpus=gpu_num)
else:
self.par_model = model
print("##### load KerasXlnet end #####")
def build(self):
from keras_xlnet import Tokenizer, ATTENTION_TYPE_BI, ATTENTION_TYPE_UNI
from keras_xlnet import load_trained_model_from_checkpoint
self.embedding_type = 'xlnet'
self.checkpoint_path = os.path.join(self.corpus_path, 'xlnet_model.ckpt')
self.config_path = os.path.join(self.corpus_path, 'xlnet_config.json')
self.spiece_model = os.path.join(self.corpus_path, 'spiece.model')
self.attention_type = self.xlnet_embed.get('attention_type', 'bi') # or 'uni'
self.attention_type = ATTENTION_TYPE_BI if self.attention_type == 'bi' else ATTENTION_TYPE_UNI
self.memory_len = self.xlnet_embed.get('memory_len', 0)
self.target_len = self.xlnet_embed.get('target_len', 5)
print('load xlnet model start!')
# 模型加载
model = load_trained_model_from_checkpoint(checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.trainable,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
# 字典加载
self.tokenizer = Tokenizer(self.spiece_model)
# debug时候查看layers
self.model_layers = model.layers
len_layers = self.model_layers.__len__()
print(len_layers)
len_couche = int((len_layers - 6) / 10)
# 一共246个layer
# 每层10个layer（MultiHeadAttention,Dropout,Add,LayerNormalization）,第一是9个layer的输入和embedding层
# 一共24层
layer_dict = [5]
layer_0 = 6
for i in range(len_couche):
layer_0 = layer_0 + 10
layer_dict.append(layer_0 - 2)
# 输出它本身
if len(self.layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，取得不正确的话就取倒数第二层
elif len(self.layer_indexes) == 1:
if self.layer_indexes[0] in [i + 1 for i in range(len_couche + 1)]:
encoder_layer = model.get_layer(index=layer_dict[self.layer_indexes[0]]).output
else:
encoder_layer = model.get_layer(index=layer_dict[-1]).output
# 否则遍历需要取的层，把所有层的weight取出来并加起来shape:768*层数
else:
# layer_indexes must be [0, 1, 2,3,......24]
all_layers = [model.get_layer(index=layer_dict[lay]).output
if lay in [i + 1 for i in range(len_couche + 1)]
else model.get_layer(index=layer_dict[-1]).output # 如果给出不正确，就默认输出倒数第一层
for lay in self.layer_indexes]
print(self.layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
self.output = NonMaskingLayer()(encoder_layer)
self.input = model.inputs
self.model = Model(model.inputs, self.output)
print("load KerasXlnetEmbedding end")
model.summary(132)
self.embedding_size = self.model.output_shape[-1]
self.vocab_size = len(self.tokenizer.sp)
def build(self):
print('load XLNet model start!')
print([
self.target_len, self.memory_len, self.attention_type,
self.batch_size
])
# 模型加载
model = load_trained_model_from_checkpoint(
checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.trainable,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
self.model_layers = model.layers
"""
# debug时候查看layers
for i in range(len(model.layers)):
print([i, model.layers[i]])
base版trainable： 129层 9 + 120 trainable=False: 126 6+120
0-8：输入 + embedding
9-128： 每10个layer一层
mid版trainable=True： 249层 9 + 240 trainable=False: 246 6+240
0-8：输入 + embedding
9-248： 每10个layer一层
"""
len_layers = self.model_layers.__len__()
len_couche = len_layers // 10
layer_0 = len_layers - len_couche * 10
layer_dict = [layer_0 - 1]
if self.trainable == False:
layer_dict[0] += 1
sub_diff = 1
else:
sub_diff = 2
for i in range(len_couche):
layer_0 += 10
layer_dict.append(layer_0 - sub_diff)
if len(self.layer_indexes) == 0:
encoder_layer = model.output
elif len(self.layer_indexes) == 1:
if abs(self.layer_indexes[0]) in [
i for i in range(len_couche + 1)
]:
encoder_layer = model.get_layer(
index=layer_dict[self.layer_indexes[0]]).get_output_at(-1)
else:
encoder_layer = model.get_layer(
index=layer_dict[-1]).get_output_at(-1)
else:
all_layers = [
model.get_layer(index=layer_dict[lay]).get_output_at(-1)
if abs(lay) in [i for i in range(len_couche + 1)] else
model.get_layer(index=layer_dict[-1]).get_output_at(-1)
for lay in self.layer_indexes
]
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
# custom
if self.merge_type == "add":
encoder_layer = Add()(all_layers_select)
elif self.merge_type == "avg":
encoder_layer = Average()(all_layers_select)
elif self.merge_type == "max":
encoder_layer = Maximum()(all_layers_select)
elif self.merge_type == "concat":
encoder_layer = Concatenate()(all_layers_select)
else:
raise RuntimeError("invalid merge type")
print(encoder_layer)
self.output = Lambda(lambda x: x,
output_shape=lambda s: s)(encoder_layer)
self.input = model.inputs
self.model = Model(model.inputs, self.output)
self.embedding_size = self.model.output_shape[-1]
self.vocab_size = len(tokenizer.sp)
self.built = True
if os.path.exists(self.path_fineture) and self.trainable:
self.model.load_weights(self.path_fineture)
print("load Keras XLNet Embedding finish")
