def __init__(self):
self.config_path, self.checkpoint_path, self.dict_path, self.max_seq_len = config_name, ckpt_name, vocab_file, max_seq_len
# 全局使用，使其可以django、flask、tornado等调用
global graph
graph = tf.get_default_graph()
global model
model = load_trained_model_from_checkpoint(self.config_path,
self.checkpoint_path,
seq_len=self.max_seq_len)
print(model.output)
print(len(model.layers))
# lay = model.layers
#一共104个layer，其中前八层包括token,pos,embed等，
# 每4层（MultiHeadAttention,Dropout,Add,LayerNormalization）
# 一共24层
layer_dict = []
layer_0 = 7
for i in range(12):
layer_0 = layer_0 + 8
layer_dict.append(layer_0)
# 输出它本身
if len(layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，就只取最后那一层的weight；取得不正确，就默认取最后一层输出
elif len(layer_indexes) == 1:
if layer_indexes[0] in [i + 1 for i in range(23)]:
encoder_layer = model.get_layer(
index=layer_dict[layer_indexes[0]]).output
else:
encoder_layer = model.get_layer(index=layer_dict[-1]).output
# 否则遍历需要取的层，把所有层的weight取出来并拼接起来shape:768*层数
else:
# layer_indexes must be [1,2,3,......12]
# all_layers = [model.get_layer(index=lay).output if lay is not 1 else model.get_layer(index=lay).output[0] for lay in layer_indexes]
all_layers = [
model.get_layer(index=layer_dict[lay - 1]).output
if lay in [i + 1 for i in range(23)] else model.get_layer(
index=layer_dict[-1]).output #如果给出不正确，就默认输出最后一层
for lay in layer_indexes
]
print(layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
print("KerasBertEmbedding:")
print(encoder_layer.shape)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(model.inputs, output_layer)
# model.summary(120)
# reader tokenizer
self.token_dict = {}
with codecs.open(self.dict_path, 'r', 'utf8') as reader:
for line in reader:
token = line.strip()
self.token_dict[token] = len(self.token_dict)
self.tokenizer = Tokenizer(self.token_dict)
def bert_encode(self):
# 全局使用，使其可以django、flask、tornado等调用
global graph
graph = tf.get_default_graph()
global model
model = load_trained_model_from_checkpoint(self.config_path,
self.checkpoint_path,
seq_len=self.max_seq_len)
print(model.output)
# 分类如果只选一层，就只取最后那一层的weight
if len(layer_indexes) == 1:
encoder_layer = model.get_layer(index=len(model.layers) - 1).output
# 否则遍历需要取的层，把所有层的weight取出来并拼接起来shape:768*层数
else:
# layer_indexes must be [1,2,3,......12]
all_layers = [
model.get_layer(index=lay).output for lay in layer_indexes
]
encoder_layer = k_keras.concatenate(all_layers, -1)
print("KerasBertEmbedding:")
print(encoder_layer.shape)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(model.inputs, output_layer)
# model.summary(120)
return model.inputs, model.output
def __init__(self):
self.config_path, self.checkpoint_path, self.dict_path, self.max_seq_len = config_name, ckpt_name, vocab_file, max_seq_len
# 全局使用，使其可以django、flask、tornado等调用
global graph
graph = tf.get_default_graph()
global model
model = load_trained_model_from_checkpoint(self.config_path, self.checkpoint_path,
seq_len=self.max_seq_len)
model.summary(120)
# 如果只选一层，就只取对应那一层的weight
if len(layer_indexes) == 1:
encoder_layer = model.get_layer(index=len(model.layers)-2).output
# 否则遍历需要取的层，把所有层的weight取出来并拼接起来shape:768*层数
else:
# layer_indexes must be [1,2,3,......12]
all_layers = [model.get_layer(index=lay).output for lay in layer_indexes]
encoder_layer = k_keras.concatenate(all_layers, -1)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(model.inputs, output_layer)
# reader tokenizer
self.token_dict = {}
with codecs.open(self.dict_path, 'r', 'utf8') as reader:
for line in reader:
token = line.strip()
self.token_dict[token] = len(self.token_dict)
self.tokenizer = Tokenizer(self.token_dict)
def bert_encode(self, layer_indexes=[12]):
# 全局使用，使其可以django、flask、tornado等调用
global graph
graph = tf.get_default_graph()
global model
model = load_trained_model_from_checkpoint(self.config_path,
self.checkpoint_path,
seq_len=self.max_seq_len)
print(model.output)
print(len(model.layers))
# lay = model.layers
#一共104个layer，其中前八层包括token,pos,embed等，
# 每8层（MultiHeadAttention,Dropout,Add,LayerNormalization）
# 一共12层
layer_dict = []
layer_0 = 7
for i in range(12):
layer_0 = layer_0 + 8
layer_dict.append(layer_0)
# 输出它本身
if len(layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，就只取最后那一层的weight；取得不正确，就默认取最后一层
elif len(layer_indexes) == 1:
if layer_indexes[0] in [i + 1 for i in range(23)]:
encoder_layer = model.get_layer(
index=layer_dict[layer_indexes[0]]).output
else:
encoder_layer = model.get_layer(index=layer_dict[-1]).output
# 否则遍历需要取的层，把所有层的weight取出来并拼接起来shape:768*层数
else:
# layer_indexes must be [1,2,3,......12]
# all_layers = [model.get_layer(index=lay).output if lay is not 1 else model.get_layer(index=lay).output[0] for lay in layer_indexes]
all_layers = [
model.get_layer(index=layer_dict[lay - 1]).output
if lay in [i + 1 for i in range(23)] else model.get_layer(
index=layer_dict[-1]).output #如果给出不正确，就默认输出最后一层
for lay in layer_indexes
]
print(layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
print("KerasBertEmbedding:")
print(encoder_layer.shape)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(model.inputs, output_layer)
# model.summary(120)
return model.inputs, model.output
def __init__(self):
self.attention_type = ATTENTION_TYPE_BI if args.attention_type[
0] == 'bi' else ATTENTION_TYPE_UNI
self.memory_len, self.target_len, self.batch_size = args.memory_len, args.target_len, args.batch_size
self.checkpoint_path, self.config_path = args.ckpt_name, args.config_name
self.layer_indexes, self.in_train_phase = args.layer_indexes, False
print("load KerasXlnetEmbedding start! ")
# 全局使用，使其可以django、flask、tornado等调用
global graph
graph = tf.get_default_graph()
global model
# 模型加载
model = load_trained_model_from_checkpoint(
checkpoint_path=self.checkpoint_path,
attention_type=self.attention_type,
in_train_phase=self.in_train_phase,
config_path=self.config_path,
memory_len=self.memory_len,
target_len=self.target_len,
batch_size=self.batch_size,
mask_index=0)
# 字典加载
self.tokenizer = Tokenizer(args.spiece_model)
# debug时候查看layers
self.model_layers = model.layers
len_layers = self.model_layers.__len__()
print(len_layers)
len_couche = int((len_layers - 6) / 10)
# 一共246个layer
# 每层10个layer（MultiHeadAttention,Dropout,Add,LayerNormalization）,第一是9个layer的输入和embedding层
# 一共24层
layer_dict = [5]
layer_0 = 6
for i in range(len_couche):
layer_0 = layer_0 + 10
layer_dict.append(layer_0 - 2)
# 输出它本身
if len(self.layer_indexes) == 0:
encoder_layer = model.output
# 分类如果只有一层，取得不正确的话就取倒数第二层
elif len(self.layer_indexes) == 1:
if self.layer_indexes[0] in [i + 1 for i in range(len_couche + 1)]:
encoder_layer = model.get_layer(
index=layer_dict[self.layer_indexes[0]]).output
else:
encoder_layer = model.get_layer(index=layer_dict[-2]).output
# 否则遍历需要取的层，把所有层的weight取出来并加起来shape:768*层数
else:
# layer_indexes must be [0, 1, 2,3,......24]
all_layers = [
model.get_layer(index=layer_dict[lay]).output if lay
in [i + 1 for i in range(len_couche + 1)] else model.get_layer(
index=layer_dict[-2]).output # 如果给出不正确，就默认输出倒数第二层
for lay in self.layer_indexes
]
print(self.layer_indexes)
print(all_layers)
all_layers_select = []
for all_layers_one in all_layers:
all_layers_select.append(all_layers_one)
encoder_layer = Add()(all_layers_select)
print(encoder_layer.shape)
output_layer = NonMaskingLayer()(encoder_layer)
model = Model(model.inputs, output_layer)
print("load KerasXlnetEmbedding end")
model.summary(132)