def test_fit(self):
current_path = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_path, 'test_bert_fit.h5')
sentence_pairs = [
[['all', 'work', 'and', 'no', 'play'], ['makes', 'jack', 'a', 'dull', 'boy']],
[['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']],
[['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more', 'power']],
]
token_dict = get_base_dict()
for pairs in sentence_pairs:
for token in pairs[0] + pairs[1]:
if token not in token_dict:
token_dict[token] = len(token_dict)
token_list = list(token_dict.keys())
if os.path.exists(model_path):
model = keras.models.load_model(
model_path,
custom_objects=get_custom_objects(),
)
else:
model = get_model(
token_num=len(token_dict),
head_num=5,
transformer_num=12,
embed_dim=25,
feed_forward_dim=100,
seq_len=20,
pos_num=20,
dropout_rate=0.05,
attention_activation=gelu,
lr=1e-3,
decay_steps=30000,
warmup_steps=10000,
weight_decay=1e-3,
)
model.summary()
def _generator():
while True:
yield gen_batch_inputs(
sentence_pairs,
token_dict,
token_list,
seq_len=20,
mask_rate=0.3,
swap_sentence_rate=1.0,
)
model.fit_generator(
generator=_generator(),
steps_per_epoch=1000,
epochs=1,
validation_data=_generator(),
validation_steps=100,
)
# model.save(model_path)
for inputs, outputs in _generator():
predicts = model.predict(inputs)
outputs = list(map(lambda x: np.squeeze(x, axis=-1), outputs))
predicts = list(map(lambda x: np.argmax(x, axis=-1), predicts))
batch_size, seq_len = inputs[-1].shape
for i in range(batch_size):
for j in range(seq_len):
if inputs[-1][i][j]:
self.assertEqual(outputs[0][i][j], predicts[0][i][j])
self.assertTrue(np.allclose(outputs[1], predicts[1]))
break
# Train & Use
from keras_bert import get_base_dict, get_model, gen_batch_inputs
import keras
# A toy input example
sentence_pairs = [
[['all', 'work', 'and', 'no', 'play'],
['makes', 'jack', 'a', 'dull', 'boy']],
[['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']],
[['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more',
'power']],
]
# Build token dictionary
token_dict = get_base_dict() # A dict that contains some special tokens
for pairs in sentence_pairs:
for token in pairs[0] + pairs[1]:
if token not in token_dict:
token_dict[token] = len(token_dict)
token_list = list(token_dict.keys()) # Used for selecting a random word
# Build & train the model
model = get_model(
token_num=len(token_dict),
head_num=5,
transformer_num=12,
embed_dim=25,
feed_forward_dim=100,
seq_len=20,
pos_num=20,
def GetModel(ucfg):
''' ucfg: user's Config for the table output: nnname, BS, BPE '''
nnname = ucfg['nnname']
isconv = True
if nnname == 'newmodel':
import sys
sys.path.append("..")
from newmodel import tfmodel
model, isconv = tfmodel()
sys.path.remove("..")
import tensorflow.keras.applications as nn
if hasattr(nn, nnname):
model = getattr(nn, nnname)(weights=None)
# efficientnet: B0-B7
elif nnname[:-2] == 'EfficientNet':
import tfmodels.efficientnet.tfkeras as nn
model = getattr(nn, nnname)(weights=None)
# TF2.x Models:
elif nnname == 'ncf':
import tfmodels.ncf as nn
name = 'ncfmodel'
model = getattr(nn, name)(istrain=False)
isconv = False
elif nnname == 'din':
import tfmodels.din as nn
name = 'din'
_, model = getattr(nn, name)(item_count=63001,
cate_count=801,
hidden_units=128)
isconv = False
# bert from bert_keras
elif nnname == 'bert':
isconv = False
from keras_bert import get_base_dict, get_model, compile_model
# Build token dictionary
token_dict = get_base_dict()
training = True
if training:
# # bert base
# embed_dim=768 # bert small
# headnum=12
# layernum=12
# bert large
embed_dim = 1024 # bert small
headnum = 16
layernum = 24
ff_dim = embed_dim * 4
token_num = 30522 # number of words from paper
model = get_model(token_num=token_num,
pos_num=512,
seq_len=512,
embed_dim=embed_dim,
transformer_num=layernum,
head_num=headnum,
feed_forward_dim=ff_dim,
training=training)
else:
# Revise lib\site-packages\keras_bert\bert.py: line164
# "return inputs, transformed" -> "return inputs, transformed,model"
_, _, model = get_model(token_num=len(token_dict),
embed_dim=1024,
head_num=16,
training=training)
compile_model(model)
if nnname == 'mymodel':
isconv = False
## ===== To add a customized model ====
# refer to: https://keras.io/guides/sequential_model/
from tensorflow.keras import layers
# Define a customized model
model = keras.Sequential()
model.add(keras.Input(shape=(250, 250, 3))) # 250x250 RGB images
model.add(layers.Conv2D(32, 5, strides=2, activation="relu"))
model.add(layers.Conv2D(32, 3, activation="relu"))
model.add(layers.MaxPooling2D(3))
model.add(layers.Conv2D(32, 3, activation="relu"))
model.add(layers.Conv2D(32, 3, activation="relu"))
model.add(layers.MaxPooling2D(3))
model.add(layers.Conv2D(32, 3, activation="relu"))
model.add(layers.Conv2D(32, 3, activation="relu"))
model.add(layers.MaxPooling2D(2))
# Now that we have 4x4 feature maps, time to apply global max pooling.
model.add(layers.GlobalMaxPooling2D())
# Finally, we add a classification layer.
model.add(layers.Dense(10))
## ===== end of your codes ======
if True:
g = keras.utils.model_to_dot(model, show_shapes=True)
if nnname == 'newmodel':
nnname = ucfg['model']
g.write_pdf(".//outputs//tf//" + nnname + '.pdf')
return model, isconv
for i in sen_n:
for ii in range(len(i) - 1):
sen.append([i[ii], i[ii + 1]])
for i in sen:
yield i
sentence_pairs = [
[['all', 'work', 'and', 'no', 'play'],
['makes', 'jack', 'a', 'dull', 'boy']],
[['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']],
[['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more',
'power']],
]
# sentence_pairs = [i for i in stp()]
token_dict = get_base_dict()
for pairs in sentence_pairs:
for token in pairs[0] + pairs[1]:
if token not in token_dict:
token_dict[token] = len(token_dict)
token_list = list(token_dict.keys())
model = get_model(token_num=len(token_dict),
head_num=5,
transformer_num=12,
embed_dim=25,
feed_forward_dim=100,
seq_len=20,
pos_num=20,
dropout_rate=0.1)
model.summary()
# 训练和使用
import keras
from keras_bert import get_base_dict, get_model, compile_model, gen_batch_inputs
# 随便的输入样例：
sentence_pairs = [
[['all', 'work', 'and', 'no', 'play'],
['makes', 'jack', 'a', 'dull', 'boy']],
[['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']],
[['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more',
'power']],
]
# 构建自定义词典
token_dict = get_base_dict() # 初始化特殊符号，如`[CLS]`
for pairs in sentence_pairs:
for token in pairs[0] + pairs[1]:
if token not in token_dict:
token_dict[token] = len(token_dict)
token_list = list(token_dict.keys()) # Used for selecting a random word
# 构建和训练模型
model = get_model(
token_num=len(token_dict),
head_num=5,
transformer_num=12,
embed_dim=25,
feed_forward_dim=100,
seq_len=20,
pos_num=20,
