def test_fit(self): current_path = os.path.dirname(os.path.abspath(__file__)) model_path = os.path.join(current_path, 'test_bert_fit.h5') sentence_pairs = [ [['all', 'work', 'and', 'no', 'play'], ['makes', 'jack', 'a', 'dull', 'boy']], [['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']], [['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more', 'power']], ] token_dict = get_base_dict() for pairs in sentence_pairs: for token in pairs[0] + pairs[1]: if token not in token_dict: token_dict[token] = len(token_dict) token_list = list(token_dict.keys()) if os.path.exists(model_path): model = keras.models.load_model( model_path, custom_objects=get_custom_objects(), ) else: model = get_model( token_num=len(token_dict), head_num=5, transformer_num=12, embed_dim=25, feed_forward_dim=100, seq_len=20, pos_num=20, dropout_rate=0.05, attention_activation=gelu, lr=1e-3, decay_steps=30000, warmup_steps=10000, weight_decay=1e-3, ) model.summary() def _generator(): while True: yield gen_batch_inputs( sentence_pairs, token_dict, token_list, seq_len=20, mask_rate=0.3, swap_sentence_rate=1.0, ) model.fit_generator( generator=_generator(), steps_per_epoch=1000, epochs=1, validation_data=_generator(), validation_steps=100, ) # model.save(model_path) for inputs, outputs in _generator(): predicts = model.predict(inputs) outputs = list(map(lambda x: np.squeeze(x, axis=-1), outputs)) predicts = list(map(lambda x: np.argmax(x, axis=-1), predicts)) batch_size, seq_len = inputs[-1].shape for i in range(batch_size): for j in range(seq_len): if inputs[-1][i][j]: self.assertEqual(outputs[0][i][j], predicts[0][i][j]) self.assertTrue(np.allclose(outputs[1], predicts[1])) break
# Train & Use from keras_bert import get_base_dict, get_model, gen_batch_inputs import keras # A toy input example sentence_pairs = [ [['all', 'work', 'and', 'no', 'play'], ['makes', 'jack', 'a', 'dull', 'boy']], [['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']], [['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more', 'power']], ] # Build token dictionary token_dict = get_base_dict() # A dict that contains some special tokens for pairs in sentence_pairs: for token in pairs[0] + pairs[1]: if token not in token_dict: token_dict[token] = len(token_dict) token_list = list(token_dict.keys()) # Used for selecting a random word # Build & train the model model = get_model( token_num=len(token_dict), head_num=5, transformer_num=12, embed_dim=25, feed_forward_dim=100, seq_len=20, pos_num=20,
def GetModel(ucfg): ''' ucfg: user's Config for the table output: nnname, BS, BPE ''' nnname = ucfg['nnname'] isconv = True if nnname == 'newmodel': import sys sys.path.append("..") from newmodel import tfmodel model, isconv = tfmodel() sys.path.remove("..") import tensorflow.keras.applications as nn if hasattr(nn, nnname): model = getattr(nn, nnname)(weights=None) # efficientnet: B0-B7 elif nnname[:-2] == 'EfficientNet': import tfmodels.efficientnet.tfkeras as nn model = getattr(nn, nnname)(weights=None) # TF2.x Models: elif nnname == 'ncf': import tfmodels.ncf as nn name = 'ncfmodel' model = getattr(nn, name)(istrain=False) isconv = False elif nnname == 'din': import tfmodels.din as nn name = 'din' _, model = getattr(nn, name)(item_count=63001, cate_count=801, hidden_units=128) isconv = False # bert from bert_keras elif nnname == 'bert': isconv = False from keras_bert import get_base_dict, get_model, compile_model # Build token dictionary token_dict = get_base_dict() training = True if training: # # bert base # embed_dim=768 # bert small # headnum=12 # layernum=12 # bert large embed_dim = 1024 # bert small headnum = 16 layernum = 24 ff_dim = embed_dim * 4 token_num = 30522 # number of words from paper model = get_model(token_num=token_num, pos_num=512, seq_len=512, embed_dim=embed_dim, transformer_num=layernum, head_num=headnum, feed_forward_dim=ff_dim, training=training) else: # Revise lib\site-packages\keras_bert\bert.py: line164 # "return inputs, transformed" -> "return inputs, transformed,model" _, _, model = get_model(token_num=len(token_dict), embed_dim=1024, head_num=16, training=training) compile_model(model) if nnname == 'mymodel': isconv = False ## ===== To add a customized model ==== # refer to: https://keras.io/guides/sequential_model/ from tensorflow.keras import layers # Define a customized model model = keras.Sequential() model.add(keras.Input(shape=(250, 250, 3))) # 250x250 RGB images model.add(layers.Conv2D(32, 5, strides=2, activation="relu")) model.add(layers.Conv2D(32, 3, activation="relu")) model.add(layers.MaxPooling2D(3)) model.add(layers.Conv2D(32, 3, activation="relu")) model.add(layers.Conv2D(32, 3, activation="relu")) model.add(layers.MaxPooling2D(3)) model.add(layers.Conv2D(32, 3, activation="relu")) model.add(layers.Conv2D(32, 3, activation="relu")) model.add(layers.MaxPooling2D(2)) # Now that we have 4x4 feature maps, time to apply global max pooling. model.add(layers.GlobalMaxPooling2D()) # Finally, we add a classification layer. model.add(layers.Dense(10)) ## ===== end of your codes ====== if True: g = keras.utils.model_to_dot(model, show_shapes=True) if nnname == 'newmodel': nnname = ucfg['model'] g.write_pdf(".//outputs//tf//" + nnname + '.pdf') return model, isconv
for i in sen_n: for ii in range(len(i) - 1): sen.append([i[ii], i[ii + 1]]) for i in sen: yield i sentence_pairs = [ [['all', 'work', 'and', 'no', 'play'], ['makes', 'jack', 'a', 'dull', 'boy']], [['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']], [['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more', 'power']], ] # sentence_pairs = [i for i in stp()] token_dict = get_base_dict() for pairs in sentence_pairs: for token in pairs[0] + pairs[1]: if token not in token_dict: token_dict[token] = len(token_dict) token_list = list(token_dict.keys()) model = get_model(token_num=len(token_dict), head_num=5, transformer_num=12, embed_dim=25, feed_forward_dim=100, seq_len=20, pos_num=20, dropout_rate=0.1) model.summary()
# 训练和使用 import keras from keras_bert import get_base_dict, get_model, compile_model, gen_batch_inputs # 随便的输入样例: sentence_pairs = [ [['all', 'work', 'and', 'no', 'play'], ['makes', 'jack', 'a', 'dull', 'boy']], [['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']], [['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more', 'power']], ] # 构建自定义词典 token_dict = get_base_dict() # 初始化特殊符号,如`[CLS]` for pairs in sentence_pairs: for token in pairs[0] + pairs[1]: if token not in token_dict: token_dict[token] = len(token_dict) token_list = list(token_dict.keys()) # Used for selecting a random word # 构建和训练模型 model = get_model( token_num=len(token_dict), head_num=5, transformer_num=12, embed_dim=25, feed_forward_dim=100, seq_len=20, pos_num=20,