def test_eager_loading(self):
print("Eager Execution:", tf.executing_eagerly())
# a temporal mini bert model_dir
model_dir = self.create_mini_bert_weights()
bert_params = loader.params_from_pretrained_ckpt(model_dir)
bert_params.adapter_size = 32
bert = BertModelLayer.from_params(bert_params, name="bert")
model = keras.models.Sequential([
keras.layers.InputLayer(input_shape=(128, )), bert,
keras.layers.Lambda(lambda x: x[:, 0, :]),
keras.layers.Dense(2)
])
model.build(input_shape=(None, 128))
model.compile(
optimizer=keras.optimizers.Adam(),
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[keras.metrics.SparseCategoricalAccuracy(name="acc")],
run_eagerly=True)
loader.load_stock_weights(bert, model_dir)
model.summary()
def load_keras_model(model_dir, max_seq_len):
from tensorflow.python import keras
from bert import BertModelLayer
from bert.loader import StockBertConfig, load_stock_weights, params_from_pretrained_ckpt
bert_config_file = os.path.join(model_dir, "bert_config.json")
bert_ckpt_file = os.path.join(model_dir, "bert_model.ckpt")
l_bert = BertModelLayer.from_params(
params_from_pretrained_ckpt(model_dir))
l_input_ids = keras.layers.Input(shape=(max_seq_len, ),
dtype='int32',
name="input_ids")
l_token_type_ids = keras.layers.Input(shape=(max_seq_len, ),
dtype='int32',
name="token_type_ids")
output = l_bert([l_input_ids, l_token_type_ids])
model = keras.Model(inputs=[l_input_ids, l_token_type_ids],
outputs=output)
model.build(input_shape=[(None, max_seq_len), (None, max_seq_len)])
load_stock_weights(l_bert, bert_ckpt_file)
return model
def test_load_pretrained(self):
print("Eager Execution:", tf.executing_eagerly())
bert_params = loader.params_from_pretrained_ckpt(self.bert_ckpt_dir)
bert_params.adapter_size = 32
bert = BertModelLayer.from_params(bert_params, name="bert")
model = keras.models.Sequential([
keras.layers.InputLayer(input_shape=(128, )), bert,
keras.layers.Lambda(lambda x: x[:, 0, :]),
keras.layers.Dense(2)
])
# we need to freeze before build/compile - otherwise keras counts the params twice
if bert_params.adapter_size is not None:
freeze_bert_layers(bert)
model.build(input_shape=(None, 128))
model.compile(
optimizer=keras.optimizers.Adam(),
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[keras.metrics.SparseCategoricalAccuracy(name="acc")])
load_stock_weights(bert, self.bert_ckpt_file)
model.summary()
def test_concat(self):
model_dir = self.create_mini_bert_weights()
bert_params = loader.params_from_pretrained_ckpt(model_dir)
bert_params.adapter_size = 32
bert = BertModelLayer.from_params(bert_params, name="bert")
max_seq_len = 4
model = keras.models.Sequential([
keras.layers.InputLayer(input_shape=(max_seq_len, )),
bert,
keras.layers.TimeDistributed(
keras.layers.Dense(bert_params.hidden_size)),
keras.layers.TimeDistributed(keras.layers.LayerNormalization()),
keras.layers.TimeDistributed(keras.layers.Activation("tanh")),
pf.Concat([
keras.layers.Lambda(lambda x: tf.math.reduce_max(x, axis=1)
), # GlobalMaxPooling1D
keras.layers.Lambda(lambda x: tf.math.reduce_mean(x, axis=1)
), # GlobalAvgPooling1
]),
keras.layers.Dense(units=bert_params.hidden_size),
keras.layers.Activation("tanh"),
keras.layers.Dense(units=2)
])
model.build(input_shape=(None, max_seq_len))
model.summary()
model.compile(
optimizer=keras.optimizers.Adam(),
loss=[
keras.losses.SparseCategoricalCrossentropy(from_logits=True)
],
metrics=[keras.metrics.SparseCategoricalAccuracy()],
run_eagerly=True)
loader.load_stock_weights(bert, model_dir)
model.summary()
if each == 't':
templabel.append(1)
else:
templabel.append(0)
y = np.array(templabel)
####################################################
# BERT TOKENIZATION
BertTokenizer = bert_tokenization.FullTokenizer
from bert.loader import params_from_pretrained_ckpt # these are necessary because of weird ImportError: cannot import name 'BertModelLayer' from 'bert' (unknown location) errors
from bert.model import BertModelLayer
bert_params = params_from_pretrained_ckpt(
'D:\\uncased_L-4_H-256_A-4') # from google, not tensorflow hub
bert_layer1 = BertModelLayer.from_params(
bert_params, name="bert") # # hidden_dropout = 0.1,
model_name = 'uncased_L-4_H-256_A-4'
vocabulary_file = os.path.join('D:\\uncased_L-4_H-256_A-4\\vocab.txt')
to_lower_case = not (model_name.find("cased") == 0
or model_name.find("multi_cased") == 0)
tokenizer = BertTokenizer(vocabulary_file, to_lower_case)
max_seq_length = 256
train_tokens = map(tokenizer.tokenize, list(
subsetdf.loc[:,
'content'])) # go all the way back to a list of raw strings
train_tokens = map(lambda tok: ["[CLS]"] + tok + ["[SEP]"], train_tokens)
