def test_eager_loading(self): print("Eager Execution:", tf.executing_eagerly()) # a temporal mini bert model_dir model_dir = self.create_mini_bert_weights() bert_params = loader.params_from_pretrained_ckpt(model_dir) bert_params.adapter_size = 32 bert = BertModelLayer.from_params(bert_params, name="bert") model = keras.models.Sequential([ keras.layers.InputLayer(input_shape=(128, )), bert, keras.layers.Lambda(lambda x: x[:, 0, :]), keras.layers.Dense(2) ]) model.build(input_shape=(None, 128)) model.compile( optimizer=keras.optimizers.Adam(), loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=[keras.metrics.SparseCategoricalAccuracy(name="acc")], run_eagerly=True) loader.load_stock_weights(bert, model_dir) model.summary()
def load_keras_model(model_dir, max_seq_len): from tensorflow.python import keras from bert import BertModelLayer from bert.loader import StockBertConfig, load_stock_weights, params_from_pretrained_ckpt bert_config_file = os.path.join(model_dir, "bert_config.json") bert_ckpt_file = os.path.join(model_dir, "bert_model.ckpt") l_bert = BertModelLayer.from_params( params_from_pretrained_ckpt(model_dir)) l_input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32', name="input_ids") l_token_type_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32', name="token_type_ids") output = l_bert([l_input_ids, l_token_type_ids]) model = keras.Model(inputs=[l_input_ids, l_token_type_ids], outputs=output) model.build(input_shape=[(None, max_seq_len), (None, max_seq_len)]) load_stock_weights(l_bert, bert_ckpt_file) return model
def test_load_pretrained(self): print("Eager Execution:", tf.executing_eagerly()) bert_params = loader.params_from_pretrained_ckpt(self.bert_ckpt_dir) bert_params.adapter_size = 32 bert = BertModelLayer.from_params(bert_params, name="bert") model = keras.models.Sequential([ keras.layers.InputLayer(input_shape=(128, )), bert, keras.layers.Lambda(lambda x: x[:, 0, :]), keras.layers.Dense(2) ]) # we need to freeze before build/compile - otherwise keras counts the params twice if bert_params.adapter_size is not None: freeze_bert_layers(bert) model.build(input_shape=(None, 128)) model.compile( optimizer=keras.optimizers.Adam(), loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=[keras.metrics.SparseCategoricalAccuracy(name="acc")]) load_stock_weights(bert, self.bert_ckpt_file) model.summary()
def test_concat(self): model_dir = self.create_mini_bert_weights() bert_params = loader.params_from_pretrained_ckpt(model_dir) bert_params.adapter_size = 32 bert = BertModelLayer.from_params(bert_params, name="bert") max_seq_len = 4 model = keras.models.Sequential([ keras.layers.InputLayer(input_shape=(max_seq_len, )), bert, keras.layers.TimeDistributed( keras.layers.Dense(bert_params.hidden_size)), keras.layers.TimeDistributed(keras.layers.LayerNormalization()), keras.layers.TimeDistributed(keras.layers.Activation("tanh")), pf.Concat([ keras.layers.Lambda(lambda x: tf.math.reduce_max(x, axis=1) ), # GlobalMaxPooling1D keras.layers.Lambda(lambda x: tf.math.reduce_mean(x, axis=1) ), # GlobalAvgPooling1 ]), keras.layers.Dense(units=bert_params.hidden_size), keras.layers.Activation("tanh"), keras.layers.Dense(units=2) ]) model.build(input_shape=(None, max_seq_len)) model.summary() model.compile( optimizer=keras.optimizers.Adam(), loss=[ keras.losses.SparseCategoricalCrossentropy(from_logits=True) ], metrics=[keras.metrics.SparseCategoricalAccuracy()], run_eagerly=True) loader.load_stock_weights(bert, model_dir) model.summary()
if each == 't': templabel.append(1) else: templabel.append(0) y = np.array(templabel) #################################################### # BERT TOKENIZATION BertTokenizer = bert_tokenization.FullTokenizer from bert.loader import params_from_pretrained_ckpt # these are necessary because of weird ImportError: cannot import name 'BertModelLayer' from 'bert' (unknown location) errors from bert.model import BertModelLayer bert_params = params_from_pretrained_ckpt( 'D:\\uncased_L-4_H-256_A-4') # from google, not tensorflow hub bert_layer1 = BertModelLayer.from_params( bert_params, name="bert") # # hidden_dropout = 0.1, model_name = 'uncased_L-4_H-256_A-4' vocabulary_file = os.path.join('D:\\uncased_L-4_H-256_A-4\\vocab.txt') to_lower_case = not (model_name.find("cased") == 0 or model_name.find("multi_cased") == 0) tokenizer = BertTokenizer(vocabulary_file, to_lower_case) max_seq_length = 256 train_tokens = map(tokenizer.tokenize, list( subsetdf.loc[:, 'content'])) # go all the way back to a list of raw strings train_tokens = map(lambda tok: ["[CLS]"] + tok + ["[SEP]"], train_tokens)