def load_tokenizer_and_model():
# move heavy imports into the load function
from transformers import BertConfig, TFBertForSequenceClassification, BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
bert_config = BertConfig.from_dict({
"attention_probs_dropout_prob": 0.1,
"directionality": "bidi",
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"hidden_size": 768,
"initializer_range": 0.02,
"intermediate_size": 3072,
"max_position_embeddings": 512,
"num_attention_heads": 12,
"num_hidden_layers": 12,
"pooler_fc_size": 768,
"pooler_num_attention_heads": 12,
"pooler_num_fc_layers": 3,
"pooler_size_per_head": 128,
"pooler_type": "first_token_transform",
"type_vocab_size": 2,
"vocab_size": 21128
})
model = TFBertForSequenceClassification(bert_config)
# Call the model once so it gets build with the right shape before loading the weights
model((np.random.random((1, 100)) * 50).astype(int))
model.load_weights('./models/sentiment_analysis_bert.hdf5')
return tokenizer, model
config = BertConfig(num_labels=3, return_dict=True, model_type='bert-base-uncased')
model = TFBertForSequenceClassification(config=config)
if save_model:
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
model.compile(optimizer=optimizer, loss=model.compute_loss, metrics=['accuracy'])
model.fit(
train_dataset[0],
np.array(y_list),
epochs=5,
batch_size=BATCH_SIZE,
callbacks=[cp_callback]
)
else:
latest = tf.train.latest_checkpoint(checkpoint_dir)
model.load_weights(latest)
preds = model.predict(val_dataset[0])["logits"]
preds_proba = tf.keras.backend.softmax(preds, axis=1)
classes = np.argmax(preds, axis=-1)
score = classification_report(y_val, classes, digits=3)
print(score)
total = time.time() - start
print(f"Done in: {total}")
