def get_input_data():
config = read_configure()
corpus = get_corpus_processor(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
tags_data = generate_tagset(corpus_meta_data['tags'])
train_data = list(train_data_generator_func())
eval_data = list(eval_data_generator_func())
tag_lookup = Lookuper({v: i for i, v in enumerate(tags_data)})
vocab_data_file = './data/unicode_char_list.txt'
vocabulary_lookup = index_table_from_file(vocab_data_file)
train_x, train_y = preprocss(train_data, tag_lookup, vocabulary_lookup)
if eval_data:
test_x, test_y = preprocss(eval_data, tag_lookup, vocabulary_lookup)
else:
test_x, test_y = None, None
return config, (train_x, train_y), (test_x,
test_y), tag_lookup, vocabulary_lookup
def main():
raw_config = read_configure()
model = Model(raw_config)
config = model.get_default_config()
config.update(raw_config)
corpus = get_corpus_processor(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
config["tags_data"] = generate_tagset(corpus_meta_data["tags"])
# train and evaluate model
train_input_func = build_input_func(train_data_generator_func, config)
eval_input_func = (build_input_func(eval_data_generator_func, config)
if eval_data_generator_func else None)
evaluate_result, export_results, final_saved_model = model.train_and_eval_then_save(
train_input_func, eval_input_func, config)
export_as_deliverable_model(create_dir_if_needed(
config["deliverable_model_dir"]),
tensorflow_saved_model=final_saved_model,
converter_for_request=converter_for_request,
converter_for_response=converter_for_response,
addition_model_dependency=["micro_toolkit"])
def train(self, addition_config=None, *args, **kwargs):
raw_config = read_configure(*args, **kwargs)
# add dynamic configure
if addition_config:
raw_config.update(addition_config)
model = Model(raw_config)
config = model.get_default_config()
config.update(raw_config)
task_status_class = get_task_status_class(config)
task_status = task_status_class(config)
# read data according configure
corpus = Corpus(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
config['tags_data'] = corpus_meta_data['labels']
config['num_classes'] = len(config['tags_data'])
# build model according configure
# send START status to monitor system
task_status.send_status(task_status.START)
# train and evaluate model
train_input_func = build_input_func(train_data_generator_func, config)
eval_input_func = build_input_func(eval_data_generator_func, config)
evaluate_result, export_results, final_saved_model = model.train_and_eval_then_save(
train_input_func, eval_input_func, config)
task_status.send_status(task_status.DONE)
if evaluate_result:
performance_metrics_class = get_performance_metrics_class(config)
performance_metrics = performance_metrics_class(config)
performance_metrics.set_metrics('test_loss',
evaluate_result['loss'])
performance_metrics.set_metrics('test_acc', evaluate_result['acc'])
model_saver = ModelSaver(config)
model_saver.save_model(final_saved_model)
return final_saved_model
def main():
raw_config = read_configure()
model = Model(raw_config)
config = model.get_default_config() # 默认的配置
config.update(raw_config)
# mlflow hyperparameter
mlflow.log_param("Batch_Size", config["batch_size"])
mlflow.log_param("Learning_Rate", config["learning_rate"])
mlflow.log_param("Epochs", config["epochs"])
mlflow.log_param("Embedding_Dim", config["embedding_dim"])
corpus = get_corpus_processor(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
config["tags_data"] = generate_tagset(corpus_meta_data["tags"])
# train and evaluate model
train_input_func = build_input_func(train_data_generator_func, config)
eval_input_func = (build_input_func(eval_data_generator_func, config)
if eval_data_generator_func else None)
evaluate_result, export_results, final_saved_model = model.train_and_eval_then_save(
train_input_func, eval_input_func, config)
# mlflow metrics
# mlflow.log_metrics(evaluate_result, step=100)
# mlflow Logging the saved model
mlflow.tensorflow.log_model(tf_saved_model_dir=final_saved_model,
tf_meta_graph_tags=[tag_constants.SERVING],
tf_signature_def_key='serving_default',
artifact_path='model')
export_as_deliverable_model(create_dir_if_needed(
config["deliverable_model_dir"]),
tensorflow_saved_model=final_saved_model,
converter_for_request=ConverterForRequest(),
converter_for_response=ConverterForResponse(),
addition_model_dependency=["micro_toolkit"])
import tensorflow as tf
from tensorflow.python.keras import models
from tensorflow.python.keras.layers import Embedding, Bidirectional, LSTM
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras import layers
from ioflow.configure import read_configure
from ioflow.corpus import get_corpus_processor
from seq2annotation.input import generate_tagset, Lookuper, \
index_table_from_file
from tf_crf_layer.layer import CRF
from tf_crf_layer.loss import crf_loss
from tf_crf_layer.metrics import crf_accuracy
from tokenizer_tools.tagset.converter.offset_to_biluo import offset_to_biluo
config = read_configure()
corpus = get_corpus_processor(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
raw_tag_data = corpus_meta_data['tags']
tags_data = generate_tagset(corpus_meta_data['tags'])
train_data = list(train_data_generator_func())
eval_data = list(eval_data_generator_func())
tag_lookuper = Lookuper({v: i for i, v in enumerate(tags_data)})
def main():
config = read_configure() # ioflow
corpus = get_corpus_processor(config)
corpus.prepare() # ?
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
tags_data = generate_tagset(corpus_meta_data["tags"]) # process entity into BIO
train_data = list(train_data_generator_func())
eval_data = list(eval_data_generator_func())
tag_lookuper = Lookuper({v: i for i, v in enumerate(tags_data)}) # tag index
vocab_data_file = config.get("vocabulary_file")
vocabulary_lookuper = index_table_from_file(vocab_data_file)
def preprocss(data, maxlen):
raw_x = []
raw_y = []
for offset_data in data:
tags = offset_to_biluo(offset_data)
words = offset_data.text
tag_ids = [tag_lookuper.lookup(i) for i in tags]
word_ids = [vocabulary_lookuper.lookup(i) for i in words]
raw_x.append(word_ids)
raw_y.append(tag_ids)
if maxlen is None:
maxlen = max(len(s) for s in raw_x)
print(">>> maxlen: {}".format(maxlen))
x = tf.keras.preprocessing.sequence.pad_sequences(
raw_x, maxlen, padding="post"
) # right padding
# lef padded with -1. Indeed, any integer works as it will be masked
# y_pos = pad_sequences(y_pos, maxlen, value=-1)
# y_chunk = pad_sequences(y_chunk, maxlen, value=-1)
y = tf.keras.preprocessing.sequence.pad_sequences(
raw_y, maxlen, value=0, padding="post"
)
return x, y
MAX_SENTENCE_LEN = config.get("max_sentence_len", 25)
train_x, train_y = preprocss(train_data, MAX_SENTENCE_LEN)
test_x, test_y = preprocss(eval_data, MAX_SENTENCE_LEN)
EPOCHS = config["epochs"]
BATCH_SIZE = config["batch_size"]
EMBED_DIM = config["embedding_dim"]
USE_ATTENTION_LAYER = config.get("use_attention_layer", False)
BiLSTM_STACK_CONFIG = config.get("bilstm_stack_config", [])
BATCH_NORMALIZATION_AFTER_EMBEDDING_CONFIG = config.get(
"use_batch_normalization_after_embedding", False
)
BATCH_NORMALIZATION_AFTER_BILSTM_CONFIG = config.get(
"use_batch_normalization_after_bilstm", False
)
CRF_PARAMS = config.get("crf_params", {})
OPTIMIZER_PARAMS = config.get("optimizer_params", {})
vacab_size = vocabulary_lookuper.size()
tag_size = tag_lookuper.size()
model = Sequential()
model.add(
Embedding(vacab_size, EMBED_DIM, embeddings_initializer='glorot_normal',
mask_zero=True, input_length=MAX_SENTENCE_LEN)
)
if BATCH_NORMALIZATION_AFTER_EMBEDDING_CONFIG:
model.add(BatchNormalization())
for bilstm_config in BiLSTM_STACK_CONFIG:
model.add(Bidirectional(LSTM(return_sequences=True, **bilstm_config)))
if BATCH_NORMALIZATION_AFTER_BILSTM_CONFIG:
model.add(BatchNormalization())
if USE_ATTENTION_LAYER:
model.add(GlobalAttentionLayer())
model.add(CRF(tag_size, name="crf", **CRF_PARAMS))
# print model summary
model.summary()
callbacks_list = []
tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=create_dir_if_needed(config["summary_log_dir"])
)
callbacks_list.append(tensorboard_callback)
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
os.path.join(create_dir_if_needed(config["model_dir"]), "cp-{epoch:04d}.ckpt"),
load_weights_on_restart=True,
verbose=1,
)
callbacks_list.append(checkpoint_callback)
metrics_list = []
metrics_list.append(SequenceCorrectness())
metrics_list.append(SequenceSpanAccuracy())
loss_func = ConditionalRandomFieldLoss()
# loss_func = crf_loss
optimizer = optimizers.Adam(**OPTIMIZER_PARAMS)
# optimizer = optimizers.Nadam(**OPTIMIZER_PARAMS)
model.compile(optimizer=optimizer, loss={"crf": loss_func}, metrics=metrics_list)
model.fit(
train_x,
train_y,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_data=[test_x, test_y],
callbacks=callbacks_list,
)
# Save the model
model.save(create_file_dir_if_needed(config["h5_model_file"]))
tf.keras.experimental.export_saved_model(
model, create_dir_if_needed(config["saved_model_dir"]))
export_as_deliverable_model(
create_dir_if_needed(config["deliverable_model_dir"]),
keras_saved_model=config["saved_model_dir"],
vocabulary_lookup_table=vocabulary_lookuper,
tag_lookup_table=tag_lookuper,
padding_parameter={"maxlen": MAX_SENTENCE_LEN, "value": 0, "padding": "post"},
addition_model_dependency=["tf-crf-layer"],
custom_object_dependency=["tf_crf_layer"],
)
def main():
# get configure
config = read_configure()
# get train/test corpus
corpus = get_corpus_processor(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
# process str data to onehot
ner_tags_data = generate_tagset(corpus_meta_data["tags"])
cls_tags_data = corpus_meta_data["labels"]
train_data = list(train_data_generator_func())
eval_data = list(eval_data_generator_func())
ner_tag_lookuper = Lookuper({v: i for i, v in enumerate(ner_tags_data)})
cls_tag_lookuper = Lookuper({v: i for i, v in enumerate(cls_tags_data)})
vocab_data_file = config.get("vocabulary_file")
if not vocab_data_file:
# load built in vocabulary file
vocab_data_file = os.path.join(
os.path.dirname(__file__), "../data/unicode_char_list.txt"
)
vocabulary_lookuper = index_table_from_file(vocab_data_file)
def preprocss(data, maxlen, **kwargs):
raw_x = []
raw_y_ner = []
raw_y_cls = []
for offset_data in data:
tags = offset_to_biluo(offset_data)
label = offset_data.label
words = offset_data.text
tag_ids = [ner_tag_lookuper.lookup(i) for i in tags]
label_id = cls_tag_lookuper.lookup(label)
word_ids = [vocabulary_lookuper.lookup(i) for i in words]
raw_x.append(word_ids)
raw_y_ner.append(tag_ids)
raw_y_cls.append(label_id)
if maxlen is None:
maxlen = max(len(s) for s in raw_x)
print(">>> maxlen: {}".format(maxlen))
x = tf.keras.preprocessing.sequence.pad_sequences(
raw_x, maxlen, padding="post"
) # right padding
y_ner = tf.keras.preprocessing.sequence.pad_sequences(
raw_y_ner, maxlen, value=0, padding="post"
)
from keras.utils import to_categorical
y_cls = np.array(raw_y_cls)
y_cls = y_cls[:, np.newaxis]
y_cls = to_categorical(y_cls, kwargs.get('cls_dims', 81))
return x, y_ner, y_cls
# get Parameters (controller)
EPOCHS = config.get("epochs", 10)
BATCHSIZE = config.get("batch_size", 32)
LEARNINGRATE = config.get("learning_rate", 0.001)
MAX_SENTENCE_LEN = config.get("max_sentence_len", 25)
# get Parameters (model structure)
EMBED_DIM = config.get("embedding_dim", 300)
USE_ATTENTION_LAYER = config.get("use_attention_layer", False)
BiLSTM_STACK_CONFIG = config.get("bilstm_stack_config", [])
BATCH_NORMALIZATION_AFTER_EMBEDDING_CONFIG = config.get(
"use_batch_normalization_after_embedding", False)
BATCH_NORMALIZATION_AFTER_BILSTM_CONFIG = config.get(
"use_batch_normalization_after_bilstm", False)
CRF_PARAMS = config.get("crf_params", {})
# get train/test data for training model
vacab_size = vocabulary_lookuper.size()
tag_size = ner_tag_lookuper.size()
label_size = cls_tag_lookuper.size()
train_x, train_y_ner, train_y_cls = preprocss(train_data, MAX_SENTENCE_LEN, **{'cls_dims':label_size})
test_x, test_y_ner, test_y_cls = preprocss(eval_data, MAX_SENTENCE_LEN, **{'cls_dims':label_size})
# build model
input_length = MAX_SENTENCE_LEN
input_layer = Input(shape=(input_length,), dtype='float', name='input_layer')
# encoder
with tf.keras.backend.name_scope("Encoder"):
embedding_layer = Embedding(vacab_size,
EMBED_DIM,
mask_zero=True,
input_length=input_length,
name='embedding')(input_layer)
# feature extractor
with tf.keras.backend.name_scope("biLSTM"):
if BATCH_NORMALIZATION_AFTER_EMBEDDING_CONFIG:
embedding_layer = BatchNormalization()(embedding_layer)
biLSTM = embedding_layer
for bilstm_config in BiLSTM_STACK_CONFIG:
biLSTM = Bidirectional(LSTM(return_sequences=True, **bilstm_config, name='biLSTM'))(biLSTM)
if BATCH_NORMALIZATION_AFTER_BILSTM_CONFIG:
biLSTM = BatchNormalization()(biLSTM)
if USE_ATTENTION_LAYER:
biLSTM = GlobalAttentionLayer()(biLSTM)
# NER branch
with tf.keras.backend.name_scope("NER_branch"):
crf = CRF(tag_size, name="crf", **CRF_PARAMS)(biLSTM)
loss_func = ConditionalRandomFieldLoss()
# classification branch
chosen = 'lstm_cls'
with tf.keras.backend.name_scope("CLS_branch"):
from tensorflow.keras.layers import Dense, Flatten, Dropout
# add paragraph vector
#paragraph_vector = get_paragraph_vector(embedding_layer)
if chosen == "lstm_cls":
cls_flat_lstm = Flatten()(biLSTM)
#cls_flat_lstm = tf.keras.layers.concatenate([cls_flat_lstm, paragraph_vector])
classification_dense = Dropout(0.2)(cls_flat_lstm)
classification_dense = SetLearningRate(Dense(label_size, activation='sigmoid', name='CLS'), lr=0.001, is_ada=True)(classification_dense)
elif chosen == "conv_cls":
from tensorflow.keras.layers import Conv1D, MaxPooling1D
embedding_layer = BatchNormalization()(embedding_layer)
cls_conv_emb = Conv1D(32, 3, activation='relu', padding='same')(embedding_layer)
cls_conv_emb = Conv1D(64, 3, activation='relu', padding='same')(cls_conv_emb)
cls_conv_emb = MaxPooling1D(2)(cls_conv_emb)
cls_conv_emb = Conv1D(128, 3, activation='relu', dilation_rate=1, padding='same')(cls_conv_emb)
cls_conv_emb = Conv1D(128, 3, activation='relu', dilation_rate=2, padding='same')(cls_conv_emb)
cls_conv_emb = Conv1D(128, 3, activation='relu', dilation_rate=5, padding='same')(cls_conv_emb)
cls_conv_emb = Conv1D(256, 1, activation='relu', padding='same')(cls_conv_emb)
cls_conv_emb = MaxPooling1D(2)(cls_conv_emb)
cls_flat = BatchNormalization()(cls_conv_emb)
cls_flat = Flatten()(cls_flat)
classification_dense = Dropout(0.2)(cls_flat)
classification_dense = Dense(label_size, activation='sigmoid', name='CLS')(classification_dense)
# merge NER and Classification
model = Model(inputs=[input_layer], outputs=[crf, classification_dense])
model.summary()
callbacks_list = []
tensorboard_callback = tf.keras.callbacks.TensorBoard(
#log_dir=create_dir_if_needed(config["summary_log_dir"])
log_dir='.\\results\\summary_log_dir',
batch_size=BATCHSIZE,
)
callbacks_list.append(tensorboard_callback)
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
os.path.join(create_dir_if_needed(config["model_dir"]), "cp-{epoch:04d}.ckpt"),
load_weights_on_restart=True,
verbose=1,
)
callbacks_list.append(checkpoint_callback)
metrics_list = []
metrics_list.append(crf_accuracy)
metrics_list.append(SequenceCorrectness())
metrics_list.append(sequence_span_accuracy)
# early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', # early stop index
# patience=3, # early stop delay epoch
# verbose=2, # display mode
# mode='auto')
# callbacks_list.append(early_stop)
from mtnlpmodel.trainer.loss_func_util import FocalLoss
adam_optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNINGRATE, beta_1=0.9, beta_2=0.999, amsgrad=False)
model.compile(optimizer=adam_optimizer,
#loss={'crf': loss_func, 'CLS': 'sparse_categorical_crossentropy'},
loss={'crf': loss_func, 'CLS': FocalLoss()},
loss_weights={'crf': 1., 'CLS': 100}, # set weight of loss
#metrics={'crf': SequenceCorrectness(), 'CLS': 'sparse_categorical_accuracy'} )
metrics={'crf': SequenceCorrectness(), 'CLS': 'categorical_accuracy'})
model.fit(
train_x,
{'crf': train_y_ner, 'CLS': train_y_cls},
epochs=EPOCHS,
batch_size=BATCHSIZE,
validation_data=[test_x, {'crf': test_y_ner, 'CLS': test_y_cls}],
callbacks=callbacks_list,
)
model.save(create_file_dir_if_needed(config["h5_model_file"]))
model.save_weights(create_file_dir_if_needed(config["h5_weights_file"]))
tf.keras.experimental.export_saved_model(
model, create_or_rm_dir_if_needed(config["saved_model_dir"])
)
mt_export_as_deliverable_model(
create_dir_if_needed(config["deliverable_model_dir"]),
keras_saved_model=config["saved_model_dir"],
converter_for_request=ConverterForRequest(),
converter_for_response=ConverterForMTResponse(),
lookup_tables={'vocab_lookup':vocabulary_lookuper,
'tag_lookup':ner_tag_lookuper,
'label_lookup':cls_tag_lookuper},
padding_parameter={"maxlen": MAX_SENTENCE_LEN, "value": 0, "padding": "post"},
addition_model_dependency=["tf-crf-layer"],
custom_object_dependency=["tf_crf_layer"],
)
import os
import tensorflow as tf
from tokenizer_tools.tagset.converter.offset_to_biluo import offset_to_biluo
from ioflow.configure import read_configure
from ioflow.corpus import get_corpus_processor
from ner_kashgari.input import generate_tagset, Lookuper, index_table_from_file
config = read_configure() # ioflow
corpus = get_corpus_processor(config)
corpus.prepare() # ?
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
tags_data = generate_tagset(
corpus_meta_data["tags"]) # process entity into BIO
train_data = list(train_data_generator_func())
eval_data = list(eval_data_generator_func())
tag_lookuper = Lookuper({v: i for i, v in enumerate(tags_data)}) # tag index
vocab_data_file = config.get("vocabulary_file")
vocabulary_lookuper = index_table_from_file(vocab_data_file) # dict index
def preprocss(data, maxlen):
raw_x = []
raw_y = []
for offset_data in data:
tags = offset_to_biluo(offset_data)
def main():
# get configure
config = read_configure()
# get train/test corpus
corpus = get_corpus_processor(config)
corpus.prepare()
train_data_generator_func = corpus.get_generator_func(corpus.TRAIN)
eval_data_generator_func = corpus.get_generator_func(corpus.EVAL)
corpus_meta_data = corpus.get_meta_info()
# process str data to onehot
ner_tags_data = generate_tagset(corpus_meta_data["tags"])
cls_tags_data = corpus_meta_data["labels"]
train_data = list(train_data_generator_func())
eval_data = list(eval_data_generator_func())
ner_tag_lookuper = Lookuper({v: i for i, v in enumerate(ner_tags_data)})
cls_tag_lookuper = Lookuper({v: i for i, v in enumerate(cls_tags_data)})
vocab_data_file = config.get("vocabulary_file")
if not vocab_data_file:
# load built in vocabulary file
vocab_data_file = os.path.join(os.path.dirname(__file__),
"../data/unicode_char_list.txt")
vocabulary_lookuper = index_table_from_file(vocab_data_file)
def preprocss(data, maxlen):
raw_x = []
raw_y_ner = []
raw_y_cls = []
for offset_data in data:
tags = offset_to_biluo(offset_data)
label = offset_data.label
words = offset_data.text
tag_ids = [ner_tag_lookuper.lookup(i) for i in tags]
label_id = cls_tag_lookuper.lookup(label)
word_ids = [vocabulary_lookuper.lookup(i) for i in words]
raw_x.append(word_ids)
raw_y_ner.append(tag_ids)
raw_y_cls.append(label_id)
if maxlen is None:
maxlen = max(len(s) for s in raw_x)
print(">>> maxlen: {}".format(maxlen))
x = tf.keras.preprocessing.sequence.pad_sequences(
raw_x, maxlen, padding="post") # right padding
y_ner = tf.keras.preprocessing.sequence.pad_sequences(raw_y_ner,
maxlen,
value=0,
padding="post")
y_cls = np.array(raw_y_cls)
y_cls = y_cls[:, np.newaxis]
return x, y_ner, y_cls
# get Parameters (controller)
EPOCHS = config.get("epochs", 10)
BATCHSIZE = config.get("batch_size", 32)
LEARNINGRATE = config.get("learning_rate", 0.0001)
MAX_SENTENCE_LEN = config.get("max_sentence_len", 25)
# get Parameters (model structure)
EMBED_DIM = config.get("embedding_dim", 300)
BiLSTM_STACK_CONFIG = config.get("bilstm_stack_config", [])
# get train/test data for training model
train_x, train_y_ner, train_y_cls = preprocss(train_data, MAX_SENTENCE_LEN)
test_x, test_y_ner, test_y_cls = preprocss(eval_data, MAX_SENTENCE_LEN)
vacab_size = vocabulary_lookuper.size()
# tag_size = ner_tag_lookuper.size()
label_size = cls_tag_lookuper.size()
# finetuning correlation code
adam_optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNINGRATE,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
index_dict = {
'optimizer': adam_optimizer,
'loss': 'sparse_categorical_crossentropy',
'metrics': ['sparse_categorical_accuracy']
}
warm_start_list = ['embedding', 'bidirectional',
'batch_normalization'] # layer in list is frozen
backbone_model_path = './mtnlpmodel/trainer/fine_tuning_trainer/save_weights/weights.h5'
output_dims = label_size
# model structure correlation code
# define new_layer for the task
new_task_output_layer = Dense(
output_dims, activation='softmax') # new softmax layer -> output
input_shape = MAX_SENTENCE_LEN
input_layer = Input(shape=(input_shape, ),
dtype='int32',
name='input_layer') # input
# backbone + transfer_learning function can use backbone to do some different job
# what you need is to define a new layer which is new_task_output_layer below
# this code is a sample for only text classification
# backbone output is biLSTM's output, so transfer_learning add a flatten layer to connect dense layer
# you can modify the structure in function transfer_learning to match your task demand
base_model = backbone_network(
BiLSTM_STACK_CONFIG,
input_layer=input_layer,
vacab_size=vacab_size,
EMBED_DIM=EMBED_DIM,
input_length=MAX_SENTENCE_LEN,
)
new_model = transfer_learning(input_shape, input_layer, base_model,
new_task_output_layer, index_dict,
backbone_model_path, warm_start_list)
# model output info correlation code
new_model.summary()
callbacks_list = []
tensorboard_callback = tf.keras.callbacks.TensorBoard(
#log_dir=create_dir_if_needed(config["summary_log_dir"])
log_dir='.\\results\\summary_log_dir',
batch_size=BATCHSIZE,
)
callbacks_list.append(tensorboard_callback)
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
os.path.join(create_dir_if_needed(config["model_dir"]),
"cp-{epoch:04d}.ckpt"),
load_weights_on_restart=True,
verbose=1,
)
callbacks_list.append(checkpoint_callback)
metrics_list = []
metrics_list.append(crf_accuracy)
metrics_list.append(SequenceCorrectness())
metrics_list.append(sequence_span_accuracy)
early_stop = tf.keras.callbacks.EarlyStopping(
monitor='val_loss', # early stop index
patience=3, # early stop delay epoch
verbose=2, # display mode
mode='auto')
callbacks_list.append(early_stop)
new_model.fit(
train_x,
train_y_cls,
epochs=EPOCHS,
batch_size=BATCHSIZE,
validation_data=[test_x, test_y_cls],
callbacks=callbacks_list,
)
new_model.save(create_file_dir_if_needed(config["h5_model_file"]))
tf.keras.experimental.export_saved_model(
new_model, create_or_rm_dir_if_needed(config["saved_model_dir"]))
mt_export_as_deliverable_model(
create_dir_if_needed(config["deliverable_model_dir"]),
keras_saved_model=config["saved_model_dir"],
vocabulary_lookup_table=vocabulary_lookuper,
tag_lookup_table=ner_tag_lookuper,
label_lookup_table=cls_tag_lookuper,
padding_parameter={
"maxlen": MAX_SENTENCE_LEN,
"value": 0,
"padding": "post"
},
addition_model_dependency=["tf-crf-layer"],
custom_object_dependency=["tf_crf_layer"],
)
