def model_fn(features, labels, mode, params):
logger.info("*** Features ***")
for name in sorted(features.keys()):
logger.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
print('shape of input_ids', input_ids.shape)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# 使用参数构建模型,input_idx 就是输入的样本idx表示，label_ids 就是标签的idx表示
total_loss, logits, trans, pred_ids = create_model(
bert_config, is_training, input_ids, input_mask, segment_ids,
label_ids, num_labels, False, FLAGS.dropout_rate, FLAGS.lstm_size,
FLAGS.cell, FLAGS.num_layers)
tvars = tf.trainable_variables()
# 加载BERT模型
if init_checkpoint:
(assignment_map, initialized_variable_names) = \
modeling.get_assignment_map_from_checkpoint(tvars,
init_checkpoint)
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, False)
hook_dict = {}
hook_dict['loss'] = total_loss
hook_dict['global_steps'] = tf.train.get_or_create_global_step()
logging_hook = tf.train.LoggingTensorHook(
hook_dict, every_n_iter=FLAGS.save_summary_steps)
output_spec = tf.estimator.EstimatorSpec(
mode=mode,
loss=total_loss,
train_op=train_op,
training_hooks=[logging_hook])
elif mode == tf.estimator.ModeKeys.EVAL:
def metric_fn(label_ids, pred_ids):
return {
"eval_loss":
tf.metrics.mean_squared_error(labels=label_ids,
predictions=pred_ids),
}
eval_metrics = metric_fn(label_ids, pred_ids)
output_spec = tf.estimator.EstimatorSpec(
mode=mode, loss=total_loss, eval_metric_ops=eval_metrics)
else:
output_spec = tf.estimator.EstimatorSpec(mode=mode,
predictions=pred_ids)
return output_spec
def bert_optimizer_layer(self):
correct_prediction = tf.equal(
tf.argmax(self.logits, 2), tf.cast(self.targets, tf.int64))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
num_train_steps = int(
self.train_length / self.batch_size * self.max_epoch)
num_warmup_steps = int(num_train_steps * 0.1)
self.train_op = create_optimizer(
self.loss, self.learning_rate, num_train_steps, num_warmup_steps, False
)
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=5)
def model_fn(features, labels, mode, params):
logger.info("*** Features ***")
for name in sorted(features.keys()):
logger.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
(total_loss, per_example_loss, logits,
probabilities) = create_model(bert_config, is_training, input_ids,
input_mask, segment_ids, label_ids,
num_labels)
# resort variable from checkpoint file to init current graph
tvars = tf.trainable_variables()
initialized_variable_names = {}
init_fn = None
if init_checkpoint:
(assignment_map, initialized_variable_names) = \
modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
#variables_to_restore = tf.contrib.framework.get_model_variables()
#init_fn = tf.contrib.framework.\
# assign_from_checkpoint_fn(init_checkpoint,
# variables_to_restore,
# ignore_missing_vars=True)
# 打印变量名称
logger.info("**** Trainable Variables ****")
for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
logger.info(" name = %s, shape = %s%s", var.name, var.shape,
init_string)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
train_op = optimization.create_optimizer(total_loss,
learning_rate,
num_train_steps,
num_warmup_steps,
use_tpu=False)
output_spec = tf.estimator.EstimatorSpec(mode=mode,
loss=total_loss,
train_op=train_op)
# training_hooks=[RestoreHook(init_fn)])
elif mode == tf.estimator.ModeKeys.EVAL:
def metric_fn(per_example_loss, label_ids, logits):
predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)
accuracy = tf.metrics.accuracy(label_ids, predictions)
loss = tf.metrics.mean(per_example_loss)
return {
"eval_accuracy": accuracy,
"eval_loss": loss,
}
eval_metrics = metric_fn(per_example_loss, label_ids, logits)
output_spec = tf.estimator.EstimatorSpec(
mode=mode,
loss=total_loss,
eval_metric_ops=eval_metrics
#evaluation_hooks=[RestoreHook(init_fn)]
)
else:
output_spec = tf.estimator.EstimatorSpec(mode=mode,
predictions=probabilities)
return output_spec
def model_fn(features, labels, mode, params):
logger.info("*** Features ***")
for name in sorted(features.keys()):
logger.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
print('shape of input_ids', input_ids.shape)
# label_mask = features["label_mask"]
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# 使用参数构建模型,input_idx 就是输入的样本idx表示，label_ids 就是标签的idx表示
total_loss, logits, trans, pred_ids = create_model(
bert_config, is_training, input_ids, input_mask, segment_ids,
label_ids, num_labels, False, args.dropout_rate, args.lstm_size,
args.cell, args.num_layers, args.crf_only)
tvars = tf.trainable_variables()
# 加载BERT模型
if init_checkpoint:
(assignment_map, initialized_variable_names) = \
modeling.get_assignment_map_from_checkpoint(tvars,
init_checkpoint)
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
# 打印变量名
# logger.info("**** Trainable Variables ****")
#
# # 打印加载模型的参数
# for var in tvars:
# init_string = ""
# if var.name in initialized_variable_names:
# init_string = ", *INIT_FROM_CKPT*"
# logger.info(" name = %s, shape = %s%s", var.name, var.shape,
# init_string)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
#train_op = optimizer.optimizer(total_loss, learning_rate, num_train_steps)
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, False)
hook_dict = {}
hook_dict['loss'] = total_loss
hook_dict['global_steps'] = tf.train.get_or_create_global_step()
logging_hook = tf.train.LoggingTensorHook(
hook_dict, every_n_iter=args.save_summary_steps)
output_spec = tf.estimator.EstimatorSpec(
mode=mode,
loss=total_loss,
train_op=train_op,
training_hooks=[logging_hook]) #如果不送值，则训练过程中不会显示字典中的数值。
elif mode == tf.estimator.ModeKeys.EVAL:
# 针对NER ,进行了修改
def metric_fn(label_ids, pred_ids):
return {
"accuracy":
tf.metrics.accuracy(labels=label_ids,
predictions=pred_ids),
'eval_loss':
tf.metrics.mean_squared_error(labels=label_ids,
predictions=pred_ids),
'f1':
tf_metrics.f1(labels=label_ids,
predictions=pred_ids,
num_classes=num_labels,
pos_indices=args.pos_indices)
# "f1score": 2 * tf.metrics.recall(labels=label_ids, predictions=pred_ids) * tf.metrics.precision(
# labels=label_ids, predictions=pred_ids) / (tf.metrics.recall(labels=label_ids,
# predictions=pred_ids) + tf.metrics.precision(
# labels=label_ids, predictions=pred_ids)),
}
eval_metrics = metric_fn(label_ids, pred_ids)
output_spec = tf.estimator.EstimatorSpec(
mode=mode, loss=total_loss, eval_metric_ops=eval_metrics)
else:
output_spec = tf.estimator.EstimatorSpec(mode=mode,
predictions=pred_ids)
return output_spec
def model_fn(features, labels, mode, params):
logger.info("*** Features ***")
for name in sorted(features.keys()):
logger.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
print('shape of input_ids', input_ids.shape)
# label_mask = features["label_mask"]
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# 使用参数构建模型,input_idx 就是输入的样本idx表示，label_ids 就是标签的idx表示
total_loss, logits, pred_ids = create_model(
bert_config, is_training, input_ids, input_mask, segment_ids,
label_ids, num_labels, False, args.dropout_rate, args.lstm_size,
args.cell, args.num_layers, args.crf_only, args.lstm_only)
tvars = tf.trainable_variables()
# 加载BERT模型
if init_checkpoint:
(assignment_map, initialized_variable_names) = \
modeling.get_assignment_map_from_checkpoint(tvars,
init_checkpoint)
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
# enable trainable on last layers, and frozen other layers
if args.trainable_last_layers:
last_layers = [
'finetune', 'bert/encoder/layer_11', 'bert/encoder/layer_10',
'bert/encoder/layer_9', 'bert/encoder/layer_8',
'bert/encoder/layer_7'
]
trainable_layers_index = [
int(ind) for ind in args.trainable_last_layers.split(',')
]
trainable_layers = [
name for index, name in enumerate(last_layers)
if index in trainable_layers_index
]
tf.logging.info("Only allow last n layers are trainable: " +
str(trainable_layers))
filter_trainable_variables(trainable_layers)
# 打印变量名
if args.verbose:
tvars = tf.trainable_variables()
logger.info("**** Trainable Variables ****")
# 打印加载模型的参数
for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
logger.info(" name = %s, shape = %s%s", var.name, var.shape,
init_string)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
#train_op = optimizer.optimizer(total_loss, learning_rate, num_train_steps)
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, False)
hook_dict = {}
hook_dict['loss'] = total_loss
hook_dict['global_steps'] = tf.train.get_or_create_global_step()
logging_hook = tf.train.LoggingTensorHook(
hook_dict, every_n_iter=args.save_summary_steps)
output_spec = tf.estimator.EstimatorSpec(
mode=mode,
loss=total_loss,
train_op=train_op,
training_hooks=[logging_hook]) #如果不送值，则训练过程中不会显示字典中的数值
elif mode == tf.estimator.ModeKeys.EVAL:
# 针对NER ,进行了修改
def metric_fn(label_ids, pred_ids, mask):
# 计算无pad的位置
mask = tf.cast(tf.reshape(mask, [-1]), dtype=tf.bool)
# 去除pad
pred_ids = tf.to_int32(tf.reshape(pred_ids, [-1]))[mask]
label_ids = tf.to_int32(tf.reshape(label_ids, [-1]))[mask]
# metrics
loss = tf.metrics.mean_squared_error(labels=label_ids,
predictions=pred_ids)
accuracy = tf.metrics.accuracy(label_ids, pred_ids)
pos_indices = [i for (i, label) in enumerate(label_list, 1)]
f1 = tf_metrics.f1(label_ids,
pred_ids,
num_labels,
pos_indices=pos_indices)
return {
"eval_loss": loss,
"eval_accuracy": accuracy,
"eval_f1": f1,
}
eval_metrics = metric_fn(label_ids, pred_ids, input_mask)
output_spec = tf.estimator.EstimatorSpec(
mode=mode, loss=total_loss, eval_metric_ops=eval_metrics)
else:
output_spec = tf.estimator.EstimatorSpec(mode=mode,
predictions=pred_ids)
return output_spec
def model_fn(features, labels, mode, params): # pylint: disable=unused-argument
"""The `model_fn` for TPUEstimator."""
tf.logging.info("*** Features ***")
for name in sorted(features.keys()):
tf.logging.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
is_real_example = None
if "is_real_example" in features:
is_real_example = tf.cast(features["is_real_example"],
dtype=tf.float32)
else:
is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
(total_loss, per_example_loss, logits,
probabilities) = create_classification_model(bert_config, is_training,
input_ids, input_mask,
segment_ids, label_ids,
num_labels,
use_one_hot_embeddings)
tvars = tf.trainable_variables()
initialized_variable_names = {}
scaffold_fn = None
if init_checkpoint:
(assignment_map, initialized_variable_names
) = modeling.get_assignment_map_from_checkpoint(
tvars, init_checkpoint)
if use_tpu:
def tpu_scaffold():
tf.train.init_from_checkpoint(init_checkpoint,
assignment_map)
return tf.train.Scaffold()
scaffold_fn = tpu_scaffold
else:
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
tf.logging.info("**** Trainable Variables ****")
for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape,
init_string)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, use_tpu)
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=total_loss,
train_op=train_op,
scaffold_fn=scaffold_fn)
elif mode == tf.estimator.ModeKeys.EVAL:
def metric_fn(per_example_loss, label_ids, logits,
is_real_example):
predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)
accuracy = tf.metrics.accuracy(labels=label_ids,
predictions=predictions,
weights=is_real_example)
loss = tf.metrics.mean(values=per_example_loss,
weights=is_real_example)
return {
"eval_accuracy": accuracy,
"eval_loss": loss,
}
eval_metrics = (metric_fn, [
per_example_loss, label_ids, logits, is_real_example
])
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=total_loss,
eval_metrics=eval_metrics,
scaffold_fn=scaffold_fn)
else:
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
predictions={"probabilities": probabilities},
scaffold_fn=scaffold_fn)
return output_spec
def model_fn(features, labels, mode, params):
"""
:param features:
:param labels:
:param mode:
:param params:
:return:
"""
logger.info('*** Features ***')
for name in sorted(features.keys()):
logger.info(' name = %s, shape = %s' %
(name, features[name].shape))
input_ids = features['input_ids']
input_mask = features['input_mask']
segment_ids = features['segment_ids']
label_ids = features['label_ids']
print('shape of input ids', input_ids.shape)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# 使用参数构造模型，input_idx就是输入的样本idx表示，label_ids就是标签的idx表示
total_loss, logits, trans, pred_ids = create_model(
bert_config=bert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
labels=label_ids,
num_labels=num_labels,
use_one_hot_embeddings=False,
dropout_rate=FLAGS.dropout_rate,
lstm_size=FLAGS.lstm_size,
cell=FLAGS.cell,
num_layers=FLAGS.num_layers)
"""
tf.trainable_variables(): 返回需要训练的变量列表
tf.all_variables(): 返回的是所有变量的列表
"""
tvars = tf.trainable_variables()
# 加载Bert模型
if init_checkpoint:
(assigment_map, initialized_variable_names
) = modeling.get_assignment_map_from_checkpoint(
tvars=tvars, init_checkpoint=init_checkpoint)
tf.train.init_from_checkpoint(init_checkpoint, assigment_map)
output_spec = None
# 分三种情况，mode分别为训练、验证、测试
if mode == tf.estimator.ModeKeys.TRAIN:
train_op = optimization.create_optimizer(
loss=total_loss,
init_lr=learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=False)
hook_dict = dict()
hook_dict['loss'] = total_loss
hook_dict['global_steps'] = tf.train.get_or_create_global_step()
logging_hook = tf.train.LoggingTensorHook(
hook_dict, every_n_iter=FLAGS.save_summary_steps)
output_spec = tf.estimator.EstimatorSpec(
mode=mode,
loss=total_loss,
train_op=train_op,
training_hooks=[logging_hook])
elif mode == tf.estimator.ModeKeys.EVAL:
def metric_fn(label_ids, pred_ids):
return {
'eval_loss':
tf.metrics.mean_squared_error(labels=label_ids,
predictions=pred_ids)
}
eval_metrics = metric_fn(label_ids, pred_ids)
output_spec = tf.estimator.EstimatorSpec(
mode=mode, loss=total_loss, eval_metric_ops=eval_metrics)
else:
output_spec = tf.estimator.EstimatorSpec(mode=mode,
predictions=pred_ids)
return output_spec
def model_fn(features, labels, mode, params):
tf.logging.info("*** Features ***")
for name in sorted(features.keys()):
tf.logging.info(" name = %s, shape = %s" %
(name, features[name].shape))
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
print('shape of input_ids', input_ids.shape)
# label_mask = features["label_mask"]
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# 使用参数构建模型,input_idx 就是输入的样本idx表示，label_ids 就是标签的idx表示
(total_loss, logits, trans,
pred_ids) = create_model(bert_config, is_training, input_ids,
input_mask, segment_ids, label_ids,
num_labels, False, args.dropout_rate,
args.lstm_size, args.cell, args.num_layers)
tvars = tf.trainable_variables()
# 加载BERT模型
if init_checkpoint:
(assignment_map, initialized_variable_names) = \
modeling.get_assignment_map_from_checkpoint(tvars,
init_checkpoint)
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
# 打印变量名
# logger.info("**** Trainable Variables ****")
#
# # 打印加载模型的参数
# for var in tvars:
# init_string = ""
# if var.name in initialized_variable_names:
# init_string = ", *INIT_FROM_CKPT*"
# logger.info(" name = %s, shape = %s%s", var.name, var.shape,
# init_string)
output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
#train_op = optimizer.optimizer(total_loss, learning_rate, num_train_steps)
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, False)
output_spec = tf.estimator.EstimatorSpec(mode=mode,
loss=total_loss,
train_op=train_op)
elif mode == tf.estimator.ModeKeys.EVAL:
# 针对NER ,进行了修改
def metric_fn(label_ids, logits, trans):
# 首先对结果进行维特比解码
# crf 解码
weight = tf.sequence_mask(args.max_seq_length)
precision = tf_metrics.precision(label_ids, pred_ids,
num_labels, None, weight)
recall = tf_metrics.recall(label_ids, pred_ids, num_labels,
None, weight)
f = tf_metrics.f1(label_ids, pred_ids, num_labels, None,
weight)
return {
"eval_precision":
precision,
"eval_recall":
recall,
"eval_f":
f,
"eval_loss":
tf.metrics.mean_squared_error(labels=label_ids,
predictions=pred_ids),
}
eval_metrics = metric_fn(label_ids, logits, trans)
output_spec = tf.estimator.EstimatorSpec(
mode=mode, loss=total_loss, eval_metric_ops=eval_metrics)
else:
output_spec = tf.estimator.EstimatorSpec(mode=mode,
predictions=pred_ids)
return output_spec
