def model_fn(features, labels, mode, params):
training = mode == tf.estimator.ModeKeys.TRAIN
spec = tfhub.create_module_spec(module_fn,
tags_and_args=[
({'train'}, {
'training': True,
'params': params
}),
(set(), {
'training': False,
'params': params
}),
])
tags = {'train'} if training else None
module = tfhub.Module(spec, trainable=training, tags=tags)
tfhub.register_module_for_export(module, 'doodle')
image = features['image']
image.shape.assert_is_compatible_with([None, 28, 28, 1])
predictions = module(image, as_dict=True)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
export_outputs={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
tf.estimator.export.PredictOutput(predictions),
})
with tf.variable_scope('losses'):
cross_entropy_loss = tf.losses.sparse_softmax_cross_entropy(
labels=labels, logits=predictions['logits'])
total_loss = tf.losses.get_total_loss()
tf.summary.image('image', image)
tf.summary.scalar('total_loss', total_loss)
metric_ops = metrics.calculate(labels, predictions['classes'],
params['num_classes'])
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode=mode,
loss=total_loss,
eval_metric_ops=metric_ops)
if mode == tf.estimator.ModeKeys.TRAIN:
global_step = tf.train.get_or_create_global_step()
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.variable_scope('optimizer'):
optimizer = tf.train.AdamOptimizer(params['learning_rate'])
with tf.control_dependencies(update_ops):
fit = optimizer.minimize(total_loss, global_step)
return tf.estimator.EstimatorSpec(mode=mode,
loss=total_loss,
train_op=fit,
eval_metric_ops=metric_ops)
def main(_):
logging.set_verbosity(logging.INFO)
processors = {"ner": NerProcessor}
if not FLAGS.do_train and not FLAGS.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file,
do_lower_case=FLAGS.do_lower_case)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_train_examples(FLAGS.data_dir)
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size *
FLAGS.num_train_epochs)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
model_fn = model_fn_builder(bert_config=bert_config,
num_labels=len(label_list),
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.eval_batch_size,
predict_batch_size=FLAGS.predict_batch_size)
if FLAGS.do_train:
train_file = os.path.join(FLAGS.output_dir, "train.tf_record")
_, _ = filed_based_convert_examples_to_features(
train_examples, label_list, FLAGS.max_seq_length, tokenizer,
train_file)
logging.info("***** Running training *****")
logging.info(" Num examples = %d", len(train_examples))
logging.info(" Batch size = %d", FLAGS.train_batch_size)
logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
if FLAGS.do_eval:
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
batch_tokens, batch_labels = filed_based_convert_examples_to_features(
eval_examples, label_list, FLAGS.max_seq_length, tokenizer,
eval_file)
logging.info("***** Running evaluation *****")
logging.info(" Num examples = %d", len(eval_examples))
logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# if FLAGS.use_tpu:
# eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size)
# eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.evaluate(input_fn=eval_input_fn)
output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as wf:
logging.info("***** Eval results *****")
confusion_matrix = result["confusion_matrix"]
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
logging.info("***********************************************")
logging.info("********************P = %s*********************",
str(p))
logging.info("********************R = %s*********************",
str(r))
logging.info("********************F = %s*********************",
str(f))
logging.info("***********************************************")
if FLAGS.do_predict:
with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as rf:
label2id = pickle.load(rf)
id2label = {value: key for key, value in label2id.items()}
predict_examples = processor.get_test_examples(FLAGS.data_dir)
predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record")
batch_tokens, batch_labels = filed_based_convert_examples_to_features(
predict_examples, label_list, FLAGS.max_seq_length, tokenizer,
predict_file)
logging.info("***** Running prediction*****")
logging.info(" Num examples = %d", len(predict_examples))
logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.predict(input_fn=predict_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt")
#here if the tag is "X" means it belong to its before token, here for convenient evaluate use
# conlleval.pl we discarding it directly
Writer(output_predict_file, result, batch_tokens, batch_labels,
id2label)
def main(_):
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
logging.set_verbosity(logging.INFO)
processors = {"ner": NerProcessor}
if not FLAGS.do_train and not FLAGS.do_eval:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(
vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
config = tf.ConfigProto()
if FLAGS.do_train:
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
config.gpu_options.per_process_gpu_memory_fraction = 0.9
else:
config.gpu_options.allow_growth = False
config.gpu_options.per_process_gpu_memory_fraction = 0.1
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
session_config=config,
keep_checkpoint_max=3,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_train_examples(FLAGS.data_dir)
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs * FLAGS.training_run_count)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
model_fn = model_fn_builder(
bert_config=bert_config,
num_labels=len(label_list),
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
predict_batch_size=FLAGS.predict_batch_size,
eval_batch_size=FLAGS.eval_batch_size
)
if FLAGS.do_train:
train_file = os.path.join(FLAGS.output_dir, "train.tf_record")
_, _, _, _ = filed_based_convert_examples_to_features(
train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file)
logging.info("***** Running training *****")
logging.info(" Num examples = %d", len(train_examples))
logging.info(" Batch size = %d", FLAGS.train_batch_size)
logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
if FLAGS.do_eval:
# # run evaluation on training set
#
# logging.info("***** Running evaluation on training *****")
# logging.info(" Num examples = %d", len(train_examples))
# logging.info(" Batch size = %d", FLAGS.train_batch_size)
# # if FLAGS.use_tpu:
# # eval_steps = int(len(eval_examples) / FLAGS.)
# # eval_drop_remainder = True if FLAGS.use_tpu else False
#
# training_result = estimator.evaluate(input_fn=train_input_fn)
# output_training_file = os.path.join(FLAGS.output_dir, "training_result.txt")
# with open(output_training_file, "w") as wf:
# logging.info("***** Training Eval results *****")
# confusion_matrix = training_result["confusion_matrix"]
# p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
# logging.info("***********************************************")
# logging.info("********************P = %s*********************", str(p))
# logging.info("********************R = %s*********************", str(r))
# logging.info("********************F = %s*********************", str(f))
# logging.info("***********************************************")
# wf.write("***********************************************")
# wf.write("********************P =" + str(p) + "*********************\n")
# wf.write("********************R =" + str(r) + "*********************\n")
# wf.write("********************F =" + str(f) + "*********************\n")
# run evaluation on dev set
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
batch_tokens, batch_labels, batch_start_positions, batch_document_ids = filed_based_convert_examples_to_features(
eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file)
logging.info("***** Running evaluation on development *****")
logging.info(" Num examples = %d", len(eval_examples))
logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# if FLAGS.use_tpu:
# eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size)
# eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.evaluate(input_fn=eval_input_fn)
output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as wf:
logging.info("***** Eval results *****")
confusion_matrix = result["confusion_matrix"]
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
logging.info("***********************************************")
logging.info("********************P = %s*********************", str(p))
logging.info("********************R = %s*********************", str(r))
logging.info("********************F = %s*********************", str(f))
logging.info("***********************************************")
wf.write("***********************************************")
wf.write("********************P =" + str(p) + "*********************")
wf.write("********************R =" + str(r) + "*********************")
wf.write("********************F =" + str(f) + "*********************")
if FLAGS.do_predict:
with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as rf:
label2id = pickle.load(rf)
id2label = {value: key for key, value in label2id.items()}
predict_examples = processor.get_test_examples(FLAGS.data_dir)
predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record")
batch_tokens, batch_labels, batch_start_positions, batch_document_ids = filed_based_convert_examples_to_features(
predict_examples,
label_list,
FLAGS.max_seq_length,
tokenizer,
predict_file)
logging.info("***** Running prediction*****")
logging.info(" Num examples = %d", len(predict_examples))
logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.predict(input_fn=predict_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt")
# here if the tag is "X" means it belong to its before token, here for convenient evaluate use
# conlleval.pl we discarding it directly
Writer(output_predict_file, result, batch_tokens, batch_labels, batch_start_positions, batch_document_ids,
id2label)
# PREDICT FOR TRAIN DATA
train_examples = processor.get_train_examples(FLAGS.data_dir)
train_predict_file = os.path.join(FLAGS.output_dir, "train.tf_record")
batch_tokens, batch_labels, batch_start_positions, batch_document_ids = filed_based_convert_examples_to_features(
train_examples,
label_list,
FLAGS.max_seq_length,
tokenizer,
train_predict_file)
logging.info("***** Running prediction for training data*****")
logging.info(" Num examples = %d", len(predict_examples))
logging.info(" Batch size = %d", FLAGS.predict_batch_size)
train_predict_input_fn = file_based_input_fn_builder(
input_file=train_predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.predict(input_fn=train_predict_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir, "label_train.txt")
# here if the tag is "X" means it belong to its before token, here for convenient evaluate use
# conlleval.pl we discarding it directly
Writer(output_predict_file, result, batch_tokens, batch_labels, batch_start_positions, batch_document_ids,
id2label)
# SAVE MODEL FOR SERVING
feature_columns_keys = ['input_ids', 'mask', 'segment_ids', 'label_ids']
feature_columns = [sequence_numeric_column(key=key) for key in feature_columns_keys]
# feature_columns["input_ids"] = sequence_numeric_column('input_ids')
# feature_columns["mask"] = sequence_numeric_column('mask')
# feature_columns["segment_ids"] = sequence_numeric_column('segment_ids')
# feature_columns["label_ids"] = sequence_numeric_column('label_ids')
def serving_input_fn():
label_ids = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='label_ids')
input_ids = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='input_ids')
input_mask = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='mask')
segment_ids = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='segment_ids')
input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({
'label_ids': label_ids,
'input_ids': input_ids,
'mask': input_mask,
'segment_ids': segment_ids,
})()
return input_fn
def serving_input_fn_prev():
with tf.variable_scope("foo"):
feature_spec = {
"input_ids": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64),
"mask": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64),
"segment_ids": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64),
"label_ids": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64),
}
serialized_tf_example = tf.placeholder(dtype=tf.string,
shape=[None],
name='input_example_tensor')
receiver_tensors = {'examples': serialized_tf_example}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
# feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
# serving_input_receiver_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
estimator._export_to_tpu = False # this is important
export_dir = estimator.export_savedmodel(FLAGS.output_dir, serving_input_fn)
print('Exported to {}'.format(export_dir))
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
processors = {
"intent_slot": IntentSlotsProcessor,
}
tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case,
FLAGS.init_checkpoint)
if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict:
raise ValueError(
"At least one of `do_train`, `do_eval` or `do_predict' must be True.")
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
tf.gfile.MakeDirs(FLAGS.output_dir)
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels(FLAGS.data_dir)
# print("******************label_list******************")
# print(label_list)
tokenizer = tokenization.FullTokenizer(
vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_train_examples(FLAGS.data_dir)
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
model_fn = model_fn_builder(
bert_config=bert_config,
num_intent_labels=len(label_list[0]),
num_slot_labels=len(label_list[1]),
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.eval_batch_size,
predict_batch_size=FLAGS.predict_batch_size)
if FLAGS.do_train:
train_file = os.path.join(FLAGS.output_dir, "train.tf_record")
file_based_convert_examples_to_features(
train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file)
tf.logging.info("***** Running training *****")
tf.logging.info(" Num examples = %d", len(train_examples))
tf.logging.info(" Batch size = %d", FLAGS.train_batch_size)
tf.logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
if FLAGS.do_eval:
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
num_actual_eval_examples = len(eval_examples)
if FLAGS.use_tpu:
# TPU requires a fixed batch size for all batches, therefore the number
# of examples must be a multiple of the batch size, or else examples
# will get dropped. So we pad with fake examples which are ignored
# later on. These do NOT count towards the metric (all tf.metrics
# support a per-instance weight, and these get a weight of 0.0).
while len(eval_examples) % FLAGS.eval_batch_size != 0:
eval_examples.append(PaddingInputExample())
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
file_based_convert_examples_to_features(
eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file)
tf.logging.info("***** Running evaluation *****")
tf.logging.info(" Num examples = %d (%d actual, %d padding)",
len(eval_examples), num_actual_eval_examples,
len(eval_examples) - num_actual_eval_examples)
tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# This tells the estimator to run through the entire set.
eval_steps = None
# However, if running eval on the TPU, you will need to specify the
# number of steps.
if FLAGS.use_tpu:
assert len(eval_examples) % FLAGS.eval_batch_size == 0
eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size)
eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=eval_drop_remainder)
result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)
#TODO
output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt")
with tf.gfile.GFile(output_eval_file, "w") as writer:
tf.logging.info("***** Eval results *****")
intent_accuracy = result["intent_eval_accuracy"]
intent_loss = result["intent_eval_loss"]
slot_confusion_matrix = result["slot_cm"]
p,r,f = metrics.calculate(slot_confusion_matrix,len(label_list))
logging.info("***********************************************")
logging.info("********************slot_P = %s*********************", str(p))
logging.info("********************slot_R = %s*********************", str(r))
logging.info("********************slot_F = %s*********************", str(f))
logging.info("******************intent_acc = %s*******************", str(intent_accuracy))
logging.info("******************intent_loss = %s******************", str(intent_loss))
logging.info("***********************************************")
if FLAGS.do_predict:
predict_examples = processor.get_test_examples(FLAGS.data_dir)
num_actual_predict_examples = len(predict_examples)
if FLAGS.use_tpu:
# TPU requires a fixed batch size for all batches, therefore the number
# of examples must be a multiple of the batch size, or else examples
# will get dropped. So we pad with fake examples which are ignored
# later on.
while len(predict_examples) % FLAGS.predict_batch_size != 0:
predict_examples.append(PaddingInputExample())
predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record")
file_based_convert_examples_to_features(predict_examples, label_list,
FLAGS.max_seq_length, tokenizer,
predict_file)
tf.logging.info("***** Running prediction*****")
tf.logging.info(" Num examples = %d (%d actual, %d padding)",
len(predict_examples), num_actual_predict_examples,
len(predict_examples) - num_actual_predict_examples)
tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_drop_remainder = True if FLAGS.use_tpu else False
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=predict_drop_remainder)
result = estimator.predict(input_fn=predict_input_fn)
output_predict_file = os.path.join(
FLAGS.output_dir, "test_results.tsv")
with tf.gfile.GFile(output_predict_file, "w") as writer:
num_written_lines = 0
tf.logging.info("***** Predict results *****")
for (i, prediction) in enumerate(result):
probabilities = prediction["probabilities"]
if i >= num_actual_predict_examples:
break
output_line = "\t".join(
str(class_probability)
for class_probability in probabilities) + "\n"
writer.write(output_line)
num_written_lines += 1
assert num_written_lines == num_actual_predict_examples
def model_fn(features, labels, mode, params):
"""
モデルを定義します
"""
#=========================================================
# ハイパーパラメータを取得します
#=========================================================
num_classes = params.get('num_classes', 10)
batch_size = params.get('batch_size', 96)
learning_rate = params.get('learning_rate', 1e-4)
init_stddev = params.get('initializer_normal_stddev', 0.09)
dropout_rate = params.get('dropout_rate', 0.4)
is_training = mode == tf.estimator.ModeKeys.TRAIN
# 第一引数のfeaturesが入力データです
image = features['image']
assert image.get_shape().as_list() == [None, 28, 28, 1]
# 変数の初期化関数です
initializer = tf.truncated_normal_initializer(stddev=init_stddev)
#=========================================================
# ニューラルネットワークを定義します
#=========================================================
with tf.variable_scope('model', initializer=initializer):
x = image
x = tf.layers.conv2d(x, 32, 5, padding='SAME', activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2, padding='SAME')
x = tf.layers.conv2d(x, 64, 5, padding='SAME', activation=tf.nn.relu)
x = tf.layers.max_pooling2d(x, 2, 2, padding='SAME')
#x = tf.layers.flatten(x)
#XXX: `tf.layers.flatten` contains unsupported op `StridedSlice` by TensorFlow.js. Use `tf.reshape` as bellow.
x = tf.reshape(x, [-1, 7*7*64])
x = tf.layers.dense(x, 1024, activation=tf.nn.relu)
x = tf.layers.dropout(x, rate=dropout_rate, training=is_training)
x = tf.layers.dense(x, 10)
logits = x
assert logits.get_shape().as_list() == [None, num_classes]
# 予測結果: クラスごとの離散確率分布、最も確率の高いクラスのインデクス
predictions = {
'probabilities': tf.nn.softmax(logits, name='probabilities'),
'classes' : tf.argmax(logits, axis=1, name='classes'),
}
#=========================================================
# 推論モードならモデルの出力を返します
#=========================================================
# `mode`は実行モードです。モードは以下の3つがあります。
# - tf.estimator.ModeKeys.PREDICT : 推論モードです
# - tf.estimator.ModeKeys.TRAIN : 学習モードです
# - tf.estimator.ModeKeys.EVAL : 評価モードです
# 推論モードの場合は、学習を実行する必要はないため、結果を返して終わります。
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
export_outputs={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
tf.estimator.export.PredictOutput(predictions),
})
#=========================================================
# モデルの誤差を定義します
#=========================================================
with tf.variable_scope('losses'):
# クロスエントロピーを計算して誤差に追加します
cross_entropy_loss = tf.losses.sparse_softmax_cross_entropy(
labels=labels, logits=logits)
# モデルで追加された全ての誤差の総和を取得します
total_loss = tf.losses.get_total_loss()
#=========================================================
# 正答率など、モデルの評価値を計算します
#=========================================================
metric_ops = metrics.calculate(labels, predictions['classes'], num_classes)
#=========================================================
# モデルを学習(=パラメータを最適化)します
#=========================================================
global_step = tf.train.get_or_create_global_step()
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.variable_scope('optimizer'), tf.control_dependencies(update_ops):
# total_loss(誤差の総和)が小さくなるように変数を更新します
optimizer = tf.train.AdamOptimizer(learning_rate)
fit = optimizer.minimize(total_loss, global_step)
#=========================================================
# 変数をサマリにまとめます
#=========================================================
# 任意の値はサマリに追加することでログとして保存できます。
# ログはTensorBoardなどでグラフ化することができるため、
# 計算した誤差やメトリクス、入力画像などをログにしておきます。
tf.summary.image('image', image)
tf.summary.scalar('total_loss', total_loss)
summary_op = tf.summary.merge_all()
#=========================================================
# SageMakerの場合は、これでモデルの定義は完了です！
#=========================================================
if params.get('sagemaker_job_name', None) is not None:
return tf.estimator.EstimatorSpec(mode=mode,
loss=total_loss,
train_op=fit,
eval_metric_ops=metric_ops)
#---------------------------------------------------------
# 以下はローカル実行テスト用です。
training_hooks = [
tf.train.SummarySaverHook(
save_steps=1,
output_dir='./test/logs/doodle.train',
summary_op=summary_op)
]
evaluation_hooks = [
tf.train.SummarySaverHook(
save_steps=5,
output_dir='./test/logs/doodle.eval',
summary_op=summary_op)
]
return tf.estimator.EstimatorSpec(mode=mode,
loss=total_loss,
train_op=fit,
eval_metric_ops=metric_ops,
evaluation_hooks=evaluation_hooks,
training_hooks=training_hooks)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
processors = {
# "cola": ColaProcessor,
# "mnli": MnliProcessor,
# "mrpc": MrpcProcessor,
# "xnli": XnliProcessor,
"ee": EEProcessor
}
tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case,
FLAGS.init_checkpoint)
if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict:
raise ValueError(
"At least one of `do_train`, `do_eval` or `do_predict' must be True."
)
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
tf.gfile.MakeDirs(FLAGS.output_dir)
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
# get labels list
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file,
do_lower_case=FLAGS.do_lower_case)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_train_examples(FLAGS.data_dir)
# 训练多少步，每一步训练数据是一个batch
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size *
FLAGS.num_train_epochs)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
model_fn = model_fn_builder(bert_config=bert_config,
num_labels=len(label_list),
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.eval_batch_size,
predict_batch_size=FLAGS.predict_batch_size)
if FLAGS.do_train:
train_file = os.path.join(FLAGS.output_dir, "train.tf_record")
file_based_convert_examples_to_features(train_examples, label_list,
FLAGS.max_seq_length,
tokenizer, train_file)
tf.logging.info("***** Running training *****")
tf.logging.info(" Num examples = %d", len(train_examples))
tf.logging.info(" Batch size = %d", FLAGS.train_batch_size)
tf.logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
if FLAGS.do_eval:
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
num_actual_eval_examples = len(eval_examples)
if FLAGS.use_tpu:
# TPU requires a fixed batch size for all batches, therefore the number
# of examples must be a multiple of the batch size, or else examples
# will get dropped. So we pad with fake examples which are ignored
# later on. These do NOT count towards the metric (all tf.metrics
# support a per-instance weight, and these get a weight of 0.0).
while len(eval_examples) % FLAGS.eval_batch_size != 0:
eval_examples.append(PaddingInputExample())
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
file_based_convert_examples_to_features(eval_examples, label_list,
FLAGS.max_seq_length,
tokenizer, eval_file)
tf.logging.info("***** Running evaluation *****")
tf.logging.info(" Num examples = %d (%d actual, %d padding)",
len(eval_examples), num_actual_eval_examples,
len(eval_examples) - num_actual_eval_examples)
tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# This tells the estimator to run through the entire set.
eval_steps = None
# However, if running eval on the TPU, you will need to specify the
# number of steps.
if FLAGS.use_tpu:
assert len(eval_examples) % FLAGS.eval_batch_size == 0
eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size)
eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=eval_drop_remainder)
result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)
output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt")
with open(output_eval_file, 'w') as f:
tf.logging.info("***** Eval results *****")
confusion_matrix = result["confusion_matrix"]
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
tf.logging.info("***************************************")
tf.logging.info("**************P = %s*************************",
str(p))
tf.logging.info("**************R = %s*************************",
str(r))
tf.logging.info("**************F = %s*************************",
str(f))
# with tf.gfile.GFile(output_eval_file, "w") as writer:
# tf.logging.info("***** Eval results *****")
# for key in sorted(result.keys()):
# tf.logging.info(" %s = %s", key, str(result[key]))
# writer.write("%s = %s\n" % (key, str(result[key])))
if FLAGS.do_predict:
with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as f:
label2id = pickle.load(f)
id2label = {v: k for k, v in label2id.items()}
predict_examples = processor.get_test_examples(FLAGS.data_dir)
num_actual_predict_examples = len(predict_examples)
if FLAGS.use_tpu:
# TPU requires a fixed batch size for all batches, therefore the number
# of examples must be a multiple of the batch size, or else examples
# will get dropped. So we pad with fake examples which are ignored
# later on.
while len(predict_examples) % FLAGS.predict_batch_size != 0:
predict_examples.append(PaddingInputExample())
predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record")
batch_tokens, batch_labels = file_based_convert_examples_to_features(
predict_examples, label_list, FLAGS.max_seq_length, tokenizer,
predict_file)
tf.logging.info("***** Running prediction*****")
tf.logging.info(" Num examples = %d (%d actual, %d padding)",
len(predict_examples), num_actual_predict_examples,
len(predict_examples) - num_actual_predict_examples)
tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_drop_remainder = True if FLAGS.use_tpu else False
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=predict_drop_remainder)
result = estimator.predict(input_fn=predict_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt")
print_predictions(output_predict_file, result, batch_tokens,
batch_labels, id2label)
def main(_):
logging.set_verbosity(logging.INFO)
processors = {"ner": NerProcessor}
if not FLAGS.do_train and not FLAGS.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file,
do_lower_case=FLAGS.do_lower_case)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_train_examples(FLAGS.data_dir)
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size *
FLAGS.num_train_epochs)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
model_fn = model_fn_builder(bert_config=bert_config,
num_labels=len(label_list),
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.eval_batch_size,
predict_batch_size=FLAGS.predict_batch_size)
if FLAGS.do_train:
train_file = join(FLAGS.output_dir, "train.tf_record")
_, _ = filed_based_convert_examples_to_features(
train_examples, label_list, FLAGS.max_seq_length, tokenizer,
train_file)
logging.info("***** Running training *****")
logging.info(" Num examples = %d", len(train_examples))
logging.info(" Batch size = %d", FLAGS.train_batch_size)
logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
if FLAGS.do_eval:
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
eval_file = join(FLAGS.output_dir, "eval.tf_record")
batch_tokens, batch_labels = filed_based_convert_examples_to_features(
eval_examples, label_list, FLAGS.max_seq_length, tokenizer,
eval_file)
logging.info("***** Running evaluation *****")
logging.info(" Num examples = %d", len(eval_examples))
logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# if FLAGS.use_tpu:
# eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size)
# eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.evaluate(input_fn=eval_input_fn)
logging.info("\n")
print(">>>>>>>>>>>>>>> evaluation",
"result type {}".format(type(result)))
print(result)
print(">>>>>>>>>>>>>>> examples: {}\n".format(len(eval_examples)))
# output_eval_file = join(FLAGS.output_dir, "eval_results.txt")
# with open(output_eval_file,"w") as wf:
logging.info("***** Eval results *****")
confusion_matrix = result["confusion_matrix"]
(rows, cols) = confusion_matrix.shape
sum_num, true_num = 0, 0
for i in range(rows):
for j in range(cols):
sum_num += confusion_matrix[i][j]
if i == j:
true_num += confusion_matrix[i][j]
print(">>>>>> type of confusion matrix", str(type(confusion_matrix)))
print(">>>>>> ", str(confusion_matrix.shape))
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
logging.info("***********************************************")
logging.info("*******************Acc = %s********************",
str(true_num / sum_num))
logging.info("********************P = %s*********************", str(p))
logging.info("********************R = %s*********************", str(r))
logging.info("********************F = %s*********************", str(f))
logging.info("***********************************************")
if FLAGS.do_predict:
# with open(FLAGS.middle_output+'/label2id.pkl', 'rb') as rf:
# label2id = pickle.load(rf)
# id2label = {value: key for key, value in label2id.items()}
predict_examples = processor.get_test_examples(FLAGS.data_dir)
predict_file = join(FLAGS.output_dir, "predict.tf_record")
batch_tokens, batch_labels = filed_based_convert_examples_to_features(
predict_examples, label_list, FLAGS.max_seq_length, tokenizer,
predict_file)
# output_predict_file = join(FLAGS.output_dir, "label_test.txt")
# here if the tag is "X" means it belong to its before token, here for convenient evaluate use
# conlleval.pl we discarding it directly
# Writer(output_predict_file,result,batch_tokens,batch_labels,id2label)
logging.info("***** Running prediction*****")
logging.info(" Num examples = %d", len(predict_examples))
logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.evaluate(input_fn=predict_input_fn)
print("\n>>>>>>>>>>>>>>> prediction",
"result type {}".format(type(result)))
print(result)
print(">>>>>>>>>>>>>>> examples: {}\n".format(len(predict_examples)))
# added by jerry
logging.info("***** Pred results *****")
confusion_matrix = result["confusion_matrix"]
sum_num, true_num = 0, 0
for i in range(rows):
for j in range(cols):
sum_num += confusion_matrix[i][j]
if i == j:
true_num += confusion_matrix[i][j]
print(">>>>>> type of confusion matrix", str(type(confusion_matrix)))
print(">>>>>> ", str(confusion_matrix.shape))
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
logging.info("***********************************************")
logging.info("*******************Acc = %s********************",
str(true_num / sum_num))
logging.info("********************P = %s*********************", str(p))
logging.info("********************R = %s*********************", str(r))
logging.info("********************F = %s*********************", str(f))
logging.info("***********************************************")
if FLAGS.do_gt:
print("\n************ Evaluation on Groundtruth ************")
# generate groundtruth
# data format: (word, label)/per line
# gt_path = 'data_new/groundtruth'
# ano_gt_path = 'data_new'
# words_list = []
# labels_list = []
# for f in listdir(gt_path):
# if isfile(join(gt_path, f)) is True:
# print('*************', f, '*************')
# with open(join(gt_path, f)) as fin:
# lines = fin.readlines()
# for idx, line in enumerate(lines):
# if idx % 3 == 1:
# words = line.strip().split(' ')
# if len(words) > 0:
# words_list.append(words)
# elif idx % 3 == 2:
# labels = line.strip().split(' ')
# if len(labels) > 0:
# labels_list.append(labels)
# print('******* operation on', f, 'finished ******')
# with open(join(ano_gt_path, 'groundtruth.txt'), 'w') as fout:
# for (words, labels) in zip(words_list, labels_list):
# for (word, label) in zip(words, labels):
# print(word, label)
# fout.write(word + ' ' + label + '\n')
# fout.write('\n')
gt_examples = processor.get_gt_examples(FLAGS.data_dir)
gt_file = join(FLAGS.output_dir, "gt.tf_record")
filed_based_convert_examples_to_features(gt_examples, label_list,
FLAGS.max_seq_length,
tokenizer, gt_file)
gt_input_fn = file_based_input_fn_builder(
input_file=gt_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.evaluate(input_fn=gt_input_fn)
print("\n>>>>>>>>>>>>>>> groundtruth:",
"result type {}".format(type(result)))
print(result)
print(">>>>>>>>>>>>>>> examples: {}\n".format(len(gt_examples)))
# added by jerry
logging.info("***** Groundtruth results *****")
confusion_matrix = result["confusion_matrix"]
sum_num, true_num = 0, 0
for i in range(rows):
for j in range(cols):
sum_num += confusion_matrix[i][j]
if i == j:
true_num += confusion_matrix[i][j]
print(">>>>>> type of confusion matrix", str(type(confusion_matrix)))
print(">>>>>> ", str(confusion_matrix.shape))
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
logging.info("***********************************************")
logging.info("*******************Acc = %s********************",
str(true_num / sum_num))
logging.info("********************P = %s*********************", str(p))
logging.info("********************R = %s*********************", str(r))
logging.info("********************F = %s*********************", str(f))
logging.info("***********************************************")
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
#### Validate flags
if FLAGS.save_steps is not None:
FLAGS.iterations = min(FLAGS.iterations, FLAGS.save_steps)
if FLAGS.do_predict:
predict_dir = FLAGS.predict_dir
if not tf.gfile.Exists(predict_dir):
tf.gfile.MakeDirs(predict_dir)
processors = {
"ner": NerProcessor,
}
if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict:
raise ValueError(
"At least one of `do_train`, `do_eval, `do_predict` or "
"`do_submit` must be True.")
if not tf.gfile.Exists(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels() if not FLAGS.is_regression else None
print('label_list')
print(label_list)
sp = spm.SentencePieceProcessor()
sp.Load(FLAGS.spiece_model_file)
def tokenize_fn(text):
text = preprocess_text(text, lower=FLAGS.uncased)
if sp.PieceToId(text) == 0:
return 99999
return sp.PieceToId(text)
run_config = model_utils.configure_tpu(FLAGS)
model_fn = get_model_fn(len(label_list))
spm_basename = os.path.basename(FLAGS.spiece_model_file)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
#if FLAGS.use_tpu:
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
predict_batch_size=FLAGS.predict_batch_size,
eval_batch_size=FLAGS.eval_batch_size)
#else:
# estimator = tf.estimator.Estimator(
# model_fn=model_fn,
# config=run_config)
if FLAGS.do_train:
train_file_base = "{}.len-{}.train.tf_record".format(
spm_basename, FLAGS.max_seq_length)
train_file = os.path.join(FLAGS.output_dir, train_file_base)
tf.logging.info("Use tfrecord file {}".format(train_file))
# print('get train examples')
train_examples = processor.get_train_examples(FLAGS.data_dir)
np.random.shuffle(train_examples)
tf.logging.info("Num of train samples: {}".format(len(train_examples)))
file_based_convert_examples_to_features(
train_examples, label_list, FLAGS.max_seq_length, tokenize_fn,
train_file, FLAGS.num_passes)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True,
batch_size=FLAGS.train_batch_size)
estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps)
# if FLAGS.do_eval or FLAGS.do_predict:
# if FLAGS.eval_split == "dev":
# eval_examples = processor.get_dev_examples(FLAGS.data_dir)
# else:
# eval_examples = processor.get_test_examples(FLAGS.data_dir)
#
# tf.logging.info("Num of eval samples: {}".format(len(eval_examples)))
if FLAGS.do_eval:
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
file_based_convert_examples_to_features(
eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn,
eval_file)
logging.info("***** Running evaluation *****")
logging.info(" Num examples = %d", len(eval_examples))
logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# if FLAGS.use_tpu:
# eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size)
# eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=True,
batch_size=FLAGS.train_batch_size)
result = estimator.evaluate(input_fn=eval_input_fn)
output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as wf:
logging.info("***** Eval results *****")
confusion_matrix = result["confusion_matrix"]
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
logging.info("***********************************************")
logging.info("********************P = %s*********************", str(p))
logging.info("********************R = %s*********************", str(r))
logging.info("********************F = %s*********************", str(f))
logging.info("***********************************************")
if FLAGS.do_predict:
# eval_file_base = "{}.len-{}.{}.predict.tf_record".format(
# spm_basename, FLAGS.max_seq_length, FLAGS.eval_split)
# eval_file = os.path.join(FLAGS.output_dir, eval_file_base)
test_file_base = "{}.len-{}.predict.tf_record".format(
spm_basename, FLAGS.max_seq_length)
test_file = os.path.join(FLAGS.output_dir, test_file_base)
test_examples = processor.get_test_examples(FLAGS.data_dir)
file_based_convert_examples_to_features(
test_examples, label_list, FLAGS.max_seq_length, tokenize_fn,
test_file)
pred_input_fn = file_based_input_fn_builder(
input_file=test_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False,
batch_size=FLAGS.train_batch_size)
predict_results = []
result = estimator.predict(input_fn=pred_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt")
# print(result)
print(list(result))
