def train_and_eval(self, local_training=True):
"""Launches train and evaluation jobs locally."""
# TODO: supports for distributed training and evaluation.
if not local_training:
raise ValueError("The non local training is not supported now!")
train_spec, eval_spec, _ = self._train_eval_specs()
tf_estimator.train_and_evaluate(self._estimator, train_spec, eval_spec)
def train_and_eval():
"""Train and Evaluate."""
train_input_fn = make_input_fn(FLAGS.train_path, FLAGS.batch_size)
eval_input_fn = make_input_fn(FLAGS.eval_path,
FLAGS.batch_size,
randomize_input=False,
num_epochs=1)
optimizer = tf.compat.v1.train.AdagradOptimizer(
learning_rate=FLAGS.learning_rate)
def _train_op_fn(loss):
"""Defines train op used in ranking head."""
update_ops = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.UPDATE_OPS)
minimize_op = optimizer.minimize(
loss=loss, global_step=tf.compat.v1.train.get_global_step())
train_op = tf.group([minimize_op, update_ops])
return train_op
ranking_head = tfr.head.create_ranking_head(
loss_fn=tfr.losses.make_loss_fn(
FLAGS.loss, weights_feature_name=FLAGS.weights_feature_name),
eval_metric_fns=eval_metric_fns(),
train_op_fn=_train_op_fn)
estimator = tf_estimator.Estimator(
model_fn=tfr.model.make_groupwise_ranking_fn(
group_score_fn=make_score_fn(),
group_size=FLAGS.group_size,
transform_fn=make_transform_fn(),
ranking_head=ranking_head),
model_dir=FLAGS.model_dir,
config=tf_estimator.RunConfig(save_checkpoints_steps=1000))
train_spec = tf_estimator.TrainSpec(input_fn=train_input_fn,
max_steps=FLAGS.num_train_steps)
exporters = tf_estimator.LatestExporter(
"saved_model_exporter",
serving_input_receiver_fn=make_serving_input_fn())
eval_spec = tf_estimator.EvalSpec(name="eval",
input_fn=eval_input_fn,
steps=1,
exporters=exporters,
start_delay_secs=0,
throttle_secs=15)
# Train and validate.
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def train_and_maybe_evaluate(hparams):
"""Run the training and evaluate using the high level API.
Args:
hparams: Holds hyperparameters used to train the model as name/value pairs.
Returns:
The estimator that was used for training (and maybe eval)
"""
schema = taxi.read_schema(hparams.schema_file)
tf_transform_output = tft.TFTransformOutput(hparams.tf_transform_dir)
train_input = lambda: model.input_fn(
hparams.train_files, tf_transform_output, batch_size=TRAIN_BATCH_SIZE)
eval_input = lambda: model.input_fn(
hparams.eval_files, tf_transform_output, batch_size=EVAL_BATCH_SIZE)
train_spec = tf_estimator.TrainSpec(
train_input, max_steps=hparams.train_steps)
serving_receiver_fn = lambda: model.example_serving_receiver_fn(
tf_transform_output, schema)
exporter = tf_estimator.FinalExporter('chicago-taxi', serving_receiver_fn)
eval_spec = tf_estimator.EvalSpec(
eval_input,
steps=hparams.eval_steps,
exporters=[exporter],
name='chicago-taxi-eval')
run_config = tf_estimator.RunConfig(
save_checkpoints_steps=999, keep_checkpoint_max=1)
serving_model_dir = os.path.join(hparams.output_dir, SERVING_MODEL_DIR)
run_config = run_config.replace(model_dir=serving_model_dir)
estimator = model.build_estimator(
tf_transform_output,
# Construct layers sizes with exponetial decay
hidden_units=[
max(2, int(FIRST_DNN_LAYER_SIZE * DNN_DECAY_FACTOR**i))
for i in range(NUM_DNN_LAYERS)
],
config=run_config)
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
return estimator
def test_model_to_estimator_missing_custom_objects(self):
keras_model = model.create_keras_model(network=self._network,
loss=self._loss,
metrics=self._eval_metrics,
optimizer=self._optimizer,
size_feature_name=_SIZE)
estimator = estimator_lib.model_to_estimator(model=keras_model,
config=self._config,
custom_objects=None)
self.assertIsInstance(estimator, tf_compat_v1_estimator.Estimator)
# Train and export model.
train_spec = tf_estimator.TrainSpec(input_fn=self._make_input_fn(),
max_steps=1)
eval_spec = tf_estimator.EvalSpec(name='eval',
input_fn=self._make_input_fn(),
steps=10)
with self.assertRaises(AttributeError):
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def test_experiment(self, listwise_inference):
tmp_dir = self.create_tempdir().full_path
data_file = os.path.join(tmp_dir, "elwc.tfrecord")
with tf.io.TFRecordWriter(data_file) as writer:
for _ in range(10):
writer.write(ELWC_PROTO.SerializeToString())
estimator = tfr_estimator.make_gam_ranking_estimator(
example_feature_columns=example_feature_columns(),
example_hidden_units=["2", "2"],
optimizer=tf.compat.v1.train.AdamOptimizer(learning_rate=0.05),
learning_rate=0.05,
loss="softmax_loss",
use_batch_norm=False,
model_dir=None)
train_spec = tf_estimator.TrainSpec(input_fn=_inner_input_fn,
max_steps=1)
eval_spec = tf_estimator.EvalSpec(name="eval",
input_fn=_inner_input_fn,
steps=10)
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def train():
encoders = get_encoders()
generator = get_generator(encoders)
checkpoint_dir = path.join(config.CHECKPOINT_DIR, "tf")
estimator = K.estimator.model_to_estimator(
keras_model=generator,
model_dir=checkpoint_dir,
checkpoint_format="saver"
) # TODO: use 'checkpoint' once object-based checkpoints supported
def input_fn():
dataset = get_dataset(encoders)
return dataset.batch(config.BATCH_SIZE)
train_spec = E.TrainSpec(input_fn=input_fn)
eval_spec = E.EvalSpec(
input_fn=input_fn,
hooks=[E.CheckpointSaverHook(checkpoint_dir, save_steps=1000)])
E.train_and_evaluate(estimator, train_spec, eval_spec)
def train_and_eval():
"""Train and Evaluate."""
train_input_fn = make_input_fn(FLAGS.train_path, FLAGS.batch_size)
eval_input_fn = make_input_fn(FLAGS.eval_path,
FLAGS.batch_size,
randomize_input=False,
num_epochs=1)
estimator = get_estimator()
train_spec = tf_estimator.TrainSpec(input_fn=train_input_fn,
max_steps=FLAGS.num_train_steps)
exporters = tf_estimator.LatestExporter(
"saved_model_exporter",
serving_input_receiver_fn=make_serving_input_fn())
eval_spec = tf_estimator.EvalSpec(name="eval",
input_fn=eval_input_fn,
steps=1,
exporters=exporters,
start_delay_secs=0,
throttle_secs=15)
# Train and validate.
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def __call__(self):
train_spec = estimator.TrainSpec(input_fn=lambda: self._input_fn(),
max_steps=self._iter_num)
eval_spec = estimator.EvalSpec(input_fn=lambda: self._input_fn(), )
return estimator.train_and_evaluate(self._estimator,
train_spec=train_spec, eval_spec=eval_spec)
def train_and_eval():
"""Train and Evaluate."""
features, labels = load_libsvm_data(FLAGS.train_path, FLAGS.list_size)
train_input_fn, train_hook = get_train_inputs(features, labels,
FLAGS.train_batch_size)
features_vali, labels_vali = load_libsvm_data(FLAGS.vali_path,
FLAGS.list_size)
vali_input_fn, vali_hook = get_eval_inputs(features_vali, labels_vali)
features_test, labels_test = load_libsvm_data(FLAGS.test_path,
FLAGS.list_size)
test_input_fn, test_hook = get_eval_inputs(features_test, labels_test)
optimizer = tf.compat.v1.train.AdagradOptimizer(
learning_rate=FLAGS.learning_rate)
def _train_op_fn(loss):
"""Defines train op used in ranking head."""
update_ops = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.UPDATE_OPS)
minimize_op = optimizer.minimize(
loss=loss, global_step=tf.compat.v1.train.get_global_step())
train_op = tf.group([minimize_op, update_ops])
return train_op
if _use_multi_head():
primary_head = tfr.head.create_ranking_head(
loss_fn=tfr.losses.make_loss_fn(FLAGS.loss),
eval_metric_fns=get_eval_metric_fns(),
train_op_fn=_train_op_fn,
name=_PRIMARY_HEAD)
secondary_head = tfr.head.create_ranking_head(
loss_fn=tfr.losses.make_loss_fn(FLAGS.secondary_loss),
eval_metric_fns=get_eval_metric_fns(),
train_op_fn=_train_op_fn,
name=_SECONDARY_HEAD)
ranking_head = tfr.head.create_multi_ranking_head(
[primary_head, secondary_head], [1.0, FLAGS.secondary_loss_weight])
else:
ranking_head = tfr.head.create_ranking_head(
loss_fn=tfr.losses.make_loss_fn(FLAGS.loss),
eval_metric_fns=get_eval_metric_fns(),
train_op_fn=_train_op_fn)
estimator = tf_estimator.Estimator(
model_fn=tfr.model.make_groupwise_ranking_fn(
group_score_fn=make_score_fn(),
group_size=FLAGS.group_size,
transform_fn=make_transform_fn(),
ranking_head=ranking_head),
config=tf_estimator.RunConfig(FLAGS.output_dir,
save_checkpoints_steps=1000))
train_spec = tf_estimator.TrainSpec(input_fn=train_input_fn,
hooks=[train_hook],
max_steps=FLAGS.num_train_steps)
# Export model to accept tf.Example when group_size = 1.
if FLAGS.group_size == 1:
vali_spec = tf_estimator.EvalSpec(
input_fn=vali_input_fn,
hooks=[vali_hook],
steps=1,
exporters=tf_estimator.LatestExporter(
"latest_exporter",
serving_input_receiver_fn=make_serving_input_fn()),
start_delay_secs=0,
throttle_secs=30)
else:
vali_spec = tf_estimator.EvalSpec(input_fn=vali_input_fn,
hooks=[vali_hook],
steps=1,
start_delay_secs=0,
throttle_secs=30)
# Train and validate
tf_estimator.train_and_evaluate(estimator, train_spec, vali_spec)
# Evaluate on the test data.
estimator.evaluate(input_fn=test_input_fn, hooks=[test_hook])
def train_bert_multitask(
problem='weibo_ner',
num_gpus=1,
num_epochs=10,
model_dir='',
params=None,
problem_type_dict={},
processing_fn_dict={},
model=None):
"""Train Multi-task Bert model
About problem:
There are two types of chaining operations can be used to chain problems.
- `&`. If two problems have the same inputs, they can be chained using `&`. Problems chained by `&` will be trained at the same time.
- `|`. If two problems don't have the same inputs, they need to be chained using `|`. Problems chained by `|` will be sampled to train at every instance.
For example, `cws|NER|weibo_ner&weibo_cws`, one problem will be sampled at each turn, say `weibo_ner&weibo_cws`, then `weibo_ner` and `weibo_cws` will trained for this turn together. Therefore, in a particular batch, some tasks might not be sampled, and their loss could be 0 in this batch.
About problem_type_dict and processing_fn_dict:
If the problem is not predefined, you need to tell the model what's the new problem's problem_type
and preprocessing function.
For example, a new problem: fake_classification
problem_type_dict = {'fake_classification': 'cls'}
processing_fn_dict = {'fake_classification': lambda: return ...}
Available problem type:
cls: Classification
seq_tag: Sequence Labeling
seq2seq_tag: Sequence to Sequence tag problem
seq2seq_text: Sequence to Sequence text generation problem
Preprocessing function example:
Please refer to https://github.com/JayYip/bert-multitask-learning/blob/master/README.md
Keyword Arguments:
problem {str} -- Problems to train (default: {'weibo_ner'})
num_gpus {int} -- Number of GPU to use (default: {1})
num_epochs {int} -- Number of epochs to train (default: {10})
model_dir {str} -- model dir (default: {''})
params {BaseParams} -- Params to define training and models (default: {DynamicBatchSizeParams()})
problem_type_dict {dict} -- Key: problem name, value: problem type (default: {{}})
processing_fn_dict {dict} -- Key: problem name, value: problem data preprocessing fn (default: {{}})
"""
if params is None:
params = DynamicBatchSizeParams()
if not os.path.exists('models'):
os.mkdir('models')
if model_dir:
base_dir, dir_name = os.path.split(model_dir)
else:
base_dir, dir_name = None, None
params.train_epoch = num_epochs
# add new problem to params if problem_type_dict and processing_fn_dict provided
if processing_fn_dict:
for new_problem, new_problem_processing_fn in processing_fn_dict.items():
print('Adding new problem {0}, problem type: {1}'.format(
new_problem, problem_type_dict[new_problem]))
params.add_problem(
problem_name=new_problem, problem_type=problem_type_dict[new_problem], processing_fn=new_problem_processing_fn)
params.assign_problem(problem, gpu=int(num_gpus),
base_dir=base_dir, dir_name=dir_name)
params.to_json()
estimator = _create_estimator(
num_gpus=num_gpus, params=params, model=model)
train_hook = RestoreCheckpointHook(params)
def train_input_fn(): return train_eval_input_fn(params)
def eval_input_fn(): return train_eval_input_fn(params, mode=EVAL)
train_spec = TrainSpec(
input_fn=train_input_fn, max_steps=params.train_steps, hooks=[train_hook])
eval_spec = EvalSpec(
eval_input_fn, throttle_secs=params.eval_throttle_secs)
# estimator.train(
# train_input_fn, max_steps=params.train_steps, hooks=[train_hook])
train_and_evaluate(estimator, train_spec, eval_spec)
return estimator
pprint(params)
print('placing model artifacts in {}'.format(model_dir))
# define model and data
model = MNISTModel()
mnist_data = Mnist(params['batch_size'])
run_config = estimator.RunConfig(
save_checkpoints_steps=params['steps_per_epoch'],
save_summary_steps=200,
keep_checkpoint_max=10)
mnist_estimator = estimator.Estimator(model_dir=model_dir,
model_fn=model.model_fn,
params=params,
config=run_config)
# training/evaluation specs for run
train_spec = estimator.TrainSpec(
input_fn=mnist_data.build_training_data,
max_steps=params['total_steps_train'],
)
eval_spec = estimator.EvalSpec(input_fn=mnist_data.build_validation_data,
steps=None,
throttle_secs=params['throttle_eval'],
start_delay_secs=0)
# run train and evaluate
estimator.train_and_evaluate(mnist_estimator, train_spec, eval_spec)
def test_model_to_estimator(self, weights_feature_name, serving_default):
keras_model = model.create_keras_model(network=self._network,
loss=self._loss,
metrics=self._eval_metrics,
optimizer=self._optimizer,
size_feature_name=_SIZE)
estimator = estimator_lib.model_to_estimator(
model=keras_model,
config=self._config,
weights_feature_name=weights_feature_name,
custom_objects=self._custom_objects,
serving_default=serving_default)
self.assertIsInstance(estimator, tf_compat_v1_estimator.Estimator)
# Train and export model.
train_spec = tf_estimator.TrainSpec(
input_fn=self._make_input_fn(weights_feature_name), max_steps=1)
eval_spec = tf_estimator.EvalSpec(
name='eval',
input_fn=self._make_input_fn(weights_feature_name),
steps=10)
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
context_feature_spec = tf.feature_column.make_parse_example_spec(
self._context_feature_columns.values())
example_feature_spec = tf.feature_column.make_parse_example_spec(
self._example_feature_columns.values())
def _make_serving_input_fn(serving_default):
if serving_default == 'predict':
return data.build_ranking_serving_input_receiver_fn(
data.ELWC,
context_feature_spec=context_feature_spec,
example_feature_spec=example_feature_spec,
size_feature_name=_SIZE)
else:
def pointwise_serving_fn():
serialized = tf.compat.v1.placeholder(
dtype=tf.string,
shape=[None],
name='input_ranking_tensor')
receiver_tensors = {'input_ranking_data': serialized}
features = data.parse_from_tf_example(
serialized,
context_feature_spec=context_feature_spec,
example_feature_spec=example_feature_spec,
size_feature_name=_SIZE)
return tf_estimator.export.ServingInputReceiver(
features, receiver_tensors)
return pointwise_serving_fn
serving_input_receiver_fn = _make_serving_input_fn(serving_default)
export_dir = os.path.join(tf.compat.v1.test.get_temp_dir(), 'export')
estimator.export_saved_model(export_dir, serving_input_receiver_fn)
# Confirm model ran and created checkpoints and saved model.
final_ckpt_path = os.path.join(estimator.model_dir,
'model.ckpt-1.meta')
self.assertTrue(tf.io.gfile.exists(final_ckpt_path))
saved_model_pb = os.path.join(export_dir,
tf.io.gfile.listdir(export_dir)[0],
'saved_model.pb')
self.assertTrue(tf.io.gfile.exists(saved_model_pb))
def train_and_eval(params,
model_fn,
input_fn,
keep_checkpoint_every_n_hours=0.5,
save_checkpoints_secs=100,
eval_steps=0,
eval_start_delay_secs=10,
eval_throttle_secs=100,
save_summary_steps=50):
"""Trains and evaluates our model.
Supports local and distributed training.
Args:
params: ConfigParams class with model training and network parameters.
model_fn: A func with prototype model_fn(features, labels, mode, hparams).
input_fn: A input function for the tf.estimator.Estimator.
keep_checkpoint_every_n_hours: Number of hours between each checkpoint to be
saved.
save_checkpoints_secs: Save checkpoints every this many seconds.
eval_steps: Number of steps to evaluate model; 0 for one epoch.
eval_start_delay_secs: Start evaluating after waiting for this many seconds.
eval_throttle_secs: Do not re-evaluate unless the last evaluation was
started at least this many seconds ago
save_summary_steps: Save summaries every this many steps.
"""
mparams = params.model_params
run_config = tf_estimator.RunConfig(
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
save_checkpoints_secs=save_checkpoints_secs,
save_summary_steps=save_summary_steps)
if run_config.model_dir:
params.model_dir = run_config.model_dir
print('\nCreating estimator with model dir %s' % params.model_dir)
estimator = tf_estimator.Estimator(model_fn=model_fn,
model_dir=params.model_dir,
config=run_config,
params=params)
print('\nCreating train_spec')
train_spec = tf_estimator.TrainSpec(input_fn=input_fn(params,
split='train'),
max_steps=params.steps)
print('\nCreating eval_spec')
def serving_input_receiver_fn():
"""Serving input_fn that builds features from placeholders.
Returns:
A tf.estimator.export.ServingInputReceiver.
"""
modelx = mparams.modelx
modely = mparams.modely
offsets = keras.Input(shape=(3, ), name='offsets', dtype='float32')
hom = keras.Input(shape=(3, 3), name='hom', dtype='float32')
to_world = keras.Input(shape=(4, 4),
name='to_world_L',
dtype='float32')
img_l = keras.Input(shape=(modely, modelx, 3),
name='img_L',
dtype='float32')
img_r = keras.Input(shape=(modely, modelx, 3),
name='img_R',
dtype='float32')
features = {
'img_L': img_l,
'img_R': img_r,
'to_world_L': to_world,
'offsets': offsets,
'hom': hom
}
return tf_estimator.export.build_raw_serving_input_receiver_fn(
features)
class SaveModel(tf_estimator.SessionRunHook):
"""Saves a model in SavedModel format."""
def __init__(self, estimator, output_dir):
self.output_dir = output_dir
self.estimator = estimator
self.save_num = 0
def begin(self):
ckpt = self.estimator.latest_checkpoint()
print('Latest checkpoint in hook:', ckpt)
ckpt_num_str = ckpt.split('.ckpt-')[1]
if (int(ckpt_num_str) - self.save_num) > 4000:
fname = os.path.join(self.output_dir,
'saved_model-' + ckpt_num_str)
print('**** Saving model in train hook: %s' % fname)
self.estimator.export_saved_model(fname,
serving_input_receiver_fn())
self.save_num = int(ckpt_num_str)
saver_hook = SaveModel(estimator, params.model_dir)
if eval_steps == 0:
eval_steps = None
eval_spec = tf_estimator.EvalSpec(input_fn=input_fn(params, split='val'),
steps=eval_steps,
hooks=[saver_hook],
start_delay_secs=eval_start_delay_secs,
throttle_secs=eval_throttle_secs)
if run_config.is_chief:
outdir = params.model_dir
if outdir is not None:
print('Writing params to %s' % outdir)
os.makedirs(outdir, exist_ok=True)
params.write_yaml(os.path.join(outdir, 'params.yaml'))
print('\nRunning estimator')
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
print('\nSaving last model')
ckpt = estimator.latest_checkpoint()
print('Last checkpoint:', ckpt)
ckpt_num_str = ckpt.split('.ckpt-')[1]
fname = os.path.join(params.model_dir, 'saved_model-' + ckpt_num_str)
print('**** Saving last model: %s' % fname)
estimator.export_saved_model(fname, serving_input_receiver_fn())
def main(_):
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "0"
# Load emotion categories
with open(FLAGS.emotion_file, "r") as f:
all_emotions = f.read().splitlines()
if FLAGS.add_neutral:
all_emotions = all_emotions + ["neutral"]
idx2emotion = {i: e for i, e in enumerate(all_emotions)}
num_labels = len(all_emotions)
print("%d labels" % num_labels)
print("Multilabel: %r" % FLAGS.multilabel)
tf.logging.set_verbosity(tf.logging.INFO)
tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case,
FLAGS.init_checkpoint)
if not FLAGS.do_train and not FLAGS.do_predict:
raise ValueError("At least one of `do_train` or `do_predict' must be True.")
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
tf.gfile.MakeDirs(FLAGS.output_dir)
processor = DataProcessor(num_labels, FLAGS.data_dir) # set up preprocessor
tokenizer = tokenization.FullTokenizer(
vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
run_config = tf_estimator.RunConfig(
model_dir=FLAGS.output_dir,
save_summary_steps=FLAGS.save_summary_steps,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
keep_checkpoint_max=FLAGS.keep_checkpoint_max)
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_examples("train", FLAGS.train_fname)
eval_examples = processor.get_examples("dev", FLAGS.dev_fname)
num_eval_examples = len(eval_examples)
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
params = {
"num_labels": num_labels,
"learning_rate": FLAGS.learning_rate,
"num_train_epochs": FLAGS.num_train_epochs,
"warmup_proportion": FLAGS.warmup_proportion,
"batch_size": FLAGS.train_batch_size,
"num_train_examples": len(train_examples),
"num_eval_examples": num_eval_examples,
"data_dir": FLAGS.data_dir,
"output_dir": FLAGS.output_dir,
"train_fname": FLAGS.train_fname,
"dev_fname": FLAGS.dev_fname,
"test_fname": FLAGS.test_fname
}
with open(os.path.join(FLAGS.output_dir, "config.json"), "w") as f:
json.dump(params, f)
model_fn = model_fn_builder(
bert_config=bert_config,
num_labels=num_labels,
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
multilabel=FLAGS.multilabel,
idx2emotion=idx2emotion)
estimator = tf_estimator.Estimator(
model_fn=model_fn,
config=run_config,
params={"batch_size": FLAGS.train_batch_size})
if FLAGS.do_train:
train_file = os.path.join(FLAGS.output_dir, "train.tf_record")
file_based_convert_examples_to_features(train_examples,
FLAGS.max_seq_length, tokenizer,
train_file)
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
file_based_convert_examples_to_features(eval_examples, FLAGS.max_seq_length,
tokenizer, eval_file)
tf.logging.info("***** Running training and evaluation *****")
tf.logging.info(" Num train examples = %d", len(train_examples))
tf.logging.info(" Num eval examples = %d", num_eval_examples)
tf.logging.info(" Batch size = %d", FLAGS.train_batch_size)
tf.logging.info(" Num training steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True,
num_labels=num_labels)
train_spec = tf_estimator.TrainSpec(
input_fn=train_input_fn, max_steps=num_train_steps)
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False,
num_labels=num_labels)
eval_spec = tf_estimator.EvalSpec(
input_fn=eval_input_fn,
steps=FLAGS.eval_steps,
start_delay_secs=0,
throttle_secs=1000)
tf_estimator.train_and_evaluate(
estimator, train_spec=train_spec, eval_spec=eval_spec)
if FLAGS.calculate_metrics:
# Setting the parameter to "dev" ensures that we get labels for the examples
eval_examples = processor.get_examples("dev", FLAGS.test_fname)
tf.logging.info("***** Running evaluation *****")
tf.logging.info(" Num eval examples = %d", len(eval_examples))
eval_file = os.path.join(FLAGS.output_dir, FLAGS.test_fname + ".tf_record")
file_based_convert_examples_to_features(eval_examples, FLAGS.max_seq_length,
tokenizer, eval_file)
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False,
num_labels=num_labels)
result = estimator.evaluate(input_fn=eval_input_fn, steps=None)
output_eval_file = os.path.join(FLAGS.output_dir,
FLAGS.test_fname + ".eval_results.txt")
with tf.gfile.GFile(output_eval_file, "w") as writer:
tf.logging.info("***** Eval results *****")
for key in sorted(result.keys()):
tf.logging.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if FLAGS.do_predict:
predict_examples = processor.get_examples("test", FLAGS.test_fname)
num_actual_predict_examples = len(predict_examples)
predict_file = os.path.join(FLAGS.output_dir,
FLAGS.test_fname + ".tf_record")
file_based_convert_examples_to_features(predict_examples,
FLAGS.max_seq_length, tokenizer,
predict_file)
tf.logging.info("***** Running prediction*****")
tf.logging.info(" Num examples = %d (%d actual, %d padding)",
len(predict_examples), num_actual_predict_examples,
len(predict_examples) - num_actual_predict_examples)
tf.logging.info(" Batch size = %d", FLAGS.train_batch_size)
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False,
num_labels=num_labels)
result = estimator.predict(input_fn=predict_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir,
FLAGS.test_fname + ".predictions.tsv")
output_labels = os.path.join(FLAGS.output_dir,
FLAGS.test_fname + ".label_predictions.tsv")
with tf.gfile.GFile(output_predict_file, "w") as writer:
with tf.gfile.GFile(output_labels, "w") as writer2:
writer.write("\t".join(all_emotions) + "\n")
writer2.write("\t".join([
"text", "emotion_1", "prob_1", "emotion_2", "prob_2", "emotion_3",
"prob_3"
]) + "\n")
tf.logging.info("***** Predict results *****")
num_written_lines = 0
for (i, prediction) in enumerate(result):
probabilities = prediction["probabilities"]
if i >= num_actual_predict_examples:
break
output_line = "\t".join(
str(class_probability)
for class_probability in probabilities) + "\n"
sorted_idx = np.argsort(-probabilities)
top_3_emotion = [idx2emotion[idx] for idx in sorted_idx[:3]]
top_3_prob = [probabilities[idx] for idx in sorted_idx[:3]]
pred_line = []
for emotion, prob in zip(top_3_emotion, top_3_prob):
if prob >= FLAGS.pred_cutoff:
pred_line.extend([emotion, "%.4f" % prob])
else:
pred_line.extend(["", ""])
writer.write(output_line)
writer2.write(predict_examples[i].text + "\t" + "\t".join(pred_line) +
"\n")
num_written_lines += 1
assert num_written_lines == num_actual_predict_examples
