def ner_eval(h_params: HParams, estimator, processor): label_list = processor.get_labels() tokenizer = get_tokenizer(h_params) eval_examples = processor.get_dev_examples(h_params.data_dir) eval_file = os.path.join(h_params.output_dir, "eval.tf_record") filed_based_convert_examples_to_features(eval_examples, label_list, h_params.max_seq_length, tokenizer, eval_file, h_params) tf.logging.info("***** Running evaluation *****") tf.logging.info(" Num examples = %d", len(eval_examples)) tf.logging.info(" Batch size = %d", h_params.eval_batch_size) eval_steps = None if h_params.use_tpu: eval_steps = int(len(eval_examples) / h_params.eval_batch_size) eval_drop_remainder = h_params.use_tpu eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=h_params.max_seq_length, is_training=False, drop_remainder=eval_drop_remainder) result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) output_eval_file = os.path.join(h_params.local_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: tf.logging.info("***** Eval results *****") for key in sorted(result.keys()): tf.logging.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key])))
def ner_train(h_params: HParams, estimator, processor): train_examples = None num_train_steps = None num_warmup_steps = None if h_params.do_train: train_examples = processor.get_train_examples(h_params.data_dir) num_train_steps = h_params.num_train_steps num_warmup_steps = int(num_train_steps * h_params.warmup_proportion) tf.gfile.MakeDirs(h_params.output_dir) label_list = processor.get_labels() tokenizer = get_tokenizer(h_params) if h_params.do_train: train_file = os.path.join(h_params.output_dir, "train.tf_record") filed_based_convert_examples_to_features(train_examples, label_list, h_params.max_seq_length, tokenizer, train_file, h_params) tf.logging.info("***** Running training *****") tf.logging.info(" Num examples = %d", len(train_examples)) tf.logging.info(" Batch size = %d", h_params.train_batch_size) tf.logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=h_params.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
def test_convert_examples_to_features(): hparams = get_debug_hparams() data_filename = get_data_filename(hparams) examples = get_examples(data_filename, SetType.train) features = convert_examples_to_features( examples=examples, label_list=get_unique_intents(data_filename), max_seq_length=hparams.max_seq_length, tokenizer=get_tokenizer(hparams)) print( examples[2].__dict__ ) # {'guid': 'train-2', 'text_a': 'Is it worth upgrading from 12.04 LTS to 13.04', 'text_b': None, 'label': 'Make Update'} print( features[2].__dict__ ) # {'input_ids': [101, 2003, 2009, 4276, 25925, 2013, 2260, 1012, 5840, 8318, 2015, 2000, 2410, 1012, 5840, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'input_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'segment_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'label_id': 1}
def ner_pred(h_params: HParams, estimator, processor): h_params = h_params._replace(use_tpu=False) label_list = processor.get_labels() tokenizer = get_tokenizer(h_params) token_path = os.path.join(h_params.local_dir, "token_test.txt") with open(os.path.join(h_params.local_dir, 'label2id.pkl'), 'rb') as rf: label2id = pickle.load(rf) id2label = {value: key for key, value in label2id.items()} if os.path.exists(token_path): os.remove(token_path) predict_examples = processor.get_test_examples(h_params.data_dir) predict_file = os.path.join(h_params.output_dir, "predict.tf_record") filed_based_convert_examples_to_features(predict_examples, label_list, h_params.max_seq_length, tokenizer, predict_file, h_params, mode="test") tf.logging.info("***** Running prediction*****") tf.logging.info(" Num examples = %d", len(predict_examples)) tf.logging.info(" Batch size = %d", h_params.predict_batch_size) if h_params.use_tpu: # Warning: According to tpu_estimator.py Prediction on TPU is an # experimental feature and hence not supported here tf.logging.warning("Prediction in TPU not supported") predict_drop_remainder = h_params.use_tpu predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=h_params.max_seq_length, is_training=False, drop_remainder=predict_drop_remainder) result = estimator.predict(input_fn=predict_input_fn) result = list(result) result = [pred['predictions'] for pred in result] return result
def predict(hparams: HParams) -> List[str]: from improv.my_classifier import get_model_fn_and_estimator, file_based_input_fn_builder from improv.run_classifier import file_based_convert_examples_to_features import os from typing import Iterable import numpy as np from improv.utils import convert_result_pred, get_rounded_f1 data_filename = hparams.data_dir.parent / (hparams.task_name + '.tsv') params = hparams._replace( use_tpu=False) # BERT code warns against using TPU for predictions. model_fn, estimator = get_model_fn_and_estimator(params) predict_examples = get_examples(data_filename, SetType.test) predict_file = os.path.join(params.output_dir, "predict.tf_record") file_based_convert_examples_to_features(predict_examples, get_unique_intents(data_filename), params.max_seq_length, get_tokenizer(params), predict_file) tf.logging.info("***** Running prediction*****") tf.logging.info(" Num examples = %d", len(predict_examples)) tf.logging.info(" Batch size = %d", params.predict_batch_size) predict_drop_remainder = params.use_tpu predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=params.max_seq_length, is_training=False, drop_remainder=predict_drop_remainder) result: Iterable[np.ndarray] = estimator.predict(input_fn=predict_input_fn) label_list = get_intents( data_filename) # used for label_list[max_class] this might be wrong y_pred = convert_result_pred(result, label_list) print('f1 score: {}'.format( get_rounded_f1(params.data_dir.parent / 'askubuntu.tsv', y_pred, average='micro'))) return y_pred
def train_and_evaluate(hparams: HParams): tf.logging.set_verbosity(tf.logging.INFO) data_filename = hparams.data_dir.parent.parent / (hparams.task_name + '.tsv') train_examples = get_examples(data_filename, SetType.train) num_train_steps = int( len(train_examples) / hparams.train_batch_size * hparams.num_train_epochs) steps_per_epoch = len(train_examples) // hparams.train_batch_size max_steps = hparams.num_train_epochs * steps_per_epoch tf.logging.info('train_batch_size=%d eval_batch_size=%d max_steps=%d', hparams.train_batch_size, hparams.eval_batch_size, max_steps) # TPU change 3 tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( hparams.tpu_name, zone=hparams.tpu_zone, project=hparams.gcp_project) config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, model_dir=hparams.output_dir, # according to my_classifier save_checkpoints_steps=hparams.save_checkpoints_steps, save_summary_steps=hparams.save_summary_steps, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=hparams.save_checkpoints_steps, per_host_input_for_training=True)) model_fn = model_fn_builder( bert_config=BertConfig.from_json_file(str(hparams.bert_config_file)), num_labels=len(get_unique_intents(data_filename)), init_checkpoint=str(hparams.init_checkpoint), learning_rate=hparams.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=int(num_train_steps * hparams.warmup_proportion), use_tpu=hparams.use_tpu, use_one_hot_embeddings=True) estimator = tf.contrib.tpu.TPUEstimator( # TPU change 4 model_fn=model_fn, config=config, # params=hparams, # model_dir=hparams.data_dir, train_batch_size=hparams.train_batch_size, eval_batch_size=hparams.eval_batch_size, use_tpu=hparams.use_tpu) train_features = convert_examples_to_features( train_examples, get_unique_intents(data_filename), hparams.max_seq_length, get_tokenizer(hparams)) tf.logging.info(" Num steps = %d", num_train_steps) train_input_fn = input_fn_builder(features=train_features, seq_length=hparams.max_seq_length, is_training=True, drop_remainder=True, use_tpu=hparams.use_tpu) eval_examples = get_examples(data_filename, SetType.dev) eval_features = convert_examples_to_features( eval_examples, get_unique_intents(data_filename), hparams.max_seq_length, get_tokenizer(hparams)) # Eval will be slightly WRONG on the TPU because it will truncate # the last batch. eval_steps = int(len(eval_examples) / hparams.eval_batch_size) eval_input_fn = input_fn_builder(features=eval_features, seq_length=hparams.max_seq_length, is_training=False, drop_remainder=True, use_tpu=hparams.use_tpu) # set up training and evaluation in a loop # def input_fn_builder(features, seq_length, is_training, drop_remainder, use_tpu): # load last checkpoint and start from there current_step = load_global_step_from_checkpoint_dir(hparams.output_dir) tf.logging.info( 'Training for %d steps (%.2f epochs in total). Current' ' step %d.', max_steps, max_steps / steps_per_epoch, current_step) start_timestamp = time.time() # This time will include compilation time while current_step < max_steps: # Train for up to steps_per_eval number of steps. # At the end of training, a checkpoint will be written to --model_dir. next_checkpoint = min(current_step + hparams.iterations_per_loop, max_steps) # possibly need to save checkpoints if hparams.do_train: estimator.train(input_fn=train_input_fn, max_steps=next_checkpoint) current_step = next_checkpoint tf.logging.info('Finished training up to step %d. Elapsed seconds %d.', next_checkpoint, int(time.time() - start_timestamp)) # Evaluate the model on the most recent model in --model_dir. # Since evaluation happens in batches of --eval_batch_size, some images # may be excluded modulo the batch size. As long as the batch size is # consistent, the evaluated images are also consistent. tf.logging.info('Starting to evaluate at step %d', next_checkpoint) eval_results = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) tf.logging.info('Eval results at step %d: %s', next_checkpoint, eval_results) elapsed_time = int(time.time() - start_timestamp) tf.logging.info('Finished training up to step %d. Elapsed seconds %d.', max_steps, elapsed_time)