def main(unused_argv): del unused_argv # Unused tf.logging.set_verbosity(tf.logging.INFO) assert FLAGS.seq_len > 0 assert FLAGS.perm_size > 0 FLAGS.n_token = data_utils.VOCAB_SIZE tf.logging.info("n_token {}".format(FLAGS.n_token)) if not tf.gfile.Exists(FLAGS.model_dir): tf.gfile.MakeDirs(FLAGS.model_dir) # Get train input function train_input_fn, train_record_info_dict = get_input_fn("train") tf.logging.info("num of batches {}".format( train_record_info_dict["num_batch"])) # Get train cache function train_cache_fn = get_cache_fn(FLAGS.mem_len) ##### Get model function model_fn = get_model_fn() ##### Create TPUEstimator # TPU Configuration run_config = model_utils.configure_tpu(FLAGS) # TPU Estimator estimator = tpu_estimator.TPUEstimator( model_fn=model_fn, train_cache_fn=train_cache_fn, use_tpu=FLAGS.use_tpu, config=run_config, params={"track_mean": FLAGS.track_mean}, train_batch_size=FLAGS.train_batch_size, eval_on_tpu=FLAGS.use_tpu) #### Training estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) tpu_config = model_utils.configure_tpu(FLAGS) model_config = xlnet.XLNetConfig(json_path=FLAGS.model_config_path) run_config = xlnet.create_run_config(False, True, FLAGS) model_builder = XLNetModelBuilder( default_model_config=model_config, default_run_config=run_config, default_init_checkpoint=FLAGS.init_checkpoint, use_tpu=FLAGS.use_tpu) model_fn = model_builder.get_model_fn(model_config, run_config, FLAGS.init_checkpoint, FLAGS.model_type) # If TPU is not available, this will fall back to normal Estimator on CPU or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=tpu_config, export_to_tpu=FLAGS.use_tpu, train_batch_size=1) tokenizer = XLNetTokenizer( sp_model_file=FLAGS.spiece_model_file, lower_case=FLAGS.lower_case) example_converter = XLNetExampleConverter( label_list=[], max_seq_length=FLAGS.max_seq_length, tokenizer=tokenizer) features = example_converter.convert_examples_to_features([PaddingInputExample()]) input_fn = XLNetInputBuilder.get_input_builder(features, FLAGS.max_seq_length, True, False) estimator.train(input_fn, max_steps=1) tf.gfile.MakeDirs(FLAGS.export_dir) serving_input_fn = XLNetInputBuilder.get_serving_input_fn(FLAGS.max_seq_length) estimator.export_savedmodel(FLAGS.export_dir, serving_input_fn, as_text=False)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) # Validate flags if FLAGS.save_steps is not None: FLAGS.iterations = min(FLAGS.iterations, FLAGS.save_steps) if FLAGS.do_predict: predict_dir = FLAGS.predict_dir if not tf.gfile.Exists(predict_dir): tf.gfile.MakeDirs(predict_dir) processor = ccfKeywordProcessor() if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError( "At least one of `do_train`, `do_eval, `do_predict` or " "`do_submit` must be True.") if not tf.gfile.Exists(FLAGS.output_dir): tf.gfile.MakeDirs(FLAGS.output_dir) label_list = processor.get_labels() if not FLAGS.is_regression else None sp = spm.SentencePieceProcessor() sp.Load(FLAGS.spiece_model_file) def tokenize_fn(text): text = preprocess_text(text, lower=FLAGS.uncased) return encode_ids(sp, text) run_config = model_utils.configure_tpu(FLAGS) model_fn = get_model_fn( len(label_list) if label_list is not None else None) spm_basename = os.path.basename(FLAGS.spiece_model_file) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. if FLAGS.use_tpu: estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size, eval_batch_size=FLAGS.eval_batch_size) else: estimator = tf.estimator.Estimator(model_fn=model_fn, config=run_config) if FLAGS.do_train: train_file_base = "{}.len-{}.train.tf_record".format( spm_basename, FLAGS.max_seq_length) train_file = os.path.join(FLAGS.output_dir, train_file_base) tf.logging.info("Use tfrecord file {}".format(train_file)) train_examples = processor.get_train_examples(FLAGS.train_data) np.random.shuffle(train_examples) tf.logging.info("Num of train samples: {}".format(len(train_examples))) file_based_convert_examples_to_features(train_examples, label_list, FLAGS.max_seq_length, tokenize_fn, train_file, FLAGS.num_passes) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.dev_data) tf.logging.info("Num of eval samples: {}".format(len(eval_examples))) while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file_base = "{}.len-{}.{}.eval.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) eval_file = os.path.join(FLAGS.output_dir, eval_file_base) file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn, eval_file) assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=True) if FLAGS.eval_only_one: result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=FLAGS.eval_ckpt) pre, rec, f1 = get_metrics(result["conf_mat"], 3) tf.logging.info("eval ckpt" + FLAGS.eval_ckpt) tf.logging.info("eval_precision: {}".format(pre)) tf.logging.info("eval_recall: {}".format(rec)) tf.logging.info("eval_f1: {}".format(f1)) tf.logging.info("eval_accuracy: {}".format( result["eval_accuracy"])) tf.logging.info("eval_loss: {}".format(result["eval_loss"])) tf.logging.info("-------------------------\n\n") else: # Filter out all checkpoints in the directory steps_and_files = [] filenames = tf.gfile.ListDirectory(FLAGS.model_dir) for filename in filenames: if filename.endswith(".index"): ckpt_name = filename[:-6] cur_filename = join(FLAGS.model_dir, ckpt_name) global_step = int(cur_filename.split("-")[-1]) tf.logging.info( "Add {} to eval list.".format(cur_filename)) steps_and_files.append([global_step, cur_filename]) steps_and_files = sorted(steps_and_files, key=lambda x: x[0]) # Decide whether to evaluate all ckpts if not FLAGS.eval_all_ckpt: steps_and_files = steps_and_files[-1:] result_list = list() for global_step, filename in sorted(steps_and_files, key=lambda x: x[0]): result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=filename) result_list.append([global_step, result]) for step, result in result_list: tf.logging.info("\n\n------ step -------" + str(step)) pre, rec, f1 = get_metrics(result["conf_mat"], 3) tf.logging.info("eval_precision: {}".format(pre)) tf.logging.info("eval_recall: {}".format(rec)) tf.logging.info("eval_f1: {}".format(f1)) tf.logging.info("eval_accuracy: {}".format( result["eval_accuracy"])) tf.logging.info("eval_loss: {}".format(result["eval_loss"])) tf.logging.info("-------------------------\n\n") if FLAGS.do_predict: """ 首先对使用的ckpt进行eval,防止加载错模型 """ eval_examples = processor.get_dev_examples(FLAGS.dev_data) tf.logging.info("Num of eval samples: {}".format(len(eval_examples))) while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file_base = "{}.len-{}.{}.eval.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) eval_file = os.path.join(FLAGS.output_dir, eval_file_base) file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn, eval_file) assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=True) result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=FLAGS.predict_ckpt) pre, rec, f1 = get_metrics(result["conf_mat"], FLAGS.label_num) tf.logging.info("\n\n\n加载的模型的效果为:") tf.logging.info("eval_precision: {}".format(pre)) tf.logging.info("eval_recall: {}".format(rec)) tf.logging.info("eval_f1: {}".format(f1)) tf.logging.info("eval_accuracy: {}".format(result["eval_accuracy"])) tf.logging.info("eval_loss: {}".format(result["eval_loss"])) tf.logging.info("-------------------------\n\n\n") """ 做完eval后,进行模型的predict """ eval_examples = processor.get_test_examples(FLAGS.test_data) num_actual_predict_examples = len(eval_examples) eval_file_base = "{}.len-{}.{}.predict.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) eval_file = os.path.join(FLAGS.output_dir, eval_file_base) file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn, eval_file) pred_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) # Decide whether to evaluate all ckpts result = estimator.predict(input_fn=pred_input_fn, checkpoint_path=FLAGS.predict_ckpt) with tf.gfile.GFile(FLAGS.predict_file_write_path, "w") as writer: num_written_lines = 0 tf.logging.info("***** Predict results *****") for (i, prediction) in enumerate(result): p = prediction["p"] if i >= num_actual_predict_examples: break output_line = "\t".join( str(class_probability) for class_probability in p) + "\n" writer.write(output_line) num_written_lines += 1 assert num_written_lines == num_actual_predict_examples tf.logging.info("模型预测完成。。。。")
def main(_): tf.logging.set_verbosity(tf.logging.INFO) layer_indexes = [int(x) for x in FLAGS.layers.split(",")] bert_config = modeling.XLNetConfig(json_path=FLAGS.bert_config_file) tokenizer = tokenization.FullTokenizer( spm_model_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config =configure_tpu(FLAGS) # tf.contrib.tpu.RunConfig( # master=FLAGS.master, # tpu_config=tf.contrib.tpu.TPUConfig( # num_shards=FLAGS.num_tpu_cores, # per_host_input_for_training=is_per_host)) # examples = read_examples(FLAGS.input_file) json_examples = [] for x in ['test', 'train', 'dev']: with open(os.path.join(FLAGS.input_file, x + '.english.jsonlines')) as f: json_examples.extend((json.loads(jsonline) for jsonline in f.readlines())) orig_examples = [] bert_examples = [] for i, json_e in enumerate(json_examples): e = process_example(json_e, i, should_filter_embedded_mentions=True) orig_examples.append(e) bert_examples.append(e.bertify(tokenizer)) model_fn = model_fn_builder( bert_config=bert_config, run_config=run_config, init_checkpoint=FLAGS.init_checkpoint, layer_indexes=layer_indexes, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_one_hot_embeddings) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, predict_batch_size=FLAGS.batch_size) input_fn = input_fn_builder( examples=bert_examples, window_size=FLAGS.window_size, stride=FLAGS.stride, tokenizer=tokenizer) writer = h5py.File(FLAGS.output_file, 'w') with tqdm(total=sum(len(e.tokens) for e in orig_examples)) as t: for result in estimator.predict(input_fn, yield_single_examples=True): document_index = int(result["unique_ids"]) bert_example = bert_examples[document_index] orig_example = orig_examples[document_index] file_key = bert_example.doc_key.replace('/', ':') t.update(n=(result['extract_indices'] >= 0).sum()) for output_index, bert_token_index in enumerate(result['extract_indices']): if bert_token_index < 0: continue token_index = bert_example.bert_to_orig_map[bert_token_index] sentence_index, token_index = orig_example.unravel_token_index(token_index) dataset_key ="{}/{}".format(file_key, sentence_index) if dataset_key not in writer: writer.create_dataset(dataset_key, (len(orig_example.sentence_tokens[sentence_index]), bert_config.hidden_size, len(layer_indexes)), dtype=np.float32) dset = writer[dataset_key] for j, layer_index in enumerate(layer_indexes): layer_output = result["layer_output_%d" % j] dset[token_index, :, j] = layer_output[output_index] writer.close()
def main(_): tf.logging.set_verbosity(tf.logging.INFO) #### Validate flags if FLAGS.save_steps is not None: FLAGS.iterations = min(FLAGS.iterations, FLAGS.save_steps) if FLAGS.do_predict: predict_dir = FLAGS.predict_dir if not tf.gfile.Exists(predict_dir): tf.gfile.MakeDirs(predict_dir) processors = { "mnli_matched": MnliMatchedProcessor, "mnli_mismatched": MnliMismatchedProcessor, 'sts-b': StsbProcessor, 'imdb': ImdbProcessor, "yelp5": Yelp5Processor } if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError( "At least one of `do_train`, `do_eval, `do_predict` or " "`do_submit` must be True.") if not tf.gfile.Exists(FLAGS.output_dir): tf.gfile.MakeDirs(FLAGS.output_dir) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels() if not FLAGS.is_regression else None sp = spm.SentencePieceProcessor() sp.Load(FLAGS.spiece_model_file) def tokenize_fn(text): text = preprocess_text(text, lower=FLAGS.uncased) return encode_ids(sp, text) run_config = model_utils.configure_tpu(FLAGS) model_fn = get_model_fn( len(label_list) if label_list is not None else None) spm_basename = os.path.basename(FLAGS.spiece_model_file) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. if FLAGS.use_tpu: estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size, eval_batch_size=FLAGS.eval_batch_size) else: estimator = tf.estimator.Estimator(model_fn=model_fn, config=run_config) if FLAGS.do_train: train_file_base = "{}.len-{}.train.tf_record".format( spm_basename, FLAGS.max_seq_length) train_file = os.path.join(FLAGS.output_dir, train_file_base) tf.logging.info("Use tfrecord file {}".format(train_file)) train_examples = processor.get_train_examples(FLAGS.data_dir) tf.logging.info("Num of train samples: {}".format(len(train_examples))) file_based_convert_examples_to_features(train_examples, label_list, FLAGS.max_seq_length, tokenize_fn, train_file, FLAGS.num_passes) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) if FLAGS.do_eval or FLAGS.do_predict: if FLAGS.eval_split == "dev": eval_examples = processor.get_dev_examples(FLAGS.data_dir) else: eval_examples = processor.get_test_examples(FLAGS.data_dir) tf.logging.info("Num of eval samples: {}".format(len(eval_examples))) if FLAGS.do_eval: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. These do NOT count towards the metric (all tf.metrics # support a per-instance weight, and these get a weight of 0.0). # # Modified in XL: We also adopt the same mechanism for GPUs. while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file_base = "{}.len-{}.{}.eval.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) eval_file = os.path.join(FLAGS.output_dir, eval_file_base) file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn, eval_file) assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=True) # Filter out all checkpoints in the directory steps_and_files = [] filenames = tf.gfile.ListDirectory(FLAGS.model_dir) for filename in filenames: if filename.endswith(".index"): ckpt_name = filename[:-6] cur_filename = join(FLAGS.model_dir, ckpt_name) global_step = int(cur_filename.split("-")[-1]) tf.logging.info("Add {} to eval list.".format(cur_filename)) steps_and_files.append([global_step, cur_filename]) steps_and_files = sorted(steps_and_files, key=lambda x: x[0]) # Decide whether to evaluate all ckpts if not FLAGS.eval_all_ckpt: steps_and_files = steps_and_files[-1:] eval_results = [] for global_step, filename in sorted(steps_and_files, key=lambda x: x[0]): ret = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=filename) ret["step"] = global_step ret["path"] = filename eval_results.append(ret) tf.logging.info("=" * 80) log_str = "Eval result | " for key, val in sorted(ret.items(), key=lambda x: x[0]): log_str += "{} {} | ".format(key, val) tf.logging.info(log_str) key_name = "eval_pearsonr" if FLAGS.is_regression else "eval_accuracy" eval_results.sort(key=lambda x: x[key_name], reverse=True) tf.logging.info("=" * 80) log_str = "Best result | " for key, val in sorted(eval_results[0].items(), key=lambda x: x[0]): log_str += "{} {} | ".format(key, val) tf.logging.info(log_str) if FLAGS.do_predict: eval_file_base = "{}.len-{}.{}.predict.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) eval_file = os.path.join(FLAGS.output_dir, eval_file_base) file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn, eval_file) pred_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) predict_results = [] with tf.gfile.Open( os.path.join(predict_dir, "{}.tsv".format(task_name)), "w") as fout: fout.write("index\tprediction\n") for pred_cnt, result in enumerate( estimator.predict(input_fn=pred_input_fn, yield_single_examples=True, checkpoint_path=FLAGS.predict_ckpt)): if pred_cnt % 1000 == 0: tf.logging.info( "Predicting submission for example: {}".format( pred_cnt)) logits = [float(x) for x in result["logits"].flat] predict_results.append(logits) if len(logits) == 1: label_out = logits[0] elif len(logits) == 2: if logits[1] - logits[0] > FLAGS.predict_threshold: label_out = label_list[1] else: label_out = label_list[0] elif len(logits) > 2: max_index = np.argmax(np.array(logits, dtype=np.float32)) label_out = label_list[max_index] else: raise NotImplementedError fout.write("{}\t{}\n".format(pred_cnt, label_out)) predict_json_path = os.path.join(predict_dir, "{}.logits.json".format(task_name)) with tf.gfile.Open(predict_json_path, "w") as fp: json.dump(predict_results, fp, indent=4)
def main(): tf.logging.set_verbosity(tf.logging.INFO) np.random.seed(cf.random_seed) processor = NerProcessor( data_dir=cf.train_data, task_name=cf.task_name.lower()) # label_list = processor.get_labels() label_list = processor.labels model_config = xlnet.XLNetConfig(json_path=cf.model_config_path) model_builder = XLNetModelBuilder( model_config=model_config, use_tpu=cf.use_tpu) model_fn = model_builder.get_model_fn(label_list) # If TPU is not available, this will fall back to normal Estimator on CPU or GPU. tpu_config = model_utils.configure_tpu(cf) estimator = tf.contrib.tpu.TPUEstimator( use_tpu=cf.use_tpu, model_fn=model_fn, config=tpu_config, export_to_tpu=cf.use_tpu, train_batch_size=cf.train_batch_size, eval_batch_size=cf.eval_batch_size, predict_batch_size=cf.predict_batch_size) tokenizer = XLNetTokenizer( sp_model_file=cf.spiece_model_file, lower_case=cf.lower_case) example_converter = XLNetExampleConverter( label_list=label_list, max_seq_length=cf.max_seq_length, tokenizer=tokenizer) if cf.do_train and cf.do_eval: # 开始训练 train_file = os.path.join(cf.output_dir, "train.tf_record") tf.logging.info("Use tfrecord samples: {}".format(len(train_file))) train_examples = processor.get_train_examples() # train data np.random.shuffle(train_examples) example_converter.file_based_convert_examples_to_features(train_examples, train_file) train_steps = int(len(train_examples) * cf.num_train_epochs / cf.train_batch_size) cf.warmup_steps = int(0.1 * train_steps) tf.logging.info("***** Run training *****") tf.logging.info(" Num examples = %d", len(train_examples)) tf.logging.info(" Batch size = %d", cf.train_batch_size) tf.logging.info(" Num steps = %d", cf.train_steps) # train_features = example_converter.convert_examples_to_features(train_examples) # if not os.path.exists(train_file): # train_features = example_converter.file_based_convert_examples_to_features(train_examples, train_file) # 读取TF_record数据 # train_input_fn = XLNetInputBuilder.get_input_builder(train_features, cf.max_seq_length, True, True) train_input_fn = XLNetInputBuilder.get_file_based_input_fn( input_file= train_file, seq_length=cf.max_seq_length, is_training=True, drop_remainder=True ) estimator.train(input_fn=train_input_fn, max_steps=train_steps) eval_examples = processor.get_dev_examples() tf.logging.info("***** Run evaluation *****") tf.logging.info(" Num examples = %d", len(eval_examples)) tf.logging.info(" Batch size = %d", cf.eval_batch_size) # early stop hook # early_stopping_hook = tf.contrib.estimator.stop_if_no_decrease_hook( # estimator=estimator, # metric_name='loss', # max_steps_without_decrease=cf.num_train_steps, # eval_dir=None, # min_steps=0, # run_every_secs=None, # run_every_steps=cf.save_checkpoints_steps # ) eval_features = example_converter.convert_examples_to_features(eval_examples) eval_input_fn = XLNetInputBuilder.get_input_builder(eval_features, cf.max_seq_length, False, False) # train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=cf.num_train_steps, # hooks=[early_stopping_hook]) # eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn) # tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) result = estimator.evaluate(input_fn=eval_input_fn) precision = result["precision"] recall = result["recall"] f1_score = 2.0 * precision * recall / (precision + recall) tf.logging.info("***** Evaluation result *****") tf.logging.info(" Precision (token-level) = %s", str(precision)) tf.logging.info(" Recall (token-level) = %s", str(recall)) tf.logging.info(" F1 score (token-level) = %s", str(f1_score)) if cf.do_predict: predict_examples = processor.get_test_examples() tf.logging.info("***** Run prediction *****") tf.logging.info(" Num examples = %d", len(predict_examples)) tf.logging.info(" Batch size = %d", cf.predict_batch_size) predict_features = example_converter.convert_examples_to_features(predict_examples) predict_input_fn = XLNetInputBuilder.get_input_builder(predict_features, cf.max_seq_length, False, False) result = estimator.predict(input_fn=predict_input_fn) predict_recorder = XLNetPredictRecorder( output_dir=cf.output_dir, label_list=label_list, max_seq_length=cf.max_seq_length, tokenizer=tokenizer, predict_tag=cf.predict_tag) predicts = [{ "input_ids": feature.input_ids, "input_masks": feature.input_masks, "label_ids": feature.label_ids, "predict_ids": predict["predict"].tolist() } for feature, predict in zip(predict_features, result)] predict_recorder.record(predicts) if cf.do_export: tf.logging.info("***** Running exporting *****") tf.io.gfile.makedirs(cf.export_dir) serving_input_fn = XLNetInputBuilder.get_serving_input_fn(cf.max_seq_length) estimator.export_saved_model(cf.export_dir, serving_input_fn, as_text=False)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) #### Validate flags if FLAGS.save_steps is not None: FLAGS.iterations = min(FLAGS.iterations, FLAGS.save_steps) if not FLAGS.do_train and not FLAGS.do_eval: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") if not tf.gfile.Exists(FLAGS.output_dir): tf.gfile.MakeDirs(FLAGS.output_dir) sp = spm.SentencePieceProcessor() sp.Load(FLAGS.spiece_model_file) def tokenize_fn(text): text = preprocess_text(text, lower=FLAGS.uncased) return encode_ids(sp, text) # TPU Configuration run_config = model_utils.configure_tpu(FLAGS) model_fn = get_model_fn() spm_basename = os.path.basename(FLAGS.spiece_model_file) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. if FLAGS.use_tpu: estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size) else: estimator = tf.estimator.Estimator(model_fn=model_fn, config=run_config) if FLAGS.do_train: train_file_base = "{}.len-{}.train.tf_record".format( spm_basename, FLAGS.max_seq_length) train_file = os.path.join(FLAGS.output_dir, train_file_base) if not tf.gfile.Exists(train_file) or FLAGS.overwrite_data: train_examples = get_examples(FLAGS.data_dir, "train") random.shuffle(train_examples) file_based_convert_examples_to_features(train_examples, tokenize_fn, train_file) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) if FLAGS.do_eval: eval_examples = get_examples(FLAGS.data_dir, FLAGS.eval_split) tf.logging.info("Num of eval samples: {}".format(len(eval_examples))) # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. These do NOT count towards the metric (all tf.metrics # support a per-instance weight, and these get a weight of 0.0). # # Modified in XL: We also adopt the same mechanism for GPUs. while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file_base = "{}.len-{}.{}.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) if FLAGS.high_only: eval_file_base = "high." + eval_file_base elif FLAGS.middle_only: eval_file_base = "middle." + eval_file_base eval_file = os.path.join(FLAGS.output_dir, eval_file_base) file_based_convert_examples_to_features(eval_examples, tokenize_fn, eval_file) assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=True) ret = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) # Log current result tf.logging.info("=" * 80) log_str = "Eval | " for key, val in ret.items(): log_str += "{} {} | ".format(key, val) tf.logging.info(log_str) tf.logging.info("=" * 80)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) np.random.seed(FLAGS.random_seed) processor = NluProcessor(data_dir=FLAGS.data_dir, task_name=FLAGS.task_name.lower()) token_label_list = processor.get_token_labels() sent_label_list = processor.get_sent_labels() model_config = xlnet.XLNetConfig(json_path=FLAGS.model_config_path) model_builder = XLNetModelBuilder(model_config=model_config, use_tpu=FLAGS.use_tpu) model_fn = model_builder.get_model_fn(token_label_list, sent_label_list) # If TPU is not available, this will fall back to normal Estimator on CPU or GPU. tpu_config = model_utils.configure_tpu(FLAGS) estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=tpu_config, export_to_tpu=FLAGS.use_tpu, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.predict_batch_size) tokenizer = XLNetTokenizer(sp_model_file=FLAGS.spiece_model_file, lower_case=FLAGS.lower_case) example_converter = XLNetExampleConverter( token_label_list=token_label_list, sent_label_list=sent_label_list, max_seq_length=FLAGS.max_seq_length, tokenizer=tokenizer) if FLAGS.do_train: train_examples = processor.get_train_examples() tf.logging.info("***** Run training *****") tf.logging.info(" Num examples = %d", len(train_examples)) tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Num steps = %d", FLAGS.train_steps) train_features = example_converter.convert_examples_to_features( train_examples) train_input_fn = XLNetInputBuilder.get_input_builder( train_features, FLAGS.max_seq_length, True, True) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples() tf.logging.info("***** Run evaluation *****") tf.logging.info(" Num examples = %d", len(eval_examples)) tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) eval_features = example_converter.convert_examples_to_features( eval_examples) eval_input_fn = XLNetInputBuilder.get_input_builder( eval_features, FLAGS.max_seq_length, False, False) result = estimator.evaluate(input_fn=eval_input_fn) token_precision = result["token_precision"] token_recall = result["token_recall"] token_f1_score = 2.0 * token_precision * token_recall / ( token_precision + token_recall) sent_accuracy = result["sent_accuracy"] tf.logging.info("***** Evaluation result *****") tf.logging.info(" Precision (token-level) = %s", str(token_precision)) tf.logging.info(" Recall (token-level) = %s", str(token_recall)) tf.logging.info(" F1 score (token-level) = %s", str(token_f1_score)) tf.logging.info(" Accuracy (sent-level) = %s", str(sent_accuracy)) if FLAGS.do_predict: predict_examples = processor.get_test_examples() tf.logging.info("***** Run prediction *****") tf.logging.info(" Num examples = %d", len(predict_examples)) tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_features = example_converter.convert_examples_to_features( predict_examples) predict_input_fn = XLNetInputBuilder.get_input_builder( predict_features, FLAGS.max_seq_length, False, False) result = estimator.predict(input_fn=predict_input_fn) predict_recorder = XLNetPredictRecorder( output_dir=FLAGS.output_dir, token_label_list=token_label_list, sent_label_list=sent_label_list, max_seq_length=FLAGS.max_seq_length, tokenizer=tokenizer, predict_tag=FLAGS.predict_tag) predicts = [{ "input_ids": feature.input_ids, "input_masks": feature.input_masks, "token_label_ids": feature.token_label_ids, "sent_label_id": feature.sent_label_id, "token_predict_ids": predict["token_predict"].tolist(), "sent_predict_id": predict["sent_predict"].tolist() } for feature, predict in zip(predict_features, result)] predict_recorder.record(predicts) if FLAGS.do_export: tf.logging.info("***** Running exporting *****") tf.gfile.MakeDirs(FLAGS.export_dir) serving_input_fn = XLNetInputBuilder.get_serving_input_fn( FLAGS.max_seq_length) estimator.export_savedmodel(FLAGS.export_dir, serving_input_fn, as_text=False)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) if not tf.gfile.Exists(FLAGS.output_dir): tf.gfile.MakeDirs(FLAGS.output_dir) if FLAGS.do_prepro: preprocess() return #### Validate flags if FLAGS.save_steps is not None: FLAGS.iterations = min(FLAGS.iterations, FLAGS.save_steps) if not FLAGS.do_train and not FLAGS.do_predict: raise ValueError( "At least one of `do_train` and `do_predict` must be True.") if FLAGS.do_predict and not tf.gfile.Exists(FLAGS.predict_dir): tf.gfile.MakeDirs(FLAGS.predict_dir) sp_model = spm.SentencePieceProcessor() sp_model.Load(FLAGS.spiece_model_file) ### TPU Configuration run_config = model_utils.configure_tpu(FLAGS) model_fn = get_model_fn() spm_basename = _get_spm_basename() # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. if FLAGS.use_tpu: estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size) else: estimator = tf.estimator.Estimator(model_fn=model_fn, config=run_config) if FLAGS.do_train: train_rec_glob = os.path.join( FLAGS.output_dir, "{}.*.slen-{}.qlen-{}.train.tf_record".format( spm_basename, FLAGS.max_seq_length, FLAGS.max_query_length)) train_input_fn = input_fn_builder(input_glob=train_rec_glob, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True, num_hosts=FLAGS.num_hosts) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) if FLAGS.do_predict: eval_examples = read_squad_examples(FLAGS.predict_file, is_training=False) with tf.gfile.Open(FLAGS.predict_file) as f: orig_data = json.load(f)["data"] eval_rec_file = os.path.join( FLAGS.output_dir, "{}.slen-{}.qlen-{}.eval.tf_record".format(spm_basename, FLAGS.max_seq_length, FLAGS.max_query_length)) eval_feature_file = os.path.join( FLAGS.output_dir, "{}.slen-{}.qlen-{}.eval.features.pkl".format( spm_basename, FLAGS.max_seq_length, FLAGS.max_query_length)) if tf.gfile.Exists(eval_rec_file) and tf.gfile.Exists( eval_feature_file) and not FLAGS.overwrite_data: tf.logging.info( "Loading eval features from {}".format(eval_feature_file)) with tf.gfile.Open(eval_feature_file, 'rb') as fin: eval_features = pickle.load(fin) else: eval_writer = FeatureWriter(filename=eval_rec_file, is_training=False) eval_features = [] def append_feature(feature): eval_features.append(feature) eval_writer.process_feature(feature) convert_examples_to_features( examples=eval_examples, sp_model=sp_model, max_seq_length=FLAGS.max_seq_length, doc_stride=FLAGS.doc_stride, max_query_length=FLAGS.max_query_length, is_training=False, output_fn=append_feature) eval_writer.close() with tf.gfile.Open(eval_feature_file, 'wb') as fout: pickle.dump(eval_features, fout) eval_input_fn = input_fn_builder(input_glob=eval_rec_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False, num_hosts=1) cur_results = [] for result in estimator.predict(input_fn=eval_input_fn, yield_single_examples=True): if len(cur_results) % 1000 == 0: tf.logging.info("Processing example: %d" % (len(cur_results))) unique_id = int(result["unique_ids"]) start_top_log_probs = ([ float(x) for x in result["start_top_log_probs"].flat ]) start_top_index = [int(x) for x in result["start_top_index"].flat] end_top_log_probs = ([ float(x) for x in result["end_top_log_probs"].flat ]) end_top_index = [int(x) for x in result["end_top_index"].flat] cls_logits = float(result["cls_logits"].flat[0]) cur_results.append( RawResult(unique_id=unique_id, start_top_log_probs=start_top_log_probs, start_top_index=start_top_index, end_top_log_probs=end_top_log_probs, end_top_index=end_top_index, cls_logits=cls_logits)) output_prediction_file = os.path.join(FLAGS.predict_dir, "predictions.json") output_nbest_file = os.path.join(FLAGS.predict_dir, "nbest_predictions.json") output_null_log_odds_file = os.path.join(FLAGS.predict_dir, "null_odds.json") ret = write_predictions(eval_examples, eval_features, cur_results, FLAGS.n_best_size, FLAGS.max_answer_length, output_prediction_file, output_nbest_file, output_null_log_odds_file, orig_data) # Log current result tf.logging.info("=" * 80) log_str = "Result | " for key, val in ret.items(): log_str += "{} {} | ".format(key, val) tf.logging.info(log_str) tf.logging.info("=" * 80)