def main(argv): processor = raw_data_utils.get_processor(FLAGS.task_name) tf.compat.v1.logging.info("loading examples") FLAGS.output_data_dir = os.path.join( FLAGS.output_data_dir, FLAGS.sub_set) if not tf.io.gfile.Exists(FLAGS.output_data_dir): tf.io.gfile.MakeDirs(FLAGS.output_data_dir) if FLAGS.sub_set == "train": examples = processor.get_train_examples(FLAGS.raw_data_dir) elif FLAGS.sub_set.startswith("unsup"): examples = processor.get_unsup_examples(FLAGS.raw_data_dir, FLAGS.sub_set) else: assert False tf.compat.v1.logging.info("finished loading examples") tf.compat.v1.logging.info("examples num: {:d}".format(len(examples))) dump_raw_examples(examples, FLAGS.separate_doc_by_newline)
def main(_): if FLAGS.max_seq_length > 512: raise ValueError( "Cannot use sequence length {:d} because the BERT model " "was only trained up to sequence length {:d}".format( FLAGS.max_seq_length, 512)) processor = raw_data_utils.get_processor(FLAGS.task_name) # Create tokenizer tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) if FLAGS.data_type == "sup": sup_out_dir = FLAGS.output_base_dir tf.logging.info("Create sup. data: subset {} => {}".format( FLAGS.sub_set, sup_out_dir)) proc_and_save_sup_data( processor, FLAGS.sub_set, FLAGS.raw_data_dir, sup_out_dir, tokenizer, FLAGS.max_seq_length, FLAGS.trunc_keep_right, FLAGS.worker_id, FLAGS.replicas, FLAGS.sup_size, ) elif FLAGS.data_type == "unsup": assert FLAGS.aug_ops is not None, \ "aug_ops is required to preprocess unsupervised data." unsup_out_dir = os.path.join(FLAGS.output_base_dir, FLAGS.aug_ops, str(FLAGS.aug_copy_num)) data_stats_dir = os.path.join(FLAGS.raw_data_dir, "data_stats") tf.logging.info("Create unsup. data: subset {} => {}".format( FLAGS.sub_set, unsup_out_dir)) proc_and_save_unsup_data(processor, FLAGS.sub_set, FLAGS.raw_data_dir, data_stats_dir, unsup_out_dir, tokenizer, FLAGS.max_seq_length, FLAGS.trunc_keep_right, FLAGS.aug_ops, FLAGS.aug_copy_num, FLAGS.worker_id, FLAGS.replicas, FLAGS.input_file)
def main(_): hvd.init() FLAGS.model_dir = FLAGS.model_dir if hvd.rank() == 0 else os.path.join( FLAGS.model_dir, str(hvd.rank())) config = tf.ConfigProto() config.gpu_options.visible_device_list = str(hvd.local_rank()) #FLAGS.num_train_steps = FLAGS.num_train_steps // hvd.size() #FLAGS.num_warmup_steps = FLAGS.num_warmup_steps // hvd.size() tf.logging.set_verbosity(tf.logging.INFO) processor = raw_data_utils.get_processor(FLAGS.task_name) label_list = processor.get_labels() bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file, FLAGS.model_dropout) tf.gfile.MakeDirs(FLAGS.model_dir) flags_dict = tf.app.flags.FLAGS.flag_values_dict() with tf.gfile.Open(os.path.join(FLAGS.model_dir, "FLAGS.json"), "w") as ouf: json.dump(flags_dict, ouf) tf.logging.info("warmup steps {}/{}".format(FLAGS.num_warmup_steps, FLAGS.num_train_steps)) save_checkpoints_steps = 500 #FLAGS.num_train_steps // FLAGS.save_checkpoints_num tf.logging.info("setting save checkpoints steps to {:d}".format( save_checkpoints_steps)) FLAGS.iterations_per_loop = min(save_checkpoints_steps, FLAGS.iterations_per_loop) if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) else: tpu_cluster_resolver = None # if not FLAGS.use_tpu and FLAGS.num_gpu > 1: # train_distribute = tf.contrib.distribute.MirroredStrategy( # num_gpus=FLAGS.num_gpu) # else: # train_distribute = None is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.model_dir, save_checkpoints_steps=save_checkpoints_steps, keep_checkpoint_max=1, # train_distribute=train_distribute, session_config=config, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, per_host_input_for_training=is_per_host)) model_fn = uda.model_fn_builder( bert_config=bert_config, init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, clip_norm=FLAGS.clip_norm, num_train_steps=FLAGS.num_train_steps, num_warmup_steps=FLAGS.num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_one_hot_embeddings, num_labels=len(label_list), unsup_ratio=FLAGS.unsup_ratio, uda_coeff=FLAGS.uda_coeff, tsa=FLAGS.tsa, print_feature=False, print_structure=False, ) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, params={"model_dir": FLAGS.model_dir}, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.eval_batch_size) if FLAGS.do_train: tf.logging.info(" >>> sup data dir : {}".format( FLAGS.sup_train_data_dir)) if FLAGS.unsup_ratio > 0: tf.logging.info(" >>> unsup data dir : {}".format( FLAGS.unsup_data_dir)) train_input_fn = proc_data_utils.training_input_fn_builder( FLAGS.sup_train_data_dir, FLAGS.unsup_data_dir, FLAGS.aug_ops, FLAGS.aug_copy, FLAGS.unsup_ratio) train_size = processor.get_train_size(FLAGS.raw_data_dir) train_steps = int(train_size / FLAGS.train_batch_size) if FLAGS.do_eval: tf.logging.info(" >>> dev data dir : {}".format(FLAGS.eval_data_dir)) eval_input_fn = proc_data_utils.evaluation_input_fn_builder( FLAGS.eval_data_dir, "clas") eval_size = processor.get_dev_size(FLAGS.raw_data_dir) eval_steps = int(eval_size / FLAGS.eval_batch_size) train_eval_input_fn = proc_data_utils.evaluation_input_fn_builder( FLAGS.sup_train_data_dir, "clas") if FLAGS.do_train and FLAGS.do_eval: hooks = [hvd.BroadcastGlobalVariablesHook(0)] tf.logging.info("***** Running training & evaluation *****") tf.logging.info(" Supervised batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Unsupervised batch size = %d", FLAGS.train_batch_size * FLAGS.unsup_ratio) tf.logging.info(" training size = %d", train_size) tf.logging.info(" training num steps = %d", train_steps) tf.logging.info(" evaluation batch size = %d", FLAGS.eval_batch_size) tf.logging.info(" dev num steps = %d", eval_steps) best_acc = 0 for _ in range(0, FLAGS.num_train_steps, save_checkpoints_steps): tf.logging.info("*** Running training ***") estimator.train(input_fn=train_input_fn, steps=save_checkpoints_steps, hooks=hooks) tf.logging.info("*** Running evaluation ***") train_result = estimator.evaluate(input_fn=train_eval_input_fn, steps=train_steps) tf.logging.info(">> Train Results:") for key in train_result.keys(): tf.logging.info(" %s = %s", key, str(train_result[key])) train_result[key] = train_result[key].item() dev_result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) tf.logging.info(">> Results:") for key in dev_result.keys(): tf.logging.info(" %s = %s", key, str(dev_result[key])) dev_result[key] = dev_result[key].item() best_acc = max(best_acc, dev_result["eval_precision"]) tf.logging.info("***** Final evaluation result *****") tf.logging.info("Best acc: {:.3f}\n\n".format(best_acc)) elif FLAGS.do_train: tf.logging.info("***** Running training *****") tf.logging.info(" Supervised batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Unsupervised batch size = %d", FLAGS.train_batch_size * FLAGS.unsup_ratio) tf.logging.info(" Num steps = %d", FLAGS.num_train_steps) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.num_train_steps) elif FLAGS.do_eval: tf.logging.info("***** Running evaluation *****") tf.logging.info(" Base evaluation batch size = %d", FLAGS.eval_batch_size) tf.logging.info(" Num steps = %d", eval_steps) checkpoint_state = tf.train.get_checkpoint_state(FLAGS.model_dir) best_acc = 0 for ckpt_path in checkpoint_state.all_model_checkpoint_paths: if not tf.gfile.Exists(ckpt_path + ".data-00000-of-00001"): tf.logging.info( "Warning: checkpoint {:s} does not exist".format( ckpt_path)) continue tf.logging.info("Evaluating {:s}".format(ckpt_path)) dev_result = estimator.evaluate( input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=ckpt_path, ) tf.logging.info(">> Results:") for key in dev_result.keys(): tf.logging.info(" %s = %s", key, str(dev_result[key])) dev_result[key] = dev_result[key].item() best_acc = max(best_acc, dev_result["eval_precision"]) tf.logging.info("***** Final evaluation result *****") tf.logging.info("Best acc: {:.3f}\n\n".format(best_acc)) from utils import tokenization tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=True) id2label = dict(zip([i for i in range(len(label_list))], label_list)) result = estimator.predict(input_fn=eval_input_fn) output_line = "" with open("label_test.txt", 'w') as writer: for re in result: sentence = re["input_ids"] gold = re["label_ids"] prediction = re["predict"] # output_line = "\n".join(id2label[id] for id in prediction if id!=0) + "\n" for gold_index, gold_item in enumerate(gold): if gold_item >= 34: gold[gold_index] = 0 for gold_index, gold_item in enumerate(prediction): if gold_item >= 34: gold[gold_index] = 0 for w, gold_label, label in zip( tokenizer.convert_ids_to_tokens([ int(s) for s in sentence ]), [id2label[id] for id in gold], [id2label[id] for id in prediction]): if w == "[PAD]": continue #if label=="NEGATIVE": # continue output_line = output_line + w + " " + gold_label + " " + label + "\n" output_line += "\n" writer.write(output_line)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) processor = raw_data_utils.get_processor(FLAGS.task_name) label_list = processor.get_labels() bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file, FLAGS.model_dropout) tf.gfile.MakeDirs(FLAGS.model_dir) flags_dict = tf.app.flags.FLAGS.flag_values_dict() with tf.gfile.Open(os.path.join(FLAGS.model_dir, "FLAGS.json"), "w") as ouf: json.dump(flags_dict, ouf) tf.logging.info("warmup steps {}/{}".format(FLAGS.num_warmup_steps, FLAGS.num_train_steps)) save_checkpoints_steps = FLAGS.num_train_steps // FLAGS.save_checkpoints_num tf.logging.info("setting save checkpoints steps to {:d}".format( save_checkpoints_steps)) FLAGS.iterations_per_loop = min(save_checkpoints_steps, FLAGS.iterations_per_loop) if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) else: tpu_cluster_resolver = None # if not FLAGS.use_tpu and FLAGS.num_gpu > 1: # train_distribute = tf.contrib.distribute.MirroredStrategy( # num_gpus=FLAGS.num_gpu) # else: # train_distribute = None is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.model_dir, save_checkpoints_steps=save_checkpoints_steps, keep_checkpoint_max=1000, # train_distribute=train_distribute, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, per_host_input_for_training=is_per_host)) model_fn = uda.model_fn_builder( bert_config=bert_config, init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, clip_norm=FLAGS.clip_norm, num_train_steps=FLAGS.num_train_steps, num_warmup_steps=FLAGS.num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_one_hot_embeddings, num_labels=len(label_list), unsup_ratio=FLAGS.unsup_ratio, uda_coeff=FLAGS.uda_coeff, tsa=FLAGS.tsa, print_feature=False, print_structure=False, ) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, params={"model_dir": FLAGS.model_dir}, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size) if FLAGS.do_train: tf.logging.info(" >>> sup data dir : {}".format( FLAGS.sup_train_data_dir)) if FLAGS.unsup_ratio > 0: tf.logging.info(" >>> unsup data dir : {}".format( FLAGS.unsup_data_dir)) train_input_fn = proc_data_utils.training_input_fn_builder( FLAGS.sup_train_data_dir, FLAGS.unsup_data_dir, FLAGS.aug_ops, FLAGS.aug_copy, FLAGS.unsup_ratio) if FLAGS.do_eval: tf.logging.info(" >>> dev data dir : {}".format(FLAGS.eval_data_dir)) eval_input_fn = proc_data_utils.evaluation_input_fn_builder( FLAGS.eval_data_dir, "clas") eval_size = processor.get_dev_size() eval_steps = int(eval_size / FLAGS.eval_batch_size) if FLAGS.do_train and FLAGS.do_eval: tf.logging.info("***** Running training & evaluation *****") tf.logging.info(" Supervised batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Unsupervised batch size = %d", FLAGS.train_batch_size * FLAGS.unsup_ratio) tf.logging.info(" Num steps = %d", FLAGS.num_train_steps) tf.logging.info(" Base evaluation batch size = %d", FLAGS.eval_batch_size) tf.logging.info(" Num steps = %d", eval_steps) best_acc = 0 for _ in range(0, FLAGS.num_train_steps, save_checkpoints_steps): tf.logging.info("*** Running training ***") estimator.train(input_fn=train_input_fn, steps=save_checkpoints_steps) tf.logging.info("*** Running evaluation ***") dev_result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) tf.logging.info(">> Results:") for key in dev_result.keys(): tf.logging.info(" %s = %s", key, str(dev_result[key])) dev_result[key] = dev_result[key].item() best_acc = max(best_acc, dev_result["eval_classify_accuracy"]) tf.logging.info("***** Final evaluation result *****") tf.logging.info("Best acc: {:.3f}\n\n".format(best_acc)) elif FLAGS.do_train: tf.logging.info("***** Running training *****") tf.logging.info(" Supervised batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Unsupervised batch size = %d", FLAGS.train_batch_size * FLAGS.unsup_ratio) tf.logging.info(" Num steps = %d", FLAGS.num_train_steps) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.num_train_steps) elif FLAGS.do_eval: tf.logging.info("***** Running evaluation *****") tf.logging.info(" Base evaluation batch size = %d", FLAGS.eval_batch_size) tf.logging.info(" Num steps = %d", eval_steps) checkpoint_state = tf.train.get_checkpoint_state(FLAGS.model_dir) best_acc = 0 for ckpt_path in checkpoint_state.all_model_checkpoint_paths: if not tf.gfile.Exists(ckpt_path + ".data-00000-of-00001"): tf.logging.info( "Warning: checkpoint {:s} does not exist".format( ckpt_path)) continue tf.logging.info("Evaluating {:s}".format(ckpt_path)) dev_result = estimator.evaluate( input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=ckpt_path, ) tf.logging.info(">> Results:") for key in dev_result.keys(): tf.logging.info(" %s = %s", key, str(dev_result[key])) dev_result[key] = dev_result[key].item() best_acc = max(best_acc, dev_result["eval_classify_accuracy"]) tf.logging.info("***** Final evaluation result *****") tf.logging.info("Best acc: {:.3f}\n\n".format(best_acc)) if FLAGS.do_predict: predict_examples = processor.get_test_examples(FLAGS.model_dir) num_actual_predict_examples = len(predict_examples) if FLAGS.use_tpu: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. while len(predict_examples) % FLAGS.predict_batch_size != 0: predict_examples.append(PaddingInputExample()) predict_file = os.path.join(FLAGS.model_dir, "predict.tf_record") file_based_convert_examples_to_features(predict_examples, label_list, FLAGS.max_seq_length, tokenizer, predict_file) tf.logging.info("***** Running prediction*****") tf.logging.info(" Num examples = %d (%d actual, %d padding)", len(predict_examples), num_actual_predict_examples, len(predict_examples) - num_actual_predict_examples) tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_drop_remainder = True if FLAGS.use_tpu else False predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=predict_drop_remainder) result = estimator.predict(input_fn=predict_input_fn) output_predict_file = os.path.join(FLAGS.model_dir, "test_results.tsv") with tf.gfile.GFile(output_predict_file, "w") as writer: num_written_lines = 0 tf.logging.info("***** Predict results *****") for (i, prediction) in enumerate(result): probabilities = prediction["probabilities"] if i >= num_actual_predict_examples: break output_line = "\t".join( str(class_probability) for class_probability in probabilities) + "\n" writer.write(output_line) num_written_lines += 1 assert num_written_lines == num_actual_predict_examples
def main(_): tf.logging.set_verbosity(tf.logging.INFO) processor = raw_data_utils.get_processor(FLAGS.task_name) label_list = processor.get_labels() bert_config = modeling.BertConfig.from_json_file( FLAGS.bert_config_file, FLAGS.model_dropout) tf.gfile.MakeDirs(FLAGS.model_dir) flags_dict = tf.app.flags.FLAGS.flag_values_dict() with tf.gfile.Open(os.path.join(FLAGS.model_dir, "FLAGS.json"), "w") as ouf: json.dump(flags_dict, ouf) tf.logging.info("warmup steps {}/{}".format( FLAGS.num_warmup_steps, FLAGS.num_train_steps)) save_checkpoints_steps = FLAGS.num_train_steps // FLAGS.save_checkpoints_num tf.logging.info("setting save checkpoints steps to {:d}".format( save_checkpoints_steps)) FLAGS.iterations_per_loop = min(save_checkpoints_steps, FLAGS.iterations_per_loop) # pdb.set_trace() if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) else: tpu_cluster_resolver = None # if not FLAGS.use_tpu and FLAGS.num_gpu > 1: # train_distribute = tf.contrib.distribute.MirroredStrategy( # num_gpus=FLAGS.num_gpu) # else: # train_distribute = None is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.model_dir, save_checkpoints_steps=save_checkpoints_steps, keep_checkpoint_max=1000, # train_distribute=train_distribute, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, per_host_input_for_training=is_per_host)) model_fn = uda.model_fn_builder( bert_config=bert_config, init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, clip_norm=FLAGS.clip_norm, num_train_steps=FLAGS.num_train_steps, num_warmup_steps=FLAGS.num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_one_hot_embeddings, num_labels=len(label_list), unsup_ratio=FLAGS.unsup_ratio, uda_coeff=FLAGS.uda_coeff, tsa=FLAGS.tsa, print_feature=False, print_structure=False, ) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, params={"model_dir": FLAGS.model_dir}, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size) if FLAGS.do_train: tf.logging.info(" >>> sup data dir : {}".format(FLAGS.sup_train_data_dir)) if FLAGS.unsup_ratio > 0: tf.logging.info(" >>> unsup data dir : {}".format( FLAGS.unsup_data_dir)) train_input_fn = proc_data_utils.training_input_fn_builder( FLAGS.sup_train_data_dir, FLAGS.unsup_data_dir, FLAGS.aug_ops, FLAGS.aug_copy, FLAGS.unsup_ratio) if FLAGS.do_eval: tf.logging.info(" >>> dev data dir : {}".format(FLAGS.eval_data_dir)) eval_input_fn = proc_data_utils.evaluation_input_fn_builder( FLAGS.eval_data_dir, "clas") eval_size = processor.get_dev_size() eval_steps = int(eval_size / FLAGS.eval_batch_size) if FLAGS.do_train and FLAGS.do_eval: tf.logging.info("***** Running training & evaluation *****") tf.logging.info(" Supervised batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Unsupervised batch size = %d", FLAGS.train_batch_size * FLAGS.unsup_ratio) tf.logging.info(" Num steps = %d", FLAGS.num_train_steps) tf.logging.info(" Base evaluation batch size = %d", FLAGS.eval_batch_size) tf.logging.info(" Num steps = %d", eval_steps) best_acc = 0 for _ in range(0, FLAGS.num_train_steps, save_checkpoints_steps): tf.logging.info("*** Running training ***") pdb.set_trace() save_checkpoints_steps = save_checkpoints_steps/15 estimator.train( input_fn=train_input_fn, steps=save_checkpoints_steps) tf.logging.info("*** Running evaluation ***") dev_result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) tf.logging.info(">> Results:") for key in dev_result.keys(): tf.logging.info(" %s = %s", key, str(dev_result[key])) dev_result[key] = dev_result[key].item() best_acc = max(best_acc, dev_result["eval_classify_accuracy"]) tf.logging.info("***** Final evaluation result *****") tf.logging.info("Best acc: {:.3f}\n\n".format(best_acc)) elif FLAGS.do_train: tf.logging.info("***** Running training *****") tf.logging.info(" Supervised batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Unsupervised batch size = %d", FLAGS.train_batch_size * FLAGS.unsup_ratio) tf.logging.info(" Num steps = %d", FLAGS.num_train_steps) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.num_train_steps) elif FLAGS.do_eval: tf.logging.info("***** Running evaluation *****") tf.logging.info(" Base evaluation batch size = %d", FLAGS.eval_batch_size) tf.logging.info(" Num steps = %d", eval_steps) checkpoint_state = tf.train.get_checkpoint_state(FLAGS.model_dir) best_acc = 0 for ckpt_path in checkpoint_state.all_model_checkpoint_paths: if not tf.gfile.Exists(ckpt_path + ".data-00000-of-00001"): tf.logging.info( "Warning: checkpoint {:s} does not exist".format(ckpt_path)) continue tf.logging.info("Evaluating {:s}".format(ckpt_path)) dev_result = estimator.evaluate( input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=ckpt_path, ) tf.logging.info(">> Results:") for key in dev_result.keys(): tf.logging.info(" %s = %s", key, str(dev_result[key])) dev_result[key] = dev_result[key].item() best_acc = max(best_acc, dev_result["eval_classify_accuracy"]) tf.logging.info("***** Final evaluation result *****") tf.logging.info("Best acc: {:.3f}\n\n".format(best_acc))
def main(_): if FLAGS.max_seq_length > 512: raise ValueError( "Cannot use sequence length {:d} because the BERT model " "was only trained up to sequence length {:d}".format( FLAGS.max_seq_length, 512)) processor = raw_data_utils.get_processor(FLAGS.task_name) if FLAGS.xlnet == False: # Create tokenizer tokenizer = tokenization.FullTokenizer( vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) else: sp = spm.SentencePieceProcessor() sp.Load(FLAGS.spiece_model_file) def tokenize_fn(text): text = preprocess_text(text, lower=False) return encode_ids(sp, text) if FLAGS.data_type == "sup": sup_out_dir = FLAGS.output_base_dir logging.info("Create sup. data: subset {} => {}".format( FLAGS.sub_set, sup_out_dir)) if FLAGS.xlnet == True: proc_and_save_sup_data_xlnet( processor, FLAGS.sub_set, FLAGS.raw_data_dir, sup_out_dir, tokenize_fn, FLAGS.max_seq_length, FLAGS.trunc_keep_right, FLAGS.worker_id, FLAGS.replicas, FLAGS.sup_size, ) else: proc_and_save_sup_data( processor, FLAGS.sub_set, FLAGS.raw_data_dir, sup_out_dir, tokenizer, FLAGS.max_seq_length, FLAGS.trunc_keep_right, FLAGS.worker_id, FLAGS.replicas, FLAGS.sup_size, ) elif FLAGS.data_type == "unsup": assert FLAGS.aug_ops is not None, \ "aug_ops is required to preprocess unsupervised data." unsup_out_dir = os.path.join( FLAGS.output_base_dir, FLAGS.aug_ops, str(FLAGS.aug_copy_num)) data_stats_dir = os.path.join(FLAGS.raw_data_dir, "data_stats") logging.info("Create unsup. data: subset {} => {}".format( FLAGS.sub_set, unsup_out_dir)) if FLAGS.xlnet == True: proc_and_save_unsup_data_xlnet( processor, FLAGS.sub_set, FLAGS.raw_data_dir, data_stats_dir, unsup_out_dir, tokenize_fn, FLAGS.max_seq_length, FLAGS.trunc_keep_right, FLAGS.aug_ops, FLAGS.aug_copy_num, FLAGS.worker_id, FLAGS.replicas) else: proc_and_save_unsup_data( processor, FLAGS.sub_set, FLAGS.raw_data_dir, data_stats_dir, unsup_out_dir, tokenizer, FLAGS.max_seq_length, FLAGS.trunc_keep_right, FLAGS.aug_ops, FLAGS.aug_copy_num, FLAGS.worker_id, FLAGS.replicas)