def model_fn(features, labels, mode, params): training = mode == tf.estimator.ModeKeys.TRAIN spec = tfhub.create_module_spec(module_fn, tags_and_args=[ ({'train'}, { 'training': True, 'params': params }), (set(), { 'training': False, 'params': params }), ]) tags = {'train'} if training else None module = tfhub.Module(spec, trainable=training, tags=tags) tfhub.register_module_for_export(module, 'doodle') image = features['image'] image.shape.assert_is_compatible_with([None, 28, 28, 1]) predictions = module(image, as_dict=True) if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec( mode=mode, predictions=predictions, export_outputs={ tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(predictions), }) with tf.variable_scope('losses'): cross_entropy_loss = tf.losses.sparse_softmax_cross_entropy( labels=labels, logits=predictions['logits']) total_loss = tf.losses.get_total_loss() tf.summary.image('image', image) tf.summary.scalar('total_loss', total_loss) metric_ops = metrics.calculate(labels, predictions['classes'], params['num_classes']) if mode == tf.estimator.ModeKeys.EVAL: return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, eval_metric_ops=metric_ops) if mode == tf.estimator.ModeKeys.TRAIN: global_step = tf.train.get_or_create_global_step() update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.variable_scope('optimizer'): optimizer = tf.train.AdamOptimizer(params['learning_rate']) with tf.control_dependencies(update_ops): fit = optimizer.minimize(total_loss, global_step) return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, train_op=fit, eval_metric_ops=metric_ops)
def main(_): logging.set_verbosity(logging.INFO) processors = {"ner": NerProcessor} if not FLAGS.do_train and not FLAGS.do_eval: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels() tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = processor.get_train_examples(FLAGS.data_dir) num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) model_fn = model_fn_builder(bert_config=bert_config, num_labels=len(label_list), init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu) estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: train_file = os.path.join(FLAGS.output_dir, "train.tf_record") _, _ = filed_based_convert_examples_to_features( train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) logging.info("***** Running training *****") logging.info(" Num examples = %d", len(train_examples)) logging.info(" Batch size = %d", FLAGS.train_batch_size) logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.data_dir) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") batch_tokens, batch_labels = filed_based_convert_examples_to_features( eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) logging.info("***** Running evaluation *****") logging.info(" Num examples = %d", len(eval_examples)) logging.info(" Batch size = %d", FLAGS.eval_batch_size) # if FLAGS.use_tpu: # eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size) # eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.evaluate(input_fn=eval_input_fn) output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") with open(output_eval_file, "w") as wf: logging.info("***** Eval results *****") confusion_matrix = result["confusion_matrix"] p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) logging.info("***********************************************") logging.info("********************P = %s*********************", str(p)) logging.info("********************R = %s*********************", str(r)) logging.info("********************F = %s*********************", str(f)) logging.info("***********************************************") if FLAGS.do_predict: with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as rf: label2id = pickle.load(rf) id2label = {value: key for key, value in label2id.items()} predict_examples = processor.get_test_examples(FLAGS.data_dir) predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") batch_tokens, batch_labels = filed_based_convert_examples_to_features( predict_examples, label_list, FLAGS.max_seq_length, tokenizer, predict_file) logging.info("***** Running prediction*****") logging.info(" Num examples = %d", len(predict_examples)) logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.predict(input_fn=predict_input_fn) output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt") #here if the tag is "X" means it belong to its before token, here for convenient evaluate use # conlleval.pl we discarding it directly Writer(output_predict_file, result, batch_tokens, batch_labels, id2label)
def main(_): if not os.path.exists(FLAGS.output_dir): os.makedirs(FLAGS.output_dir) logging.set_verbosity(logging.INFO) processors = {"ner": NerProcessor} if not FLAGS.do_train and not FLAGS.do_eval: raise ValueError("At least one of `do_train` or `do_eval` must be True.") bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels() tokenizer = tokenization.FullTokenizer( vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 config = tf.ConfigProto() if FLAGS.do_train: os.environ["CUDA_VISIBLE_DEVICES"] = "0" config.gpu_options.allow_growth = True config.allow_soft_placement = True config.log_device_placement = False config.gpu_options.per_process_gpu_memory_fraction = 0.9 else: config.gpu_options.allow_growth = False config.gpu_options.per_process_gpu_memory_fraction = 0.1 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, session_config=config, keep_checkpoint_max=3, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = processor.get_train_examples(FLAGS.data_dir) num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs * FLAGS.training_run_count) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) model_fn = model_fn_builder( bert_config=bert_config, num_labels=len(label_list), init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu) estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size, eval_batch_size=FLAGS.eval_batch_size ) if FLAGS.do_train: train_file = os.path.join(FLAGS.output_dir, "train.tf_record") _, _, _, _ = filed_based_convert_examples_to_features( train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) logging.info("***** Running training *****") logging.info(" Num examples = %d", len(train_examples)) logging.info(" Batch size = %d", FLAGS.train_batch_size) logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_eval: # # run evaluation on training set # # logging.info("***** Running evaluation on training *****") # logging.info(" Num examples = %d", len(train_examples)) # logging.info(" Batch size = %d", FLAGS.train_batch_size) # # if FLAGS.use_tpu: # # eval_steps = int(len(eval_examples) / FLAGS.) # # eval_drop_remainder = True if FLAGS.use_tpu else False # # training_result = estimator.evaluate(input_fn=train_input_fn) # output_training_file = os.path.join(FLAGS.output_dir, "training_result.txt") # with open(output_training_file, "w") as wf: # logging.info("***** Training Eval results *****") # confusion_matrix = training_result["confusion_matrix"] # p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) # logging.info("***********************************************") # logging.info("********************P = %s*********************", str(p)) # logging.info("********************R = %s*********************", str(r)) # logging.info("********************F = %s*********************", str(f)) # logging.info("***********************************************") # wf.write("***********************************************") # wf.write("********************P =" + str(p) + "*********************\n") # wf.write("********************R =" + str(r) + "*********************\n") # wf.write("********************F =" + str(f) + "*********************\n") # run evaluation on dev set eval_examples = processor.get_dev_examples(FLAGS.data_dir) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") batch_tokens, batch_labels, batch_start_positions, batch_document_ids = filed_based_convert_examples_to_features( eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) logging.info("***** Running evaluation on development *****") logging.info(" Num examples = %d", len(eval_examples)) logging.info(" Batch size = %d", FLAGS.eval_batch_size) # if FLAGS.use_tpu: # eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size) # eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.evaluate(input_fn=eval_input_fn) output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") with open(output_eval_file, "w") as wf: logging.info("***** Eval results *****") confusion_matrix = result["confusion_matrix"] p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) logging.info("***********************************************") logging.info("********************P = %s*********************", str(p)) logging.info("********************R = %s*********************", str(r)) logging.info("********************F = %s*********************", str(f)) logging.info("***********************************************") wf.write("***********************************************") wf.write("********************P =" + str(p) + "*********************") wf.write("********************R =" + str(r) + "*********************") wf.write("********************F =" + str(f) + "*********************") if FLAGS.do_predict: with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as rf: label2id = pickle.load(rf) id2label = {value: key for key, value in label2id.items()} predict_examples = processor.get_test_examples(FLAGS.data_dir) predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") batch_tokens, batch_labels, batch_start_positions, batch_document_ids = filed_based_convert_examples_to_features( predict_examples, label_list, FLAGS.max_seq_length, tokenizer, predict_file) logging.info("***** Running prediction*****") logging.info(" Num examples = %d", len(predict_examples)) logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.predict(input_fn=predict_input_fn) output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt") # here if the tag is "X" means it belong to its before token, here for convenient evaluate use # conlleval.pl we discarding it directly Writer(output_predict_file, result, batch_tokens, batch_labels, batch_start_positions, batch_document_ids, id2label) # PREDICT FOR TRAIN DATA train_examples = processor.get_train_examples(FLAGS.data_dir) train_predict_file = os.path.join(FLAGS.output_dir, "train.tf_record") batch_tokens, batch_labels, batch_start_positions, batch_document_ids = filed_based_convert_examples_to_features( train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_predict_file) logging.info("***** Running prediction for training data*****") logging.info(" Num examples = %d", len(predict_examples)) logging.info(" Batch size = %d", FLAGS.predict_batch_size) train_predict_input_fn = file_based_input_fn_builder( input_file=train_predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.predict(input_fn=train_predict_input_fn) output_predict_file = os.path.join(FLAGS.output_dir, "label_train.txt") # here if the tag is "X" means it belong to its before token, here for convenient evaluate use # conlleval.pl we discarding it directly Writer(output_predict_file, result, batch_tokens, batch_labels, batch_start_positions, batch_document_ids, id2label) # SAVE MODEL FOR SERVING feature_columns_keys = ['input_ids', 'mask', 'segment_ids', 'label_ids'] feature_columns = [sequence_numeric_column(key=key) for key in feature_columns_keys] # feature_columns["input_ids"] = sequence_numeric_column('input_ids') # feature_columns["mask"] = sequence_numeric_column('mask') # feature_columns["segment_ids"] = sequence_numeric_column('segment_ids') # feature_columns["label_ids"] = sequence_numeric_column('label_ids') def serving_input_fn(): label_ids = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='label_ids') input_ids = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='input_ids') input_mask = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='mask') segment_ids = tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='segment_ids') input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({ 'label_ids': label_ids, 'input_ids': input_ids, 'mask': input_mask, 'segment_ids': segment_ids, })() return input_fn def serving_input_fn_prev(): with tf.variable_scope("foo"): feature_spec = { "input_ids": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64), "mask": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64), "segment_ids": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64), "label_ids": tf.FixedLenFeature([FLAGS.max_seq_length], tf.int64), } serialized_tf_example = tf.placeholder(dtype=tf.string, shape=[None], name='input_example_tensor') receiver_tensors = {'examples': serialized_tf_example} features = tf.parse_example(serialized_tf_example, feature_spec) return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) # feature_spec = tf.feature_column.make_parse_example_spec(feature_columns) # serving_input_receiver_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec) estimator._export_to_tpu = False # this is important export_dir = estimator.export_savedmodel(FLAGS.output_dir, serving_input_fn) print('Exported to {}'.format(export_dir))
def main(_): tf.logging.set_verbosity(tf.logging.INFO) processors = { "intent_slot": IntentSlotsProcessor, } tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, FLAGS.init_checkpoint) if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError( "At least one of `do_train`, `do_eval` or `do_predict' must be True.") bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) tf.gfile.MakeDirs(FLAGS.output_dir) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels(FLAGS.data_dir) # print("******************label_list******************") # print(label_list) tokenizer = tokenization.FullTokenizer( vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = processor.get_train_examples(FLAGS.data_dir) num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) model_fn = model_fn_builder( bert_config=bert_config, num_intent_labels=len(label_list[0]), num_slot_labels=len(label_list[1]), init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: train_file = os.path.join(FLAGS.output_dir, "train.tf_record") file_based_convert_examples_to_features( train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) tf.logging.info("***** Running training *****") tf.logging.info(" Num examples = %d", len(train_examples)) tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.data_dir) num_actual_eval_examples = len(eval_examples) if FLAGS.use_tpu: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. These do NOT count towards the metric (all tf.metrics # support a per-instance weight, and these get a weight of 0.0). while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") file_based_convert_examples_to_features( eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) tf.logging.info("***** Running evaluation *****") tf.logging.info(" Num examples = %d (%d actual, %d padding)", len(eval_examples), num_actual_eval_examples, len(eval_examples) - num_actual_eval_examples) tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) # This tells the estimator to run through the entire set. eval_steps = None # However, if running eval on the TPU, you will need to specify the # number of steps. if FLAGS.use_tpu: assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=eval_drop_remainder) result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) #TODO output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") with tf.gfile.GFile(output_eval_file, "w") as writer: tf.logging.info("***** Eval results *****") intent_accuracy = result["intent_eval_accuracy"] intent_loss = result["intent_eval_loss"] slot_confusion_matrix = result["slot_cm"] p,r,f = metrics.calculate(slot_confusion_matrix,len(label_list)) logging.info("***********************************************") logging.info("********************slot_P = %s*********************", str(p)) logging.info("********************slot_R = %s*********************", str(r)) logging.info("********************slot_F = %s*********************", str(f)) logging.info("******************intent_acc = %s*******************", str(intent_accuracy)) logging.info("******************intent_loss = %s******************", str(intent_loss)) logging.info("***********************************************") if FLAGS.do_predict: predict_examples = processor.get_test_examples(FLAGS.data_dir) num_actual_predict_examples = len(predict_examples) if FLAGS.use_tpu: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. while len(predict_examples) % FLAGS.predict_batch_size != 0: predict_examples.append(PaddingInputExample()) predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") file_based_convert_examples_to_features(predict_examples, label_list, FLAGS.max_seq_length, tokenizer, predict_file) tf.logging.info("***** Running prediction*****") tf.logging.info(" Num examples = %d (%d actual, %d padding)", len(predict_examples), num_actual_predict_examples, len(predict_examples) - num_actual_predict_examples) tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_drop_remainder = True if FLAGS.use_tpu else False predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=predict_drop_remainder) result = estimator.predict(input_fn=predict_input_fn) output_predict_file = os.path.join( FLAGS.output_dir, "test_results.tsv") with tf.gfile.GFile(output_predict_file, "w") as writer: num_written_lines = 0 tf.logging.info("***** Predict results *****") for (i, prediction) in enumerate(result): probabilities = prediction["probabilities"] if i >= num_actual_predict_examples: break output_line = "\t".join( str(class_probability) for class_probability in probabilities) + "\n" writer.write(output_line) num_written_lines += 1 assert num_written_lines == num_actual_predict_examples
def model_fn(features, labels, mode, params): """ モデルを定義します """ #========================================================= # ハイパーパラメータを取得します #========================================================= num_classes = params.get('num_classes', 10) batch_size = params.get('batch_size', 96) learning_rate = params.get('learning_rate', 1e-4) init_stddev = params.get('initializer_normal_stddev', 0.09) dropout_rate = params.get('dropout_rate', 0.4) is_training = mode == tf.estimator.ModeKeys.TRAIN # 第一引数のfeaturesが入力データです image = features['image'] assert image.get_shape().as_list() == [None, 28, 28, 1] # 変数の初期化関数です initializer = tf.truncated_normal_initializer(stddev=init_stddev) #========================================================= # ニューラルネットワークを定義します #========================================================= with tf.variable_scope('model', initializer=initializer): x = image x = tf.layers.conv2d(x, 32, 5, padding='SAME', activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2, padding='SAME') x = tf.layers.conv2d(x, 64, 5, padding='SAME', activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2, padding='SAME') #x = tf.layers.flatten(x) #XXX: `tf.layers.flatten` contains unsupported op `StridedSlice` by TensorFlow.js. Use `tf.reshape` as bellow. x = tf.reshape(x, [-1, 7*7*64]) x = tf.layers.dense(x, 1024, activation=tf.nn.relu) x = tf.layers.dropout(x, rate=dropout_rate, training=is_training) x = tf.layers.dense(x, 10) logits = x assert logits.get_shape().as_list() == [None, num_classes] # 予測結果: クラスごとの離散確率分布、最も確率の高いクラスのインデクス predictions = { 'probabilities': tf.nn.softmax(logits, name='probabilities'), 'classes' : tf.argmax(logits, axis=1, name='classes'), } #========================================================= # 推論モードならモデルの出力を返します #========================================================= # `mode`は実行モードです。モードは以下の3つがあります。 # - tf.estimator.ModeKeys.PREDICT : 推論モードです # - tf.estimator.ModeKeys.TRAIN : 学習モードです # - tf.estimator.ModeKeys.EVAL : 評価モードです # 推論モードの場合は、学習を実行する必要はないため、結果を返して終わります。 if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, export_outputs={ tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(predictions), }) #========================================================= # モデルの誤差を定義します #========================================================= with tf.variable_scope('losses'): # クロスエントロピーを計算して誤差に追加します cross_entropy_loss = tf.losses.sparse_softmax_cross_entropy( labels=labels, logits=logits) # モデルで追加された全ての誤差の総和を取得します total_loss = tf.losses.get_total_loss() #========================================================= # 正答率など、モデルの評価値を計算します #========================================================= metric_ops = metrics.calculate(labels, predictions['classes'], num_classes) #========================================================= # モデルを学習(=パラメータを最適化)します #========================================================= global_step = tf.train.get_or_create_global_step() update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.variable_scope('optimizer'), tf.control_dependencies(update_ops): # total_loss(誤差の総和)が小さくなるように変数を更新します optimizer = tf.train.AdamOptimizer(learning_rate) fit = optimizer.minimize(total_loss, global_step) #========================================================= # 変数をサマリにまとめます #========================================================= # 任意の値はサマリに追加することでログとして保存できます。 # ログはTensorBoardなどでグラフ化することができるため、 # 計算した誤差やメトリクス、入力画像などをログにしておきます。 tf.summary.image('image', image) tf.summary.scalar('total_loss', total_loss) summary_op = tf.summary.merge_all() #========================================================= # SageMakerの場合は、これでモデルの定義は完了です! #========================================================= if params.get('sagemaker_job_name', None) is not None: return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, train_op=fit, eval_metric_ops=metric_ops) #--------------------------------------------------------- # 以下はローカル実行テスト用です。 training_hooks = [ tf.train.SummarySaverHook( save_steps=1, output_dir='./test/logs/doodle.train', summary_op=summary_op) ] evaluation_hooks = [ tf.train.SummarySaverHook( save_steps=5, output_dir='./test/logs/doodle.eval', summary_op=summary_op) ] return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, train_op=fit, eval_metric_ops=metric_ops, evaluation_hooks=evaluation_hooks, training_hooks=training_hooks)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) processors = { # "cola": ColaProcessor, # "mnli": MnliProcessor, # "mrpc": MrpcProcessor, # "xnli": XnliProcessor, "ee": EEProcessor } tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, FLAGS.init_checkpoint) if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError( "At least one of `do_train`, `do_eval` or `do_predict' must be True." ) bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) tf.gfile.MakeDirs(FLAGS.output_dir) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() # get labels list label_list = processor.get_labels() tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = processor.get_train_examples(FLAGS.data_dir) # 训练多少步,每一步训练数据是一个batch num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) model_fn = model_fn_builder(bert_config=bert_config, num_labels=len(label_list), init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: train_file = os.path.join(FLAGS.output_dir, "train.tf_record") file_based_convert_examples_to_features(train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) tf.logging.info("***** Running training *****") tf.logging.info(" Num examples = %d", len(train_examples)) tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.data_dir) num_actual_eval_examples = len(eval_examples) if FLAGS.use_tpu: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. These do NOT count towards the metric (all tf.metrics # support a per-instance weight, and these get a weight of 0.0). while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) tf.logging.info("***** Running evaluation *****") tf.logging.info(" Num examples = %d (%d actual, %d padding)", len(eval_examples), num_actual_eval_examples, len(eval_examples) - num_actual_eval_examples) tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) # This tells the estimator to run through the entire set. eval_steps = None # However, if running eval on the TPU, you will need to specify the # number of steps. if FLAGS.use_tpu: assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=eval_drop_remainder) result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") with open(output_eval_file, 'w') as f: tf.logging.info("***** Eval results *****") confusion_matrix = result["confusion_matrix"] p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) tf.logging.info("***************************************") tf.logging.info("**************P = %s*************************", str(p)) tf.logging.info("**************R = %s*************************", str(r)) tf.logging.info("**************F = %s*************************", str(f)) # with tf.gfile.GFile(output_eval_file, "w") as writer: # tf.logging.info("***** Eval results *****") # for key in sorted(result.keys()): # tf.logging.info(" %s = %s", key, str(result[key])) # writer.write("%s = %s\n" % (key, str(result[key]))) if FLAGS.do_predict: with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as f: label2id = pickle.load(f) id2label = {v: k for k, v in label2id.items()} predict_examples = processor.get_test_examples(FLAGS.data_dir) num_actual_predict_examples = len(predict_examples) if FLAGS.use_tpu: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. while len(predict_examples) % FLAGS.predict_batch_size != 0: predict_examples.append(PaddingInputExample()) predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") batch_tokens, batch_labels = file_based_convert_examples_to_features( predict_examples, label_list, FLAGS.max_seq_length, tokenizer, predict_file) tf.logging.info("***** Running prediction*****") tf.logging.info(" Num examples = %d (%d actual, %d padding)", len(predict_examples), num_actual_predict_examples, len(predict_examples) - num_actual_predict_examples) tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_drop_remainder = True if FLAGS.use_tpu else False predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=predict_drop_remainder) result = estimator.predict(input_fn=predict_input_fn) output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt") print_predictions(output_predict_file, result, batch_tokens, batch_labels, id2label)
def main(_): logging.set_verbosity(logging.INFO) processors = {"ner": NerProcessor} if not FLAGS.do_train and not FLAGS.do_eval: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels() tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = processor.get_train_examples(FLAGS.data_dir) num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) model_fn = model_fn_builder(bert_config=bert_config, num_labels=len(label_list), init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu) estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: train_file = join(FLAGS.output_dir, "train.tf_record") _, _ = filed_based_convert_examples_to_features( train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) logging.info("***** Running training *****") logging.info(" Num examples = %d", len(train_examples)) logging.info(" Batch size = %d", FLAGS.train_batch_size) logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.data_dir) eval_file = join(FLAGS.output_dir, "eval.tf_record") batch_tokens, batch_labels = filed_based_convert_examples_to_features( eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) logging.info("***** Running evaluation *****") logging.info(" Num examples = %d", len(eval_examples)) logging.info(" Batch size = %d", FLAGS.eval_batch_size) # if FLAGS.use_tpu: # eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size) # eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.evaluate(input_fn=eval_input_fn) logging.info("\n") print(">>>>>>>>>>>>>>> evaluation", "result type {}".format(type(result))) print(result) print(">>>>>>>>>>>>>>> examples: {}\n".format(len(eval_examples))) # output_eval_file = join(FLAGS.output_dir, "eval_results.txt") # with open(output_eval_file,"w") as wf: logging.info("***** Eval results *****") confusion_matrix = result["confusion_matrix"] (rows, cols) = confusion_matrix.shape sum_num, true_num = 0, 0 for i in range(rows): for j in range(cols): sum_num += confusion_matrix[i][j] if i == j: true_num += confusion_matrix[i][j] print(">>>>>> type of confusion matrix", str(type(confusion_matrix))) print(">>>>>> ", str(confusion_matrix.shape)) p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) logging.info("***********************************************") logging.info("*******************Acc = %s********************", str(true_num / sum_num)) logging.info("********************P = %s*********************", str(p)) logging.info("********************R = %s*********************", str(r)) logging.info("********************F = %s*********************", str(f)) logging.info("***********************************************") if FLAGS.do_predict: # with open(FLAGS.middle_output+'/label2id.pkl', 'rb') as rf: # label2id = pickle.load(rf) # id2label = {value: key for key, value in label2id.items()} predict_examples = processor.get_test_examples(FLAGS.data_dir) predict_file = join(FLAGS.output_dir, "predict.tf_record") batch_tokens, batch_labels = filed_based_convert_examples_to_features( predict_examples, label_list, FLAGS.max_seq_length, tokenizer, predict_file) # output_predict_file = join(FLAGS.output_dir, "label_test.txt") # here if the tag is "X" means it belong to its before token, here for convenient evaluate use # conlleval.pl we discarding it directly # Writer(output_predict_file,result,batch_tokens,batch_labels,id2label) logging.info("***** Running prediction*****") logging.info(" Num examples = %d", len(predict_examples)) logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_input_fn = file_based_input_fn_builder( input_file=predict_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.evaluate(input_fn=predict_input_fn) print("\n>>>>>>>>>>>>>>> prediction", "result type {}".format(type(result))) print(result) print(">>>>>>>>>>>>>>> examples: {}\n".format(len(predict_examples))) # added by jerry logging.info("***** Pred results *****") confusion_matrix = result["confusion_matrix"] sum_num, true_num = 0, 0 for i in range(rows): for j in range(cols): sum_num += confusion_matrix[i][j] if i == j: true_num += confusion_matrix[i][j] print(">>>>>> type of confusion matrix", str(type(confusion_matrix))) print(">>>>>> ", str(confusion_matrix.shape)) p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) logging.info("***********************************************") logging.info("*******************Acc = %s********************", str(true_num / sum_num)) logging.info("********************P = %s*********************", str(p)) logging.info("********************R = %s*********************", str(r)) logging.info("********************F = %s*********************", str(f)) logging.info("***********************************************") if FLAGS.do_gt: print("\n************ Evaluation on Groundtruth ************") # generate groundtruth # data format: (word, label)/per line # gt_path = 'data_new/groundtruth' # ano_gt_path = 'data_new' # words_list = [] # labels_list = [] # for f in listdir(gt_path): # if isfile(join(gt_path, f)) is True: # print('*************', f, '*************') # with open(join(gt_path, f)) as fin: # lines = fin.readlines() # for idx, line in enumerate(lines): # if idx % 3 == 1: # words = line.strip().split(' ') # if len(words) > 0: # words_list.append(words) # elif idx % 3 == 2: # labels = line.strip().split(' ') # if len(labels) > 0: # labels_list.append(labels) # print('******* operation on', f, 'finished ******') # with open(join(ano_gt_path, 'groundtruth.txt'), 'w') as fout: # for (words, labels) in zip(words_list, labels_list): # for (word, label) in zip(words, labels): # print(word, label) # fout.write(word + ' ' + label + '\n') # fout.write('\n') gt_examples = processor.get_gt_examples(FLAGS.data_dir) gt_file = join(FLAGS.output_dir, "gt.tf_record") filed_based_convert_examples_to_features(gt_examples, label_list, FLAGS.max_seq_length, tokenizer, gt_file) gt_input_fn = file_based_input_fn_builder( input_file=gt_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) result = estimator.evaluate(input_fn=gt_input_fn) print("\n>>>>>>>>>>>>>>> groundtruth:", "result type {}".format(type(result))) print(result) print(">>>>>>>>>>>>>>> examples: {}\n".format(len(gt_examples))) # added by jerry logging.info("***** Groundtruth results *****") confusion_matrix = result["confusion_matrix"] sum_num, true_num = 0, 0 for i in range(rows): for j in range(cols): sum_num += confusion_matrix[i][j] if i == j: true_num += confusion_matrix[i][j] print(">>>>>> type of confusion matrix", str(type(confusion_matrix))) print(">>>>>> ", str(confusion_matrix.shape)) p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) logging.info("***********************************************") logging.info("*******************Acc = %s********************", str(true_num / sum_num)) logging.info("********************P = %s*********************", str(p)) logging.info("********************R = %s*********************", str(r)) logging.info("********************F = %s*********************", str(f)) logging.info("***********************************************")
def main(_): tf.logging.set_verbosity(tf.logging.INFO) #### Validate flags if FLAGS.save_steps is not None: FLAGS.iterations = min(FLAGS.iterations, FLAGS.save_steps) if FLAGS.do_predict: predict_dir = FLAGS.predict_dir if not tf.gfile.Exists(predict_dir): tf.gfile.MakeDirs(predict_dir) processors = { "ner": NerProcessor, } if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError( "At least one of `do_train`, `do_eval, `do_predict` or " "`do_submit` must be True.") if not tf.gfile.Exists(FLAGS.output_dir): tf.gfile.MakeDirs(FLAGS.output_dir) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels() if not FLAGS.is_regression else None print('label_list') print(label_list) sp = spm.SentencePieceProcessor() sp.Load(FLAGS.spiece_model_file) def tokenize_fn(text): text = preprocess_text(text, lower=FLAGS.uncased) if sp.PieceToId(text) == 0: return 99999 return sp.PieceToId(text) run_config = model_utils.configure_tpu(FLAGS) model_fn = get_model_fn(len(label_list)) spm_basename = os.path.basename(FLAGS.spiece_model_file) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. #if FLAGS.use_tpu: estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size, eval_batch_size=FLAGS.eval_batch_size) #else: # estimator = tf.estimator.Estimator( # model_fn=model_fn, # config=run_config) if FLAGS.do_train: train_file_base = "{}.len-{}.train.tf_record".format( spm_basename, FLAGS.max_seq_length) train_file = os.path.join(FLAGS.output_dir, train_file_base) tf.logging.info("Use tfrecord file {}".format(train_file)) # print('get train examples') train_examples = processor.get_train_examples(FLAGS.data_dir) np.random.shuffle(train_examples) tf.logging.info("Num of train samples: {}".format(len(train_examples))) file_based_convert_examples_to_features( train_examples, label_list, FLAGS.max_seq_length, tokenize_fn, train_file, FLAGS.num_passes) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True, batch_size=FLAGS.train_batch_size) estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps) # if FLAGS.do_eval or FLAGS.do_predict: # if FLAGS.eval_split == "dev": # eval_examples = processor.get_dev_examples(FLAGS.data_dir) # else: # eval_examples = processor.get_test_examples(FLAGS.data_dir) # # tf.logging.info("Num of eval samples: {}".format(len(eval_examples))) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.data_dir) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") file_based_convert_examples_to_features( eval_examples, label_list, FLAGS.max_seq_length, tokenize_fn, eval_file) logging.info("***** Running evaluation *****") logging.info(" Num examples = %d", len(eval_examples)) logging.info(" Batch size = %d", FLAGS.eval_batch_size) # if FLAGS.use_tpu: # eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size) # eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=True, batch_size=FLAGS.train_batch_size) result = estimator.evaluate(input_fn=eval_input_fn) output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") with open(output_eval_file, "w") as wf: logging.info("***** Eval results *****") confusion_matrix = result["confusion_matrix"] p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1) logging.info("***********************************************") logging.info("********************P = %s*********************", str(p)) logging.info("********************R = %s*********************", str(r)) logging.info("********************F = %s*********************", str(f)) logging.info("***********************************************") if FLAGS.do_predict: # eval_file_base = "{}.len-{}.{}.predict.tf_record".format( # spm_basename, FLAGS.max_seq_length, FLAGS.eval_split) # eval_file = os.path.join(FLAGS.output_dir, eval_file_base) test_file_base = "{}.len-{}.predict.tf_record".format( spm_basename, FLAGS.max_seq_length) test_file = os.path.join(FLAGS.output_dir, test_file_base) test_examples = processor.get_test_examples(FLAGS.data_dir) file_based_convert_examples_to_features( test_examples, label_list, FLAGS.max_seq_length, tokenize_fn, test_file) pred_input_fn = file_based_input_fn_builder( input_file=test_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False, batch_size=FLAGS.train_batch_size) predict_results = [] result = estimator.predict(input_fn=pred_input_fn) output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt") # print(result) print(list(result))