def main(_): tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) processors = { "cola": ColaProcessor, "mnli": MnliProcessor, "mrpc": MrpcProcessor, "xnli": XnliProcessor, } tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, FLAGS.init_checkpoint) bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) tf.compat.v1.gfile.MakeDirs(FLAGS.output_dir) task_name = FLAGS.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() label_list = processor.get_labels() tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.compat.v1.estimator.tpu.InputPipelineConfig.PER_HOST_V2 session_config = tf.compat.v1.ConfigProto( inter_op_parallelism_threads=FLAGS.num_inter_threads, intra_op_parallelism_threads=FLAGS.num_intra_threads) run_config = tf.compat.v1.estimator.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=tf.compat.v1.estimator.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host), session_config=session_config) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = processor.get_train_examples(FLAGS.data_dir) num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) model_fn = model_fn_builder( bert_config=bert_config, num_labels=len(label_list), init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu, ) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.compat.v1.estimator.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, eval_batch_size=FLAGS.eval_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: train_file = os.path.join(FLAGS.output_dir, "train.tf_record") file_based_convert_examples_to_features(train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) tf.compat.v1.logging.info("***** Running training *****") tf.compat.v1.logging.info(" Num examples = %d", len(train_examples)) tf.compat.v1.logging.info(" Batch size = %d", FLAGS.train_batch_size) tf.compat.v1.logging.info(" Num steps = %d", num_train_steps) train_input_fn = file_based_input_fn_builder( input_file=train_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_eval: eval_examples = processor.get_dev_examples(FLAGS.data_dir) num_actual_eval_examples = len(eval_examples) if FLAGS.use_tpu: # TPU requires a fixed batch size for all batches, therefore the number # of examples must be a multiple of the batch size, or else examples # will get dropped. So we pad with fake examples which are ignored # later on. These do NOT count towards the metric (all tf.metrics # support a per-instance weight, and these get a weight of 0.0). while len(eval_examples) % FLAGS.eval_batch_size != 0: eval_examples.append(PaddingInputExample()) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") file_based_convert_examples_to_features(eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) tf.compat.v1.logging.info("***** Running evaluation *****") tf.compat.v1.logging.info( " Num examples = %d (%d actual, %d padding)", len(eval_examples), num_actual_eval_examples, len(eval_examples) - num_actual_eval_examples) tf.compat.v1.logging.info(" Batch size = %d", FLAGS.eval_batch_size) # This tells the estimator to run through the entire set. eval_steps = None # However, if running eval on the TPU, you will need to specify the # number of steps. if FLAGS.use_tpu: assert len(eval_examples) % FLAGS.eval_batch_size == 0 eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) + 1 eval_drop_remainder = True if FLAGS.use_tpu else False eval_input_fn = file_based_input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=eval_drop_remainder) start = time.time() result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps, hooks=[LoggerHook()]) end = time.time() - start result['global_step'] = str(eval_steps) result['latency_total'] = str(end) result['latency_per_step'] = str(end / eval_steps) if FLAGS.eval_batch_size != 1: result['samples_per_sec'] = str(FLAGS.eval_batch_size / (end / eval_steps)) output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") with tf.compat.v1.gfile.GFile(output_eval_file, "w") as writer: tf.compat.v1.logging.info("***** Eval results *****") for key in sorted(result.keys()): tf.compat.v1.logging.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) # BELOW IS LPOT TUNING AND BENCHMARK CODE class Dataset(object): def __init__(self, file_name, batch_size): self.file_name = file_name self.batch_size = batch_size def __getitem__(self, idx): return (self.file_name, self.batch_size), 0 def __len__(self): return 1 def collate_fn(batch): """Puts each data field into a pd frame with outer dimension batch size""" elem = batch[0] return elem from lpot.metric import METRICS class Accuracy(object): def __init__(self): self.metric = METRICS('tensorflow')['Accuracy']() # it's ugly that the label is in the iterator def update(self, preds, label): logits, labels = preds self.metric.update(logits, labels) def reset(self): self.metric.reset() def result(self): return self.metric.result() if FLAGS.tune: eval_examples = processor.get_dev_examples(FLAGS.data_dir) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") convert_examples_to_features(examples=eval_examples, label_list=label_list, max_seq_length=FLAGS.max_seq_length, tokenizer=tokenizer, output_file=eval_file) estimator_input_fn = input_fn_builder(input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) from lpot.experimental import Quantization, common quantizer = Quantization(FLAGS.config) dataset = Dataset(eval_file, FLAGS.eval_batch_size) quantizer.model = common.Model(estimator, input_fn=estimator_input_fn) quantizer.calib_dataloader = common.DataLoader(dataset, collate_fn=collate_fn) quantizer.eval_dataloader = common.DataLoader(dataset, collate_fn=collate_fn) quantizer.metric = common.Metric(metric_cls=Accuracy) q_model = quantizer() q_model.save(FLAGS.output_model) if FLAGS.benchmark: eval_examples = processor.get_dev_examples(FLAGS.data_dir) eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") from lpot.experimental import Benchmark, common evaluator = Benchmark(FLAGS.config) dataset = Dataset(eval_file, FLAGS.eval_batch_size) evaluator.b_dataloader = common.DataLoader(\ dataset, batch_size=FLAGS.eval_batch_size, collate_fn=collate_fn) evaluator.metric = common.Metric(metric_cls=Accuracy) from lpot.model.model import get_model_type model_type = get_model_type(FLAGS.input_model) if model_type == 'frozen_pb': evaluator.model = FLAGS.input_model else: estimator_input_fn = input_fn_builder( input_file=eval_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) evaluator.model = common.Model(estimator, input_fn=estimator_input_fn) evaluator(FLAGS.mode)
pass def reset(self): self.pred_list = [] self.label_list = [] self.samples = 0 pass def result(self): correct_num = np.sum( np.array(self.pred_list) == np.array(self.label_list)) return correct_num / self.samples # Quantize with customized dataloader and metric quantizer = Quantization('./conf.yaml') dataset = Dataset() quantizer.metric = common.Metric(MyMetric, 'hello_metric') quantizer.calib_dataloader = common.DataLoader(dataset, batch_size=1) quantizer.eval_dataloader = common.DataLoader(dataset, batch_size=1) quantizer.model = common.Model('../models/saved_model') q_model = quantizer() # Optional, run quantized model import tensorflow as tf with tf.compat.v1.Graph().as_default(), tf.compat.v1.Session() as sess: tf.compat.v1.import_graph_def(q_model.graph_def, name='') styled_image = sess.run(['output:0'], feed_dict={'input:0': dataset.test_images}) print("Inference is done.")