def __init__(self, model_fn, params): self._model_dir = params.model_dir # Sets up evaluator. self._evaluator = factory.evaluator_generator(params.eval) input_partition_dims = None num_cores_per_replica = None if params.use_tpu: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( params.platform.tpu, zone=params.platform.tpu_zone, project=params.platform.gcp_project) tpu_grpc_url = tpu_cluster_resolver.get_master() tf.Session.reset(tpu_grpc_url) # If the input image is transposed (from NHWC to HWCN), the partition # dimensions also need to be transposed the same way. def _maybe_transpose(input_partition_dims): if input_partition_dims and params.train.transpose_input: return [input_partition_dims[i] for i in [1, 2, 3, 0]] else: return input_partition_dims if params.train.input_partition_dims is not None: num_cores_per_replica = params.train.num_cores_per_replica input_partition_dims = params.train.input_partition_dims # Parse 'None' into None. input_partition_dims = [ None if x == 'None' else _maybe_transpose(x) for x in input_partition_dims ] else: tpu_cluster_resolver = None # Sets up config for TPUEstimator. tpu_config = tf.contrib.tpu.TPUConfig( params.train.iterations_per_loop, num_cores_per_replica=num_cores_per_replica, input_partition_dims=input_partition_dims, per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig. PER_HOST_V2 # pylint: disable=line-too-long ) run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, evaluation_master=params.platform.eval_master, model_dir=params.model_dir, log_step_count_steps=params.train.iterations_per_loop, tpu_config=tpu_config, ) self._estimator = tf.contrib.tpu.TPUEstimator( model_fn=model_fn, use_tpu=params.use_tpu, train_batch_size=params.train.train_batch_size, eval_batch_size=params.eval.eval_batch_size, predict_batch_size=params.predict.predict_batch_size, config=run_config, params=params.as_dict())
def prepare_evaluation(self): """Preapre for evaluation.""" val_json_file = os.path.join(self._params.model_dir, 'eval_annotation_file.json') if self._params.eval.val_json_file: tf.gfile.Copy(self._params.eval.val_json_file, val_json_file) else: coco_utils.scan_and_generator_annotation_file( self._params.eval.eval_file_pattern, self._params.eval.eval_samples, include_mask=False, annotation_file=val_json_file) eval_params = params_dict.ParamsDict(self._params.eval) eval_params.override({'val_json_file': val_json_file}) self._evaluator = factory.evaluator_generator(eval_params)
def prepare_evaluation(self): """Preapre for evaluation.""" eval_params = params_dict.ParamsDict(self._params.eval) if self._params.eval.use_json_file: val_json_file = os.path.join( self._params.model_dir, "eval_annotation_file.json" ) if self._params.eval.val_json_file: tf.io.gfile.copy( self._params.eval.val_json_file, val_json_file, overwrite=True ) else: coco_utils.scan_and_generator_annotation_file( self._params.eval.eval_file_pattern, self._params.eval.eval_samples, include_mask=False, annotation_file=val_json_file, dataset_type=self._params.eval.eval_dataset_type, ) eval_params.override({"val_json_file": val_json_file}) self._evaluator = factory.evaluator_generator(eval_params)
def __init__(self, model_fn, params): self._model_dir = params.model_dir # Sets up evaluator. self._evaluator = factory.evaluator_generator(params.eval) if params.use_tpu: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( params.platform.tpu, zone=params.platform.tpu_zone, project=params.platform.gcp_project) tpu_grpc_url = tpu_cluster_resolver.get_master() tf.Session.reset(tpu_grpc_url) else: tpu_cluster_resolver = None # Sets up config for TPUEstimator. tpu_config = tf.contrib.tpu.TPUConfig( params.train.iterations_per_loop, num_shards=params.train.num_shards, per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig. PER_HOST_V2 # pylint: disable=line-too-long ) run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, evaluation_master=params.platform.eval_master, model_dir=params.model_dir, log_step_count_steps=params.train.iterations_per_loop, tpu_config=tpu_config, ) self._estimator = tf.contrib.tpu.TPUEstimator( model_fn=model_fn, use_tpu=params.use_tpu, train_batch_size=params.train.train_batch_size, eval_batch_size=params.eval.eval_batch_size, predict_batch_size=params.predict.predict_batch_size, config=run_config, params=params.as_dict())
def prepare_evaluation(self): """Preapre for evaluation.""" eval_params = params_dict.ParamsDict(self._params.eval) if self._params.eval.type == 'box_and_mask': if (not self._params.eval.use_json_file or not self._params.eval.val_json_file): raise ValueError('If `eval.type` == `box_and_mask`, ' '`eval.val_json_file` is required.') if self._params.eval.use_json_file: val_json_file = os.path.join(self._params.model_dir, 'eval_annotation_file.json') if self._params.eval.val_json_file: tf.gfile.Copy( self._params.eval.val_json_file, val_json_file, overwrite=True) else: coco_utils.scan_and_generator_annotation_file( self._params.eval.eval_file_pattern, self._params.eval.eval_samples, include_mask=False, annotation_file=val_json_file, dataset_type=self._params.eval.eval_dataset_type) eval_params.override({'val_json_file': val_json_file}) self._evaluator = factory.evaluator_generator(eval_params)
def main(unused_argv): del unused_argv params = config_factory.config_generator(FLAGS.model) if FLAGS.config_file: params = params_dict.override_params_dict( params, FLAGS.config_file, is_strict=True) params = params_dict.override_params_dict( params, FLAGS.params_override, is_strict=True) # We currently only support batch_size = 1 to evaluate images one by one. # Override the `eval_batch_size` = 1 here. params.override({ 'eval': { 'eval_batch_size': 1, }, }) params.validate() params.lock() model = model_factory.model_generator(params) evaluator = evaluator_factory.evaluator_generator(params.eval) parse_fn = functools.partial(parse_single_example, params=params) with tf.Graph().as_default(): dataset = tf.data.Dataset.list_files( params.eval.eval_file_pattern, shuffle=False) dataset = dataset.apply( tf.data.experimental.parallel_interleave( lambda filename: tf.data.TFRecordDataset(filename).prefetch(1), cycle_length=32, sloppy=False)) dataset = dataset.map(parse_fn, num_parallel_calls=64) dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) dataset = dataset.batch(1, drop_remainder=False) images, labels, groundtruths = dataset.make_one_shot_iterator().get_next() images.set_shape([ 1, params.retinanet_parser.output_size[0], params.retinanet_parser.output_size[1], 3]) # model inference outputs = model.build_outputs(images, labels, mode=mode_keys.PREDICT) predictions = outputs predictions.update({ 'source_id': groundtruths['source_id'], 'image_info': labels['image_info'], }) # Create a saver in order to load the pre-trained checkpoint. saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, FLAGS.checkpoint_path) num_batches = params.eval.eval_samples // params.eval.eval_batch_size for i in range(num_batches): if i % 100 == 0: print('{}/{} batches...'.format(i, num_batches)) predictions_np, groundtruths_np = sess.run([predictions, groundtruths]) evaluator.update(predictions_np, groundtruths_np) if FLAGS.dump_predictions_only: print('Dumping the predction results...') evaluator.dump_predictions(FLAGS.predictions_path) print('Done!') else: print('Evaluating the prediction results...') metrics = evaluator.evaluate() print('Eval results: {}'.format(metrics))
def eval_metrics(self): return eval_factory.evaluator_generator(self._params.eval)
def __init__(self, model_fn, params): self._model_dir = params.model_dir # Sets up evaluator. self._evaluator = factory.evaluator_generator(params.eval) input_partition_dims = None num_cores_per_replica = None tpu_address = '' if 'COLAB_TPU_ADDR' not in os.environ: print('ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!') else: tpu_address = 'grpc://' + os.environ['COLAB_TPU_ADDR'] with tf.Session(tpu_address) as sess: with open('/content/adc.json', 'r') as f: auth_info = json.load(f) tf.contrib.cloud.configure_gcs(sess, credentials=auth_info) if params.use_tpu: # tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( # params.platform.tpu, # zone=params.platform.tpu_zone, # project=params.platform.gcp_project) # tpu_grpc_url = tpu_cluster_resolver.get_master() # tf.Session.reset(tpu_grpc_url) if params.train.input_partition_dims is not None: num_cores_per_replica = params.train.num_cores_per_replica input_partition_dims = params.train.input_partition_dims # Parse 'None' into None. input_partition_dims = [ None if x == 'None' else x for x in input_partition_dims ] # else: # tpu_cluster_resolver = None # Sets up config for TPUEstimator. tpu_config = tf.contrib.tpu.TPUConfig( params.train.iterations_per_loop, num_cores_per_replica=num_cores_per_replica, input_partition_dims=input_partition_dims, per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 # pylint: disable=line-too-long ) run_config = tf.contrib.tpu.RunConfig( # cluster=tpu_cluster_resolver, master=tpu_address, evaluation_master=params.platform.eval_master, model_dir=params.model_dir, log_step_count_steps=params.train.iterations_per_loop, tpu_config=tpu_config, ) self._estimator = tf.contrib.tpu.TPUEstimator( model_fn=model_fn, use_tpu=params.use_tpu, train_batch_size=params.train.train_batch_size, eval_batch_size=params.eval.eval_batch_size, predict_batch_size=params.predict.predict_batch_size, config=run_config, params=params.as_dict())