def __init__(self, model_fn, params):
self._model_dir = params.model_dir
# Sets up evaluator.
self._evaluator = factory.evaluator_generator(params.eval)
input_partition_dims = None
num_cores_per_replica = None
if params.use_tpu:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
params.platform.tpu,
zone=params.platform.tpu_zone,
project=params.platform.gcp_project)
tpu_grpc_url = tpu_cluster_resolver.get_master()
tf.Session.reset(tpu_grpc_url)
# If the input image is transposed (from NHWC to HWCN), the partition
# dimensions also need to be transposed the same way.
def _maybe_transpose(input_partition_dims):
if input_partition_dims and params.train.transpose_input:
return [input_partition_dims[i] for i in [1, 2, 3, 0]]
else:
return input_partition_dims
if params.train.input_partition_dims is not None:
num_cores_per_replica = params.train.num_cores_per_replica
input_partition_dims = params.train.input_partition_dims
# Parse 'None' into None.
input_partition_dims = [
None if x == 'None' else _maybe_transpose(x)
for x in input_partition_dims
]
else:
tpu_cluster_resolver = None
# Sets up config for TPUEstimator.
tpu_config = tf.contrib.tpu.TPUConfig(
params.train.iterations_per_loop,
num_cores_per_replica=num_cores_per_replica,
input_partition_dims=input_partition_dims,
per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.
PER_HOST_V2 # pylint: disable=line-too-long
)
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
evaluation_master=params.platform.eval_master,
model_dir=params.model_dir,
log_step_count_steps=params.train.iterations_per_loop,
tpu_config=tpu_config,
)
self._estimator = tf.contrib.tpu.TPUEstimator(
model_fn=model_fn,
use_tpu=params.use_tpu,
train_batch_size=params.train.train_batch_size,
eval_batch_size=params.eval.eval_batch_size,
predict_batch_size=params.predict.predict_batch_size,
config=run_config,
params=params.as_dict())
def prepare_evaluation(self):
"""Preapre for evaluation."""
val_json_file = os.path.join(self._params.model_dir,
'eval_annotation_file.json')
if self._params.eval.val_json_file:
tf.gfile.Copy(self._params.eval.val_json_file, val_json_file)
else:
coco_utils.scan_and_generator_annotation_file(
self._params.eval.eval_file_pattern,
self._params.eval.eval_samples,
include_mask=False,
annotation_file=val_json_file)
eval_params = params_dict.ParamsDict(self._params.eval)
eval_params.override({'val_json_file': val_json_file})
self._evaluator = factory.evaluator_generator(eval_params)
def prepare_evaluation(self):
"""Preapre for evaluation."""
eval_params = params_dict.ParamsDict(self._params.eval)
if self._params.eval.use_json_file:
val_json_file = os.path.join(
self._params.model_dir, "eval_annotation_file.json"
)
if self._params.eval.val_json_file:
tf.io.gfile.copy(
self._params.eval.val_json_file, val_json_file, overwrite=True
)
else:
coco_utils.scan_and_generator_annotation_file(
self._params.eval.eval_file_pattern,
self._params.eval.eval_samples,
include_mask=False,
annotation_file=val_json_file,
dataset_type=self._params.eval.eval_dataset_type,
)
eval_params.override({"val_json_file": val_json_file})
self._evaluator = factory.evaluator_generator(eval_params)
def __init__(self, model_fn, params):
self._model_dir = params.model_dir
# Sets up evaluator.
self._evaluator = factory.evaluator_generator(params.eval)
if params.use_tpu:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
params.platform.tpu,
zone=params.platform.tpu_zone,
project=params.platform.gcp_project)
tpu_grpc_url = tpu_cluster_resolver.get_master()
tf.Session.reset(tpu_grpc_url)
else:
tpu_cluster_resolver = None
# Sets up config for TPUEstimator.
tpu_config = tf.contrib.tpu.TPUConfig(
params.train.iterations_per_loop,
num_shards=params.train.num_shards,
per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.
PER_HOST_V2 # pylint: disable=line-too-long
)
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
evaluation_master=params.platform.eval_master,
model_dir=params.model_dir,
log_step_count_steps=params.train.iterations_per_loop,
tpu_config=tpu_config,
)
self._estimator = tf.contrib.tpu.TPUEstimator(
model_fn=model_fn,
use_tpu=params.use_tpu,
train_batch_size=params.train.train_batch_size,
eval_batch_size=params.eval.eval_batch_size,
predict_batch_size=params.predict.predict_batch_size,
config=run_config,
params=params.as_dict())
def prepare_evaluation(self):
"""Preapre for evaluation."""
eval_params = params_dict.ParamsDict(self._params.eval)
if self._params.eval.type == 'box_and_mask':
if (not self._params.eval.use_json_file or
not self._params.eval.val_json_file):
raise ValueError('If `eval.type` == `box_and_mask`, '
'`eval.val_json_file` is required.')
if self._params.eval.use_json_file:
val_json_file = os.path.join(self._params.model_dir,
'eval_annotation_file.json')
if self._params.eval.val_json_file:
tf.gfile.Copy(
self._params.eval.val_json_file, val_json_file, overwrite=True)
else:
coco_utils.scan_and_generator_annotation_file(
self._params.eval.eval_file_pattern,
self._params.eval.eval_samples,
include_mask=False,
annotation_file=val_json_file,
dataset_type=self._params.eval.eval_dataset_type)
eval_params.override({'val_json_file': val_json_file})
self._evaluator = factory.evaluator_generator(eval_params)
def main(unused_argv):
del unused_argv
params = config_factory.config_generator(FLAGS.model)
if FLAGS.config_file:
params = params_dict.override_params_dict(
params, FLAGS.config_file, is_strict=True)
params = params_dict.override_params_dict(
params, FLAGS.params_override, is_strict=True)
# We currently only support batch_size = 1 to evaluate images one by one.
# Override the `eval_batch_size` = 1 here.
params.override({
'eval': {
'eval_batch_size': 1,
},
})
params.validate()
params.lock()
model = model_factory.model_generator(params)
evaluator = evaluator_factory.evaluator_generator(params.eval)
parse_fn = functools.partial(parse_single_example, params=params)
with tf.Graph().as_default():
dataset = tf.data.Dataset.list_files(
params.eval.eval_file_pattern, shuffle=False)
dataset = dataset.apply(
tf.data.experimental.parallel_interleave(
lambda filename: tf.data.TFRecordDataset(filename).prefetch(1),
cycle_length=32,
sloppy=False))
dataset = dataset.map(parse_fn, num_parallel_calls=64)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
dataset = dataset.batch(1, drop_remainder=False)
images, labels, groundtruths = dataset.make_one_shot_iterator().get_next()
images.set_shape([
1,
params.retinanet_parser.output_size[0],
params.retinanet_parser.output_size[1],
3])
# model inference
outputs = model.build_outputs(images, labels, mode=mode_keys.PREDICT)
predictions = outputs
predictions.update({
'source_id': groundtruths['source_id'],
'image_info': labels['image_info'],
})
# Create a saver in order to load the pre-trained checkpoint.
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, FLAGS.checkpoint_path)
num_batches = params.eval.eval_samples // params.eval.eval_batch_size
for i in range(num_batches):
if i % 100 == 0:
print('{}/{} batches...'.format(i, num_batches))
predictions_np, groundtruths_np = sess.run([predictions, groundtruths])
evaluator.update(predictions_np, groundtruths_np)
if FLAGS.dump_predictions_only:
print('Dumping the predction results...')
evaluator.dump_predictions(FLAGS.predictions_path)
print('Done!')
else:
print('Evaluating the prediction results...')
metrics = evaluator.evaluate()
print('Eval results: {}'.format(metrics))
def eval_metrics(self):
return eval_factory.evaluator_generator(self._params.eval)
def __init__(self, model_fn, params):
self._model_dir = params.model_dir
# Sets up evaluator.
self._evaluator = factory.evaluator_generator(params.eval)
input_partition_dims = None
num_cores_per_replica = None
tpu_address = ''
if 'COLAB_TPU_ADDR' not in os.environ:
print('ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!')
else:
tpu_address = 'grpc://' + os.environ['COLAB_TPU_ADDR']
with tf.Session(tpu_address) as sess:
with open('/content/adc.json', 'r') as f:
auth_info = json.load(f)
tf.contrib.cloud.configure_gcs(sess, credentials=auth_info)
if params.use_tpu:
# tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
# params.platform.tpu,
# zone=params.platform.tpu_zone,
# project=params.platform.gcp_project)
# tpu_grpc_url = tpu_cluster_resolver.get_master()
# tf.Session.reset(tpu_grpc_url)
if params.train.input_partition_dims is not None:
num_cores_per_replica = params.train.num_cores_per_replica
input_partition_dims = params.train.input_partition_dims
# Parse 'None' into None.
input_partition_dims = [
None if x == 'None' else x for x in input_partition_dims
]
# else:
# tpu_cluster_resolver = None
# Sets up config for TPUEstimator.
tpu_config = tf.contrib.tpu.TPUConfig(
params.train.iterations_per_loop,
num_cores_per_replica=num_cores_per_replica,
input_partition_dims=input_partition_dims,
per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 # pylint: disable=line-too-long
)
run_config = tf.contrib.tpu.RunConfig(
# cluster=tpu_cluster_resolver,
master=tpu_address,
evaluation_master=params.platform.eval_master,
model_dir=params.model_dir,
log_step_count_steps=params.train.iterations_per_loop,
tpu_config=tpu_config,
)
self._estimator = tf.contrib.tpu.TPUEstimator(
model_fn=model_fn,
use_tpu=params.use_tpu,
train_batch_size=params.train.train_batch_size,
eval_batch_size=params.eval.eval_batch_size,
predict_batch_size=params.predict.predict_batch_size,
config=run_config,
params=params.as_dict())
