def main(unused_argv):
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu
if FLAGS.clear:
if os.path.exists(FLAGS.eval_dir):
shutil.rmtree(FLAGS.eval_dir)
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
if FLAGS.pipeline_config_path:
model_config, eval_config, input_config = get_configs_from_pipeline_file()
else:
model_config, eval_config, input_config = get_configs_from_multiple_files()
model_fn = functools.partial(
model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(
input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(
model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
graph_rewriter_fn = None
if 'graph_rewriter_config' in configs:
graph_rewriter_fn = graph_rewriter_builder.build(
configs['graph_rewriter_config'], is_training=False)
evaluator.evaluate(
create_input_dict_fn,
model_fn,
eval_config,
categories,
FLAGS.checkpoint_dir,
FLAGS.eval_dir,
graph_hook_fn=graph_rewriter_fn)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
if FLAGS.pipeline_config_path:
model_config, eval_config, input_config = get_configs_from_pipeline_file(
)
else:
model_config, eval_config, input_config = get_configs_from_multiple_files(
)
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
mitosis_map = [0, 0]
mitosis_class = 'mitosis'
evaluator.evaluate(FLAGS.training_process, mitosis_map, mitosis_class,
create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
if FLAGS.pipeline_config_path:
model_config, eval_config, input_config = get_configs_from_pipeline_file(
)
else:
model_config, eval_config, input_config = get_configs_from_multiple_files(
)
if os.path.exists(eval_config.visualization_export_dir) == False:
print(eval_config.visualization_export_dir)
os.makedirs(eval_config.visualization_export_dir)
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
if FLAGS.run_on_CPU_only == True:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
graph_rewriter_fn = None
if 'graph_rewriter_config' in configs:
graph_rewriter_fn = graph_rewriter_builder.build(
configs['graph_rewriter_config'], is_training=False)
evaluator.evaluate(create_input_dict_fn,
model_fn,
eval_config,
categories,
FLAGS.checkpoint_dir,
FLAGS.eval_dir,
graph_hook_fn=graph_rewriter_fn)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
#evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
# FLAGS.checkpoint_dir, FLAGS.eval_dir)
evaluator.evaluate(FLAGS.report_filename, create_input_dict_fn, model_fn,
eval_config, categories, FLAGS.checkpoint_dir,
FLAGS.eval_dir)
#writing the sorted report and best model statistics
df = pd.read_csv(FLAGS.report_filename)
maxval = df.loc[df['mean_ap'].idxmax()]
df1 = df.sort_values('model-iter')
names = FLAGS.report_filename.split('/')
dir_path = ''
for i in range(len(names) - 1):
if i > 0:
dir_path = dir_path + '/' + names[i]
df1.to_csv(dir_path + '/sorted-' + names[-1])
maxval.to_csv(dir_path + '/best' + names[-1])
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
if len(FLAGS.convDict_path) != 0:
convDict = pickle.load(open(FLAGS.convDict_path, 'rb'))
else:
convDict = None
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False,
add_summaries=True,
convDict=convDict)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
else:
input_config = configs['eval_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
print(label_map)
#max_num_classes = max([item.id for item in label_map.item])
max_num_classes = 764
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def evaluate(self, eval_pipeline_file, model_dir, eval_dir):
configs = self._get_configs_from_pipeline_file(eval_pipeline_file)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=True)
create_input_dict_fn = functools.partial(self.get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config,
categories, model_dir, eval_dir)
def main(unused_argv):
assert FLAGS.train_dir, '`train_dir` is missing.'
eval_dir = os.path.join(FLAGS.train_dir, 'eval_logs')
ckpt_dir = os.path.join(FLAGS.train_dir, 'train_logs')
tf.gfile.MakeDirs(eval_dir)
pipeline_config_path = os.path.join(FLAGS.train_dir, 'model.config')
configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
tf.gfile.Copy(pipeline_config_path,
os.path.join(eval_dir, 'pipeline.config'),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
best_eval_metrics = {'best_mAP': 0.0, 'best_tot_loss': 999.9}
evaluator.evaluate(create_input_dict_fn,
model_fn,
eval_config,
categories,
ckpt_dir,
eval_dir,
best_eval_metrics=best_eval_metrics)
def eval_main(max_number_of_evaluations=None):
if FLAGS.eval_label:
if FLAGS.pipeline_config_path == '':
FLAGS.pipeline_config_path = '../configs/test/' + FLAGS.eval_label + '.config'
if FLAGS.checkpoint_dir == '':
FLAGS.checkpoint_dir = '../checkpoints/train/' + FLAGS.eval_label
FLAGS.eval_dir = '../checkpoints/eval/' + FLAGS.eval_label
FLAGS.eval_tag = FLAGS.eval_label
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
if FLAGS.pipeline_config_path:
model_config, eval_config, input_config = get_configs_from_pipeline_file(
)
elif FLAGS.eval_config_path:
model_config, eval_config, input_config = get_configs_from_multiple_files(
)
else:
model_config, eval_config, input_config = get_configs_from_checkpoint_dir(
)
if not FLAGS.eval_dir:
if not FLAGS.eval_tag:
FLAGS.eval_tag = time.strftime("%Y%m%d-%H%M%S")
FLAGS.eval_dir = utils.mkdir_p(FLAGS.checkpoint_dir + '_eval_' +
FLAGS.eval_tag)
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
input_path = input_config.tf_record_input_reader.input_path
num_examples = sum(1 for _ in tf.python_io.tf_record_iterator(input_path))
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir, num_examples,
FLAGS.gpu_fraction, max_number_of_evaluations)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
else:
input_config = configs['eval_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
categories = []
categories.append({'id': 1, 'name': 'Daisy'})
if FLAGS.run_once:
eval_config.max_evals = 1
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
print "Hello WOrld!"
def run_eval(checkpoint_dir, eval_dir, pipeline_config_path, num_examples):
run_once = False
assert checkpoint_dir, '`checkpoint_dir` is missing.'
assert eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(eval_dir)
if pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
pipeline_config_path)
tf.gfile.Copy(pipeline_config_path,
os.path.join(eval_dir, 'pipeline.config'),
overwrite=True)
else:
return
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
# if eval_training_data:
# input_config = configs['train_input_config']
eval_config.num_examples = num_examples
eval_config.max_evals = 1
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_builder.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if run_once:
eval_config.max_evals = 1
graph_rewriter_fn = None
if 'graph_rewriter_config' in configs:
graph_rewriter_fn = graph_rewriter_builder.build(
configs['graph_rewriter_config'], is_training=False)
result = evaluator.evaluate(create_input_dict_fn,
model_fn,
eval_config,
categories,
checkpoint_dir,
eval_dir,
graph_hook_fn=graph_rewriter_fn)
return result
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.mini_batch:
input_config.shuffle = True
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
if FLAGS.mini_batch:
eval_config.max_evals = 1
eval_config.num_visualizations = 100
eval_config.num_examples = 100
eval_config.visualization_export_dir = os.path.join(
FLAGS.eval_dir, 'images')
os.makedirs(eval_config.visualization_export_dir, exist_ok=True)
metrics = evaluator.evaluate(create_input_dict_fn, model_fn, eval_config,
categories, FLAGS.checkpoint_dir,
FLAGS.eval_dir)
process_metrics(metrics)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(FLAGS.pipeline_config_path)
tf.gfile.Copy(
FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True
)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path
)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config, os.path.join(FLAGS.eval_dir, name), overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(
model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
metrics = evaluator.evaluate(
create_input_dict_fn,
model_fn,
eval_config,
categories,
FLAGS.checkpoint_dir,
FLAGS.eval_dir
)
print(json.dumps({'PascalBoxes_Precision/[email protected]': str(metrics['PascalBoxes_Precision/[email protected]'])}))
def evaluate_step():
print("Evaluating")
assert FLAGS.eval_dir, '`eval_dir` is missing.'
model_config, eval_config, input_config = get_eval_configs_from_pipeline_file(
)
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.train_dir, FLAGS.eval_dir)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
model_config, train_config, input_config, eval_config = get_configs_from_pipeline_file()
model_fn = functools.partial(
build_man_model,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(
input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
if FLAGS.pipeline_config_path:
model_config, eval_config, input_config = get_configs_from_pipeline_file()
else:
model_config, eval_config, input_config = get_configs_from_multiple_files()
model_fn = functools.partial(
model_builder.build,
model_config=model_config,
is_training=False)
create_input_dict_fn = functools.partial(
input_reader_builder.build,
input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def evaluate():
return evaluator.evaluate(eval_input_dict_fn, eval_model_fn, eval_config,
categories, FLAGS.train_dir, FLAGS.eval_dir,
graph_hook_fn=eval_graph_rewriter_fn)
def main(pipeline_config_path, checkpoint_dir, eval_dir, eval_training_data=False,
eval_config_path="", input_config_path="",
model_config_path="", run_once=False):
"""
DEFINE_boolean('eval_training_data', False,
'If training data should be evaluated for this job.')
DEFINE_string('checkpoint_dir', '',
'Directory containing checkpoints to evaluate, typically '
'set to `train_dir` used in the training job.')
DEFINE_string('eval_dir', '',
'Directory to write eval summaries to.')
DEFINE_string('pipeline_config_path', '',
'Path to a pipeline_pb2.TrainEvalPipelineConfig config '
'file. If provided, other configs are ignored')
DEFINE_string('eval_config_path', '',
'Path to an eval_pb2.EvalConfig config file.')
DEFINE_string('input_config_path', '',
'Path to an input_reader_pb2.InputReader config file.')
DEFINE_string('model_config_path', '',
'Path to a model_pb2.DetectionModel config file.')
DEFINE_boolean('run_once', False, 'Option to only run a single pass of '
'evaluation. Overrides the `max_evals` parameter in the '
'provided config.')
"""
tf.logging.set_verbosity(tf.logging.INFO)
tf.gfile.MakeDirs(eval_dir)
if pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
pipeline_config_path)
tf.gfile.Copy(pipeline_config_path,
os.path.join(eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=model_config_path,
eval_config_path=eval_config_path,
eval_input_config_path=input_config_path)
for name, config in [('model.config', model_config_path),
('eval.config', eval_config_path),
('input.config', input_config_path)]:
tf.gfile.Copy(config,
os.path.join(eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(
model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_builder.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if run_once:
eval_config.max_evals = 1
graph_rewriter_fn = None
if 'graph_rewriter_config' in configs:
graph_rewriter_fn = graph_rewriter_builder.build(
configs['graph_rewriter_config'], is_training=False)
evaluator.evaluate(
create_input_dict_fn,
model_fn,
eval_config,
categories,
checkpoint_dir,
eval_dir,
graph_hook_fn=graph_rewriter_fn)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
wait_time = 300
while wait_time > 0:
latest_checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if latest_checkpoint:
num_steps = latest_checkpoint.split('-')[-1]
if int(num_steps) > 0:
wait_time = 0
if wait_time > 0:
tf.logging.info("waiting for checkpoint...")
time.sleep(wait_time)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
do_augmentation = False
if input_config.WhichOneof('input_reader') == 'tf_record_input_reader':
input_reader_config = input_config.tf_record_input_reader
input_path = input_reader_config.input_path
if not input_path or not input_path[0]:
do_augmentation = True
train_input_config = configs['train_input_config']
train_input_reader_config = train_input_config.tf_record_input_reader
input_reader_config.input_path[:] = train_input_reader_config.input_path[:]
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
evaluator.evaluate(create_input_dict_fn,
model_fn,
eval_config,
categories,
FLAGS.checkpoint_dir,
FLAGS.eval_dir,
do_augmentation=do_augmentation)
def main(unused_argv):
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_builder.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
graph_rewriter_fn = None
if 'graph_rewriter_config' in configs:
graph_rewriter_fn = graph_rewriter_builder.build(
configs['graph_rewriter_config'], is_training=False)
metadata = {
'outputs': [{
'type': 'tensorboard',
'source': FLAGS.eval_dir,
}]
}
with open('/mlpipeline-ui-metadata.json', 'w') as f:
json.dump(metadata, f)
evaluator.evaluate(create_input_dict_fn,
model_fn,
eval_config,
categories,
FLAGS.checkpoint_dir,
FLAGS.eval_dir,
graph_hook_fn=graph_rewriter_fn)
def main(unused_argv):
checkpoint_dir = './train'
eval_dir = './eval'
pipeline_config_path = './pipeline_config.config'
model_config_path = './train'
eval_config_path = './train'
input_config_path = './train'
run_once = False
eval_training_data = False
assert checkpoint_dir, '`checkpoint_dir` is missing.'
assert eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(eval_dir)
if pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
pipeline_config_path)
tf.gfile.Copy(pipeline_config_path,
os.path.join(eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=model_config_path,
eval_config_path=eval_config_path,
eval_input_config_path=input_config_path)
for name, config in [('model.config', model_config_path),
('eval.config', eval_config_path),
('input.config', input_config_path)]:
tf.gfile.Copy(config, os.path.join(eval_dir, name), overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if run_once:
eval_config.max_evals = 1
graph_rewriter_fn = None
if 'graph_rewriter_config' in configs:
graph_rewriter_fn = graph_rewriter_builder.build(
configs['graph_rewriter_config'], is_training=False)
evaluator.evaluate(create_input_dict_fn,
model_fn,
eval_config,
categories,
checkpoint_dir,
eval_dir,
graph_hook_fn=graph_rewriter_fn)
def main(unused_argv):
# Uncomment the next lines on Linux to run the evaluation on the CPU
# config = tf.ConfigProto(
# device_count={'GPU': 0}
# )
# sess = tf.Session(config=config)
# Uncomment the next line on Windows to run the evaluation on the CPU
# os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
# Use the following lines to restrict this process to only 30% of the GPU VRAM
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
assert FLAGS.checkpoint_dir, '`checkpoint_dir` is missing.'
assert FLAGS.eval_dir, '`eval_dir` is missing.'
tf.gfile.MakeDirs(FLAGS.eval_dir)
if FLAGS.pipeline_config_path:
configs = config_util.get_configs_from_pipeline_file(
FLAGS.pipeline_config_path)
tf.gfile.Copy(FLAGS.pipeline_config_path,
os.path.join(FLAGS.eval_dir, 'pipeline.config'),
overwrite=True)
else:
configs = config_util.get_configs_from_multiple_files(
model_config_path=FLAGS.model_config_path,
eval_config_path=FLAGS.eval_config_path,
eval_input_config_path=FLAGS.input_config_path)
for name, config in [('model.config', FLAGS.model_config_path),
('eval.config', FLAGS.eval_config_path),
('input.config', FLAGS.input_config_path)]:
tf.gfile.Copy(config,
os.path.join(FLAGS.eval_dir, name),
overwrite=True)
model_config = configs['model']
eval_config = configs['eval_config']
input_config = configs['eval_input_config']
if FLAGS.eval_training_data:
input_config = configs['train_input_config']
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=False)
def get_next(config):
return dataset_util.make_initializable_iterator(
dataset_builder.build(config)).get_next()
create_input_dict_fn = functools.partial(get_next, input_config)
label_map = label_map_util.load_labelmap(input_config.label_map_path)
max_num_classes = max([item.id for item in label_map.item])
categories = label_map_util.convert_label_map_to_categories(
label_map, max_num_classes)
if FLAGS.run_once:
eval_config.max_evals = 1
evaluator.evaluate(create_input_dict_fn, model_fn, eval_config, categories,
FLAGS.checkpoint_dir, FLAGS.eval_dir)
def _run_checkpoint_once(tensor_dict,
evaluators=None,
batch_processor=None,
checkpoint_dirs=None,
variables_to_restore=None,
restore_fn=None,
num_batches=1,
master='',
save_graph=False,
save_graph_dir='',
losses_dict=None):
"""Evaluates metrics defined in evaluators and returns summaries.
This function loads the latest checkpoint in checkpoint_dirs and evaluates
all metrics defined in evaluators. The metrics are processed in batch by the
batch_processor.
Args:
tensor_dict: a dictionary holding tensors representing a batch of detections
and corresponding groundtruth annotations.
evaluators: a list of object of type DetectionEvaluator to be used for
evaluation. Note that the metric names produced by different evaluators
must be unique.
batch_processor: a function taking four arguments:
1. tensor_dict: the same tensor_dict that is passed in as the first
argument to this function.
2. sess: a tensorflow session
3. batch_index: an integer representing the index of the batch amongst
all batches
By default, batch_processor is None, which defaults to running:
return sess.run(tensor_dict)
To skip an image, it suffices to return an empty dictionary in place of
result_dict.
checkpoint_dirs: list of directories to load into an EnsembleModel. If it
has only one directory, EnsembleModel will not be used --
a DetectionModel
will be instantiated directly. Not used if restore_fn is set.
variables_to_restore: None, or a dictionary mapping variable names found in
a checkpoint to model variables. The dictionary would normally be
generated by creating a tf.train.ExponentialMovingAverage object and
calling its variables_to_restore() method. Not used if restore_fn is set.
restore_fn: None, or a function that takes a tf.Session object and correctly
restores all necessary variables from the correct checkpoint file. If
None, attempts to restore from the first directory in checkpoint_dirs.
num_batches: the number of batches to use for evaluation.
master: the location of the Tensorflow session.
save_graph: whether or not the Tensorflow graph is stored as a pbtxt file.
save_graph_dir: where to store the Tensorflow graph on disk. If save_graph
is True this must be non-empty.
losses_dict: optional dictionary of scalar detection losses.
Returns:
global_step: the count of global steps.
all_evaluator_metrics: A dictionary containing metric names and values.
Raises:
ValueError: if restore_fn is None and checkpoint_dirs doesn't have at least
one element.
ValueError: if save_graph is True and save_graph_dir is not defined.
"""
global result_losses_dict
if save_graph and not save_graph_dir:
raise ValueError( '`save_graph_dir` must be defined.' )
sess: Session = tf.Session( master, graph=tf.get_default_graph() )
sess.run( tf.global_variables_initializer() )
sess.run( tf.local_variables_initializer() )
sess.run( tf.tables_initializer() )
if restore_fn:
restore_fn( sess )
else:
if not checkpoint_dirs:
raise ValueError( '`checkpoint_dirs` must have at least one entry.' )
checkpoint_file = tf.train.latest_checkpoint( checkpoint_dirs[0] )
saver = tf.train.Saver( variables_to_restore )
saver.restore( sess, checkpoint_file )
if save_graph:
tf.train.write_graph( sess.graph_def, save_graph_dir, 'eval.pbtxt' )
counters = {'skipped': 0, 'success': 0}
aggregate_result_losses_dict = collections.defaultdict( list )
with tf.contrib.slim.queues.QueueRunners( sess ):
try:
for batch in range( int( num_batches ) ):
if (batch + 1) % 100 == 0:
logging.info( 'Running eval ops batch %d/%d', batch + 1, num_batches )
if not batch_processor:
try:
if not losses_dict:
losses_dict = {}
result_dict, result_losses_dict = sess.run( [tensor_dict,
losses_dict] )
counters['success'] += 1
except tf.errors.InvalidArgumentError:
logging.info( 'Skipping image' )
counters['skipped'] += 1
result_dict = {}
else:
result_dict, result_losses_dict = batch_processor(
tensor_dict, sess, batch, counters, losses_dict=losses_dict )
if not result_dict:
continue
for key, value in iter( result_losses_dict.items() ):
aggregate_result_losses_dict[key].append( value )
for evaluator in evaluators:
# TODO(b/65130867): Use image_id tensor once we fix the input data
# decoders to return correct image_id.
# TODO(akuznetsa): result_dict contains batches of images, while
# add_single_ground_truth_image_info expects a single image. Fix
evaluator.add_single_ground_truth_image_info(
image_id=batch, groundtruth_dict=result_dict )
evaluator.add_single_detected_image_info(
image_id=batch, detections_dict=result_dict )
logging.info( 'Running eval batches done.' )
except tf.errors.OutOfRangeError:
logging.info( 'Done evaluating -- epoch limit reached' )
finally:
# When done, ask the threads to stop.
logging.info( '# success: %d', counters['success'] )
logging.info( '# skipped: %d', counters['skipped'] )
all_evaluator_metrics = {}
for evaluator in evaluators:
metrics = evaluator.evaluate()
evaluator.clear()
if any( key in all_evaluator_metrics for key in metrics ):
raise ValueError( 'Metric names between evaluators must not collide.' )
all_evaluator_metrics.update( metrics )
with sess.graph.as_default():
# global_step = tf.train.global_step( sess, tf.train.get_global_step() )
global_step = 200000
for key, value in iter( aggregate_result_losses_dict.items() ):
all_evaluator_metrics['Losses/' + key] = np.mean( value )
sess.close()
# print( all_evaluator_metrics )
return all_evaluator_metrics
