def main(_):
# Print settings
print("Model frozen graph: " + params["FROZEN_GRAPH"])
print("Model label map file: " + params["LABELMAP_FILE"])
print("Model is for dataset: " + params["MODEL_DATASET_TYPE"])
print("Dataset images: " + params["IMAGES_DIR"])
print("Dataset annotations: " + params["ANNOTATIONS_PATH"])
print("Dataset type: " + params["DATASET_TYPE"])
print('Image count: {}'.format(params["BATCH_COUNT"] *
params["BATCH_SIZE"]))
print("Metric type: " + params["METRIC_TYPE"])
print('Results directory: {}'.format(params["RESULTS_OUT_DIR"]))
print("Temporary annotations directory: " + params["ANNOTATIONS_OUT_DIR"])
print("Detections directory: " + params["DETECTIONS_OUT_DIR"])
print("Result images directory: " + params["IMAGES_OUT_DIR"])
print('Save result images: {}'.format(params["SAVE_IMAGES"]))
# Create category index
category_index = label_map_util.create_category_index_from_labelmap(
params["LABELMAP_FILE"], use_display_name=True)
categories_list = category_index.values(
) # array: [{"id": 88, "name": "teddy bear"}, ...]
print('Categories: {}'.format(categories_list))
# Init function: set the correct imports and function pointers.
func_defs = init(params)
# Run detection if needed
ck_utils.print_header('Process images')
if params["SKIP_DETECTION"]:
print('\nSkip detection, evaluate previous results')
else:
processed_image_ids = detect(category_index, func_defs)
ENV = {}
ENV['PYTHONPATH'] = os.getenv('PYTHONPATH')
ENV['LABELMAP_FILE'] = params["LABELMAP_FILE"]
ENV['MODEL_DATASET_TYPE'] = params["MODEL_DATASET_TYPE"]
ENV['DATASET_TYPE'] = params["DATASET_TYPE"]
ENV['ANNOTATIONS_PATH'] = params["ANNOTATIONS_PATH"]
ENV['METRIC_TYPE'] = params["METRIC_TYPE"]
ENV['IMAGES_OUT_DIR'] = params["IMAGES_OUT_DIR"]
ENV['DETECTIONS_OUT_DIR'] = params["DETECTIONS_OUT_DIR"]
ENV['ANNOTATIONS_OUT_DIR'] = params["ANNOTATIONS_OUT_DIR"]
ENV['RESULTS_OUT_DIR'] = params["RESULTS_OUT_DIR"]
ENV['FULL_REPORT'] = params["FULL_REPORT"]
ENV['IMAGE_LIST_FILE'] = params["IMAGE_LIST_FILE"]
ENV['TIMER_JSON'] = params["TIMER_JSON"]
with open(params["ENV_JSON"], 'w') as o:
json.dump(ENV, o, indent=2, sort_keys=True)
def main(_):
# Print settings
print("Model frozen graph: " + FROZEN_GRAPH)
print("Model label map file: " + LABELMAP_FILE)
print("Model is for dataset: " + MODEL_DATASET_TYPE)
print("Dataset images: " + IMAGES_DIR)
print("Dataset annotations: " + ANNOTATIONS_PATH)
print("Dataset type: " + DATASET_TYPE)
print('Image count: {}'.format(IMAGE_COUNT))
print("Metric type: " + METRIC_TYPE)
print('Results directory: {}'.format(RESULTS_OUT_DIR))
print("Temporary annotations directory: " + ANNOTATIONS_OUT_DIR)
print("Detections directory: " + DETECTIONS_OUT_DIR)
print("Result images directory: " + IMAGES_OUT_DIR)
print('Save result images: {}'.format(SAVE_IMAGES))
# Create category index
category_index = label_map_util.create_category_index_from_labelmap(
LABELMAP_FILE, use_display_name=True)
categories_list = category_index.values(
) # array: [{"id": 88, "name": "teddy bear"}, ...]
print('Categories: {}'.format(categories_list))
# Run detection if needed
ck_utils.print_header('Process images')
if SKIP_DETECTION:
print('\nSkip detection, evaluate previous results')
else:
processed_image_ids = detect(category_index)
ENV = {}
ENV['PYTHONPATH'] = os.getenv('PYTHONPATH')
ENV['LABELMAP_FILE'] = LABELMAP_FILE
ENV['MODEL_DATASET_TYPE'] = MODEL_DATASET_TYPE
ENV['DATASET_TYPE'] = DATASET_TYPE
ENV['ANNOTATIONS_PATH'] = ANNOTATIONS_PATH
ENV['METRIC_TYPE'] = METRIC_TYPE
ENV['IMAGES_OUT_DIR'] = IMAGES_OUT_DIR
ENV['DETECTIONS_OUT_DIR'] = DETECTIONS_OUT_DIR
ENV['ANNOTATIONS_OUT_DIR'] = ANNOTATIONS_OUT_DIR
ENV['RESULTS_OUT_DIR'] = RESULTS_OUT_DIR
ENV['FULL_REPORT'] = FULL_REPORT
ENV['IMAGE_LIST_FILE'] = IMAGE_LIST_FILE
ENV['TIMER_JSON'] = TIMER_JSON
with open(ENV_JSON, 'w') as o:
json.dump(ENV, o, indent=2, sort_keys=True)
def ck_postprocess(i):
def evaluate(processed_image_ids, categories_list):
# Convert annotations from original format of the dataset
# to a format specific for a tool that will calculate metrics
if DATASET_TYPE != METRIC_TYPE:
print('\nConvert annotations from {} to {} ...'.format(
DATASET_TYPE, METRIC_TYPE))
annotations = converter_annotations.convert(
ANNOTATIONS_PATH, ANNOTATIONS_OUT_DIR, DATASET_TYPE,
METRIC_TYPE)
else:
annotations = ANNOTATIONS_PATH
# Convert detection results from our universal text format
# to a format specific for a tool that will calculate metrics
print('\nConvert results to {} ...'.format(METRIC_TYPE))
results = converter_results.convert(DETECTIONS_OUT_DIR,
RESULTS_OUT_DIR, DATASET_TYPE,
MODEL_DATASET_TYPE, METRIC_TYPE)
# Run evaluation tool
print('\nEvaluate metrics as {} ...'.format(METRIC_TYPE))
if METRIC_TYPE == ck_utils.COCO:
mAP, recall, all_metrics = calc_metrics_coco.evaluate_via_pycocotools(
processed_image_ids, results, annotations)
elif METRIC_TYPE == ck_utils.COCO_TF:
mAP, recall, all_metrics = calc_metrics_coco.evaluate_via_tf(
categories_list, results, annotations, FULL_REPORT)
elif METRIC_TYPE == ck_utils.OID:
mAP, _, all_metrics = calc_metrics_oid.evaluate(
results, annotations, LABELMAP_FILE, FULL_REPORT)
recall = 'N/A'
else:
raise ValueError(
'Metrics type is not supported: {}'.format(METRIC_TYPE))
OPENME['mAP'] = mAP
OPENME['recall'] = recall
OPENME['metrics'] = all_metrics
return
OPENME = {}
with open(IMAGE_LIST_FILE, 'r') as f:
processed_image_ids = json.load(f)
if os.path.isfile(TIMER_JSON):
with open(TIMER_JSON, 'r') as f:
OPENME = json.load(f)
# Run evaluation
ck_utils.print_header('Process results')
category_index = label_map_util.create_category_index_from_labelmap(
LABELMAP_FILE, use_display_name=True)
categories_list = category_index.values()
evaluate(processed_image_ids, categories_list)
# Store benchmark results
with open(TIMER_JSON, 'w') as o:
json.dump(OPENME, o, indent=2, sort_keys=True)
# Print metrics
print('\nSummary:')
print('-------------------------------')
print('Graph loaded in {:.6f}s'.format(OPENME.get('graph_load_time_s', 0)))
print('All images loaded in {:.6f}s'.format(
OPENME.get('images_load_time_total_s', 0)))
print('All images detected in {:.6f}s'.format(
OPENME.get('detection_time_total_s', 0)))
print('Average detection time: {:.6f}s'.format(
OPENME.get('detection_time_avg_s', 0)))
print('mAP: {}'.format(OPENME['mAP']))
print('Recall: {}'.format(OPENME['recall']))
print('--------------------------------\n')
return {'return': 0}
def ck_postprocess(i):
def my_env(var):
return i['env'].get(var)
def dep_env(dep, var):
return i['deps'][dep]['dict']['env'].get(var)
def set_in_my_env(var):
return my_env(var) and my_env(var).lower() in [
'yes', 'true', 'on', '1'
]
def set_in_dep_env(dep, var):
return dep_env(dep, var) and dep_env(
dep, var).lower() in ['yes', 'true', 'on', '1']
def has_dep_env(dep, var):
return var in i['deps'][dep]['dict']['env']
def has_dep(dep):
return dep in i['deps']
import os
import json
import sys
import inspect
# gain access to other scripts in the same directory as the postprocess.py :
#
SCRIPT_DIR = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
sys.path.append(SCRIPT_DIR)
# gain access to the some python dependencies :
PYTHONPATH = ''
for dep_name in ['tool-coco', 'lib-python-matplotlib']:
PYTHONPATH = dep_env(dep_name, 'PYTHONPATH') + ':' + PYTHONPATH
split_path = set()
for p in PYTHONPATH.split(":"):
if p in ["${PYTHONPATH}", "$PYTHONPATH", ""]:
continue
split_path.add(p)
sys.path.extend(list(
split_path)) # allow THIS SCRIPT to be able to use numpy, pillow, etc.
# get some parameters directly from the deps' environment:
#
MODEL_ROOT = dep_env('weights', "CK_ENV_TENSORFLOW_MODEL_ROOT")
if MODEL_ROOT:
LABELMAP_FILE = os.path.join(
MODEL_ROOT,
dep_env('weights', 'CK_ENV_TENSORFLOW_MODEL_LABELMAP_FILE') or "")
MODEL_DATASET_TYPE = dep_env('weights',
"CK_ENV_TENSORFLOW_MODEL_DATASET_TYPE")
else:
MODEL_ROOT = dep_env('weights', "CK_ENV_ONNX_MODEL_ROOT")
LABELMAP_FILE = os.path.join(
MODEL_ROOT,
dep_env('weights', 'CK_ENV_ONNX_MODEL_CLASSES_LABELS') or "")
MODEL_DATASET_TYPE = dep_env('weights',
"CK_ENV_ONNX_MODEL_DATASET_TYPE")
# Annotations can be a directory or a single file, depending on dataset type:
ANNOTATIONS_PATH = dep_env('dataset', "CK_ENV_DATASET_ANNOTATIONS")
TIMER_JSON = my_env('CK_TIMER_FILE')
PREPROCESSED_FILES = dep_env(
'dataset', 'CK_ENV_DATASET_OBJ_DETECTION_PREPROCESSED_SUBSET_FOF'
) or my_env('CK_PREPROCESSED_FOF_WITH_ORIGINAL_DIMENSIONS')
DETECTIONS_OUT_DIR = my_env('CK_DETECTIONS_OUT_DIR')
RESULTS_OUT_DIR = my_env('CK_RESULTS_OUT_DIR')
ANNOTATIONS_OUT_DIR = my_env('CK_ANNOTATIONS_OUT_DIR')
DATASET_TYPE = dep_env('dataset', "CK_ENV_DATASET_TYPE")
METRIC_TYPE = (my_env("CK_METRIC_TYPE") or DATASET_TYPE).lower()
FULL_REPORT = not set_in_my_env("CK_SILENT_MODE")
import ck_utils
import converter_annotations
import converter_results
ck_utils.prepare_dir(RESULTS_OUT_DIR)
ck_utils.prepare_dir(ANNOTATIONS_OUT_DIR)
if METRIC_TYPE != ck_utils.COCO:
import calc_metrics_coco_tf
import calc_metrics_kitti
import calc_metrics_oid
from object_detection.utils import label_map_util
else:
import calc_metrics_coco_pycocotools
def evaluate(processed_image_ids, categories_list):
# Convert annotations from original format of the dataset
# to a format specific for a tool that will calculate metrics
if DATASET_TYPE != METRIC_TYPE:
print('\nConvert annotations from {} to {} ...'.format(
DATASET_TYPE, METRIC_TYPE))
annotations = converter_annotations.convert(
ANNOTATIONS_PATH, ANNOTATIONS_OUT_DIR, DATASET_TYPE,
METRIC_TYPE)
else:
annotations = ANNOTATIONS_PATH
# Convert detection results from our universal text format
# to a format specific for a tool that will calculate metrics
print('\nConvert results to {} ...'.format(METRIC_TYPE))
results = converter_results.convert(DETECTIONS_OUT_DIR,
RESULTS_OUT_DIR, DATASET_TYPE,
MODEL_DATASET_TYPE, METRIC_TYPE)
# Run evaluation tool
print('\nEvaluate metrics as {} ...'.format(METRIC_TYPE))
if METRIC_TYPE == ck_utils.COCO:
mAP, recall, all_metrics = calc_metrics_coco_pycocotools.evaluate(
processed_image_ids, results, annotations)
elif METRIC_TYPE == ck_utils.COCO_TF:
mAP, recall, all_metrics = calc_metrics_coco_tf.evaluate(
categories_list, results, annotations, FULL_REPORT)
elif METRIC_TYPE == ck_utils.OID:
mAP, _, all_metrics = calc_metrics_oid.evaluate(
results, annotations, LABELMAP_FILE, FULL_REPORT)
recall = 'N/A'
else:
raise ValueError(
'Metrics type is not supported: {}'.format(METRIC_TYPE))
OPENME['mAP'] = mAP
OPENME['recall'] = recall
OPENME['metrics'] = all_metrics
return
OPENME = {}
with open(PREPROCESSED_FILES, 'r') as f:
processed_image_filenames = [x.split(';')[0] for x in f.readlines()]
processed_image_ids = [
ck_utils.filename_to_id(image_filename, DATASET_TYPE)
for image_filename in processed_image_filenames
]
if os.path.isfile(TIMER_JSON):
with open(TIMER_JSON, 'r') as f:
OPENME = json.load(f)
# Run evaluation
ck_utils.print_header('Process results')
if METRIC_TYPE != ck_utils.COCO:
category_index = label_map_util.create_category_index_from_labelmap(
LABELMAP_FILE, use_display_name=True)
categories_list = category_index.values()
else:
categories_list = []
evaluate(processed_image_ids, categories_list)
OPENME[
'frame_predictions'] = converter_results.convert_to_frame_predictions(
DETECTIONS_OUT_DIR)
OPENME['execution_time'] = OPENME['run_time_state'].get(
'test_time_s', 0) + OPENME['run_time_state'].get('setup_time_s', 0)
# Store benchmark results
with open(TIMER_JSON, 'w') as o:
json.dump(OPENME, o, indent=2, sort_keys=True)
# Print metrics
print('\nSummary:')
print('-------------------------------')
print('All images loaded in {:.6f}s'.format(OPENME['run_time_state'].get(
'load_images_time_total_s', 0)))
print('Average image load time: {:.6f}s'.format(
OPENME['run_time_state'].get('load_images_time_avg_s', 0)))
print('All images detected in {:.6f}s'.format(OPENME['run_time_state'].get(
'prediction_time_total_s', 0)))
print('Average detection time: {:.6f}s'.format(
OPENME['run_time_state'].get('prediction_time_avg_s', 0)))
print('mAP: {}'.format(OPENME['mAP']))
print('Recall: {}'.format(OPENME['recall']))
print('--------------------------------\n')
return {'return': 0}
