def compute_metrics(images, gt_boxes, gt_class_ids, gt_masks,
results_list) -> List[Tuple]:
mAPs = []
APs_75 = []
APs_5 = []
recalls_75 = []
recalls_5 = []
roundness_list = []
for image, gt_bbox, gt_class_id, gt_mask, results in zip(
images, gt_boxes, gt_class_ids, gt_masks, results_list):
# Compute metrics
r = results
AP_75, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r['rois'], r['class_ids'], r['scores'], r['masks'], iou_threshold=0.75)
AP_5, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r['rois'], r['class_ids'], r['scores'], r['masks'], iou_threshold=0.5)
mAP = utils.compute_ap_range(gt_bbox,
gt_class_id,
gt_mask,
r['rois'],
r['class_ids'],
r['scores'],
r['masks'],
verbose=False)
recall_75, _ = utils.compute_recall(r['rois'], gt_bbox, iou=0.75)
recall_5, _ = utils.compute_recall(r['rois'], gt_bbox, iou=0.5)
# Roundness
contours = get_contours(r)
img_roundness = avg_roundness(contours)
mAPs.append(mAP)
APs_75.append(AP_75)
APs_5.append(AP_5)
recalls_75.append(recall_75)
recalls_5.append(recall_5)
roundness_list.append(img_roundness)
names = [
'1. mAP@IoU Rng', '2. mAP@IoU=75', '3. mAP@IoU=50',
'4. Recall @ IoU=75', '5. Recall @ IoU=50', '6. Roundness'
]
values_list = [mAPs, APs_75, APs_5, recalls_75, recalls_5, roundness_list]
avg_values = [np.mean(values) for values in values_list]
return list(zip(names, avg_values))
def compute_batch_metrics(dataset):
APs50 = []
APs75 = []
APs_range = []
ARs50 = []
ARs75 = []
ARs_range = []
APs_range_small = []
APs_range_medium = []
APs_range_large = []
ARs_range_small = []
ARs_range_medium = []
ARs_range_large = []
'''for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox =\
modellib.load_image_gt(dataset, config,
image_id, use_mini_mask=False)'''
for image_id in dataset.image_ids:
image = dataset.load_image(image_id)
'''gt_bbox=[]
gt_class_id=[]
for dict in dataset.image_info[image_id]['annotations']:
gt_bbox.append(dict['bbox'])
gt_class_id.append(dict['category_id'])
cv2.rectangle(image, (dict['bbox'][1], dict['bbox'][0]), (dict['bbox'][3], dict['bbox'][2]), (255, 0, 0), 2)
gt_bbox = np.array(gt_bbox)
print('gt_bbox:,', gt_bbox)
#print('gt_bbox.shape:', gt_bbox.shape)
gt_class_id = np.array(gt_class_id)
#zerod = np.zeros(len(gt_class_id))
#gt_class_id = np.vstack((gt_class_id, zerod))'''
gt_bbox, gt_class_id = dataset.load_mask(image_id)
'''print('gt_bbox:,', gt_bbox)
if len(gt_bbox[0]) == 4:
for i in range(len(gt_bbox)):
cv2.rectangle(image, (gt_bbox[i][1], gt_bbox[i][0]), (gt_bbox[i][3], gt_bbox[i][2]), (255, 0, 0), 2)'''
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
faces = faceCascade.detectMultiScale(gray,
scaleFactor=1.03,
minNeighbors=6,
minSize=(24, 24))
# Draw a rectangle around the faces
pred_bbox = []
pred_scores = []
pred_class_id = []
count = 0
for (x, y, w, h) in faces:
#cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
pred_bbox.append([y, x, y + h, x + w])
pred_scores.append(1.)
pred_class_id.append(1)
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
count += 1
if count == 0:
pred_bbox.append([])
pred_scores.append(1.)
pred_class_id.append(1)
pred_bbox = np.array(pred_bbox)
pred_scores = np.array(pred_scores)
pred_class_id = np.array(pred_class_id)
#zerod = np.zeros(len(pred_scores))
#pred_scores = np.vstack((pred_scores, zerod))
#pred_class_id = np.vstack((pred_class_id, zerod))
#print(pred_bbox)
r = {
'rois': pred_bbox,
'class_ids': pred_class_id,
'scores': pred_scores
}
'''print("r['rois']:", r['rois'])
img = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
cv2.imshow('Video', img)
cv2.waitKey(0)'''
# Run object detection
#results = model.detect([image], verbose=0)
# Compute AP
#r = results[0]
if pred_bbox.shape != (1, 0):
AP50, precisions50, recalls50, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id,
r['rois'], r['class_ids'], r['scores'], config=config, iou_threshold=0.5)
#print('AP50:', AP50)
#print('pred_bbox.shape:', pred_bbox.shape)
AP75, precisions75, recalls75, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id,
r['rois'], r['class_ids'], r['scores'], config=config, iou_threshold=0.75)
AP_range =\
utils.compute_ap_range(gt_bbox, gt_class_id,
r['rois'], r['class_ids'], r['scores'], config=config)
AR50, positive_ids50 =\
utils.compute_recall(r['rois'], gt_bbox, iou=0.5)
AR75, positive_ids75 =\
utils.compute_recall(r['rois'], gt_bbox, iou=0.75)
AR_range =\
utils.compute_ar_range(r['rois'], gt_bbox)
else:
AP50 = 0.
AP75 = 0.
AP_range = 0.
AR50 = 0.
AR75 = 0.
AR_range = 0.
#AR50 = recalls50[-2]
#AR75 = recalls75[-2]
#AR_range = recalls_range[-2]
APs50.append(AP50)
APs75.append(AP75)
APs_range.append(AP_range)
ARs50.append(AR50)
ARs75.append(AR75)
ARs_range.append(AR_range)
gt_ind_small, gt_ind_medium, gt_ind_large = find_bbox_indices_by_size(
gt_bbox)
gt_bbox_small = gt_bbox[gt_ind_small, :]
gt_bbox_medium = gt_bbox[gt_ind_medium, :]
gt_bbox_large = gt_bbox[gt_ind_large, :]
gt_class_id_small = gt_class_id[gt_ind_small]
gt_class_id_medium = gt_class_id[gt_ind_medium]
gt_class_id_large = gt_class_id[gt_ind_large]
if pred_bbox.shape != (1, 0):
pred_ind_small, pred_ind_medium, pred_ind_large = find_bbox_indices_by_size(
r['rois'])
pred_bbox_small = r['rois'][pred_ind_small, :]
pred_bbox_medium = r['rois'][pred_ind_medium, :]
pred_bbox_large = r['rois'][pred_ind_large, :]
pred_class_id_small = r['class_ids'][pred_ind_small]
pred_class_id_medium = r['class_ids'][pred_ind_medium]
pred_class_id_large = r['class_ids'][pred_ind_large]
pred_score_small = r['scores'][pred_ind_small]
pred_score_medium = r['scores'][pred_ind_medium]
pred_score_large = r['scores'][pred_ind_large]
if gt_bbox_small.size != 0:
if pred_bbox.shape != (1, 0):
AP_range_small =\
utils.compute_ap_range(gt_bbox_small, gt_class_id_small,
pred_bbox_small, pred_class_id_small, pred_score_small, config=config)
AR_range_small =\
utils.compute_ar_range(pred_bbox_small, gt_bbox_small)
else:
AP_range_small = 0.
AR_range_small = 0.
APs_range_small.append(AP_range_small)
ARs_range_small.append(AR_range_small)
if gt_bbox_medium.size != 0:
if pred_bbox.shape != (1, 0):
AP_range_medium =\
utils.compute_ap_range(gt_bbox_medium, gt_class_id_medium,
pred_bbox_medium, pred_class_id_medium, pred_score_medium, config=config)
AR_range_medium =\
utils.compute_ar_range(pred_bbox_medium, gt_bbox_medium)
else:
AP_range_medium = 0.
AR_range_medium = 0.
APs_range_medium.append(AP_range_medium)
ARs_range_medium.append(AR_range_medium)
if gt_bbox_large.size != 0:
if pred_bbox.shape != (1, 0):
AP_range_large =\
utils.compute_ap_range(gt_bbox_large, gt_class_id_large,
pred_bbox_large, pred_class_id_large, pred_score_large, config=config)
AR_range_large =\
utils.compute_ar_range(pred_bbox_large, gt_bbox_large)
else:
AP_range_large = 0.
AR_range_large = 0.
APs_range_large.append(AP_range_large)
ARs_range_large.append(AR_range_large)
return APs50, APs75, APs_range, ARs50, ARs75, ARs_range, APs_range_small, APs_range_medium,\
APs_range_large, ARs_range_small, ARs_range_medium, ARs_range_large
# after NMS) but with coordinates normalized to [0, 1] range.
limit = 50
# Convert back to image coordinates for display
h, w = config.IMAGE_SHAPE[:2]
proposals = rpn['proposals'][0, :limit] * np.array([h, w, h, w])
visualize.draw_boxes(image, refined_boxes=proposals, ax=get_ax())
#%%
# Measure the RPN recall (percent of objects covered by anchors)
# Here we measure recall for 3 different methods:
# - All anchors
# - All refined anchors
# - Refined anchors after NMS
iou_threshold = 0.7
recall, positive_anchor_ids = utils.compute_recall(model.anchors, gt_bbox,
iou_threshold)
print("All Anchors ({:5}) Recall: {:.3f} Positive anchors: {}".format(
model.anchors.shape[0], recall, len(positive_anchor_ids)))
recall, positive_anchor_ids = utils.compute_recall(rpn['refined_anchors'][0],
gt_bbox, iou_threshold)
print("Refined Anchors ({:5}) Recall: {:.3f} Positive anchors: {}".format(
rpn['refined_anchors'].shape[1], recall, len(positive_anchor_ids)))
recall, positive_anchor_ids = utils.compute_recall(proposals, gt_bbox,
iou_threshold)
print("Post NMS Anchors ({:5}) Recall: {:.3f} Positive anchors: {}".format(
proposals.shape[0], recall, len(positive_anchor_ids)))
#%% Stage 2: Proposal Classification
# This stage takes the region proposals from the RPN and classifies them.
def execute_eval(detect,
model,
dataset,
datacfg,
dnncfg,
class_names,
reportcfg,
get_mask=True):
"""Execute the evaluation and generates the evaluation reports
- classification report with differet scores
- confusion matrix
- summary report
"""
log.info("execute_eval---------------------------->")
save_viz_and_json = reportcfg['save_viz_and_json']
evaluate_no_of_result = reportcfg['evaluate_no_of_result']
filepath = reportcfg['filepath']
evaluate_run_summary = reportcfg['evaluate_run_summary']
log.info("evaluate_no_of_result: {}".format(evaluate_no_of_result))
detection_on_dataset = []
## TODO: put at right place
iou_threshold = reportcfg['iou_threshold']
# iou_thresholds = None
# iou_thresholds = iou_thresholds or np.arange(0.5, 1.0, 0.05)
iou_thresholds = np.arange(0.5, 1.0, 0.05)
gt_total_annotation = 0
pred_total_annotation = 0
remaining_num_images = -1
image_ids = dataset.image_ids
num_images = len(image_ids)
pred_match_total_annotation = []
log.info("class_names: {}".format(class_names))
colors = viz.random_colors(len(class_names))
log.info("len(colors), colors: {},{}".format(len(colors), colors))
cc = dict(zip(class_names, colors))
## for some reasons if gets an error in iterating through dataset, all the hardwork is lost,
## therefore, save the data to disk in finally clause
via_jsonres = {}
imagelist = []
T0 = time.time()
try:
for i, image_id in enumerate(image_ids):
log.debug("-------")
filepath_image_in = dataset.image_reference(image_id)
image_filename = filepath_image_in.split(os.path.sep)[-1]
image_name_without_ext = image_filename.split('.')[0]
imagelist.append(filepath_image_in)
log.debug("Running on {}".format(image_filename))
if evaluate_no_of_result == i:
log.info("evaluate_no_of_result reached: i: {}\n".format(i))
break
remaining_num_images = evaluate_no_of_result if evaluate_no_of_result and evaluate_no_of_result > 0 else num_images
remaining_num_images = remaining_num_images - i - 1
log.info(
"To be evaluated remaining_num_images:...................{}".
format(remaining_num_images))
t0 = time.time()
im, gt_class_ids, gt_boxes, gt_masks, gt_active_class_ids = load_image_gt_without_resizing(
dataset, datacfg, dnncfg, image_id)
t1 = time.time()
time_taken_imread = (t1 - t0)
log.debug('Total time taken in time_taken_imread: %f seconds' %
(time_taken_imread))
gt_total_annotation += len(gt_class_ids)
log.info("\nGround Truth-------->")
log.info("i,image_id:{},{}".format(i, image_id))
log.info(
"len(gt_active_class_ids),gt_active_class_ids: {},{}".format(
len(gt_active_class_ids), gt_active_class_ids))
log.info(
"len(gt_class_ids): {}\nTotal Unique classes: len(set(gt_class_ids)): {}\ngt_class_ids: {}"
.format(len(gt_class_ids), len(set(gt_class_ids)),
gt_class_ids))
log.info("len(gt_boxes), gt_boxes.shape, type(gt_boxes): {},{},{}".
format(len(gt_boxes), gt_boxes.shape, type(gt_boxes)))
log.info("len(gt_masks), gt_masks.shape, type(gt_masks): {},{},{}".
format(len(gt_masks), gt_masks.shape, type(gt_masks)))
log.debug("gt_boxes: {}".format(gt_boxes))
log.debug("gt_masks: {}".format(gt_masks))
log.info("--------")
# Detect objects
##---------------------------------------------
t2 = time.time()
r = detect(model, im=im, verbose=1)[0]
pred_boxes = r['rois']
pred_masks = r['masks']
pred_class_ids = r['class_ids']
pred_scores = r['scores']
pred_total_annotation += len(pred_class_ids)
log.debug("Prediction on Groud Truth-------->")
log.debug('len(r): {}'.format(len(r)))
log.debug(
"len(pred_class_ids), pred_class_ids, type(pred_class_ids): {},{},{}"
.format(len(pred_class_ids), pred_class_ids,
type(pred_class_ids)))
log.debug(
"len(pred_boxes), pred_boxes.shape, type(pred_boxes): {},{},{}"
.format(len(pred_boxes), pred_boxes.shape, type(pred_boxes)))
log.debug(
"len(pred_masks), pred_masks.shape, type(pred_masks): {},{},{}"
.format(len(pred_masks), pred_masks.shape, type(pred_masks)))
log.debug("--------")
t3 = time.time()
time_taken_in_detect = (t3 - t2)
log.debug('Total time taken in detect: %f seconds' %
(time_taken_in_detect))
t4 = time.time()
## TODO: gt via_json resp and pred via jsn res separate data strucure
## TODO: mAP calculation for per image, per class and enitre dataset
##TODO: this does not help; need to flatten the all ground truts for eniter dataset.
## np.zeros(len(gt_for_all_images)), ideally same number of predictions sohuld be there
## Insort; have to re-write the compute_matches function for entire dataset
evaluate_run_summary['images'].append(image_filename)
# evaluate_run_summary['gt_boxes'].append(gt_boxes)
evaluate_run_summary['gt_class_ids'].append(list(gt_class_ids))
# evaluate_run_summary['gt_masks'].append(gt_masks)
# evaluate_run_summary['pred_boxes'].append(pred_boxes)
evaluate_run_summary['pred_class_ids'].append(list(pred_class_ids))
evaluate_run_summary['pred_scores'].append(list(pred_scores))
# evaluate_run_summary['pred_masks'].append(pred_masks)
evaluate_run_summary['gt_total_annotation_per_image'].append(
len(gt_class_ids))
evaluate_run_summary['pred_total_annotation_per_image'].append(
len(pred_class_ids))
detection_on_dataset_item = defaultdict(list)
__pred_match_total_annotation = np.zeros([len(iou_thresholds)],
dtype=int)
for count, iou_threshold in enumerate(iou_thresholds):
log.info("count, iou_threshold: {}, {}".format(
count, iou_threshold))
## Compute Average Precision at a set IoU threshold
## --------------------------------------------
AP_per_image, precisions, recalls, gt_match, pred_match, overlaps, pred_match_scores, pred_match_class_ids = utils.compute_ap(
gt_boxes,
gt_class_ids,
gt_masks,
pred_boxes,
pred_class_ids,
pred_scores,
pred_masks,
iou_threshold=iou_threshold)
__pred_match_total_annotation[count] += len(
pred_match_class_ids)
## compute and returns f1 score metric
## --------------------------------------------
f1_per_image = utils.compute_f1score(precisions, recalls)
## Compute the recall at the given IoU threshold. It's an indication
## of how many GT boxes were found by the given prediction boxes.
## --------------------------------------------
recall_bbox, positive_ids_bbox = utils.compute_recall(
gt_boxes, pred_boxes, iou_threshold)
# log.info("len(precisions),precisions: {},{}".format(len(precisions), precisions))
# log.info("len(recalls),recalls: {},{}".format(len(recalls), recalls))
# log.info("len(pred_match_class_ids),pred_match_class_ids: {},{}".format(len(pred_match_class_ids), pred_match_class_ids))
# log.info("AP_per_image: {}".format(AP_per_image))
# log.info("len(overlaps),overlaps: {},{}".format(len(overlaps), overlaps))
class_names = np.array(class_names)
pred_match_class_names = list(class_names[np.where(
np.in1d(dataset.class_ids, pred_match_class_ids))[0]])
class_names = list(class_names)
detection_on_dataset_item['ap_per_image'].append(AP_per_image)
detection_on_dataset_item['f1_per_image'].append(f1_per_image)
detection_on_dataset_item['precisions'].append(precisions)
detection_on_dataset_item['recalls'].append(list(recalls))
detection_on_dataset_item['recall_bbox'].append(recall_bbox)
detection_on_dataset_item['positive_ids_bbox'].append(
list(positive_ids_bbox))
detection_on_dataset_item['gt_match'].append(list(gt_match))
detection_on_dataset_item['pred_match'].append(
list(pred_match))
detection_on_dataset_item['pred_match_scores'].append(
list(pred_match_scores))
detection_on_dataset_item['overlaps_mask_iou'].append(
list(overlaps))
detection_on_dataset_item['pred_match_class_ids'].append(
list(pred_match_class_ids))
detection_on_dataset_item['pred_match_class_names'].append(
pred_match_class_names)
detection_on_dataset_item[
'pred_match_total_annotation'].append(
len(pred_match_class_ids))
detection_on_dataset_item['iou_thresholds'].append(
iou_threshold)
## TODO: ref temp-evaluate-viz.code.py
detection_on_dataset.append(detection_on_dataset_item)
pred_match_total_annotation.append(__pred_match_total_annotation)
log.info("---x-x---")
except Exception as e:
log.info("Exception: {}".format(e))
raise
finally:
log.info("--------X--------X--------X--------")
T1 = time.time()
evaluate_run_summary['total_execution_time'] = T1 - T0
evaluate_run_summary[
'pred_match_total_annotation'] = pred_match_total_annotation
evaluate_run_summary['gt_total_annotation'] = gt_total_annotation
evaluate_run_summary['pred_total_annotation'] = pred_total_annotation
evaluate_run_summary['iou_thresholds'] = iou_thresholds
evaluate_run_summary['execution_end_time'] = "{:%d%m%y_%H%M%S}".format(
datetime.datetime.now())
# evaluate_run_summary['detection_min_confidence'] = dnncfg.config['DETECTION_MIN_CONFIDENCE']
evaluate_run_summary['remaining_num_images'] = remaining_num_images
# evaluate_run_summary['total_images'] = num_images
log.debug("evaluate_run_summary: {}".format(evaluate_run_summary))
## Save the image list for loading the response in VIA along with the images
imagelist_filepath = os.path.join(filepath, 'annotations',
"imagelist.csv")
pd.DataFrame(imagelist).to_csv(imagelist_filepath)
classification_reportfile_path = reportcfg[
'classification_reportfile'] + '-per_dataset.json'
with open(classification_reportfile_path, 'w') as fw:
fw.write(common.numpy_to_json(detection_on_dataset))
evaluate_run_summary_reportfile_path = reportcfg[
'evaluate_run_summary_reportfile'] + '.json'
with open(evaluate_run_summary_reportfile_path, 'w') as fw:
fw.write(common.numpy_to_json(evaluate_run_summary))
print("EVALUATE_REPORT:IMAGELIST:{}".format(imagelist_filepath))
print(
"EVALUATE_REPORT:METRIC:{}".format(classification_reportfile_path))
print("EVALUATE_REPORT:SUMMARY:{}".format(
evaluate_run_summary_reportfile_path))
log.info("--------")
return evaluate_run_summary
def evaluate(mode,
cmdcfg,
appcfg,
modelcfg,
dataset,
datacfg,
class_names,
reportcfg,
get_mask=True,
auto_show=False):
"""API
Execute the evaluation and generates the evaluation reports classification report
with differet scores confusion matrix summary report
Ref:
https://github.com/matterport/Mask_RCNN/blob/master/samples/shapes/train_shapes.ipynb
"""
log.info("---------------------------->")
dnncfg = get_dnncfg(cmdcfg.config)
log_dir_path = apputil.get_abs_path(appcfg, cmdcfg, 'AI_LOGS')
cmdcfg['log_dir_path'] = log_dir_path
model = load_model_and_weights(mode, cmdcfg, appcfg)
save_viz_and_json = reportcfg['save_viz_and_json']
evaluate_no_of_result = reportcfg['evaluate_no_of_result']
filepath = reportcfg['filepath']
evaluate_run_summary = reportcfg['evaluate_run_summary']
log.info("evaluate_no_of_result: {}".format(evaluate_no_of_result))
detection_on_dataset = []
## TODO: put at right place
iou_threshold_input = reportcfg['iou_threshold']
# iou_thresholds = None
# iou_thresholds = iou_thresholds or np.arange(0.5, 1.0, 0.05)
iou_thresholds = np.arange(0.5, 1.0, 0.05)
size = len(iou_thresholds)
_mAPs, _precisions, _recalls = size * [None], size * [None], size * [None]
gt_total_annotation = 0
pred_total_annotation = 0
remaining_num_images = -1
image_ids = dataset.image_ids
num_images = len(image_ids)
pred_match_total_annotation = []
colors = viz.random_colors(len(class_names))
log.info("len(colors), colors: {},{}".format(len(colors), colors))
cc = dict(zip(class_names, colors))
## for some reasons if gets an error in iterating through dataset, all the hardwork is lost,
## therefore, save the data to disk in finally clause
via_jsonres = {}
imagelist = []
class_ids_dataset = dataset.class_ids
class_names_model = modelcfg['classes']
class_ids_model = np.arange(len(class_names_model))
log.debug("class_names dataset: {}".format(class_names))
log.debug("class_names_model: {}".format(class_names_model))
log.debug("class_ids_dataset: {}".format(class_ids_dataset))
log.debug("class_ids_model: {}".format(class_ids_model))
## class_names consists of BG at index 0
## class_names_model consists of BG at index 0
class_ids_map = False
class_names_common = class_names.copy()
if class_names != modelcfg['classes']:
class_ids_map = True
class_names_common, class_ids_common_dataset, class_ids_common_model, gt_to_model_map = class_ids_of_model_to_dataset(
np.array(class_names), class_ids_dataset,
np.array(class_names_model), class_ids_model)
log.info("class_names_common: {}".format(class_names_common))
log.info(
"class_ids_common_dataset: {}".format(class_ids_common_dataset))
log.info("class_ids_common_model: {}".format(class_ids_common_model))
## TODO: Exception handling: if class_ids_of_model_to_dataset length is 1 then only BG is common
class_names = np.array(class_names)
T0 = time.time()
try:
for i, image_id in enumerate(image_ids):
log.debug("-------")
filepath_image_in = dataset.image_reference(image_id)
image_filename = filepath_image_in.split(os.path.sep)[-1]
image_name_without_ext = image_filename.split('.')[0]
imagelist.append(filepath_image_in)
log.debug("Running on {}".format(image_filename))
if evaluate_no_of_result == i:
log.info("evaluate_no_of_result reached: i: {}\n".format(i))
break
remaining_num_images = evaluate_no_of_result if evaluate_no_of_result and evaluate_no_of_result > 0 else num_images
remaining_num_images = remaining_num_images - i - 1
log.info(
"To be evaluated remaining_num_images:...................{}".
format(remaining_num_images))
t0 = time.time()
# im, gt_class_ids, gt_boxes, gt_masks, gt_active_class_ids = load_image_gt_without_resizing(dataset, datacfg, dnncfg, image_id)
im, gt_image_meta, gt_class_ids, gt_boxes, gt_masks = modellib.load_image_gt(
dataset, datacfg, dnncfg, image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(im, dnncfg), 0)
if class_ids_map:
log.debug("Before gt_class_id_map...:")
log.debug(
"len(gt_class_ids): {}\nTotal Unique classes: len(set(gt_class_ids)): {}\ngt_class_ids: {}"
.format(len(gt_class_ids), len(set(gt_class_ids)),
gt_class_ids))
for _i, gt_id in enumerate(gt_class_ids):
gt_class_ids[_i] = gt_to_model_map[gt_id]
log.debug("After gt_class_id_map...:")
log.debug(
"len(gt_class_ids): {}\nTotal Unique classes: len(set(gt_class_ids)): {}\ngt_class_ids: {}"
.format(len(gt_class_ids), len(set(gt_class_ids)),
gt_class_ids))
t1 = time.time()
time_taken_imread = (t1 - t0)
log.debug('Total time taken in time_taken_imread: %f seconds' %
(time_taken_imread))
gt_total_annotation += len(gt_class_ids)
log.info("\nGround Truth-------->")
log.info("i,image_id:{},{}".format(i, image_id))
log.debug(
"len(gt_boxes), gt_boxes.shape, type(gt_boxes): {},{},{}".
format(len(gt_boxes), gt_boxes.shape, type(gt_boxes)))
log.debug(
"len(gt_masks), gt_masks.shape, type(gt_masks): {},{},{}".
format(len(gt_masks), gt_masks.shape, type(gt_masks)))
log.debug("gt_boxes: {}".format(gt_boxes))
log.debug("gt_masks: {}".format(gt_masks))
log.info("--------")
# Detect objects
##---------------------------------------------
t2 = time.time()
r = detect(model, im=im, verbose=1)[0]
## N - total number of predictions
## (N, 4)
pred_boxes = r['rois']
## (H, W, N)
pred_masks = r['masks']
## N
pred_class_ids = r['class_ids']
## N
pred_scores = r['scores']
log.debug("Prediction on Groud Truth-------->")
log.debug('len(r): {}'.format(len(r)))
log.debug("len(gt_class_ids), gt_class_ids: {},{}".format(
len(gt_class_ids), gt_class_ids))
log.debug(
"len(pred_class_ids), pred_class_ids, type(pred_class_ids): {},{},{}"
.format(len(pred_class_ids), pred_class_ids,
type(pred_class_ids)))
log.debug("len(pred_scores), pred_scores: {},{}".format(
len(pred_scores), pred_scores))
log.debug(
"len(pred_boxes), pred_boxes.shape, type(pred_boxes): {},{},{}"
.format(len(pred_boxes), pred_boxes.shape, type(pred_boxes)))
log.debug(
"len(pred_masks), pred_masks.shape, type(pred_masks): {},{},{}"
.format(len(pred_masks), pred_masks.shape, type(pred_masks)))
log.debug("--------")
if class_ids_map:
pred_class_ids_common_model_indices = np.where(
np.in1d(pred_class_ids, class_ids_common_model))[0]
class_ids_common_model_pred_class_ids_indices = np.where(
np.in1d(class_ids_common_model, pred_class_ids))[0]
# pred_class_ids_common_dataset_indices = np.where(np.in1d(class_ids_common_dataset, pred_class_ids_common_model_indices))[0]
pred_boxes = pred_boxes[pred_class_ids_common_model_indices]
pred_masks = pred_masks[...,
pred_class_ids_common_model_indices]
pred_class_ids = pred_class_ids[
pred_class_ids_common_model_indices]
pred_scores = pred_scores[pred_class_ids_common_model_indices]
log.debug(
"Prediction on Groud Truth: After Model class filtering-------->"
)
log.debug('len(r): {}'.format(len(r)))
log.debug(
"len(pred_class_ids_common_model_indices), pred_class_ids_common_model_indices: {},{}"
.format(len(pred_class_ids_common_model_indices),
pred_class_ids_common_model_indices))
log.debug(
"len(class_ids_common_model_pred_class_ids_indices), class_ids_common_model_pred_class_ids_indices: {},{}"
.format(len(class_ids_common_model_pred_class_ids_indices),
class_ids_common_model_pred_class_ids_indices))
log.debug(
"len(class_ids_common_dataset[class_ids_common_model_pred_class_ids_indices]), class_ids_common_dataset[class_ids_common_model_pred_class_ids_indices]: {},{}"
.format(
len(class_ids_common_dataset[
class_ids_common_model_pred_class_ids_indices]),
class_ids_common_dataset[
class_ids_common_model_pred_class_ids_indices]))
log.debug("len(gt_class_ids), gt_class_ids: {},{}".format(
len(gt_class_ids), gt_class_ids))
log.debug(
"len(pred_class_ids), pred_class_ids, type(pred_class_ids): {},{},{}"
.format(len(pred_class_ids), pred_class_ids,
type(pred_class_ids)))
log.debug("len(pred_scores), pred_scores: {},{}".format(
len(pred_scores), pred_scores))
log.debug(
"len(pred_boxes), pred_boxes.shape, type(pred_boxes): {},{},{}"
.format(len(pred_boxes), pred_boxes.shape,
type(pred_boxes)))
log.debug(
"len(pred_masks), pred_masks.shape, type(pred_masks): {},{},{}"
.format(len(pred_masks), pred_masks.shape,
type(pred_masks)))
log.debug("--------")
pred_total_annotation += len(pred_class_ids)
t3 = time.time()
time_taken_in_detect = (t3 - t2)
log.info('TIME_TAKEN_IN_DETECT:%f seconds' %
(time_taken_in_detect))
t4 = time.time()
## TODO: gt via_json resp and pred via jsn res separate data strucure
## TODO: mAP calculation for per class and over enitre dataset
## TODO: this does not help; need to flatten the all ground truts for eniter dataset.
## np.zeros(len(gt_for_all_images)), ideally same number of predictions sohuld be there
## Insort; have to re-write the compute_matches function for entire dataset
evaluate_run_summary['images'].append(image_filename)
# evaluate_run_summary['gt_boxes'].append(gt_boxes)
evaluate_run_summary['gt_class_ids'].append(list(gt_class_ids))
# evaluate_run_summary['gt_masks'].append(gt_masks)
# evaluate_run_summary['pred_boxes'].append(pred_boxes)
evaluate_run_summary['pred_class_ids'].append(list(pred_class_ids))
evaluate_run_summary['pred_scores'].append(list(pred_scores))
# evaluate_run_summary['pred_masks'].append(pred_masks)
evaluate_run_summary['gt_total_annotation_per_image'].append(
len(gt_class_ids))
evaluate_run_summary['pred_total_annotation_per_image'].append(
len(pred_class_ids))
detection_on_dataset_item = defaultdict(list)
__pred_match_total_annotation = np.zeros([len(iou_thresholds)],
dtype=int)
for count, iou_threshold in enumerate(iou_thresholds):
log.info("count, iou_threshold: {}, {}".format(
count, iou_threshold))
## Compute Average Precision at a set IoU threshold
## --------------------------------------------
AP_per_image, precisions, recalls, gt_match, pred_match, overlaps, pred_match_scores, pred_match_class_ids = utils.compute_ap(
gt_boxes,
gt_class_ids,
gt_masks,
pred_boxes,
pred_class_ids,
pred_scores,
pred_masks,
iou_threshold=iou_threshold)
__pred_match_total_annotation[count] += len(
pred_match_class_ids)
## compute and returns f1 score metric
## --------------------------------------------
f1_per_image = utils.compute_f1score(precisions, recalls)
## Compute the recall at the given IoU threshold. It's an indication
## of how many GT boxes were found by the given prediction boxes.
## --------------------------------------------
recall_bbox, positive_ids_bbox = utils.compute_recall(
gt_boxes, pred_boxes, iou_threshold)
# log.info("len(precisions),precisions: {},{}".format(len(precisions), precisions))
# log.info("len(recalls),recalls: {},{}".format(len(recalls), recalls))
# log.info("len(pred_match_class_ids),pred_match_class_ids: {},{}".format(len(pred_match_class_ids), pred_match_class_ids))
# log.info("AP_per_image: {}".format(AP_per_image))
# log.info("len(overlaps),overlaps: {},{}".format(len(overlaps), overlaps))
pred_match_class_names = class_names[np.where(
np.in1d(dataset.class_ids, pred_match_class_ids))[0]]
detection_on_dataset_item['ap_per_image'].append(AP_per_image)
detection_on_dataset_item['f1_per_image'].append(f1_per_image)
detection_on_dataset_item['precisions'].append(precisions)
detection_on_dataset_item['recalls'].append(list(recalls))
detection_on_dataset_item['recall_bbox'].append(recall_bbox)
detection_on_dataset_item['positive_ids_bbox'].append(
list(positive_ids_bbox))
detection_on_dataset_item['gt_match'].append(list(gt_match))
detection_on_dataset_item['pred_match'].append(
list(pred_match))
detection_on_dataset_item['pred_match_scores'].append(
list(pred_match_scores))
detection_on_dataset_item['overlaps_mask_iou'].append(
list(overlaps))
detection_on_dataset_item['pred_match_class_ids'].append(
list(pred_match_class_ids))
detection_on_dataset_item['pred_match_class_names'].append(
list(pred_match_class_names))
detection_on_dataset_item[
'pred_match_total_annotation'].append(
len(pred_match_class_ids))
detection_on_dataset_item['iou_thresholds'].append(
iou_threshold)
if save_viz_and_json and iou_threshold == float(
iou_threshold_input):
fext = ".png"
file_name = image_filename + fext
log.info("@IoU, SAVED_FILE_NAME: {},{}".format(
iou_threshold, file_name))
jsonres = viz.get_display_instances(im,
pred_boxes,
pred_masks,
pred_class_ids,
class_names_model,
pred_scores,
colors=cc,
show_bbox=True,
show_mask=True,
get_mask=get_mask,
filepath=filepath,
filename=file_name,
auto_show=auto_show)
## Convert Json response to VIA Json response
##---------------------------------------------
# size_image = 0
size_image = os.path.getsize(filepath_image_in)
jsonres["filename"] = image_filename
jsonres["size"] = size_image
jsonres['file_attributes']['iou'] = iou_threshold
## TODO: if want to store in mongoDB, '.' (dot) should not be present in the key in the json data
## but, to visualize the results in VIA tool, this (dot) and size is expected
# via_jsonres[image_filename.replace('.','-')+str(size_image)] = json.loads(common.numpy_to_json(jsonres))
via_jsonres[image_filename + str(size_image)] = json.loads(
common.numpy_to_json(jsonres))
# log.debug("jsonres: {}".format(jsonres))
# log.debug("via_jsonres[image_filename+str(size_image)]: {}".format(via_jsonres[image_filename+str(size_image)]))
detection_on_dataset.append(detection_on_dataset_item)
pred_match_total_annotation.append(__pred_match_total_annotation)
mean_ap_of_per_image, mean_f1_of_per_image, mean_recall_bbox_of_per_image, total_pred_match_total_annotation_of_per_image = compute_ap_of_per_image_over_dataset(
detection_on_dataset)
## TODO: use detection_on_dataset for evaluation over entire dataset and per class
## fix the TODO items within compute_ap_dataset
# _mAPs, _precisions, _recalls = compute_ap_dataset(detection_on_dataset, iou_thresholds)
log.info("---x-x---")
except Exception as e:
log.info("Exception: {}".format(e))
log.error("Fatal error in main loop".format(e), exc_info=True)
# log.error('Error occurred ' + str(e))
raise
finally:
log.info("--------X--------X--------X--------")
T1 = time.time()
evaluate_run_summary['total_execution_time'] = T1 - T0
evaluate_run_summary['mAP'] = _mAPs
evaluate_run_summary['precision'] = _precisions
evaluate_run_summary['recall'] = _recalls
evaluate_run_summary['class_names_dataset'] = class_names
evaluate_run_summary['class_ids_dataset'] = dataset.class_ids
evaluate_run_summary['class_names_model'] = class_names_model
evaluate_run_summary['class_ids_model'] = class_ids_model
evaluate_run_summary['class_names_common'] = class_names_common
evaluate_run_summary['mean_ap_of_per_image'] = mean_ap_of_per_image
evaluate_run_summary['mean_f1_of_per_image'] = mean_f1_of_per_image
evaluate_run_summary[
'mean_recall_bbox_of_per_image'] = mean_recall_bbox_of_per_image
evaluate_run_summary[
'total_pred_match_total_annotation_of_per_image'] = total_pred_match_total_annotation_of_per_image
evaluate_run_summary[
'pred_match_total_annotation'] = pred_match_total_annotation
evaluate_run_summary['gt_total_annotation'] = gt_total_annotation
evaluate_run_summary['pred_total_annotation'] = pred_total_annotation
evaluate_run_summary['iou_thresholds'] = iou_thresholds
evaluate_run_summary['execution_end_time'] = "{:%d%m%y_%H%M%S}".format(
datetime.datetime.now())
# evaluate_run_summary['detection_min_confidence'] = dnncfg.config['DETECTION_MIN_CONFIDENCE']
evaluate_run_summary['remaining_num_images'] = remaining_num_images
log.debug("evaluate_run_summary: {}".format(evaluate_run_summary))
classification_reportfile_path = reportcfg[
'classification_reportfile'] + '-per_dataset.json'
with open(classification_reportfile_path, 'w') as fw:
fw.write(common.numpy_to_json(detection_on_dataset))
evaluate_run_summary_reportfile_path = reportcfg[
'evaluate_run_summary_reportfile'] + '.json'
with open(evaluate_run_summary_reportfile_path, 'w') as fw:
fw.write(common.numpy_to_json(evaluate_run_summary))
## Save the image list for loading the response in VIA along with the images
imagelist_filepath = os.path.join(filepath, 'annotations',
"imagelist.csv")
pd.DataFrame(imagelist).to_csv(imagelist_filepath)
## https://stackoverflow.com/questions/12309269/how-do-i-write-json-data-to-a-file
via_jsonres_filepath = os.path.join(filepath, 'annotations',
"annotations.json")
if via_jsonres and len(via_jsonres) > 0:
with open(via_jsonres_filepath, 'w') as fw:
fw.write(json.dumps(via_jsonres))
print("EVALUATE_REPORT:ANNOTATION:{}".format(via_jsonres_filepath))
print("EVALUATE_REPORT:IMAGELIST:{}".format(imagelist_filepath))
print(
"EVALUATE_REPORT:METRIC:{}".format(classification_reportfile_path))
print("EVALUATE_REPORT:SUMMARY:{}".format(
evaluate_run_summary_reportfile_path))
log.info("--------")
return evaluate_run_summary