def score_bboxes(hypo_bboxes, gt_bboxes):
if len(hypo_bboxes) != len(gt_bboxes):
print('Error: the number of bboxes in GT and hypothesis differ!')
sys.exit()
bboxTP = 0
bboxFN = 0
bboxFP = 0
bbox_precision = 0
bbox_accuracy = 0
iou = 0
for ii in range(len(gt_bboxes)):
[local_bboxTP, local_bboxFN, local_bboxFP, local_iou
] = evalf.performance_accumulation_window(hypo_bboxes[ii],
gt_bboxes[ii])
bboxTP = bboxTP + local_bboxTP
bboxFN = bboxFN + local_bboxFN
bboxFP = bboxFP + local_bboxFP
iou = iou + local_iou
# Plot performance evaluation
[bbox_precision, bbox_sensitivity, bbox_accuracy
] = evalf.performance_evaluation_window(bboxTP, bboxFN, bboxFP)
iou = iou / bboxTP
bboxF1 = 0
if (bbox_precision + bbox_sensitivity) != 0:
bbox_f1 = 2 * ((bbox_precision * bbox_sensitivity) /
(bbox_precision + bbox_sensitivity))
return bbox_precision, bbox_sensitivity, bbox_f1, bbox_accuracy, iou
def compute_metrics_general(gt, bboxes, k=10, iou_thresh=0.5):
"""
Compute Fscore, IoU, Map of two lists of Bounding Boxes.
:param gt: list of GT bounding boxes
:param bboxes: list of result Bboxes
:param k: Map at k
:return: Noisy Bboxes, Fscore, IoU, MaP
"""
bboxTP, bboxFN, bboxFP = evalf.performance_accumulation_window(
bboxes, gt, iou_thresh)
#print(bboxTP, bboxFN, bboxFP )
"""
Compute F-score of GT against modified bboxes PER FRAME NUMBER
"""
fscore_val = ut.fscore(bboxTP, bboxFN, bboxFP)
"""
Compute IoU of GT against modified Bboxes PER FRAME NUMBER:
"""
iou = ut.iterate_iou(bboxes, gt)
"""
Compute mAP of GT against modified bboxes PER FRAME NUMBER:
"""
map = ut.mapk(bboxes, gt, k)
return (fscore_val, iou, map, bboxTP, bboxFN, bboxFP)
def compute_metrics(gt, img_shape, noise_size=True, noise_size_factor=5, noise_position=True,
noise_position_factor=5, create_bbox_proba=0.5, destroy_bbox_proba=0.5, k=10, iou_thresh=0.5):
"""
1. Add noise to ground truth Bounding Boxes.
2.Compute Fscore, IoU, Map of two lists of Bounding Boxes.
:param gt: list of GT bounding boxes
:param img_shape: Original image shape
:param noise_size: Change bbox size param
:param noise_position: Increase bbox size param
:param destroy_bbox_proba: Proba of destroying Bboxes
:param create_bbox_proba: Proba of creating Bboxes
:param k: Map at k
:return: Noisy Bboxes, Fscore, IoU, MaP
"""
# Add noise to GT depending on noise parameter
bboxes = add_noise_to_bboxes(gt, img_shape,
noise_size=noise_size,
noise_size_factor=noise_size_factor,
noise_position=noise_position,
noise_position_factor=noise_position_factor)
# Randomly create and destroy bounding boxes depending
# on probability parameter
bboxes = create_bboxes(bboxes, img_shape, prob=create_bbox_proba)
bboxes = destroy_bboxes(bboxes, prob=destroy_bbox_proba)
bboxTP, bboxFN, bboxFP = evalf.performance_accumulation_window(bboxes, gt, iou_thresh)
print(bboxTP, bboxFN, bboxFP )
"""
Compute F-score of GT against modified bboxes PER FRAME NUMBER
"""
# ToDo: Add dependency on frame number
fscore_val = fscore(bboxTP, bboxFN, bboxFP)
"""
Compute IoU of GT against modified Bboxes PER FRAME NUMBER:
"""
iou = iterate_iou(bboxes, gt)
"""
Compute mAP of GT against modified bboxes PER FRAME NUMBER:
"""
map = mapk(bboxes, gt, k)
return (bboxes, fscore_val, iou, map)
pixelTN = pixelTN + localPixelTN
if window_evaluation == 1:
# Read .pkl file
name_r, ext_r = os.path.splitext(result_files[ii])
pkl_name = '{}/{}/{}.pkl'.format(results_dir, method, name_r)
with open(pkl_name, "rb") as fp: # Unpickling
windowCandidates = pickle.load(fp)
gt_annotations_name = '{}/gt/gt.{}.txt'.format(test_dir, name)
windowAnnotations = load_annotations(gt_annotations_name)
[localWindowTP, localWindowFN, localWindowFP] = evalf.performance_accumulation_window(windowCandidates, windowAnnotations)
windowTP = windowTP + localWindowTP
windowFN = windowFN + localWindowFN
windowFP = windowFP + localWindowFP
# Plot performance evaluation
[pixelPrecision, pixelAccuracy, pixelSpecificity, pixelSensitivity] = evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN, pixelTN)
pixelF1 = 0
if (pixelPrecision + pixelSensitivity) != 0:
pixelF1 = 2*((pixelPrecision*pixelSensitivity)/(pixelPrecision + pixelSensitivity))
print ('Team {:02d} pixel, method {} : {:.2f}, {:.2f}, {:.2f}\n'.format(team, method, pixelPrecision, pixelSensitivity, pixelF1))
if window_evaluation == 1:
[windowPrecision, windowSensitivity, windowAccuracy] = evalf.performance_evaluation_window(windowTP, windowFN, windowFP) # (Needed after Week 3)
windowF1 = 0
def traffic_sign_detection(directory, output_dir, pixel_method, window_method):
pixelTP = 0
pixelFN = 0
pixelFP = 0
pixelTN = 0
windowTP = 0
windowFN = 0
windowFP = 0
window_precision = 0
window_accuracy = 0
# Load image names in the given directory
file_names = sorted(fnmatch.filter(os.listdir(directory), '*.jpg'))
pixel_time = 0
for name in file_names:
base, extension = os.path.splitext(name)
# Read file
im = imageio.imread('{}/{}'.format(directory, name))
print('{}/{}'.format(directory, name))
# Candidate Generation (pixel) ######################################
start = time.time()
pixel_candidates = candidate_generation_pixel(im, pixel_method)
end = time.time()
pixel_time += (end - start)
fd = '{}/{}_{}'.format(output_dir, pixel_method, window_method)
if not os.path.exists(fd):
os.makedirs(fd)
out_mask_name = '{}/{}.png'.format(fd, base)
imageio.imwrite(out_mask_name, np.uint8(np.round(pixel_candidates)))
if window_method != 'None':
window_candidates = candidate_generation_window(
im, pixel_candidates, window_method)
out_list_name = '{}/{}.pkl'.format(fd, base)
with open(out_list_name, "wb") as fp: #Pickling
pickle.dump(window_candidates, fp)
# Accumulate pixel performance of the current image #################
pixel_annotation = imageio.imread('{}/mask/mask.{}.png'.format(
directory, base)) > 0
[localPixelTP, localPixelFP, localPixelFN, localPixelTN
] = evalf.performance_accumulation_pixel(pixel_candidates,
pixel_annotation)
pixelTP = pixelTP + localPixelTP
pixelFP = pixelFP + localPixelFP
pixelFN = pixelFN + localPixelFN
pixelTN = pixelTN + localPixelTN
[
pixel_precision, pixel_accuracy, pixel_recall, pixel_specificity,
pixel_sensitivity, pixel_F1, pixel_TP, pixel_FP, pixel_FN
] = evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN,
pixelTN)
if window_method != 'None':
# Accumulate object performance of the current image ################
window_annotationss = load_annotations('{}/gt/gt.{}.txt'.format(
directory, base))
[localWindowTP, localWindowFN, localWindowFP
] = evalf.performance_accumulation_window(window_candidates,
window_annotationss)
windowTP = windowTP + localWindowTP
windowFN = windowFN + localWindowFN
windowFP = windowFP + localWindowFP
# Plot performance evaluation
[window_precision, window_sensitivity, window_accuracy
] = evalf.performance_evaluation_window(windowTP, windowFN,
windowFP)
pixel_time /= len(file_names)
return [
pixel_precision, pixel_accuracy, pixel_recall, pixel_specificity,
pixel_sensitivity, pixel_F1, pixel_TP, pixel_FP, pixel_FN, pixel_time
]
def traffic_sign_detection_test(directory, output_dir, pixel_method,
window_method):
"""
Calculates all statistical evaluation metrics of different pixel selector method (TRAINING AND VALIDATION)
* Inputs:
- directory = path to train images
- outpit_dir = Directory where to store output masks, etc. For instance '~/m1-results/week1/test'
- pixel_method = pixel method that will segmentate the image
- window_method = -------
*Outputs:
- pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity, window_precision, window_accuracy
"""
from main import CONSOLE_ARGUMENTS
pixelTP = 0
pixelFN = 0
pixelFP = 0
pixelTN = 0
windowTP = 0
windowFN = 0
windowFP = 0
window_precision = 0
window_accuracy = 0
print("splitting in trainning test")
# Load image names in the given directory
file_names = sorted(fnmatch.filter(os.listdir(directory), '*.jpg'))
signals_type_dict = get_dictionary()
training, validation = [], []
for key in signals_type_dict:
sig_subdict = signals_type_dict[key]
training_type, validation_type = divide_training_validation_SL(
sig_subdict['signal_list'])
training.extend(training_type)
validation.extend(validation_type)
print("extracting mask")
dataset = training
if (CONSOLE_ARGUMENTS.use_validation):
dataset = validation
# if(CONSOLE_ARGUMENTS.use_test):
totalTime = 0
for signal in dataset:
signal_path = signal.img_orig_path
_, name = signal_path.rsplit('/', 1)
base, extension = os.path.splitext(name)
# Read file
im = imageio.imread('{}/{}'.format(directory, name))
print('{}/{}'.format(directory, name))
# Candidate Generation (pixel) ######################################
start = time.time()
pixel_candidates = candidate_generation_pixel(im, pixel_method)
totalTime += time.time() - start
fd = '{}/{}_{}'.format(output_dir, pixel_method, window_method)
if not os.path.exists(fd):
os.makedirs(fd)
out_mask_name = '{}/{}.png'.format(fd, base)
imageio.imwrite(out_mask_name, np.uint8(np.round(pixel_candidates)))
if window_method != 'None':
window_candidates = candidate_generation_window(
im, pixel_candidates, window_method)
# Accumulate pixel performance of the current image #################
pixel_annotation = imageio.imread('{}/mask/mask.{}.png'.format(
directory, base)) > 0
[localPixelTP, localPixelFP, localPixelFN, localPixelTN
] = evalf.performance_accumulation_pixel(pixel_candidates,
pixel_annotation)
pixelTP = pixelTP + localPixelTP
pixelFP = pixelFP + localPixelFP
pixelFN = pixelFN + localPixelFN
pixelTN = pixelTN + localPixelTN
[
pixel_precision, pixel_accuracy, pixel_specificity,
pixel_sensitivity
] = evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN,
pixelTN)
if window_method != 'None':
# Accumulate object performance of the current image ################
window_annotationss = load_annotations('{}/gt/gt.{}.txt'.format(
directory, base))
[localWindowTP, localWindowFN, localWindowFP
] = evalf.performance_accumulation_window(window_candidates,
window_annotationss)
windowTP = windowTP + localWindowTP
windowFN = windowFN + localWindowFN
windowFP = windowFP + localWindowFP
# Plot performance evaluation
[window_precision, window_sensitivity, window_accuracy
] = evalf.performance_evaluation_window(windowTP, windowFN,
windowFP)
print("meanTime", totalTime / len(dataset))
print("pixelTP", pixelTP, "\t", pixelFP, "\t", pixelFN)
return [
pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity,
window_precision, window_accuracy
]
def traffic_sign_detection(directory, output_dir, pixel_method, window_method,
calculate_metrics):
pixelTP = 0
pixelFN = 0
pixelFP = 0
pixelTN = 0
pixel_F1 = 0
windowTP = 0
windowFN = 0
windowFP = 0
window_precision = 0
window_accuracy = 0
window_F1 = 0
counter = 0
# Load image names in the given directory
file_names = sorted(fnmatch.filter(os.listdir(directory), '*.jpg'))
for name in file_names:
counter += 1
base, extension = os.path.splitext(name)
# Read file
im = imageio.imread('{}/{}'.format(directory, name))
#print ('{}/{}'.format(directory,name))
# Candidate Generation (pixel) ######################################
pixel_candidates = candidate_generation_pixel(im, pixel_method)
# pixel_candidates = morphological_operators(pixel_candidates)
pixel_candidates = connected_labels_pixel_cand(im, pixel_candidates)
fd = '{}/{}_{}'.format(output_dir, pixel_method, window_method)
if not os.path.exists(fd):
os.makedirs(fd)
out_mask_name = '{}/{}.png'.format(fd, base)
if window_method != 'None':
window_candidates = candidate_generation_window(
im, pixel_candidates, window_method)
window_mask = np.zeros(pixel_candidates.shape)
for window_candidate in window_candidates:
window_mask[window_candidate[0]:window_candidate[2],
window_candidate[1]:
window_candidate[3]] = pixel_candidates[
window_candidate[0]:window_candidate[2],
window_candidate[1]:window_candidate[3]]
out_list_name = '{}/{}.pkl'.format(fd, base)
pixel_candidates = window_mask
with open(out_list_name, "wb") as fp: #Pickling
pickle.dump(window_candidates, fp)
imageio.imwrite(out_mask_name, np.uint8(np.round(pixel_candidates)))
pixel_precision = 0
pixel_accuracy = 0
pixel_specificity = 0
pixel_sensitivity = 0
window_precision = 0
window_accuracy = 0
if (calculate_metrics):
# Accumulate pixel performance of the current image #################
pixel_annotation = imageio.imread('{}/mask/mask.{}.png'.format(
directory, base)) > 0
[localPixelTP, localPixelFP, localPixelFN, localPixelTN
] = evalf.performance_accumulation_pixel(pixel_candidates,
pixel_annotation)
pixelTP = pixelTP + localPixelTP
pixelFP = pixelFP + localPixelFP
pixelFN = pixelFN + localPixelFN
pixelTN = pixelTN + localPixelTN
# [pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity, pixel_F1] = evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN, pixelTN)
if window_method != 'None':
# Accumulate object performance of the current image ################
window_annotationss = load_annotations(
'{}/gt/gt.{}.txt'.format(directory, base))
[localWindowTP, localWindowFN,
localWindowFP] = evalf.performance_accumulation_window(
window_candidates, window_annotationss)
windowTP = windowTP + localWindowTP
windowFN = windowFN + localWindowFN
windowFP = windowFP + localWindowFP
# Plot performance evaluation
# [window_precision, window_sensitivity, window_accuracy, window_F1] = evalf.performance_evaluation_window(windowTP, windowFN, windowFP)
if (calculate_metrics):
[
pixel_precision, pixel_accuracy, pixel_specificity,
pixel_sensitivity, pixel_F1
] = evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN,
pixelTN)
print("Pixel precision: " + str(pixel_precision))
print("Pixel accuracy: " + str(pixel_accuracy))
print("Pixel recall: " + str(pixel_sensitivity))
print("Pixel F1-measure: " + str(pixel_F1))
print("Pixel TP: " + str(pixelTP))
print("Pixel FP: " + str(pixelFP))
print("Pixel FN: " + str(pixelFN))
print("Pixel TN: " + str(pixelTN))
if window_method != 'None':
[window_precision, window_sensitivity, window_accuracy, window_F1
] = evalf.performance_evaluation_window(windowTP, windowFN,
windowFP)
print("Window precision: " + str(window_precision))
print("Window accuracy: " + str(window_accuracy))
print("Window F1-measure: " + str(window_F1))
return [
pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity,
pixel_F1, window_precision, window_accuracy, window_F1, counter
]
pixelTN = pixelTN + localPixelTN
if window_evaluation == 1:
# Read .pkl file
name_r, ext_r = os.path.splitext(result_files[ii])
pkl_name = '{}/{}/{}.pkl'.format(results_dir, method, name_r)
with open(pkl_name, "rb") as fp: # Unpickling
windowCandidates = pickle.load(fp)
gt_annotations_name = '{}/gt/gt.{}.txt'.format(test_dir, name)
windowAnnotations = load_annotations(gt_annotations_name)
[localWindowTP, localWindowFN, localWindowFP
] = evalf.performance_accumulation_window(windowCandidates,
windowAnnotations)
windowTP = windowTP + localWindowTP
windowFN = windowFN + localWindowFN
windowFP = windowFP + localWindowFP
# Plot performance evaluation
[pixelPrecision, pixelAccuracy, pixelSpecificity, pixelSensitivity
] = evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN, pixelTN)
pixelF1 = 2 * ((pixelPrecision * pixelSensitivity) /
max(pixelPrecision + pixelSensitivity, 1))
print('Team {:02d} pixel, method {} : {:.2f}, {:.2f}, {:.2f}\n'.format(
team, method, pixelPrecision, pixelSensitivity, pixelF1))
if window_evaluation == 1:
[windowPrecision, windowSensitivity,
def traffic_sign_detection_test(directory, output_dir, pixel_method, window_method, use_dataset="training"):
"""
Calculates all statistical evaluation metrics of different pixel selector method (TRAINING AND VALIDATION)
* Inputs:
- directory = path to train images
- outpit_dir = Directory where to store output masks, etc. For instance '~/m1-results/week1/test'
- pixel_method = pixel method that will segmentate the image
- window_method = -------
*Outputs:
- pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity, window_precision, window_accuracy
"""
pixelTP = 0
pixelFN = 0
pixelFP = 0
pixelTN = 0
windowTP = 0
windowFN = 0
windowFP = 0
window_precision = 0
window_accuracy = 0
window_sensitivity = 0
# print("splitting in trainning test")
# Load image names in the given directory
# file_names = sorted(fnmatch.filter(os.listdir(directory), '*.jpg'))
signals_type_dict = get_dictionary()
training, validation = [], []
for key in signals_type_dict:
sig_subdict = signals_type_dict[key]
training_type, validation_type = divide_training_validation_SL(sig_subdict['signal_list'])
training.extend(training_type)
validation.extend(validation_type)
# print("extracting mask")
dataset = training
if(use_dataset == 'validation'):
dataset = validation
# if(CONSOLE_ARGUMENTS.use_test):
totalTime = 0
dataset_paths = [signal.img_orig_path for signal in dataset]
for signal_path in tqdm(dataset_paths, ascii=True, desc="Calculating Statistics"):
startTime = time.time()
rgb_mask, bb_list = get_pixel_candidates(signal_path)
totalTime = time.time() - startTime
if(bb_list is not None): bb_list = convertBBFormat(bb_list)
_, name = signal_path.rsplit('/', 1)
base, extension = os.path.splitext(name)
# Accumulate pixel performance of the current image #################
pixel_annotation = imageio.imread('{}/mask/mask.{}.png'.format(directory,base)) > 0
[localPixelTP, localPixelFP, localPixelFN, localPixelTN] =\
evalf.performance_accumulation_pixel(rgb_mask, pixel_annotation)
pixelTP = pixelTP + localPixelTP
pixelFP = pixelFP + localPixelFP
pixelFN = pixelFN + localPixelFN
pixelTN = pixelTN + localPixelTN
[pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity] =\
evalf.performance_evaluation_pixel(pixelTP, pixelFP, pixelFN, pixelTN)
if(bb_list != None):
# Accumulate object performance of the current image ################
window_annotationss = load_annotations('{}/gt/gt.{}.txt'.format(directory, base))
[localWindowTP, localWindowFN, localWindowFP] = \
evalf.performance_accumulation_window(bb_list, window_annotationss)
windowTP = windowTP + localWindowTP
windowFN = windowFN + localWindowFN
windowFP = windowFP + localWindowFP
# Plot performance evaluation
[window_precision, window_sensitivity, window_accuracy] = \
evalf.performance_evaluation_window(windowTP, windowFN, windowFP)
print("meanTime", totalTime/len(dataset))
print("pixelTP", pixelTP, "\t", pixelFP, "\t", pixelFN)
print("windowTP", windowTP, "\t", windowFP, "\t", windowFN)
return [pixel_precision, pixel_accuracy, pixel_specificity, pixel_sensitivity,\
window_precision, window_accuracy, window_sensitivity]
