def calculate_map(gt, det): # Get groundtruth boxes allBoundingBoxes, allClasses = getBoundingBoxes(gt, True, gtFormat, gtCoordType, imgSize=imgSize) # Get detected boxes allBoundingBoxes, allClasses = getBoundingBoxes(det, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve(allBoundingBoxes, IOUThreshold=0.5, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, ) # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}%".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) mAP = acc_AP / validClasses mAP_str = "{0:.2f}%".format(mAP * 100) return mAP
detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision= False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=showPlot) f = open(os.path.join(savePath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write('https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') f.write('Average Precision (AP), Precision and Recall per class:') # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class']
f = open(os.path.join(savePath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write('https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') f.write('Average Precision (AP), Precision and Recall per class:') evaluator = Evaluator() acc_AP = 0 validClasses = 0 # for each class for c in allClasses: # Plot Precision x Recall curve metricsPerClass = evaluator.PlotPrecisionRecallCurve( c, # Class to show allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision= False, # Don't plot the interpolated precision curve savePath=os.path.join(savePath, c + '.png'), showGraphic=showPlot) # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1
def evaluation(gtFolder, detFolder, iouThreshold, gtFormat, detFormat, savePath, confidence_TH, range=None): # Get current path to set default folders currentPath = os.path.dirname(os.path.abspath(__file__)) #confidence_TH = 0.8 VERSION = '0.1 (beta)' gtCoordType = 'abs' detCoordType = 'abs' showPlot = False # parser = argparse.ArgumentParser( # prog='Object Detection Metrics - Pascal VOC', # description='This project applies the most popular metrics used to evaluate object detection ' # 'algorithms.\nThe current implemention runs the Pascal VOC metrics.\nFor further references, ' # 'please check:\nhttps://github.com/rafaelpadilla/Object-Detection-Metrics', # epilog="Developed by: Rafael Padilla ([email protected])") # # formatter_class=RawTextHelpFormatter) # parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + VERSION) # # Positional arguments # # Mandatory # parser.add_argument( # '-gt', # '--gtfolder', # dest='gtFolder', # default=os.path.join(currentPath, 'groundtruths'), # metavar='', # help='folder containing your ground truth bounding boxes') # parser.add_argument( # '-det', # '--detfolder', # dest='detFolder', # default=os.path.join(currentPath, 'detections'), # metavar='', # help='folder containing your detected bounding boxes') # # Optional # parser.add_argument( # '-t', # '--threshold', # dest='iouThreshold', # type=float, # default=0.5, # metavar='', # help='IOU threshold. Default 0.5') # parser.add_argument( # '-gtformat', # dest='gtFormat', # metavar='', # default='xywh', # help='format of the coordinates of the ground truth bounding boxes: ' # '(\'xywh\': <left> <top> <width> <height>)' # ' or (\'xyrb\': <left> <top> <right> <bottom>)') # parser.add_argument( # '-detformat', # dest='detFormat', # metavar='', # default='xywh', # help='format of the coordinates of the detected bounding boxes ' # '(\'xywh\': <left> <top> <width> <height>) ' # 'or (\'xyrb\': <left> <top> <right> <bottom>)') # parser.add_argument( # '-gtcoords', # dest='gtCoordinates', # default='abs', # metavar='', # help='reference of the ground truth bounding box coordinates: absolute ' # 'values (\'abs\') or relative to its image size (\'rel\')') # parser.add_argument( # '-detcoords', # default='abs', # dest='detCoordinates', # metavar='', # help='reference of the ground truth bounding box coordinates: ' # 'absolute values (\'abs\') or relative to its image size (\'rel\')') # parser.add_argument( # '-imgsize', # dest='imgSize', # metavar='', # help='image size. Required if -gtcoords or -detcoords are \'rel\'') # parser.add_argument( # '-sp', '--savepath', dest='savePath', metavar='', help='folder where the plots are saved') # parser.add_argument( # '-np', # '--noplot', # dest='showPlot', # action='store_false', # help='no plot is shown during execution') # args = parser.parse_args() # # iouThreshold = args.iouThreshold # Arguments validation errors = [] # # Validate formats gtFormat = ValidateFormats(gtFormat, '-gtformat', errors) detFormat = ValidateFormats(detFormat, '-detformat', errors) # # Groundtruth folder # if ValidateMandatoryArgs(args.gtFolder, '-gt/--gtfolder', errors): # gtFolder = ValidatePaths(args.gtFolder, '-gt/--gtfolder', errors) # else: # # errors.pop() # gtFolder = os.path.join(currentPath, 'groundtruths') # if os.path.isdir(gtFolder) is False: # errors.append('folder %s not found' % gtFolder) # # Coordinates types gtCoordType = ValidateCoordinatesTypes(gtCoordType, '-gtCoordinates', errors) detCoordType = ValidateCoordinatesTypes(gtCoordType, '-detCoordinates', errors) imgSize = (0, 0) # if gtCoordType == CoordinatesType.Relative: # Image size is required # imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-gtCoordinates', errors) # if detCoordType == CoordinatesType.Relative: # Image size is required # imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-detCoordinates', errors) # # Detection folder # if ValidateMandatoryArgs(args.detFolder, '-det/--detfolder', errors): # detFolder = ValidatePaths(args.detFolder, '-det/--detfolder', errors) # else: # # errors.pop() # detFolder = os.path.join(currentPath, 'detections') # if os.path.isdir(detFolder) is False: # errors.append('folder %s not found' % detFolder) # print(args.savePath) if savePath is not None: print(savePath) if not os.path.exists(savePath): os.makedirs(savePath) #savePath = ValidatePaths(savePath, '-sp/--savepath', errors) else: savePath = os.path.join(currentPath, 'results') # Validate savePath # If error, show error messages # if len(errors) is not 0: # print("""usage: Object Detection Metrics [-h] [-v] [-gt] [-det] [-t] [-gtformat] # [-detformat] [-save]""") # print('Object Detection Metrics: error(s): ') # # print(e) for e in errors # for e in errors: # print(e) # sys.exit() # Create directory to save results shutil.rmtree(savePath, ignore_errors=True) # Clear folder os.makedirs(savePath) # Show plot during execution #showPlot = args.showPlot # print('iouThreshold= %f' % iouThreshold) # print('savePath = %s' % savePath) # print('gtFormat = %s' % gtFormat) # print('detFormat = %s' % detFormat) # print('gtFolder = %s' % gtFolder) # print('detFolder = %s' % detFolder) # print('gtCoordType = %s' % gtCoordType) # print('detCoordType = %s' % detCoordType) # print('showPlot %s' % showPlot) # Get groundtruth boxes allBoundingBoxes, allClasses = getBoundingBoxes(gtFolder, True, gtFormat, gtCoordType, imgSize=imgSize, range=range) # Get detected boxes allBoundingBoxes, allClasses = getBoundingBoxes( detFolder, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize, confidence_TH=confidence_TH, range=range) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 output_str = '' if iouThreshold == 0: AP_sum = 0 for iouTH in np.arange(0.5, 0.96, 0.05): detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouTH, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision= False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=showPlot) for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}%".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) #print('AP: %s (%s)' % (ap_str, cl)) mAP = acc_AP / validClasses print('mAP: {0:.2f}%'.format(mAP * 100)) AP_sum += mAP mAP_str = "{0:.2f}%".format(AP_sum * 10) print('Total mAP: %s' % mAP_str) else: # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision= False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=showPlot) f = open(os.path.join(savePath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write( 'https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') f.write('Average Precision (AP), Precision and Recall per class:') # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] print(cl) ap = metricsPerClass['AP'] print(ap) precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap print(acc_AP) prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) prec_1 = 0 rec_1 = 0 if (len(precision) > 0): prec_1 = precision[-1] if (len(recall) > 0): rec_1 = recall[-1] fscore = 0 if (prec_1 != 0 and rec_1 != 0): fscore = (2 * prec_1 * rec_1) / (prec_1 + rec_1) rec_str = '{0:.2f}'.format(rec_1 * 100) prec_str = '{0:.2f}'.format(prec_1 * 100) fscore_str = '{0:.2f}'.format(fscore * 100) #print('precision {0:.2f}%' .format( precision[-1]*100)) #print('recall {0:.2f}%' .format(recall[-1]*100)) #print('fscore={0:.2f}%'.format(fscore * 100)) #print('AP: %s (%s)' % (ap_str, cl)) #output_str = ap_str+','+prec_str+','+rec_str+','+fscore_str output_str = ap_str + ',' + fscore_str f.write('\n\nClass: %s' % cl) f.write('\nAP: %s' % ap_str) f.write('\nPrecision: %s' % prec) f.write('\nRecall: %s' % rec) mAP = acc_AP / validClasses mAP_str = "{0:.2f}%".format(mAP * 100) #print('mAP: %s' % mAP_str) f.write('\n\n\nmAP: %s' % mAP_str) return output_str
# Read txt files containing bounding boxes (ground truth and detections) boundingboxes = getBoundingBoxes() # Uncomment the line below to generate images based on the bounding boxes # createImages(dictGroundTruth, dictDetected) # Create an evaluator object in order to obtain the metrics evaluator = Evaluator() ############################################################## # VOC PASCAL Metrics ############################################################## # Plot Precision x Recall curve evaluator.PlotPrecisionRecallCurve( boundingboxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=0.3, # IOU threshold method=MethodAveragePrecision. EveryPointInterpolation, # As the official matlab code showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision=True) # Plot the interpolated precision curve # Get metrics with PASCAL VOC metrics metricsPerClass = evaluator.GetPascalVOCMetrics( boundingboxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=0.3, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation ) # As the official matlab code print("Average precision values per class:\n") # Loop through classes to obtain their metrics for mc in metricsPerClass: # Get metric values per each class c = mc['class'] precision = mc['precision']
def get_mAP(basedir, gt_dir, det_dir, sp, threshold=0.5): VERSION = '0.1 (beta)' global currentPath currentPath = basedir parser = argparse.ArgumentParser( prog='Object Detection Metrics - Pascal VOC', description='This project applies the most popular metrics used to evaluate object detection ' 'algorithms.\nThe current implemention runs the Pascal VOC metrics.\nFor further references, ' 'please check:\nhttps://github.com/rafaelpadilla/Object-Detection-Metrics', epilog="Developed by: Rafael Padilla ([email protected])") # formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + VERSION) # Positional arguments # Mandatory parser.add_argument( '-gt', '--gtfolder', dest='gtFolder', default=gt_dir, metavar='', help='folder containing your ground truth bounding boxes') parser.add_argument( '-det', '--detfolder', dest='detFolder', default=det_dir, metavar='', help='folder containing your detected bounding boxes') # Optional parser.add_argument( '-t', '--threshold', dest='iouThreshold', type=float, default=threshold, metavar='', help='IOU threshold. Default 0.5') parser.add_argument( '-gtformat', dest='gtFormat', metavar='', default='xyrb', help='format of the coordinates of the ground truth bounding boxes: ' '(\'xywh\': <left> <top> <width> <height>)' ' or (\'xyrb\': <left> <top> <right> <bottom>)') parser.add_argument( '-detformat', dest='detFormat', metavar='', default='xyrb', help='format of the coordinates of the detected bounding boxes ' '(\'xywh\': <left> <top> <width> <height>) ' 'or (\'xyrb\': <left> <top> <right> <bottom>)') parser.add_argument( '-gtcoords', dest='gtCoordinates', default='abs', metavar='', help='reference of the ground truth bounding box coordinates: absolute ' 'values (\'abs\') or relative to its image size (\'rel\')') parser.add_argument( '-detcoords', default='abs', dest='detCoordinates', metavar='', help='reference of the ground truth bounding box coordinates: ' 'absolute values (\'abs\') or relative to its image size (\'rel\')') parser.add_argument( '-imgsize', dest='imgSize', default=300, metavar='', help='image size. Required if -gtcoords or -detcoords are \'rel\'') parser.add_argument( '-sp', '--savepath', dest='savePath', metavar='', default=sp, help='folder where the plots are saved') parser.add_argument( '-np', '--noplot', dest='showPlot', action='store_false', help='no plot is shown during execution') args, unknown = parser.parse_known_args() iouThreshold = args.iouThreshold # Arguments validation errors = [] # Validate formats gtFormat = ValidateFormats(args.gtFormat, '-gtformat', errors) detFormat = ValidateFormats(args.detFormat, '-detformat', errors) # Groundtruth folder if ValidateMandatoryArgs(args.gtFolder, '-gt/--gtfolder', errors): gtFolder = ValidatePaths(args.gtFolder, '-gt/--gtfolder', errors) else: # errors.pop() gtFolder = os.path.join(currentPath, 'groundtruths') if os.path.isdir(gtFolder) is False: errors.append('folder %s not found' % gtFolder) # Coordinates types gtCoordType = ValidateCoordinatesTypes(args.gtCoordinates, '-gtCoordinates', errors) detCoordType = ValidateCoordinatesTypes(args.detCoordinates, '-detCoordinates', errors) imgSize = (0, 0) if gtCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-gtCoordinates', errors) if detCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-detCoordinates', errors) # Detection folder if ValidateMandatoryArgs(args.detFolder, '-det/--detfolder', errors): detFolder = ValidatePaths(args.detFolder, '-det/--detfolder', errors) else: # errors.pop() detFolder = os.path.join(currentPath, 'detections') if os.path.isdir(detFolder) is False: errors.append('folder %s not found' % detFolder) if args.savePath is not None: savePath = ValidatePaths(args.savePath, '-sp/--savepath', errors) else: savePath = os.path.join(currentPath, 'results') # Validate savePath # If error, show error messages if len(errors) is not 0: print("""usage: Object Detection Metrics [-h] [-v] [-gt] [-det] [-t] [-gtformat] [-detformat] [-save]""") print('Object Detection Metrics: error(s): ') [print(e) for e in errors] sys.exit() # Create directory to save results shutil.rmtree(savePath, ignore_errors=True) # Clear folder os.makedirs(savePath) # Show plot during execution showPlot = args.showPlot # print('iouThreshold= %f' % iouThreshold) # print('savePath = %s' % savePath) # print('gtFormat = %s' % gtFormat) # print('detFormat = %s' % detFormat) # print('gtFolder = %s' % gtFolder) # print('detFolder = %s' % detFolder) # print('gtCoordType = %s' % gtCoordType) # print('detCoordType = %s' % detCoordType) # print('showPlot %s' % showPlot) # Get groundtruth boxes allBoundingBoxes, allClasses = getBoundingBoxes( gtFolder, True, gtFormat, gtCoordType, imgSize=imgSize) # Get detected boxes allBoundingBoxes, allClasses = getBoundingBoxes( detFolder, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision=False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=showPlot) f = open(os.path.join(savePath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write('https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') f.write('Average Precision (AP), Precision and Recall per class:') # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}%".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) print('AP: %s (%s)' % (ap_str, cl)) f.write('\n\nClass: %s' % cl) f.write('\nAP: %s' % ap_str) f.write('\nPrecision: %s' % prec) f.write('\nRecall: %s' % rec) mAP = acc_AP / validClasses mAP_str = "{0:.2f}%".format(mAP * 100) print('mAP: %s' % mAP_str) f.write('\n\n\nmAP: %s' % mAP_str) return mAP
# Read txt files containing bounding boxes (ground truth and detections) boundingboxes = getBoundingBoxes() # Uncomment the line below to generate images based on the bounding boxes #createImages(dictGroundTruth, dictDetected) # Create an evaluator object in order to obtain the metrics evaluator = Evaluator() ############################################################## # VOC PASCAL Metrics ############################################################## # Plot Precision x Recall curve evaluator.PlotPrecisionRecallCurve( 'object', # Class to show boundingboxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=0.3, # IOU threshold showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision=False ) # Don't plot the interpolated precision curve # Get metrics with PASCAL VOC metrics metricsPerClass = evaluator.GetPascalVOCMetrics( boundingboxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=0.3) # IOU threshold print("Average precision values per class:\n") # Loop through classes to obtain their metrics for mc in metricsPerClass: # Get metric values per each class c = mc['class'] precision = mc['precision'] recall = mc['recall'] average_precision = mc['AP']
def compute_metrics(gtFolder, detFolder, iouThreshold=0.5, gtFormat='xyrb', detFormat='xyrb', gtCoordinates='abs', detCoordinates='abs', showPlot=False, savePath=None): currentPath = os.path.dirname(os.path.abspath(__file__)) # Arguments validation errors = [] # Validate formats gtFormat = ValidateFormats(gtFormat, '-gtformat', errors) detFormat = ValidateFormats(detFormat, '-detformat', errors) # Coordinates types gtCoordType = ValidateCoordinatesTypes(gtCoordinates, '-gtCoordinates', errors) detCoordType = ValidateCoordinatesTypes(detCoordinates, '-detCoordinates', errors) imgSize = (0, 0) if gtCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(imgSize, '-imgsize', '-gtCoordinates', errors) if detCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(imgSize, '-imgsize', '-detCoordinates', errors) if savePath is not None: savePath = ValidatePaths(savePath, '-sp/--savepath', errors) else: savePath = os.path.join(currentPath, 'results') # Validate savePath # If error, show error messages if len(errors) != 0: print( """usage: Object Detection Metrics [-h] [-v] [-gt] [-det] [-t] [-gtformat] [-detformat] [-save]""") print('Object Detection Metrics: error(s): ') [print(e) for e in errors] sys.exit() # Create directory to save results shutil.rmtree(savePath, ignore_errors=True) # Clear folder os.makedirs(savePath) # Show plot during execution showPlot = showPlot # print('iouThreshold= %f' % iouThreshold) # print('savePath = %s' % savePath) # print('gtFormat = %s' % gtFormat) # print('detFormat = %s' % detFormat) # print('gtFolder = %s' % gtFolder) # print('detFolder = %s' % detFolder) # print('gtCoordType = %s' % gtCoordType) # print('detCoordType = %s' % detCoordType) # print('showPlot %s' % showPlot) # Get groundtruth boxes allBoundingBoxes, allClasses = getBoundingBoxes(gtFolder, True, gtFormat, gtCoordType, imgSize=imgSize) # Get detected boxes allBoundingBoxes, allClasses = getBoundingBoxes(detFolder, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision= False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=showPlot) f = open(os.path.join(savePath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write('https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') f.write('Average Precision (AP), Precision and Recall per class:') # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = np.array(metricsPerClass['precision']) recall = np.array(metricsPerClass['recall']) totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}%".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) print('AP: %s (%s)' % (ap_str, cl)) f.write('\n\nClass: %s' % cl) f.write('\nAP: %s' % ap_str) f.write('\nPrecision: %s' % prec) f.write('\nRecall: %s' % rec) # print(recall) # print(precision) precision = precision[-1] recall = recall[-1] print(recall) print(precision) f1 = 2 * precision * recall / (precision + recall) print('mean f1 ', f1) mAP = acc_AP / validClasses mAP_str = "{0:.2f}%".format(mAP * 100) print('mAP: %s' % mAP_str) f.write('\n\n\nmAP: %s' % mAP_str) return (mAP)
def Pascal(): # Validate formats def ValidateFormats(argFormat, argName, errors): if argFormat == 'xywh': return BBFormat.XYWH elif argFormat == 'xyrb': return BBFormat.XYX2Y2 elif argFormat is None: return BBFormat.XYWH # default when nothing is passed else: errors.append( 'argument %s: invalid value. It must be either \'xywh\' or \'xyrb\'' % argName) # Validate mandatory args def ValidateMandatoryArgs(arg, argName, errors): if arg is None: errors.append('argument %s: required argument' % argName) else: return True def ValidateImageSize(arg, argName, argInformed, errors): errorMsg = 'argument %s: required argument if %s is relative' % (argName, argInformed) ret = None if arg is None: errors.append(errorMsg) else: arg = arg.replace('(', '').replace(')', '') args = arg.split(',') if len(args) != 2: errors.append( '%s. It must be in the format \'width,height\' (e.g. \'600,400\')' % errorMsg) else: if not args[0].isdigit() or not args[1].isdigit(): errors.append( '%s. It must be in INdiaTEGER the format \'width,height\' (e.g. \'600,400\')' % errorMsg) else: ret = (int(args[0]), int(args[1])) return ret # Validate coordinate types def ValidateCoordinatesTypes(arg, argName, errors): if arg == 'abs': return CoordinatesType.Absolute elif arg == 'rel': return CoordinatesType.Relative elif arg is None: return CoordinatesType.Absolute # default when nothing is passed errors.append('argument %s: invalid value. It must be either \'rel\' or \'abs\'' % argName) def ValidatePaths(arg, nameArg, errors): if arg is None: errors.append('argument %s: invalid directory' % nameArg) elif os.path.isdir(arg) is False and os.path.isdir(os.path.join(currentPath, arg)) is False: errors.append('argument %s: directory does not exist \'%s\'' % (nameArg, arg)) # elif os.path.isdir(os.path.join(currentPath, arg)) is True: # arg = os.path.join(currentPath, arg) else: arg = os.path.join(currentPath, arg) return arg def getBoundingBoxes(directory, isGT, bbFormat, coordType, allBoundingBoxes=None, allClasses=None, imgSize=(0, 0)): """Read txt files containing bounding boxes (ground truth and ss).""" if allBoundingBoxes is None: allBoundingBoxes = BoundingBoxes() if allClasses is None: allClasses = [] # Read ground truths os.chdir(directory) files = glob.glob("*.txt") files.sort() # Read GT detections from txt file # Each line of the files in the groundtruths folder represents a ground truth bounding box # (bounding boxes that a detector should detect) # Each value of each line is "class_id, x, y, width, height" respectively # Class_id represents the class of the bounding box # x, y represents the most top-left coordinates of the bounding box # x2, y2 represents the most bottom-right coordinates of the bounding box for f in files: nameOfImage = f.replace(".txt", "") fh1 = open(f, "r") for line in fh1: line = line.replace("\n", "") if line.replace(' ', '') == '': continue splitLine = line.split(" ") if isGT: # idClass = int(splitLine[0]) #class idClass = (splitLine[0]) # class x = float(splitLine[1]) y = float(splitLine[2]) w = float(splitLine[3]) h = float(splitLine[4]) bb = BoundingBox( nameOfImage, idClass, x, y, w, h, coordType, imgSize, BBType.GroundTruth, format=bbFormat) else: # idClass = int(splitLine[0]) #class idClass = (splitLine[0]) # class confidence = float(splitLine[1]) x = float(splitLine[2]) y = float(splitLine[3]) w = float(splitLine[4]) h = float(splitLine[5]) bb = BoundingBox( nameOfImage, idClass, x, y, w, h, coordType, imgSize, BBType.Detected, confidence, format=bbFormat) allBoundingBoxes.addBoundingBox(bb) if idClass not in allClasses: allClasses.append(idClass) fh1.close() return allBoundingBoxes, allClasses # Get current path to set default folders currentPath = os.path.dirname(os.path.abspath(__file__)) VERSION = '0.1 (beta)' parser = argparse.ArgumentParser( prog='Object Detection Metrics - Pascal VOC', description='This project applies the most popular metrics used to evaluate object detection ' 'algorithms.\nThe current implemention runs the Pascal VOC metrics.\nFor further references, ' 'please check:\nhttps://github.com/rafaelpadilla/Object-Detection-Metrics', epilog="Developed by: Rafael Padilla ([email protected])") # formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + VERSION) # Positional arguments # Mandatory parser.add_argument( '-gt', '--gtfolder', dest='gtFolder', default=os.path.join(currentPath, 'groundtruths'), metavar='', help='folder containing your ground truth bounding boxes') parser.add_argument( '-det', '--detfolder', dest='detFolder', default=os.path.join(currentPath, 'detections'), metavar='', help='folder containing your detected bounding boxes') # Optional parser.add_argument( '-t', '--threshold', dest='iouThreshold', type=float, default=0.5, metavar='', help='IOU threshold. Default 0.3') parser.add_argument( '-gtformat', dest='gtFormat', metavar='', default='xyrb', help='format of the coordinates of the ground truth bounding boxes: ' '(\'xywh\': <left> <top> <width> <height>)' ' or (\'xyrb\': <left> <top> <right> <bottom>)') parser.add_argument( '-detformat', dest='detFormat', metavar='', default='xyrb', help='format of the coordinates of the detected bounding boxes ' '(\'xywh\': <left> <top> <width> <height>) ' 'or (\'xyrb\': <left> <top> <right> <bottom>)') parser.add_argument( '-gtcoords', dest='gtCoordinates', default='abs', metavar='', help='reference of the ground truth bounding box coordinates: absolute ' 'values (\'abs\') or relative to its image size (\'rel\')') parser.add_argument( '-detcoords', default='abs', dest='detCoordinates', metavar='', help='reference of the ground truth bounding box coordinates: ' 'absolute values (\'abs\') or relative to its image size (\'rel\')') parser.add_argument( '-imgsize', dest='imgSize', metavar='', help='image size. Required if -gtcoords or -detcoords are \'rel\'') parser.add_argument( '-sp', '--savepath', dest='savePath', metavar='', help='folder where the plots are saved') parser.add_argument( '-np', '--noplot', dest='showPlot', action='store_false', help='no plot is shown during execution') args = parser.parse_args() iouThreshold = args.iouThreshold # Arguments validation errors = [] # Validate formats gtFormat = ValidateFormats(args.gtFormat, '-gtformat', errors) detFormat = ValidateFormats(args.detFormat, '-detformat', errors) # Groundtruth folder if ValidateMandatoryArgs(args.gtFolder, '-gt/--gtfolder', errors): gtFolder = ValidatePaths(args.gtFolder, '-gt/--gtfolder', errors) else: # errors.pop() gtFolder = os.path.join(currentPath, 'groundtruths') if os.path.isdir(gtFolder) is False: errors.append('folder %s not found' % gtFolder) # Coordinates types gtCoordType = ValidateCoordinatesTypes(args.gtCoordinates, '-gtCoordinates', errors) detCoordType = ValidateCoordinatesTypes(args.detCoordinates, '-detCoordinates', errors) imgSize = (0, 0) if gtCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-gtCoordinates', errors) if detCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-detCoordinates', errors) # Detection folder if ValidateMandatoryArgs(args.detFolder, '-det/--detfolder', errors): detFolder = ValidatePaths(args.detFolder, '-det/--detfolder', errors) else: # errors.pop() detFolder = os.path.join(currentPath, 'detections') if os.path.isdir(detFolder) is False: errors.append('folder %s not found' % detFolder) if args.savePath is not None: savePath = ValidatePaths(args.savePath, '-sp/--savepath', errors) else: savePath = os.path.join(currentPath, 'results') # Validate savePath # If error, show error messages if len(errors) != 0: print("""usage: Object Detection Metrics [-h] [-v] [-gt] [-det] [-t] [-gtformat] [-detformat] [-save]""") print('Object Detection Metrics: error(s): ') [print(e) for e in errors] sys.exit() # Create directory to save results shutil.rmtree(savePath, ignore_errors=True) # Clear folder os.makedirs(savePath) # Show plot during execution showPlot = args.showPlot # print('iouThreshold= %f' % iouThreshold) # print('savePath = %s' % savePath) # print('gtFormat = %s' % gtFormat) # print('detFormat = %s' % detFormat) # print('gtFolder = %s' % gtFolder) # print('detFolder = %s' % detFolder) # print('gtCoordType = %s' % gtCoordType) # print('detCoordType = %s' % detCoordType) # print('showPlot %s' % showPlot) # Get groundtruth boxes allBoundingBoxes, allClasses = getBoundingBoxes( gtFolder, True, gtFormat, gtCoordType, imgSize=imgSize) # Get detected boxes allBoundingBoxes, allClasses = getBoundingBoxes( detFolder, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision=False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=showPlot) f = open(os.path.join(savePath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write('https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') f.write('Average Precision (AP), Precision and Recall per class:') # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}%".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) print('AP: %s (%s)' % (ap_str, cl)) f.write('\n\nClass: %s' % cl) f.write('\nAP: %s' % ap_str) f.write('\nPrecision: %s' % prec) f.write('\nRecall: %s' % rec) mAP = acc_AP / validClasses mAP_str = "{0:.2f}".format(mAP * 100) print('mAP: %s' % mAP_str) f.write('\n\n\nmAP: %s' % mAP_str) # Copy results to result_path return(mAP_str)
allBoundingBoxes, allClasses = getBoundingBoxes(detFolder, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation) f = open(os.path.join(currentPath, 'results.txt'), 'w') f.write('Object Detection Metrics\n') f.write('Average Precision (AP), Precision and Recall per class:') # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives']
ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: ap_str = "{0:.2f}%".format(ap * 100) print('AP: %s (%s)' % (ap_str, cl)) mAP_str = "{0:.2f}%".format(mAP * 100) print('\nmAP: %s' % mAP_str) save_dir = os.path.join(root, r'model_predict\plot') if not os.path.exists(save_dir): os.mkdir(save_dir) evaluator.PlotPrecisionRecallCurve( gt_lst, det_lst, method='EveryPointInterpolation', showAP=True, showInterpolatedPrecision=False, savePath=save_dir, showGraphic=False ) # x = list(range(2000)) # y = [IOU_THRESHOLD(i) for i in x] # plt.plot(x, y) # plt.show()
def evaluate_agent(experiment_path, n_samples=100, agent_dir='best', visualize_episodes=True ): logging.basicConfig(level=logging.INFO, stream=sys.stdout, format='') print_config() dataset = load_dataset(CONFIG['dataset'], CONFIG['dataset_path']) # Always playout full episodes during testing CONFIG['playout_episode'] = True CONFIG['premasking'] = False env = create_env(dataset, CONFIG, mode='test') # Load agent from given path agent_path = os.path.join(experiment_path, agent_dir) agent = create_agent(env, CONFIG, from_path=agent_path) # Plot training summary (if it doesn't exist yet) if not os.path.exists(os.path.join(experiment_path, 'training')): plot_training_summary(experiment_path) # Create new evaluation folder eval_dirname = 'evaluation' eval_path = os.path.join(experiment_path, eval_dirname) ensure_folder(eval_path) # Use sampling to speed up evaluation if needed sample_size = len(dataset) if n_samples is not None and n_samples > -1: sample_size = min(sample_size, n_samples) collector = DetectionMetrics(eval_path) hooks = [] hooks.append(collector) if visualize_episodes: gif_path = os.path.join(eval_path, 'episodes') hooks.append(EpisodeRenderer(gif_path)) run_agent(agent, env, sample_size, hooks=hooks) print("Write bbox files") def _write_bbox_file(name, bbox_map): dir_path = os.path.join(eval_path, name) ensure_folder(dir_path) for image_idx, bboxes in bbox_map.items(): image_name = dataset.get_image_name(image_idx) print(image_name) image_txt = '' for bbox in bboxes: image_txt += 'text ' # object class name # Ensure bounding boxes are saved as integers image_txt += str(int(bbox[0])) + ' ' image_txt += str(int(bbox[1])) + ' ' image_txt += str(int(bbox[2])) + ' ' image_txt += str(int(bbox[3])) + ' ' image_txt += '\n' print(image_txt) txt_fpath = os.path.join(dir_path, f'{image_name}.txt') with open(txt_fpath, 'w+') as f: f.write(image_txt) _write_bbox_file('predictions', collector.image_pred_bboxes) _write_bbox_file('groundtruths', collector.image_true_bboxes) print("Evaluating predictions against ground truth") def _generate_lib_bboxes(bb_type, bbox_map, confidence=None): boxes = [] for image_idx, bboxes in bbox_map.items(): image_name = dataset.get_image_name(image_idx) for bbox in bboxes: box = BoundingBox( image_name, 'text', # object class name int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3]), typeCoordinates=CoordinatesType.Absolute, classConfidence=confidence, bbType=bb_type, format=BBFormat.XYX2Y2 ) boxes.append(box) return boxes true_boxes = _generate_lib_bboxes(BBType.GroundTruth, collector.image_true_bboxes) # Set default confidence as .01 for now (since agent doesn't score regions) pred_boxes = _generate_lib_bboxes( BBType.Detected, collector.image_pred_bboxes, confidence=.01 ) all_boxes = BoundingBoxes() for bbox in pred_boxes: all_boxes.addBoundingBox(bbox) for bbox in true_boxes: all_boxes.addBoundingBox(bbox) evaluator = Evaluator() # Mapping from IoU treshold to metrics calculated at this threshold iou_metrics = {} iou_thresholds = [round(x, 2) for x in np.arange(0, 1, .05)] all_actions = list(itertools.chain(*collector.image_actions.values())) action_counter = Counter(all_actions) n_actions = len(action_counter.keys()) for iou_threshold in iou_thresholds: metrics_per_class = evaluator.GetPascalVOCMetrics( all_boxes, IOUThreshold=iou_threshold, method=MethodAveragePrecision.EveryPointInterpolation ) text_metrics = metrics_per_class[0] # class = 'text' metrics = { 'precision': text_metrics['precision'][-1], 'recall': text_metrics['recall'][-1], 'ap': text_metrics['AP'], 'num_p_total': text_metrics['total positives'], 'num_tp': text_metrics['total TP'], 'num_fp': text_metrics['total FP'], } metrics['f1'] = f1(metrics['precision'], metrics['recall']) if len(collector.image_avg_iou) > 0: metrics['avg_iou'] = sum(list(collector.image_avg_iou.values())) / len(collector.image_avg_iou) else: metrics['avg_iou'] = 0 metrics['total_actions'] = sum(list(collector.image_num_actions.values())) if len(collector.image_num_actions) > 0: metrics['avg_actions'] = sum(list(collector.image_num_actions.values())) / len(collector.image_num_actions) else: metrics['avg_actions'] = 0 print(collector.image_num_actions_per_subepisode) avg_actions_subepisode = [sum(x) / len(x) if len(x) else 0 for x in collector.image_num_actions_per_subepisode.values()] print(avg_actions_subepisode) metrics['mean_avg_actions_subepisode'] = sum(avg_actions_subepisode) / len(avg_actions_subepisode) print(metrics['mean_avg_actions_subepisode']) for action, count in action_counter.items(): action_name = str(action) metrics[f'total_action_{action_name}'] = count iou_metrics[iou_threshold] = metrics # Save metrics as CSV iou_metrics_df = pd.DataFrame.from_dict(iou_metrics, orient='index') iou_metrics_df.index.name = 'iou_threshold' iou_metrics_df.to_csv(os.path.join(eval_path, 'metrics.csv')) print("Generating plots") plots_path = os.path.join(eval_path, 'plots') ensure_folder(plots_path) # Histogram of agent's actions fig, ax = plt.subplots() ax.hist(all_actions, bins=n_actions, orientation='horizontal', color='#0504aa') ax.set(xlabel='Frequency (Total)', ylabel='Action', title='Agent Actions') fig.savefig(os.path.join(plots_path, 'action_hist.png')) # Precision-Recall curves at different IoU thresholds for iou_threshold in [0.5, 0.75]: iou_fname_str = str(iou_threshold).replace('.', '') plot_path = os.path.join(plots_path, f'ap_{iou_fname_str}') ensure_folder(plot_path) evaluator.PlotPrecisionRecallCurve( all_boxes, IOUThreshold=iou_threshold, method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, showInterpolatedPrecision=True, savePath=plot_path, showGraphic=False ) # Recall-IoU curve x = iou_metrics_df.index.values y = iou_metrics_df['recall'].values fig, ax = plt.subplots() ax.plot(x, y, '-o') ax.set(xlabel='Intersection over Union (IoU)', ylabel='Recall', title='Recall-IoU') ax.grid() fig.savefig(os.path.join(plots_path, 'recall_iou.png')) # Precision-IoU curve x = iou_metrics_df.index.values y = iou_metrics_df['precision'].values fig, ax = plt.subplots() ax.plot(x, y, '-o') ax.set(xlabel='Intersection over Union (IoU)', ylabel='Precision', title='Precision-IoU') ax.grid() fig.savefig(os.path.join(plots_path, 'precision_iou.png')) print("Drawing images with predictions and ground truths") images_path = os.path.join(eval_path, 'images') ensure_folder(images_path) for image_idx in range(sample_size): image_path, _ = dataset.get(image_idx, as_image=False) image_name = dataset.get_image_name(image_idx) image = cv2.imread(image_path) image = all_boxes.drawAllBoundingBoxes(image, image_name) image_fname = Path(image_path).name cv2.imwrite(os.path.join(images_path, image_fname), image) print('Image %s created successfully!' % image_name)
def pascalvoc_eval(): iouThreshold = args.iouThreshold # Arguments validation errors = [] # Validate formats gtFormat = ValidateFormats(args.gtFormat, '-gtformat', errors) detFormat = ValidateFormats(args.detFormat, '-detformat', errors) # Groundtruth folder if ValidateMandatoryArgs(args.gtFolder, '-gt/--gtfolder', errors): gtFolder = ValidatePaths(args.gtFolder, '-gt/--gtfolder', errors) else: # errors.pop() gtFolder = os.path.join(currentPath, 'groundtruths') if os.path.isdir(gtFolder) is False: errors.append('folder %s not found' % gtFolder) # Coordinates types gtCoordType = ValidateCoordinatesTypes(args.gtCoordinates, '-gtCoordinates', errors) detCoordType = ValidateCoordinatesTypes(args.detCoordinates, '-detCoordinates', errors) imgSize = (0, 0) if gtCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-gtCoordinates', errors) if detCoordType == CoordinatesType.Relative: # Image size is required imgSize = ValidateImageSize(args.imgSize, '-imgsize', '-detCoordinates', errors) # Detection folder if ValidateMandatoryArgs(args.detFolder, '-det/--detfolder', errors): detFolder = ValidatePaths(args.detFolder, '-det/--detfolder', errors) else: # errors.pop() detFolder = os.path.join(currentPath, 'detections') if os.path.isdir(detFolder) is False: errors.append('folder %s not found' % detFolder) if args.savePath is not None: savePath = ValidatePaths(args.savePath, '-sp/--savepath', errors) else: savePath = os.path.join(currentPath, 'results') # Validate savePath # If error, show error messages if len(errors) is not 0: print( """usage: Object Detection Metrics [-h] [-v] [-gt] [-det] [-t] [-gtformat] [-detformat] [-save]""") print('Object Detection Metrics: error(s): ') [print(e) for e in errors] sys.exit() # Create directory to save results #shutil.rmtree(savePath, ignore_errors=True) # Clear folder #os.makedirs(savePath) # Show plot during execution #showPlot = args.showPlot # print('iouThreshold= %f' % iouThreshold) # print('savePath = %s' % savePath) # print('gtFormat = %s' % gtFormat) # print('detFormat = %s' % detFormat) # print('gtFolder = %s' % gtFolder) # print('detFolder = %s' % detFolder) # print('gtCoordType = %s' % gtCoordType) # print('detCoordType = %s' % detCoordType) # print('showPlot %s' % showPlot) # Get groundtruth boxes allBoundingBoxes, allClasses = getBoundingBoxes(gtFolder, True, gtFormat, gtCoordType, imgSize=imgSize) # Get detected boxes allBoundingBoxes, allClasses = getBoundingBoxes(detFolder, False, detFormat, detCoordType, allBoundingBoxes, allClasses, imgSize=imgSize) allClasses.sort() evaluator = Evaluator() acc_AP = 0 validClasses = 0 # Plot Precision x Recall curve detections = evaluator.PlotPrecisionRecallCurve( allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold method=MethodAveragePrecision.EveryPointInterpolation, showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision= False, # Don't plot the interpolated precision curve savePath=savePath, showGraphic=None) #f = open(os.path.join(savePath, 'results.txt'), 'w') #f.write('Object Detection Metrics\n') #f.write('https://github.com/rafaelpadilla/Object-Detection-Metrics\n\n\n') #f.write('Average Precision (AP), Precision and Recall per class:') AP_eval = [] # each detection is a class for metricsPerClass in detections: # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}%".format(ap * 100) # ap_str = "{0:.4f}%".format(ap * 100) print('AP: %s (%s)' % (ap_str, cl)) #f.write('\n\nClass: %s' % cl) #f.write('\nAP: %s' % ap_str) #f.write('\nPrecision: %s' % prec) #f.write('\nRecall: %s' % rec) AP_eval.append([cl, ap_str]) mAP = acc_AP / validClasses mAP_str = "{0:.2f}%".format(mAP * 100) print('mAP: %s' % mAP_str) #f.write('\n\n\nmAP: %s' % mAP_str) return AP_eval, mAP
def com_mAP_own_folder(dict_mAP,detections_foldername='detections'): ##### Arguments validation ##### errors = [] # Validate formats gtFormat = ValidateFormats(args.gtFormat, '-gtformat', errors) detFormat = ValidateFormats(args.detFormat, '-detformat', errors) # Validate mandatory (paths) # currentPath = os.path.dirname(os.path.abspath(__file__)) # Groundtruth folder if ValidateMandatoryArgs(args.gtFolder, '-gt/--gtfolder', errors): gtFolder = ValidatePaths(args.gtFolder, '-gt/--gtfolder', errors) else: errors.pop() gtFolder = os.path.join(currentPath, 'groundtruths') if os.path.isdir(gtFolder) == False: errors.append('folder %s not found' % gtFolder) # Coordinates types gtCoordType = ValidateCoordinatesTypes(args.gtCoordinates, '-gtCoordinates', errors) detCoordType = ValidateCoordinatesTypes(args.detCoordinates, '-detCoordinates', errors) if gtCoordType == CoordinatesType.Relative: # Image size is required ValidateImageSize(args.imgSize, '-imgsize', '-gtCoordinates', errors) if detCoordType == CoordinatesType.Relative: # Image size is required ValidateImageSize(args.imgSize, '-imgsize', '-detCoordinates', errors) # Detection folder if ValidateMandatoryArgs(args.detFolder, '-det/--detfolder', errors): detFolder = ValidatePaths(args.detFolder, '-det/--detfolder', errors) else: errors.pop() detFolder = os.path.join(currentPath, detections_foldername) if os.path.isdir(detFolder) == False: errors.append('folder %s not found' % detFolder) # Validate savePath if args.savePath != None: savePath = ValidatePaths(args.savePath, '-sp/--savepath', errors) savePath = os.path.join(args.savePath, 'results') else: savePath = os.path.join(currentPath, 'results') # If error, show error messages if len(errors) != 0: print("""usage: Object Detection Metrics [-h] [-v] [-gt] [-det] [-t] [-gtformat] [-detformat] [-save]""") print('Object Detection Metrics: error(s): ') [print(e) for e in errors] sys.exit() # Create directory to save results shutil.rmtree(savePath, ignore_errors=True) # Clear folder # os.makedirs(savePath) # Show plot during execution showPlot = args.showPlot showPlot = False # print('iouThreshold= %f' % iouThreshold) # print('savePath = %s' % savePath) # print('gtFormat = %s' % gtFormat) # print('detFormat = %s' % detFormat) # print('gtFolder = %s' % gtFolder) # print('detFolder = %s' % detFolder) # print('gtCoordType = %s' % gtCoordType) # print('detCoordType = %s' % detCoordType) # print('showPlot %s' % showPlot) allBoundingBoxes, allClasses = getBoundingBoxes(gtFolder, True, gtFormat) allBoundingBoxes, allClasses = getBoundingBoxes(detFolder, False, detFormat, allBoundingBoxes, allClasses) allClasses.sort() if (not os.path.exists(savePath)): os.mkdir(savePath) f = open(os.path.join(savePath, 'results.txt'), 'w') evaluator = Evaluator() acc_AP = 0 validClasses = 0 # for each class for c in allClasses: # Plot Precision x Recall curve metricsPerClass = evaluator.PlotPrecisionRecallCurve(c, # Class to show allBoundingBoxes, # Object containing all bounding boxes (ground truths and detections) IOUThreshold=iouThreshold, # IOU threshold showAP=True, # Show Average Precision in the title of the plot showInterpolatedPrecision=False, # Don't plot the interpolated precision curve savePath=os.path.join(savePath, c + '.png'), showGraphic=showPlot) # Get metric values per each class cl = metricsPerClass['class'] ap = metricsPerClass['AP'] precision = metricsPerClass['precision'] recall = metricsPerClass['recall'] totalPositives = metricsPerClass['total positives'] total_TP = metricsPerClass['total TP'] total_FP = metricsPerClass['total FP'] if totalPositives > 0: validClasses = validClasses + 1 acc_AP = acc_AP + ap prec = ['%.2f' % p for p in precision] rec = ['%.2f' % r for r in recall] ap_str = "{0:.2f}".format(ap * 100) # ap_str = str('%.2f' % ap) #AQUI # print('AP: %s (%s)' % (ap_str, cl)) dict_mAP[cl] = round(ap * 100, 3) f.write('\n\nClass: %s' % cl) f.write('\nAP: %s' % ap_str) f.write('\nPrecision: %s' % prec) f.write('\nRecall: %s' % rec) mAP = acc_AP / validClasses mAP_str = "{0:.2f}%".format(mAP * 100) # print('mAP: %s' % mAP_str) f.write('\n\n\nmAP: %s' % mAP_str) f.close() ap_str = "{0:.2f}".format(mAP * 100) dict_mAP['mAP'] = round(mAP * 100, 3) return dict_mAP