def eval_road(li_pred, li_gt): print "Starting evaluation ..." thresh = np.array(range(0, 256)) / 255.0 # recall thresh # init result totalFP = np.zeros(thresh.shape) totalFN = np.zeros(thresh.shape) totalPosNum = 0 totalNegNum = 0 for i in range(len(li_pred)): pred = li_pred[i] # gt = li_gt[i] > 0 gt = li_gt[i] == 1 validArea = li_gt[i] < 250 FN, FP, posNum, negNum = evalExp(gt, pred, thresh, validMap=None, validArea=validArea) assert FN.max() <= posNum, 'BUG @ poitive samples' assert FP.max() <= negNum, 'BUG @ negative samples' # collect results for whole category totalFP += FP totalFN += FN totalPosNum += posNum totalNegNum += negNum # if category_ok: print "Computing evaluation scores..." # Compute eval scores! eval_dict = pxEval_maximizeFMeasure(totalPosNum, totalNegNum, totalFN, totalFP, thresh=thresh) for property in li_property: print '%s: %4.2f ' % (property, eval_dict[property] * 100,) print "Finished evaluating!" return eval_dict['MaxF']
totalNegNum[label_class[j]] += negNum # calculate measures globally FN, FP, posNum, negNum = calcClassMeasures(np.ones([im.shape[0],im.shape[1]]),(net.blobs['score'].data[0].argmax(axis=0)==tmp).astype(np.float32),tmp!=255,thresh) totalFP[11,:] += FP totalFN[11,:] += FN totalPosNum[11] += posNum totalNegNum[11] += negNum #groundtruth = np.uint8(np.zeros((im.shape[0],im.shape[1]))) #for i in range(0,len(label_colors)): # groundtruth[:,:] = groundtruth[:,:] + label_class[i]*np.prod(np.equal(im,label_colors[i]),2) # #prediction=net.blobs['score'].data[0].argmax(axis=0) #prediction = prediction.reshape([np.prod(prediction.shape),1]) #groundtruth = groundtruth.reshape([np.prod(groundtruth.shape),1]) #prediction = prediction[groundtruth!=255] #groundtruth = groundtruth[groundtruth!=255] #confmat = confusion_matrix(groundtruth,prediction) #confmat = confmat.astype('float') / confmat.sum(axis=1)[:, np.newaxis] # # accuracy as in Segmentation and Recognition Using Structure from Motion Point Clouds, ECCV 2008: # # per-class accuracies (the normalized diagonal of the pixel-wise confusion matrix) # is the same as REC_wp for i in range(0,12): out = pxEval_maximizeFMeasure(totalPosNum[i,:], totalNegNum[i,:], totalFN[i,:], totalFP[i,:], thresh) print 'Klasse: ' + label_name[i] +'\t', for property in ['REC_wp', 'MaxF']: print '%s: %4.2f \t' %(property, out[property]*100,), print ' '
def main(result_dir, train_dir, debug=False): ''' main method of evaluateRoad :param result_dir: directory with the result propability maps, e.g., /home/elvis/kitti_road/my_results :param gt_dir: training directory (has to contain gt_image_2) e.g., /home/elvis/kitti_road/training :param debug: debug flag (OPTIONAL) ''' print "Starting evaluation ..." print "Available categories are: %s" % dataStructure.cats thresh = np.array(range(0, 256)) / 255.0 trainData_subdir_gt = 'gt_image_2/' gt_dir = os.path.join(train_dir, trainData_subdir_gt) print os.getcwd() assert os.path.isdir( result_dir), 'Cannot find result_dir: %s ' % result_dir # In the submission_dir we expect the probmaps! submission_dir = result_dir assert os.path.isdir(submission_dir), 'Cannot find %s, ' % submission_dir # init result prob_eval_scores = [] # the eval results in a dict eval_cats = [] # saves al categories at were evaluated outputline = [] for cat in dataStructure.cats: print "Execute evaluation for category %s ..." % cat fn_search = '%s*%s' % (cat, dataStructure.gt_end) prob_fileList = glob(os.path.join(result_dir, fn_search)) assert len(prob_fileList) > 0, 'Error reading ground truth' gt_fileList = glob(os.path.join(gt_dir, fn_search)) assert len(gt_fileList) > 0, 'Error reading ground truth' # Init data for categgory category_ok = True # Flag for each cat totalFP = np.zeros(thresh.shape) totalFN = np.zeros(thresh.shape) totalPosNum = 0 totalNegNum = 0 # firstFile = gt_fileList[0] firstFile = prob_fileList[0] file_key = firstFile.split('/')[-1].split('.')[0] tags = file_key.split('_') ts_tag = tags[2] dataset_tag = tags[0] class_tag = tags[1] submission_tag = dataset_tag + '_' + class_tag + '_' print "Searching for submitted files with prefix: %s" % submission_tag for prob_cur in prob_fileList: file_key = prob_cur.split('/')[-1].split('.')[0] file_key_1, file_key_2, file_key_3 = file_key.split('_') _file_key = file_key_1 + '_' + file_key_2 + '_' + file_key_3 if debug: print "Processing file: %s " % file_key # get tags tags = file_key.split('_') ts_tag = tags[2] dataset_tag = tags[0] class_tag = tags[1] cur_prob = cv2.imread(prob_cur, 0) fn_curGt = os.path.join(gt_dir, _file_key + dataStructure.prob_end) if not os.path.isfile(fn_curGt): print "Cannot find file: %s for category %s." % (fn_curGt, cat) print "--> Will now abort evaluation for this particular category." category_ok = False break cur_gt, validArea = getGroundTruth(fn_curGt) cur_prob = np.clip( (cur_prob.astype('f4')) / (np.iinfo(cur_prob.dtype).max), 0., 1.) FN, FP, posNum, negNum = evalExp(cur_gt, cur_prob, thresh, validMap=None, validArea=validArea) assert FN.max() <= posNum, 'BUG @ poitive samples' assert FP.max() <= negNum, 'BUG @ negative samples' # collect results for whole category totalFP += FP totalFN += FN totalPosNum += posNum totalNegNum += negNum if category_ok: print "Computing evaluation scores..." # Compute eval scores! prob_eval_scores.append( pxEval_maximizeFMeasure(totalPosNum, totalNegNum, totalFN, totalFP, thresh=thresh)) eval_cats.append(cat) factor = 100 for property in dataStructure.eval_propertyList: print '%s: %4.2f ' % ( property, prob_eval_scores[-1][property] * factor, ) print "Finished evaluating category: %s " % (eval_cats[-1], ) if len(eval_cats) > 0: print "Successfully finished evaluation for %d categories: %s " % ( len(eval_cats), eval_cats) return True else: print "No categories have been evaluated!" return False
def main(result_dir, train_dir, debug=False): ''' main method of evaluateRoad :param result_dir: directory with the result propability maps, e.g., /home/elvis/kitti_road/my_results :param gt_dir: training directory (has to contain gt_image_2) e.g., /home/elvis/kitti_road/training :param debug: debug flag (OPTIONAL) ''' print("Starting evaluation ...") thresh = np.array(range(0, 256)) / 255.0 trainData_subdir_gt = 'origin/' gt_dir = os.path.join(train_dir, trainData_subdir_gt) assert os.path.isdir( result_dir), 'Cannot find result_dir: %s ' % result_dir # In the submission_dir we expect the probmaps! submission_dir = result_dir assert os.path.isdir(submission_dir), 'Cannot find %s, ' % submission_dir # init result prob_eval_scores = [] # the eval results in a dict gt_fileList = glob(os.path.join(gt_dir, '*')) assert len(gt_fileList) > 0, 'Error reading ground truth' # Init data for categgory category_ok = True # Flag for each cat totalFP = np.zeros(thresh.shape) totalFN = np.zeros(thresh.shape) totalPosNum = 0 totalNegNum = 0 for fn_curGt in gt_fileList: fn_curGt = fn_curGt.replace('\\', '/') file_key = fn_curGt.split('/')[-1].split('.')[0] print("Processing file: %s " % file_key) # Read GT cur_gt, validArea = getGroundTruth(fn_curGt) # Read probmap and normalize fn_curProb = os.path.join(submission_dir, file_key + dataStructure.prob_end) if not os.path.isfile(fn_curProb): print( "--> Will now abort evaluation for this particular category.") category_ok = False break cur_prob = cv2.imread(fn_curProb, 0) cur_prob = np.clip( (cur_prob.astype('f4')) / (np.iinfo(cur_prob.dtype).max), 0., 1.) FN, FP, posNum, negNum = evalExp(cur_gt, cur_prob, thresh, validMap=None, validArea=validArea) assert FN.max() <= posNum, 'BUG @ poitive samples' assert FP.max() <= negNum, 'BUG @ negative samples' # collect results for whole category totalFP += FP totalFN += FN totalPosNum += posNum totalNegNum += negNum if category_ok: print("Computing evaluation scores...") # Compute eval scores! print( pxEval_maximizeFMeasure(totalPosNum, totalNegNum, totalFN, totalFP, thresh=thresh)) prob_eval_scores.append( pxEval_maximizeFMeasure(totalPosNum, totalNegNum, totalFN, totalFP, thresh=thresh)) factor = 100 for property in dataStructure.eval_propertyList: print('%s: %4.2f ' % ( property, prob_eval_scores[-1][property] * factor, ))
def main(result_dir, train_dir, debug = False): ''' main method of evaluateRoad :param result_dir: directory with the result propability maps, e.g., /home/elvis/kitti_road/my_results :param gt_dir: training directory (has to contain gt_image_2) e.g., /home/elvis/kitti_road/training :param debug: debug flag (OPTIONAL) ''' print "Starting evaluation ..." print "Available categories are: %s" %dataStructure.cats thresh = np.array(range(0,256))/255.0 trainData_subdir_gt = 'image/' gt_dir = os.path.join(train_dir,trainData_subdir_gt) assert os.path.isdir(result_dir), 'Cannot find result_dir: %s ' %result_dir # In the submission_dir we expect the probmaps! submission_dir = result_dir assert os.path.isdir(submission_dir), 'Cannot find %s, ' %submission_dir # init result prob_eval_scores = [] # the eval results in a dict eval_cats = [] # saves al categories at were evaluated outputline = [] for cat in dataStructure.cats: print "Execute evaluation for category %s ..." %cat fn_search = '%s*%s' %(cat, dataStructure.gt_end) gt_fileList = glob(os.path.join(gt_dir, fn_search)) assert len(gt_fileList)>0, 'Error reading ground truth' # Init data for categgory category_ok = True # Flag for each cat totalFP = np.zeros( thresh.shape ) totalFN = np.zeros( thresh.shape ) totalPosNum = 0 totalNegNum = 0 firstFile = gt_fileList[0] file_key = firstFile.split('/')[-1].split('.')[0] tags = file_key.split('_') print tags ts_tag = tags[2] dataset_tag = tags[0] class_tag = tags[1] submission_tag = dataset_tag + '_' + class_tag + '_' print "Searching for submitted files with prefix: %s" %submission_tag for fn_curGt in gt_fileList: file_key = fn_curGt.split('/')[-1].split('.')[0] if debug: print "Processing file: %s " %file_key # get tags tags = file_key.split('_') ts_tag = tags[2] dataset_tag = tags[0] class_tag = tags[1] # Read GT cur_gt, validArea = getGroundTruth(fn_curGt) # Read probmap and normalize fn_curProb = os.path.join(submission_dir, file_key + dataStructure.prob_end) if not os.path.isfile(fn_curProb): print "Cannot find file: %s for category %s." %(file_key, cat) print "--> Will now abort evaluation for this particular category." category_ok = False break cur_prob = cv2.imread(fn_curProb,0) cur_prob = np.clip( (cur_prob.astype('f4'))/(np.iinfo(cur_prob.dtype).max),0.,1.) FN, FP, posNum, negNum = evalExp(cur_gt, cur_prob, thresh, validMap = None, validArea=validArea) assert FN.max()<=posNum, 'BUG @ poitive samples' assert FP.max()<=negNum, 'BUG @ negative samples' # collect results for whole category totalFP += FP totalFN += FN totalPosNum += posNum totalNegNum += negNum if category_ok: print "Computing evaluation scores..." # Compute eval scores! prob_eval_scores.append(pxEval_maximizeFMeasure(totalPosNum, totalNegNum, totalFN, totalFP, thresh = thresh)) eval_cats.append(cat) factor = 100 for property in dataStructure.eval_propertyList: print '%s: %4.2f ' %(property, prob_eval_scores[-1][property]*factor,) print "Finished evaluating category: %s " %(eval_cats[-1],) if len(eval_cats)>0: print "Successfully finished evaluation for %d categories: %s " %(len(eval_cats),eval_cats) return True else: print "No categories have been evaluated!" return False
tmp != 255, thresh) totalFP[11, :] += FP totalFN[11, :] += FN totalPosNum[11] += posNum totalNegNum[11] += negNum #groundtruth = np.uint8(np.zeros((im.shape[0],im.shape[1]))) #for i in range(0,len(label_colors)): # groundtruth[:,:] = groundtruth[:,:] + label_class[i]*np.prod(np.equal(im,label_colors[i]),2) # #prediction=net.blobs['score'].data[0].argmax(axis=0) #prediction = prediction.reshape([np.prod(prediction.shape),1]) #groundtruth = groundtruth.reshape([np.prod(groundtruth.shape),1]) #prediction = prediction[groundtruth!=255] #groundtruth = groundtruth[groundtruth!=255] #confmat = confusion_matrix(groundtruth,prediction) #confmat = confmat.astype('float') / confmat.sum(axis=1)[:, np.newaxis] # # accuracy as in Segmentation and Recognition Using Structure from Motion Point Clouds, ECCV 2008: # # per-class accuracies (the normalized diagonal of the pixel-wise confusion matrix) # is the same as REC_wp for i in range(0, 12): out = pxEval_maximizeFMeasure(totalPosNum[i, :], totalNegNum[i, :], totalFN[i, :], totalFP[i, :], thresh) print 'Klasse: ' + label_name[i] + '\t', for property in ['REC_wp', 'MaxF', 'IoU']: print '%s: %4.2f \t' % ( property, out[property] * 100, ), print ' '
def main(result_dir, train_dir, file_path="./data1", debug=False): ''' main method of evaluateRoad :param result_dir: directory with the result propability maps, e.g., /home/user/lane/eval/eval_images/tmp :param gt_dir: training directory (has to contain groundtruth) e.g., /home/user/lane/eval/eval_images :param file_path: path pointing to the (not yet existing) measurement file e.g. /home/user/lane/eval/eval_results/dataxxx :param debug: debug flag (OPTIONAL) ''' if (debug): print("Starting evaluation ...") print("Available categories are: %s" % (dataStructure.cats)) thresh = np.array(range(0, 256)) / 255.0 trainData_subdir_gt = 'groundtruth/' gt_dir = os.path.join(train_dir, trainData_subdir_gt) assert os.path.isdir( result_dir), 'Cannot find result_dir: %s ' % result_dir # In the submission_dir we expect the probmaps! submission_dir = result_dir assert os.path.isdir(submission_dir), 'Cannot find %s, ' % submission_dir # init result prob_eval_scores = [] # the eval results in a dict eval_cats = [] # saves al categories at were evaluated outputline = [] path = Path(file_path) with path.open(mode="w") as f: f.write('#') for property in dataStructure.eval_propertyList: f.write('%s ' % (property)) f.write("\n") totalFP = np.zeros(thresh.shape) totalFN = np.zeros(thresh.shape) totalPosNum = 0 totalNegNum = 0 for cat in dataStructure.cats: print() if (debug): print("Execute evaluation for category %s ..." % (cat)) fn_search = '%s*%s' % (cat, dataStructure.gt_end) gt_fileList = sorted(glob(os.path.join(gt_dir, fn_search))) assert len(gt_fileList) > 0, 'Error reading ground truth' # Init data for categgory category_ok = True # Flag for each cat #totalFP = np.zeros(thresh.shape) #totalFN = np.zeros(thresh.shape) #totalPosNum = 0 #totalNegNum = 0 firstFile = gt_fileList[0] file_key = firstFile.split('/')[-1].split('.')[0] tags = file_key.split('_') ts_tag = tags[2] dataset_tag = tags[0] class_tag = tags[1] submission_tag = dataset_tag + '_' + class_tag + '_' if (debug): print("Searching for submitted files with prefix: %s" % (submission_tag)) for fn_curGt in gt_fileList: file_key = fn_curGt.split('/')[-1].split('.')[0] if debug: print("Processing file: %s " % (file_key)) # get tags tags = file_key.split('_') ts_tag = tags[2] dataset_tag = tags[0] class_tag = tags[1] # Read GT cur_gt, validArea = getGroundTruth(fn_curGt) # Read probmap and normalize fn_curProb = os.path.join(submission_dir, file_key + dataStructure.prob_end) if not os.path.isfile(fn_curProb): print("Cannot find file: %s for category %s." % (file_key, cat)) print( "--> Will now abort evaluation for this particular category." ) category_ok = False break cur_prob = cv2.imread(fn_curProb, 0) cur_prob = np.clip( (cur_prob.astype('f4')) / (np.iinfo(cur_prob.dtype).max), 0., 1.) FN, FP, posNum, negNum = evalExp(cur_gt, cur_prob, thresh, validMap=None, validArea=validArea) assert FN.max() <= posNum, 'BUG @ poitive samples' assert FP.max() <= negNum, 'BUG @ negative samples' prob_eval_scores.append( pxEval_maximizeFMeasure(posNum, negNum, FN, FP, thresh=thresh)) factor = 100 with path.open(mode="a") as f: for property in dataStructure.eval_propertyList: f.write('%4.2f ' % (prob_eval_scores[-1][property] * factor)) f.write("\n") prob_eval_scores = [] # collect results for whole category totalFP += FP totalFN += FN totalPosNum += posNum totalNegNum += negNum if category_ok: if (debug): print("Computing evaluation scores...") # Compute eval scores! prob_eval_scores.append( pxEval_maximizeFMeasure(totalPosNum, totalNegNum, totalFN, totalFP, thresh=thresh)) eval_cats.append(cat) factor = 100 with path.open(mode="a") as f: for property in dataStructure.eval_propertyList: f.write('%4.2f ' % (prob_eval_scores[-1][property] * factor)) f.write('\n') if (debug): print("Finished evaluating category: %s " % (eval_cats[-1], )) if len(eval_cats) > 0: if (debug): print("Successfully finished evaluation for %d categories: %s " % (len(eval_cats), eval_cats)) return True else: print("No categories have been evaluated!") return False