def cleval_evaluation(gt_file, submit_file): """ evaluate and returns the results Returns with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - per_sample (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 }, 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ # storing overall result overall_result = GlobalResult(PARAMS.E2E) # to store per sample evaluation results per_sample_metrics = {} gt_files = load_zip_file(gt_file, PARAMS.GT_SAMPLE_NAME_2_ID) submission_files = load_zip_file(submit_file, PARAMS.DET_SAMPLE_NAME_2_ID, True) # prepare ThreadPool for multi-process executor = concurrent.futures.ProcessPoolExecutor( max_workers=PARAMS.NUM_WORKERS) futures = {} bar_len = len(gt_files) for file_idx in gt_files: gt_file = rrc_evaluation_funcs.decode_utf8(gt_files[file_idx]) if file_idx in submission_files: det_file = decode_utf8(submission_files[file_idx]) if det_file is None: det_file = "" else: det_file = "" future = executor.submit(eval_single_result, gt_file, det_file) futures[future] = file_idx with tqdm(total=bar_len) as pbar: pbar.set_description("Integrating results...") for future in concurrent.futures.as_completed(futures): file_idx = futures[future] result = future.result() per_sample_metrics[file_idx] = result overall_result.accumulate_stats(result['Rawdata']) pbar.update(1) executor.shutdown() resDict = { 'calculated': True, 'Message': '', 'method': overall_result.to_dict(), 'per_sample': per_sample_metrics } return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().items(): globals()[alias] = importlib.import_module(module) def polygon_from_points(points): """ Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4 """ resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(points[0]) resBoxes[0, 4] = int(points[1]) resBoxes[0, 1] = int(points[2]) resBoxes[0, 5] = int(points[3]) resBoxes[0, 2] = int(points[4]) resBoxes[0, 6] = int(points[5]) resBoxes[0, 3] = int(points[6]) resBoxes[0, 7] = int(points[7]) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_polygon(rect): resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(rect.xmin) resBoxes[0, 4] = int(rect.ymax) resBoxes[0, 1] = int(rect.xmin) resBoxes[0, 5] = int(rect.ymin) resBoxes[0, 2] = int(rect.xmax) resBoxes[0, 6] = int(rect.ymin) resBoxes[0, 3] = int(rect.xmax) resBoxes[0, 7] = int(rect.ymax) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_points(rect): points = [ int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin) ] return points def get_union(pD, pG): areaA = pD.area() areaB = pG.area() return areaA + areaB - get_intersection(pD, pG) def get_intersection_over_union(pD, pG): try: return get_intersection(pD, pG) / get_union(pD, pG) except: return 0 def get_intersection(pD, pG): pInt = pD & pG if len(pInt) == 0: return 0 return pInt.area() def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP perSampleMetrics = {} matchedSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') gt = rrc_evaluation_funcs.load_zip_file( gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file( submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGlobalCareGt = 0 numGlobalCareDet = 0 arrGlobalConfidences = [] arrGlobalMatches = [] for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 detMatched = 0 iouMat = np.empty([1, 1]) gtPols = [] detPols = [] gtPolPoints = [] detPolPoints = [] #Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [] #Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [] pairs = [] detMatchedNums = [] arrSampleConfidences = [] arrSampleMatch = [] sampleAP = 0 evaluationLog = "" pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" if evaluationParams['LTRB']: gtRect = Rectangle(*points) gtPol = rectangle_to_polygon(gtRect) else: gtPol = polygon_from_points(points) gtPols.append(gtPol) gtPolPoints.append(points) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) evaluationLog += "GT polygons: " + str(len(gtPols)) + ( " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( detFile, evaluationParams['CRLF'], evaluationParams['LTRB'], evaluationParams['TRANSCRIPTIONS'], evaluationParams['CONFIDENCES']) for n in range(len(pointsList)): points = pointsList[n] if evaluationParams['LTRB']: detRect = Rectangle(*points) detPol = rectangle_to_polygon(detRect) else: detPol = polygon_from_points(points) detPols.append(detPol) detPolPoints.append(points) if len(gtDontCarePolsNum) > 0: for dontCarePol in gtDontCarePolsNum: dontCarePol = gtPols[dontCarePol] intersected_area = get_intersection( dontCarePol, detPol) pdDimensions = detPol.area() precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']): detDontCarePolsNum.append(len(detPols) - 1) break evaluationLog += "DET polygons: " + str(len(detPols)) + ( " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n") if len(gtPols) > 0 and len(detPols) > 0: #Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] iouMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): if gtRectMat[gtNum] == 0 and detRectMat[ detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: if iouMat[gtNum, detNum] > evaluationParams[ 'IOU_CONSTRAINT']: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 detMatched += 1 pairs.append({'gt': gtNum, 'det': detNum}) detMatchedNums.append(detNum) evaluationLog += "Match GT #" + str( gtNum) + " with Det #" + str(detNum) + "\n" if evaluationParams['CONFIDENCES']: for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: #we exclude the don't care detections match = detNum in detMatchedNums arrSampleConfidences.append(confidencesList[detNum]) arrSampleMatch.append(match) arrGlobalConfidences.append(confidencesList[detNum]) arrGlobalMatches.append(match) numGtCare = (len(gtPols) - len(gtDontCarePolsNum)) numDetCare = (len(detPols) - len(detDontCarePolsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if numDetCare > 0 else float(1) sampleAP = precision else: recall = float(detMatched) / numGtCare precision = 0 if numDetCare == 0 else float( detMatched) / numDetCare if evaluationParams['CONFIDENCES'] and evaluationParams[ 'PER_SAMPLE_RESULTS']: sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare) hmean = 0 if ( precision + recall) == 0 else 2.0 * precision * recall / (precision + recall) matchedSum += detMatched numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare if evaluationParams['PER_SAMPLE_RESULTS']: perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'pairs': pairs, 'AP': sampleAP, 'iouMat': [] if len(detPols) > 100 else iouMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtDontCare': gtDontCarePolsNum, 'detDontCare': detDontCarePolsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } # Compute MAP and MAR AP = 0 if evaluationParams['CONFIDENCES']: AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt) methodRecall = 0 if numGlobalCareGt == 0 else float( matchedSum) / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else float( matchedSum) / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodMetrics = { 'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP } resDict = { 'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics } return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ if evaluationParams['E2E']: from hanziconv import HanziConv import editdistance for module, alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) def polygon_from_points(points): """ Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4 """ resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(points[0]) resBoxes[0, 4] = int(points[1]) resBoxes[0, 1] = int(points[2]) resBoxes[0, 5] = int(points[3]) resBoxes[0, 2] = int(points[4]) resBoxes[0, 6] = int(points[5]) resBoxes[0, 3] = int(points[6]) resBoxes[0, 7] = int(points[7]) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_polygon(rect): resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(rect.xmin) resBoxes[0, 4] = int(rect.ymax) resBoxes[0, 1] = int(rect.xmin) resBoxes[0, 5] = int(rect.ymin) resBoxes[0, 2] = int(rect.xmax) resBoxes[0, 6] = int(rect.ymin) resBoxes[0, 3] = int(rect.xmax) resBoxes[0, 7] = int(rect.ymax) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_points(rect): points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin)] return points def get_union(pD, pG): areaA = pD.area() areaB = pG.area() return areaA + areaB - get_intersection(pD, pG) def get_intersection_over_union(pD, pG): try: return get_intersection(pD, pG) / get_union(pD, pG) except: return 0 def get_intersection(pD, pG): pInt = pD & pG if len(pInt) == 0: return 0 return pInt.area() def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP #from RTWC17 def normalize_txt(st): """ Normalize Chinese text strings by: - remove puncutations and other symbols - convert traditional Chinese to simplified - convert English characters to lower cases """ st = ''.join(st.split(' ')) st = re.sub("\"", "", st) # remove any this not one of Chinese character, ascii 0-9, and ascii a-z and A-Z new_st = re.sub(ur'[^\u4e00-\u9fa5\u0041-\u005a\u0061-\u007a0-9]+', '', st) # convert Traditional Chinese to Simplified Chinese new_st = HanziConv.toSimplified(new_st) # convert uppercase English letters to lowercase new_st = new_st.lower() return new_st def text_distance(str1, str2): str1 = normalize_txt(str1) str2 = normalize_txt(str2) return editdistance.eval(str1, str2) perSampleMetrics = {} matchedSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') gt = rrc_evaluation_funcs.load_zip_file(gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file(submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGlobalCareGt = 0 numGlobalCareDet = 0 arrGlobalConfidences = [] arrGlobalMatches = [] #total edit distance total_dist = 0 for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 detMatched = 0 iouMat = np.empty([1, 1]) gtPols = [] detPols = [] gtTrans = [] detTrans = [] gtPolPoints = [] detPolPoints = [] # Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [] # Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [] pairs = [] detMatchedNums = [] arrSampleConfidences = [] arrSampleMatch = [] sampleAP = 0 example_dist = 0 match_tuples = [] evaluationLog = "" pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile,evaluationParams['CRLF'],evaluationParams['LTRB'],True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = (transcription == "###") or (transcription=="?") if evaluationParams['LTRB']: gtRect = Rectangle(*points) gtPol = rectangle_to_polygon(gtRect) else: gtPol = polygon_from_points(points) gtPols.append(gtPol) gtPolPoints.append(points) gtTrans.append(transcription) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) evaluationLog += "GT polygons: " + str(len(gtPols)) + ( " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, confidencesList, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile,evaluationParams['CRLF'],evaluationParams['LTRB'],evaluationParams['E2E'],evaluationParams['CONFIDENCES']) for n in range(len(pointsList)): points = pointsList[n] if evaluationParams['LTRB']: detRect = Rectangle(*points) detPol = rectangle_to_polygon(detRect) else: detPol = polygon_from_points(points) detPols.append(detPol) detPolPoints.append(points) if evaluationParams['E2E']: transcription = transcriptionsList[n] detTrans.append(transcription) if len(gtDontCarePolsNum) > 0: for dontCarePol in gtDontCarePolsNum: dontCarePol = gtPols[dontCarePol] intersected_area = get_intersection(dontCarePol, detPol) pdDimensions = detPol.area() precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']): detDontCarePolsNum.append(len(detPols) - 1) break evaluationLog += "DET polygons: " + str(len(detPols)) + ( " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n") if len(gtPols) > 0 and len(detPols) > 0: # Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] iouMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG) # match dt index of every gt gtMatch = np.empty(len(gtPols), np.int8) gtMatch.fill(-1) # match gt index of every dt dtMatch = np.empty(len(detPols), dtype=np.int8) dtMatch.fill(-1) for gtNum in range(len(gtPols)): max_iou = 0 match_dt_idx = -1 for detNum in range(len(detPols)): if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0\ and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT']: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 detMatched += 1 pairs.append({'gt': gtNum, 'det': detNum}) detMatchedNums.append(detNum) evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n" if evaluationParams['E2E'] and gtMatch[gtNum] == -1 and dtMatch[detNum] == -1\ and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT'] and iouMat[gtNum, detNum] > max_iou: max_iou = iouMat[gtNum, detNum] match_dt_idx = detNum if evaluationParams['E2E'] and match_dt_idx >= 0: gtMatch[gtNum] = match_dt_idx dtMatch[match_dt_idx] = gtNum if evaluationParams['E2E']: for gtNum in range(len(gtPols)): if gtNum in gtDontCarePolsNum: continue gt_text = gtTrans[gtNum] if gtMatch[gtNum] >= 0: dt_text = detTrans[gtMatch[gtNum]] else: dt_text = u'' dist = text_distance(gt_text, dt_text) example_dist += dist match_tuples.append((gt_text, dt_text, dist)) match_tuples.append(("===============","==============", -1)) for detNum in range(len(detPols)): if detNum in detDontCarePolsNum: continue if dtMatch[detNum] == -1: gt_text = u'' dt_text = detTrans[detNum] dist = text_distance(gt_text, dt_text) example_dist += dist match_tuples.append((gt_text, dt_text, dist)) if evaluationParams['CONFIDENCES']: for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: # we exclude the don't care detections match = detNum in detMatchedNums arrSampleConfidences.append(confidencesList[detNum]) arrSampleMatch.append(match) arrGlobalConfidences.append(confidencesList[detNum]) arrGlobalMatches.append(match) #avoid when det file don't exist, example_dist=0 elif evaluationParams['E2E']: match_tuples.append(("===============", "==============", -1)) dt_text = u'' for gtNum in range(len(gtPols)): if gtNum in gtDontCarePolsNum: continue gt_text = gtTrans[gtNum] dist = text_distance(gt_text, dt_text) example_dist += dist match_tuples.append((gt_text, dt_text, dist)) total_dist += example_dist if evaluationParams['E2E']: logger.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') logger.debug("file:{}".format(resFile)) for tp in match_tuples: gt_text, dt_text, dist = tp logger.debug(u'GT: "{}" matched to DT: "{}", distance = {}'.format(gt_text, dt_text, dist)) logger.debug('Distance = {:f}'.format(example_dist)) logger.debug('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<') numGtCare = (len(gtPols) - len(gtDontCarePolsNum)) numDetCare = (len(detPols) - len(detDontCarePolsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if numDetCare > 0 else float(1) sampleAP = precision else: recall = float(detMatched) / numGtCare precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare if evaluationParams['CONFIDENCES'] and evaluationParams['PER_SAMPLE_RESULTS']: sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare) hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / (precision + recall) matchedSum += detMatched numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare if evaluationParams['PER_SAMPLE_RESULTS']: perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'pairs': pairs, 'AP': sampleAP, 'iouMat': [] if len(detPols) > 100 else iouMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtDontCare': gtDontCarePolsNum, 'detDontCare': detDontCarePolsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } if evaluationParams['E2E']: perSampleMetrics[resFile]['exampleDistance'] = example_dist # print("file:{} exampleDistance:{}".format(resFile,example_dist)) # Compute MAP and MAR AP = 0 if evaluationParams['CONFIDENCES']: AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt) methodRecall = 0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodDistance = 0 if len(gt) == 0 else float(total_dist)/len(gt) methodMetrics = {'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP, 'distance': methodDistance} resDict = {'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics} return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) def polygon_from_points(points, correctOffset=False): """ Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4 """ if correctOffset: #this will substract 1 from the coordinates that correspond to the xmax and ymax points[2] -= 1 points[4] -= 1 points[5] -= 1 points[7] -= 1 resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(points[0]) resBoxes[0, 4] = int(points[1]) resBoxes[0, 1] = int(points[2]) resBoxes[0, 5] = int(points[3]) resBoxes[0, 2] = int(points[4]) resBoxes[0, 6] = int(points[5]) resBoxes[0, 3] = int(points[6]) resBoxes[0, 7] = int(points[7]) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_polygon(rect): resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(rect.xmin) resBoxes[0, 4] = int(rect.ymax) resBoxes[0, 1] = int(rect.xmin) resBoxes[0, 5] = int(rect.ymin) resBoxes[0, 2] = int(rect.xmax) resBoxes[0, 6] = int(rect.ymin) resBoxes[0, 3] = int(rect.xmax) resBoxes[0, 7] = int(rect.ymax) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_points(rect): points = [ int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin) ] return points def get_union(pD, pG): areaA = pD.area() areaB = pG.area() return areaA + areaB - get_intersection(pD, pG) def get_intersection_over_union(pD, pG): try: return get_intersection(pD, pG) / get_union(pD, pG) except: return 0 def get_intersection(pD, pG): pInt = pD & pG if len(pInt) == 0: return 0 return pInt.area() def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP def transcription_match(transGt, transDet, specialCharacters=unicode('!?.:,*"()·[]/\'', 'utf-8'), onlyRemoveFirstLastCharacterGT=True): if onlyRemoveFirstLastCharacterGT: #special characters in GT are allowed only at initial or final position if (transGt == transDet): return True if specialCharacters.find(transGt[0]) > -1: if transGt[1:] == transDet: return True if specialCharacters.find(transGt[-1]) > -1: if transGt[0:len(transGt) - 1] == transDet: return True if specialCharacters.find( transGt[0]) > -1 and specialCharacters.find( transGt[-1]) > -1: if transGt[1:len(transGt) - 1] == transDet: return True return False else: #Special characters are removed from the begining and the end of both Detection and GroundTruth while len(transGt) > 0 and specialCharacters.find(transGt[0]) > -1: transGt = transGt[1:] while len(transDet) > 0 and specialCharacters.find( transDet[0]) > -1: transDet = transDet[1:] while len(transGt) > 0 and specialCharacters.find( transGt[-1]) > -1: transGt = transGt[0:len(transGt) - 1] while len(transDet) > 0 and specialCharacters.find( transDet[-1]) > -1: transDet = transDet[0:len(transDet) - 1] return transGt == transDet def include_in_dictionary(transcription): """ Function used in Word Spotting that finds if the Ground Truth transcription meets the rules to enter into the dictionary. If not, the transcription will be cared as don't care """ #special case 's at final if transcription[len(transcription) - 2:] == "'s" or transcription[len(transcription) - 2:] == "'S": transcription = transcription[0:len(transcription) - 2] #hypens at init or final of the word transcription = transcription.strip('-') specialCharacters = unicode("'!?.:,*\"()·[]/", "utf-8") for character in specialCharacters: transcription = transcription.replace(character, ' ') transcription = transcription.strip() if len(transcription) != len(transcription.replace(" ", "")): return False if len(transcription) < evaluationParams['MIN_LENGTH_CARE_WORD']: return False notAllowed = unicode("×÷·", "utf-8") range1 = [ord(u'a'), ord(u'z')] range2 = [ord(u'A'), ord(u'Z')] range3 = [ord(u'À'), ord(u'ƿ')] range4 = [ord(u'DŽ'), ord(u'ɿ')] range5 = [ord(u'Ά'), ord(u'Ͽ')] range6 = [ord(u'-'), ord(u'-')] for char in transcription: charCode = ord(char) if (notAllowed.find(char) != -1): return False valid = (charCode >= range1[0] and charCode <= range1[1]) or ( charCode >= range2[0] and charCode <= range2[1] ) or (charCode >= range3[0] and charCode <= range3[1]) or ( charCode >= range4[0] and charCode <= range4[1]) or ( charCode >= range5[0] and charCode <= range5[1]) or (charCode >= range6[0] and charCode <= range6[1]) if valid == False: return False return True def include_in_dictionary_transcription(transcription): """ Function applied to the Ground Truth transcriptions used in Word Spotting. It removes special characters or terminations """ #special case 's at final if transcription[len(transcription) - 2:] == "'s" or transcription[len(transcription) - 2:] == "'S": transcription = transcription[0:len(transcription) - 2] #hypens at init or final of the word transcription = transcription.strip('-') specialCharacters = unicode("'!?.:,*\"()·[]/", "utf-8") for character in specialCharacters: transcription = transcription.replace(character, ' ') transcription = transcription.strip() return transcription perSampleMetrics = {} matchedSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') gt = rrc_evaluation_funcs.load_zip_file( gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file( submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGlobalCareGt = 0 numGlobalCareDet = 0 arrGlobalConfidences = [] arrGlobalMatches = [] for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) if (gtFile is None): raise Exception("The file %s is not UTF-8" % resFile) recall = 0 precision = 0 hmean = 0 detCorrect = 0 iouMat = np.empty([1, 1]) gtPols = [] detPols = [] gtTrans = [] detTrans = [] gtPolPoints = [] detPolPoints = [] gtDontCarePolsNum = [ ] #Array of Ground Truth Polygons' keys marked as don't Care detDontCarePolsNum = [ ] #Array of Detected Polygons' matched with a don't Care GT detMatchedNums = [] pairs = [] arrSampleConfidences = [] arrSampleMatch = [] sampleAP = 0 evaluationLog = "" pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" if evaluationParams['LTRB']: gtRect = Rectangle(*points) gtPol = rectangle_to_polygon(gtRect) else: gtPol = polygon_from_points(points) gtPols.append(gtPol) gtPolPoints.append(points) #On word spotting we will filter some transcriptions with special characters if evaluationParams['WORD_SPOTTING']: if dontCare == False: if include_in_dictionary(transcription) == False: dontCare = True else: transcription = include_in_dictionary_transcription( transcription) gtTrans.append(transcription) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) evaluationLog += "GT polygons: " + str(len(gtPols)) + ( " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, confidencesList, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( detFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True, evaluationParams['CONFIDENCES']) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] if evaluationParams['LTRB']: detRect = Rectangle(*points) detPol = rectangle_to_polygon(detRect) else: detPol = polygon_from_points(points) detPols.append(detPol) detPolPoints.append(points) detTrans.append(transcription) if len(gtDontCarePolsNum) > 0: for dontCarePol in gtDontCarePolsNum: dontCarePol = gtPols[dontCarePol] intersected_area = get_intersection( dontCarePol, detPol) pdDimensions = detPol.area() precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']): detDontCarePolsNum.append(len(detPols) - 1) break evaluationLog += "DET polygons: " + str(len(detPols)) + ( " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n") if len(gtPols) > 0 and len(detPols) > 0: #Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] iouMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): if gtRectMat[gtNum] == 0 and detRectMat[ detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: if iouMat[gtNum, detNum] > evaluationParams[ 'IOU_CONSTRAINT']: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 #detection matched only if transcription is equal if evaluationParams['WORD_SPOTTING']: correct = gtTrans[gtNum].upper( ) == detTrans[detNum].upper() else: correct = transcription_match( gtTrans[gtNum].upper(), detTrans[detNum].upper(), evaluationParams['SPECIAL_CHARACTERS'], evaluationParams[ 'ONLY_REMOVE_FIRST_LAST_CHARACTER'] ) == True detCorrect += (1 if correct else 0) if correct: detMatchedNums.append(detNum) pairs.append({ 'gt': gtNum, 'det': detNum, 'correct': correct }) evaluationLog += "Match GT #" + str( gtNum) + " with Det #" + str( detNum) + " trans. correct: " + str( correct) + "\n" if evaluationParams['CONFIDENCES']: for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: #we exclude the don't care detections match = detNum in detMatchedNums arrSampleConfidences.append(confidencesList[detNum]) arrSampleMatch.append(match) arrGlobalConfidences.append(confidencesList[detNum]) arrGlobalMatches.append(match) numGtCare = (len(gtPols) - len(gtDontCarePolsNum)) numDetCare = (len(detPols) - len(detDontCarePolsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if numDetCare > 0 else float(1) sampleAP = precision else: recall = float(detCorrect) / numGtCare precision = 0 if numDetCare == 0 else float( detCorrect) / numDetCare if evaluationParams['CONFIDENCES']: sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare) hmean = 0 if ( precision + recall) == 0 else 2.0 * precision * recall / (precision + recall) matchedSum += detCorrect numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'pairs': pairs, 'AP': sampleAP, 'iouMat': [] if len(detPols) > 100 else iouMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtTrans': gtTrans, 'detTrans': detTrans, 'gtDontCare': gtDontCarePolsNum, 'detDontCare': detDontCarePolsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } # Compute AP AP = 0 if evaluationParams['CONFIDENCES']: AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt) methodRecall = 0 if numGlobalCareGt == 0 else float( matchedSum) / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else float( matchedSum) / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodMetrics = { 'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP } resDict = { 'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics } return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) def one_to_one_match(row, col): cont = 0 for j in range(len(recallMat[0])): if recallMat[row, j] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row, j] >= \ evaluationParams['AREA_PRECISION_CONSTRAINT']: cont = cont + 1 if (cont != 1): return False cont = 0 for i in range(len(recallMat)): if recallMat[i, col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[i, col] >= \ evaluationParams['AREA_PRECISION_CONSTRAINT']: cont = cont + 1 if (cont != 1): return False if recallMat[row, col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row, col] >= \ evaluationParams['AREA_PRECISION_CONSTRAINT']: return True return False def one_to_many_match(gtNum): many_sum = 0 detRects = [] for detNum in range(len(recallMat[0])): if gtRectMat[gtNum] == 0 and detRectMat[ detNum] == 0 and detNum not in detDontCareRectsNum: if precisionMat[gtNum, detNum] >= evaluationParams[ 'AREA_PRECISION_CONSTRAINT']: many_sum += recallMat[gtNum, detNum] detRects.append(detNum) if many_sum >= evaluationParams['AREA_RECALL_CONSTRAINT']: return True, detRects else: return False, [] def many_to_one_match(detNum): many_sum = 0 gtRects = [] for gtNum in range(len(recallMat)): if gtRectMat[gtNum] == 0 and detRectMat[ detNum] == 0 and gtNum not in gtDontCareRectsNum: if recallMat[ gtNum, detNum] >= evaluationParams['AREA_RECALL_CONSTRAINT']: many_sum += precisionMat[gtNum, detNum] gtRects.append(gtNum) if many_sum >= evaluationParams['AREA_PRECISION_CONSTRAINT']: return True, gtRects else: return False, [] def area(a, b): dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin) + 1 dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin) + 1 if (dx >= 0) and (dy >= 0): return dx * dy else: return 0. def center(r): x = float(r.xmin) + float(r.xmax - r.xmin + 1) / 2. y = float(r.ymin) + float(r.ymax - r.ymin + 1) / 2. return Point(x, y) def point_distance(r1, r2): distx = math.fabs(r1.x - r2.x) disty = math.fabs(r1.y - r2.y) return math.sqrt(distx * distx + disty * disty) def center_distance(r1, r2): return point_distance(center(r1), center(r2)) def diag(r): w = (r.xmax - r.xmin + 1) h = (r.ymax - r.ymin + 1) return math.sqrt(h * h + w * w) perSampleMetrics = {} methodRecallSum = 0 methodPrecisionSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') Point = namedtuple('Point', 'x y') gt = rrc_evaluation_funcs.load_zip_file( gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file( submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGt = 0 numDet = 0 for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 recallAccum = 0. precisionAccum = 0. gtRects = [] detRects = [] gtPolPoints = [] detPolPoints = [] gtDontCareRectsNum = [ ] # Array of Ground Truth Rectangles' keys marked as don't Care detDontCareRectsNum = [ ] # Array of Detected Rectangles' matched with a don't Care GT pairs = [] evaluationLog = "" recallMat = np.empty([1, 1]) precisionMat = np.empty([1, 1]) pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( gtFile, evaluationParams['CRLF'], True, True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" # convert x1,y1,x2,y2,x3,y3,x4,y4 to xmin,ymin,xmax,ymax if len(points) == 8: points_tmp = np.array(points).reshape(4, 2) points_x = points_tmp[:, 0] points_y = points_tmp[:, 1] xmin = points_x[np.argmin(points_x)] xmax = points_x[np.argmax(points_x)] ymin = points_y[np.argmin(points_y)] ymax = points_y[np.argmax(points_y)] points = [xmin, ymin, xmax, ymax] gtRect = Rectangle(*points) gtRects.append(gtRect) gtPolPoints.append(points) if dontCare: gtDontCareRectsNum.append(len(gtRects) - 1) evaluationLog += "GT rectangles: " + str(len(gtRects)) + ( " (" + str(len(gtDontCareRectsNum)) + " don't care)\n" if len(gtDontCareRectsNum) > 0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, _, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( detFile, evaluationParams['CRLF'], True, True, False) for n in range(len(pointsList)): points = pointsList[n] # print points detRect = Rectangle(*points) detRects.append(detRect) detPolPoints.append(points) if len(gtDontCareRectsNum) > 0: for dontCareRectNum in gtDontCareRectsNum: dontCareRect = gtRects[dontCareRectNum] intersected_area = area(dontCareRect, detRect) rdDimensions = ((detRect.xmax - detRect.xmin + 1) * (detRect.ymax - detRect.ymin + 1)) if (rdDimensions == 0): precision = 0 else: precision = intersected_area / rdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']): detDontCareRectsNum.append(len(detRects) - 1) break evaluationLog += "DET rectangles: " + str(len(detRects)) + ( " (" + str(len(detDontCareRectsNum)) + " don't care)\n" if len(detDontCareRectsNum) > 0 else "\n") if len(gtRects) == 0: recall = 1 precision = 0 if len(detRects) > 0 else 1 if len(detRects) > 0: # Calculate recall and precision matrixs outputShape = [len(gtRects), len(detRects)] recallMat = np.empty(outputShape) precisionMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtRects), np.int8) detRectMat = np.zeros(len(detRects), np.int8) for gtNum in range(len(gtRects)): for detNum in range(len(detRects)): rG = gtRects[gtNum] rD = detRects[detNum] intersected_area = area(rG, rD) rgDimensions = ((rG.xmax - rG.xmin + 1) * (rG.ymax - rG.ymin + 1)) rdDimensions = ((rD.xmax - rD.xmin + 1) * (rD.ymax - rD.ymin + 1)) recallMat[ gtNum, detNum] = 0 if rgDimensions == 0 else intersected_area / rgDimensions precisionMat[ gtNum, detNum] = 0 if rdDimensions == 0 else intersected_area / rdDimensions # Find one-to-one matches evaluationLog += "Find one-to-one matches\n" for gtNum in range(len(gtRects)): for detNum in range(len(detRects)): if gtRectMat[gtNum] == 0 and detRectMat[ detNum] == 0 and gtNum not in gtDontCareRectsNum and detNum not in detDontCareRectsNum: match = one_to_one_match(gtNum, detNum) if match is True: rG = gtRects[gtNum] rD = detRects[detNum] normDist = center_distance(rG, rD) normDist /= diag(rG) + diag(rD) normDist *= 2.0 if normDist < evaluationParams[ 'EV_PARAM_IND_CENTER_DIFF_THR']: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 recallAccum += evaluationParams[ 'MTYPE_OO_O'] precisionAccum += evaluationParams[ 'MTYPE_OO_O'] pairs.append({ 'gt': gtNum, 'det': detNum, 'type': 'OO' }) evaluationLog += "Match GT #" + str( gtNum) + " with Det #" + str( detNum) + "\n" else: evaluationLog += "Match Discarded GT #" + str( gtNum) + " with Det #" + str( detNum) + " normDist: " + str( normDist) + " \n" # Find one-to-many matches evaluationLog += "Find one-to-many matches\n" for gtNum in range(len(gtRects)): if gtNum not in gtDontCareRectsNum: match, matchesDet = one_to_many_match(gtNum) if match is True: gtRectMat[gtNum] = 1 recallAccum += evaluationParams['MTYPE_OM_O'] precisionAccum += evaluationParams[ 'MTYPE_OM_O'] * len(matchesDet) pairs.append({ 'gt': gtNum, 'det': matchesDet, 'type': 'OM' }) for detNum in matchesDet: detRectMat[detNum] = 1 evaluationLog += "Match GT #" + str( gtNum) + " with Det #" + str(matchesDet) + "\n" # Find many-to-one matches evaluationLog += "Find many-to-one matches\n" for detNum in range(len(detRects)): if detNum not in detDontCareRectsNum: match, matchesGt = many_to_one_match(detNum) if match is True: detRectMat[detNum] = 1 recallAccum += evaluationParams[ 'MTYPE_OM_M'] * len(matchesGt) precisionAccum += evaluationParams['MTYPE_OM_M'] pairs.append({ 'gt': matchesGt, 'det': detNum, 'type': 'MO' }) for gtNum in matchesGt: gtRectMat[gtNum] = 1 evaluationLog += "Match GT #" + str( matchesGt) + " with Det #" + str(detNum) + "\n" numGtCare = (len(gtRects) - len(gtDontCareRectsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if len(detRects) > 0 else float(1) else: recall = float(recallAccum) / numGtCare precision = float(0) if ( len(detRects) - len(detDontCareRectsNum) ) == 0 else float(precisionAccum) / ( len(detRects) - len(detDontCareRectsNum)) hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / ( precision + recall) evaluationLog += "Recall = " + str(recall) + "\n" evaluationLog += "Precision = " + str(precision) + "\n" methodRecallSum += recallAccum methodPrecisionSum += precisionAccum numGt += len(gtRects) - len(gtDontCareRectsNum) numDet += len(detRects) - len(detDontCareRectsNum) perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'pairs': pairs, 'recallMat': [] if len(detRects) > 100 else recallMat.tolist(), 'precisionMat': [] if len(detRects) > 100 else precisionMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtDontCare': gtDontCareRectsNum, 'detDontCare': detDontCareRectsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } methodRecall = 0 if numGt == 0 else methodRecallSum / numGt methodPrecision = 0 if numDet == 0 else methodPrecisionSum / numDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodMetrics = { 'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean } resDict = { 'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics } return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) def polygon_from_points(points): """ Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4 """ resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(points[0]) resBoxes[0, 4] = int(points[1]) resBoxes[0, 1] = int(points[2]) resBoxes[0, 5] = int(points[3]) resBoxes[0, 2] = int(points[4]) resBoxes[0, 6] = int(points[5]) resBoxes[0, 3] = int(points[6]) resBoxes[0, 7] = int(points[7]) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_polygon(rect): resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(rect.xmin) resBoxes[0, 4] = int(rect.ymax) resBoxes[0, 1] = int(rect.xmin) resBoxes[0, 5] = int(rect.ymin) resBoxes[0, 2] = int(rect.xmax) resBoxes[0, 6] = int(rect.ymin) resBoxes[0, 3] = int(rect.xmax) resBoxes[0, 7] = int(rect.ymax) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_points(rect): points = [ int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin) ] return points def get_union(pD, pG): areaA = pD.area() areaB = pG.area() return areaA + areaB - get_intersection(pD, pG) def get_intersection_over_union(pD, pG): try: return get_intersection(pD, pG) / get_union(pD, pG) except: return 0 def funcCt(x): if x <= 0.01: return 1 else: return 1 - x def get_text_intersection_over_union_recall(pD, pG): ''' Ct (cut): Area of ground truth that is not covered by detection bounding box. ''' try: Ct = pG.area() - get_intersection(pD, pG) assert (Ct >= 0 and Ct <= pG.area()), 'Invalid Ct value' assert (pG.area() > 0), 'Invalid Gt' return (get_intersection(pD, pG) * funcCt(Ct * 1.0 / pG.area())) / get_union(pD, pG) except Exception as e: return 0 def funcOt(x): if x <= 0.01: return 1 else: return 1 - x def get_text_intersection_over_union_precision(pD, pG, gtNum, gtPolys, gtDontCarePolsNum): ''' Ot: Outlier gt area ''' Ot = 0 try: inside_pG = pD & pG gt_union_inside_pD = None gt_union_inside_pD_and_pG = None count_initial = 0 for i in xrange(len(gtPolys)): if i != gtNum and gtNum not in gtDontCarePolsNum: # ignore don't care regions if not get_intersection(pD, gtPolys[i]) == 0: if count_initial == 0: # initial gt_union_inside_pD = gtPolys[i] gt_union_inside_pD_and_pG = inside_pG & gtPolys[i] count_initial = 1 continue gt_union_inside_pD = gt_union_inside_pD | gtPolys[i] inside_pG_i = inside_pG & gtPolys[i] gt_union_inside_pD_and_pG = gt_union_inside_pD_and_pG | inside_pG_i if not gt_union_inside_pD == None: pD_union_with_other_gt = pD & gt_union_inside_pD Ot = pD_union_with_other_gt.area( ) - gt_union_inside_pD_and_pG.area() if Ot <= 1.0e-10: Ot = 0 else: Ot = 0 assert (Ot >= 0 and Ot <= pD.area() + 1 ), ' Invalid Ot value: ' + str(Ot) + ' ' + str(pD.area()) assert (pD.area() > 0), ' Invalid pD area: ' + str(pD.area()) return (get_intersection(pD, pG) * funcOt(Ot * 1.0 / pD.area())) / get_union(pD, pG) except Exception as e: # print(e) return 0 def get_intersection(pD, pG): pInt = pD & pG if len(pInt) == 0: return 0 return pInt.area() def get_intersection_three(pD, pG, pGi): pInt = pD & pG pInt_3 = pInt & pGi if len(pInt_3) == 0: return 0 return pInt_3.area() def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP perSampleMetrics = {} matchedSum = 0 matchedSum_iou = 0 matchedSum_tiouGt = 0 matchedSum_tiouDt = 0 matchedSum_cutGt = 0 matchedSum_coverOtherGt = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') gt = rrc_evaluation_funcs.load_zip_file( gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file( submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGlobalCareGt = 0 numGlobalCareDet = 0 arrGlobalConfidences = [] arrGlobalMatches = [] totalNumGtPols = 0 totalNumDetPols = 0 # fper_ = open('per_samle_result.txt', 'w') for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 detMatched = 0 detMatched_iou = 0 detMatched_tiouGt = 0 detMatched_tiouDt = 0 detMatched_cutGt = 0 detMatched_coverOtherGt = 0 iouMat = np.empty([1, 1]) gtPols = [] detPols = [] gtPolPoints = [] detPolPoints = [] #Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [] #Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [] pairs = [] detMatchedNums = [] arrSampleConfidences = [] arrSampleMatch = [] sampleAP = 0 evaluationLog = "" pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" if evaluationParams['LTRB']: gtRect = Rectangle(*points) gtPol = rectangle_to_polygon(gtRect) else: gtPol = polygon_from_points(points) gtPols.append(gtPol) gtPolPoints.append(points) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) evaluationLog += "GT polygons: " + str(len(gtPols)) + ( " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( detFile, evaluationParams['CRLF'], evaluationParams['LTRB'], False, evaluationParams['CONFIDENCES']) for n in range(len(pointsList)): points = pointsList[n] if evaluationParams['LTRB']: detRect = Rectangle(*points) detPol = rectangle_to_polygon(detRect) else: detPol = polygon_from_points(points) detPols.append(detPol) detPolPoints.append(points) if len(gtDontCarePolsNum) > 0: for dontCarePol in gtDontCarePolsNum: dontCarePol = gtPols[dontCarePol] intersected_area = get_intersection( dontCarePol, detPol) pdDimensions = detPol.area() precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']): detDontCarePolsNum.append(len(detPols) - 1) break evaluationLog += "DET polygons: " + str(len(detPols)) + ( " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n") if len(gtPols) > 0 and len(detPols) > 0: #Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] iouMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) tiouRecallMat = np.empty(outputShape) tiouPrecisionMat = np.empty(outputShape) tiouGtRectMat = np.zeros(len(gtPols), np.int8) tiouDetRectMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG) tiouRecallMat[ gtNum, detNum] = get_text_intersection_over_union_recall( pD, pG) tiouPrecisionMat[ gtNum, detNum] = get_text_intersection_over_union_precision( pD, pG, gtNum, gtPols, gtDontCarePolsNum) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): if gtRectMat[gtNum] == 0 and detRectMat[ detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: if iouMat[gtNum, detNum] > evaluationParams[ 'IOU_CONSTRAINT']: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 detMatched += 1 detMatched_iou += iouMat[gtNum, detNum] detMatched_tiouGt += tiouRecallMat[gtNum, detNum] detMatched_tiouDt += tiouPrecisionMat[gtNum, detNum] if iouMat[gtNum, detNum] != tiouRecallMat[gtNum, detNum]: detMatched_cutGt += 1 if iouMat[gtNum, detNum] != tiouPrecisionMat[gtNum, detNum]: detMatched_coverOtherGt += 1 pairs.append({'gt': gtNum, 'det': detNum}) detMatchedNums.append(detNum) evaluationLog += "Match GT #" + str( gtNum) + " with Det #" + str(detNum) + "\n" if evaluationParams['CONFIDENCES']: for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: #we exclude the don't care detections match = detNum in detMatchedNums arrSampleConfidences.append(confidencesList[detNum]) arrSampleMatch.append(match) arrGlobalConfidences.append(confidencesList[detNum]) arrGlobalMatches.append(match) numGtCare = (len(gtPols) - len(gtDontCarePolsNum)) numDetCare = (len(detPols) - len(detDontCarePolsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if numDetCare > 0 else float(1) sampleAP = precision tiouRecall = float(1) tiouPrecision = float(0) if numDetCare > 0 else float(1) else: recall = float(detMatched) / numGtCare precision = 0 if numDetCare == 0 else float( detMatched) / numDetCare iouRecall = float(detMatched_iou) / numGtCare iouPrecision = 0 if numDetCare == 0 else float( detMatched_iou) / numDetCare tiouRecall = float(detMatched_tiouGt) / numGtCare tiouPrecision = 0 if numDetCare == 0 else float( detMatched_tiouDt) / numDetCare if evaluationParams['CONFIDENCES'] and evaluationParams[ 'PER_SAMPLE_RESULTS']: sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare) hmean = 0 if ( precision + recall) == 0 else 2.0 * precision * recall / (precision + recall) tiouHmean = 0 if ( tiouPrecision + tiouRecall) == 0 else 2.0 * tiouPrecision * tiouRecall / ( tiouPrecision + tiouRecall) iouHmean = 0 if ( iouPrecision + iouRecall ) == 0 else 2.0 * iouPrecision * iouRecall / (iouPrecision + iouRecall) matchedSum += detMatched matchedSum_iou += detMatched_iou matchedSum_tiouGt += detMatched_tiouGt matchedSum_tiouDt += detMatched_tiouDt matchedSum_cutGt += detMatched_cutGt matchedSum_coverOtherGt += detMatched_coverOtherGt numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare if evaluationParams['PER_SAMPLE_RESULTS']: perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'iouPrecision': iouPrecision, 'iouRecall': iouRecall, 'iouHmean': iouHmean, 'tiouPrecision': tiouPrecision, 'tiouRecall': tiouRecall, 'tiouHmean': tiouHmean, 'pairs': pairs, 'AP': sampleAP, 'iouMat': [] if len(detPols) > 100 else iouMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtDontCare': gtDontCarePolsNum, 'detDontCare': detDontCarePolsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } # fper_.writelines(resFile+'\t"IoU: (P: {:.3f}. R: {:.3f}. F: {:.3f})",\t"TIoU: (P: {:.3f}. R: {:.3f}. F: {:.3f})".\n'.format(precision, recall, hmean, tiouPrecision, tiouRecall, tiouHmean)) try: totalNumGtPols += len(gtPols) totalNumDetPols += len(detPols) except Exception as e: raise e # fper_.close() # Compute MAP and MAR AP = 0 if evaluationParams['CONFIDENCES']: AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt) print('num_gt, num_det: ', numGlobalCareGt, totalNumDetPols) methodRecall = 0 if numGlobalCareGt == 0 else float( matchedSum) / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else float( matchedSum) / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodRecall_iou = 0 if numGlobalCareGt == 0 else float( matchedSum_iou) / numGlobalCareGt methodPrecision_iou = 0 if numGlobalCareDet == 0 else float( matchedSum_iou) / numGlobalCareDet iouMethodHmean = 0 if methodRecall_iou + methodPrecision_iou == 0 else 2 * methodRecall_iou * methodPrecision_iou / ( methodRecall_iou + methodPrecision_iou) methodRecall_tiouGt = 0 if numGlobalCareGt == 0 else float( matchedSum_tiouGt) / numGlobalCareGt methodPrecision_tiouDt = 0 if numGlobalCareDet == 0 else float( matchedSum_tiouDt) / numGlobalCareDet tiouMethodHmean = 0 if methodRecall_tiouGt + methodPrecision_tiouDt == 0 else 2 * methodRecall_tiouGt * methodPrecision_tiouDt / ( methodRecall_tiouGt + methodPrecision_tiouDt) methodMetrics = { 'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean } iouMethodMetrics = { 'iouPrecision': methodPrecision_iou, 'iouRecall': methodRecall_iou, 'iouHmean': iouMethodHmean } tiouMethodMetrics = { 'tiouPrecision': methodPrecision_tiouDt, 'tiouRecall': methodRecall_tiouGt, 'tiouHmean': tiouMethodHmean } print("recall: ", round(methodRecall, 3), "precision: ", round(methodPrecision, 3), "hmean: ", round(methodHmean, 3)) print("tiouRecall:", round(methodRecall_tiouGt, 3), "tiouPrecision:", round(methodPrecision_tiouDt, 3), "tiouHmean:", round(tiouMethodHmean, 3)) resDict = { 'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics, 'iouMethod': iouMethodMetrics, 'tiouMethod': tiouMethodMetrics } return resDict
def validate_data(gtFilePath, submFilePath, evaluationParams): """ Method validate_data: validates that all files in the results folder are correct (have the correct name contents). Validates also that there are no missing files in the folder. If some error detected, the method raises the error """ gtFile = rrc_evaluation_funcs.decode_utf8(open(gtFilePath, 'rb').read()) if (gtFile is None): raise Exception("The GT file is not UTF-8") gtLines = gtFile.split("\r\n" if evaluationParams['CRLF'] else "\n") ids = {} for line in gtLines: line = line.replace("\r", "").replace("\n", "") if (line != ""): if (evaluationParams['DOUBLE_QUOTES']): m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?\"(.*)\"\s*\t?$', line) else: m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.*)$', line) if m == None: if (evaluationParams['DOUBLE_QUOTES']): raise Exception( ("Line in GT not valid.Found: %s should be: %s" % (line, evaluationParams['SAMPLE_NAME_2_ID'] + ',transcription')).encode('utf-8', 'replace')) else: raise Exception( ("Line in GT not valid.Found: %s should be: %s" % (line, evaluationParams['SAMPLE_NAME_2_ID'] + ',"transcription"')).encode('utf-8', 'replace')) ids[m.group(1)] = {'gt': m.group(2), 'det': ''} submFile = rrc_evaluation_funcs.decode_utf8( open(submFilePath, 'rb').read()) if (submFile is None): raise Exception("The Det file is not UTF-8") submLines = submFile.split("\r\n" if evaluationParams['CRLF'] else "\n") for line in submLines: line = line.replace("\r", "").replace("\n", "") if (line != ""): if (evaluationParams['DOUBLE_QUOTES']): m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?\"(.*)\"\s*\t?$', line) else: m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.*)$', line) if m == None: if (evaluationParams['DOUBLE_QUOTES']): raise Exception( ("Line in results not valid.Found: %s should be: %s" % (line, evaluationParams['SAMPLE_NAME_2_ID'] + ',transcription')).encode('utf-8', 'replace')) else: raise Exception( ("Line in results not valid.Found: %s should be: %s" % (line, evaluationParams['SAMPLE_NAME_2_ID'] + ',"transcription"')).encode('utf-8', 'replace')) try: ids[m.group(1)]['det'] = m.group(2) except Exception, e: raise Exception(( "Line in results not valid. Line: %s Sample item not valid: %s" % (line, m.group(1))).encode('utf-8', 'replace'))
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) gtFile = rrc_evaluation_funcs.decode_utf8(open(gtFilePath, 'rb').read()) gtLines = gtFile.split("\r\n" if evaluationParams['CRLF'] else "\n") ids = {} for line in gtLines: line = line.replace("\r", "").replace("\n", "") if (line != ""): if (evaluationParams['DOUBLE_QUOTES']): m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?\"(.+)\"$', line) ids[m.group(1)] = { "gt": m.group(2).replace("\\\\", "\\").replace("\\\"", "\""), "det": "" } else: m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.+)$', line) ids[m.group(1)] = {"gt": m.group(2), "det": ""} totalDistance = 0.0 totalLength = 0.0 totalDistanceUpper = 0.0 totalLengthUpper = 0.0 numWords = 0 correctWords = 0.0 correctWordsUpper = 0.0 perSampleMetrics = {} submFile = rrc_evaluation_funcs.decode_utf8( open(submFilePath, 'rb').read()) if (submFile is None): raise Exception("The file is not UTF-8") xls_output = StringIO() workbook = xlsxwriter.Workbook(xls_output) worksheet = workbook.add_worksheet() worksheet.write(1, 1, "sample") worksheet.write(1, 2, "gt") worksheet.write(1, 3, "E.D.") worksheet.write(1, 4, "normalized") worksheet.write(1, 5, "E.D. upper") worksheet.write(1, 6, "normalized upper") submLines = submFile.split("\r\n" if evaluationParams['CRLF'] else "\n") for line in submLines: line = line.replace("\r", "").replace("\n", "") if (line != ""): numWords = numWords + 1 if (evaluationParams['DOUBLE_QUOTES']): m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?\"(.*)\"\s*\t?$', line) detected = m.group(2).replace("\\\\", "\\").replace("\\\"", "\"") else: m = re.match( r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.*)$', line) detected = m.group(2) ids[m.group(1)]['det'] = detected row = 1 for k, v in ids.iteritems(): gt = v['gt'] detected = v['det'] if gt == detected: correctWords = correctWords + 1 if gt.upper() == detected.upper(): correctWordsUpper = correctWordsUpper + 1 distance = editdistance.eval(gt, detected) length = float(distance) / len(gt) distance_up = editdistance.eval(gt.upper(), detected.upper()) length_up = float(distance_up) / len(gt) totalDistance += distance totalLength += length totalDistanceUpper += distance_up totalLengthUpper += length_up perSampleMetrics[k] = { 'gt': gt, 'det': detected, 'edist': distance, 'norm': length, 'edistUp': distance_up, 'normUp': length_up } row = row + 1 worksheet.write(row, 1, k) worksheet.write(row, 2, gt) worksheet.write(row, 3, detected) worksheet.write(row, 4, distance) worksheet.write(row, 5, length) worksheet.write(row, 6, distance_up) worksheet.write(row, 7, length_up) methodMetrics = { 'totalWords': len(ids), 'detWords': numWords, 'crwN': correctWords, 'crwupN': correctWordsUpper, 'ted': totalDistance, 'tedL': totalLength, 'crw': 0 if numWords == 0 else correctWords / numWords, 'crwN': correctWords, 'tedup': totalDistanceUpper, 'tedupL': totalLengthUpper, 'crwup': 0 if numWords == 0 else correctWordsUpper / numWords, 'crwupN': correctWordsUpper } workbook.close() output_items = {'samples.xlsx': xls_output.getvalue()} xls_output.close() resDict = { 'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics, 'output_items': output_items } return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().items(): globals()[alias] = importlib.import_module(module) def polygon_from_points(points): """ Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4 """ resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(points[0]) resBoxes[0, 4] = int(points[1]) resBoxes[0, 1] = int(points[2]) resBoxes[0, 5] = int(points[3]) resBoxes[0, 2] = int(points[4]) resBoxes[0, 6] = int(points[5]) resBoxes[0, 3] = int(points[6]) resBoxes[0, 7] = int(points[7]) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_polygon(rect): resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(rect.xmin) resBoxes[0, 4] = int(rect.ymin) resBoxes[0, 1] = int(rect.xmax) resBoxes[0, 5] = int(rect.ymin) resBoxes[0, 2] = int(rect.xmax) resBoxes[0, 6] = int(rect.ymax) resBoxes[0, 3] = int(rect.xmin) resBoxes[0, 7] = int(rect.ymax) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_points(rect): points = [ int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin) ] return points def polygon_to_points(pol): pointMat = [] for p in pol: for i in range(len(p)): pointMat.extend(p[i]) return pointMat def get_intersection(pD, pG): pInt = pD & pG if len(pInt) == 0: return 0 return pInt.area() def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP def point_distance(a, b): distx = math.fabs(a[0] - b[0]) disty = math.fabs(a[1] - b[1]) return math.sqrt(distx * distx + disty * disty) def diag(points): diag1 = point_distance((points[0], points[1]), (points[4], points[5])) diag2 = point_distance((points[2], points[3]), (points[6], points[7])) return (diag1 + diag2) / 2 def center_distance(p1, p2): return point_distance(p1.center(), p2.center()) def get_midpoints(p1, p2): return ((p1[0] + p2[0]) / 2, (p1[1] + p2[1]) / 2) def get_angle_3pt(a, b, c): """Counterclockwise angle in degrees by turning from a to c around b Returns a float between 0.0 and 360.0""" ang = math.degrees( math.atan2(c[1] - b[1], c[0] - b[0]) - math.atan2(a[1] - b[1], a[0] - b[0])) return ang + 360 if ang < 0 else ang def gtBoxtoChars(num, points): chars = [] assert len(points) == 8 p1 = get_midpoints([points[0], points[1]], [points[6], points[7]]) p2 = get_midpoints([points[2], points[3]], [points[4], points[5]]) unitx = (p2[0] - p1[0]) / num unity = (p2[1] - p1[1]) / num for i in range(num): x = p1[0] + unitx / 2 + unitx * i y = p1[1] + unity / 2 + unity * i chars.append((x, y)) return chars def char_fill(detNums, matchMat): for detNum in detNums: detPol = detPols[detNum] for gtNum, gtChars in enumerate(gtCharPoints): if matchMat[gtNum, detNum] == 1: for gtCharNum, gtChar in enumerate(gtChars): if detPol.isInside(gtChar[0], gtChar[1]): gtCharCounts[gtNum][detNum][gtCharNum] = 1 def one_to_one_match(row, col): cont = 0 for j in range(len(recallMat[0])): if recallMat[row, j] >= evaluationParams[ 'AREA_RECALL_CONSTRAINT'] and precisionMat[ row, j] >= evaluationParams['AREA_PRECISION_CONSTRAINT']: cont = cont + 1 if (cont != 1): return False cont = 0 for i in range(len(recallMat)): if recallMat[i, col] >= evaluationParams[ 'AREA_RECALL_CONSTRAINT'] and precisionMat[ i, col] >= evaluationParams['AREA_PRECISION_CONSTRAINT']: cont = cont + 1 if (cont != 1): return False if recallMat[row, col] >= evaluationParams[ 'AREA_RECALL_CONSTRAINT'] and precisionMat[ row, col] >= evaluationParams['AREA_PRECISION_CONSTRAINT']: return True return False def one_to_many_match(gtNum): many_sum = 0 detRects = [] for detNum in range(len(recallMat[0])): if detNum not in detDontCarePolsNum and gtExcludeMat[ gtNum] == 0 and detExcludeMat[detNum] == 0: if precisionMat[gtNum, detNum] >= evaluationParams[ 'AREA_PRECISION_CONSTRAINT']: many_sum += recallMat[gtNum, detNum] detRects.append(detNum) if many_sum >= evaluationParams['AREA_RECALL_CONSTRAINT'] and len( detRects) >= 2: pivots = [] for matchDet in detRects: pD = polygon_from_points(detPolPoints[matchDet]) pivots.append([get_midpoints(pD[0][0], pD[0][3]), pD.center()]) for i in range(len(pivots)): for k in range(len(pivots)): if k == i: continue angle = get_angle_3pt(pivots[i][0], pivots[k][1], pivots[i][1]) if angle > 180: angle = 360 - angle if min(angle, 180 - angle) >= 45: return False, [] return True, detRects else: return False, [] def many_to_one_match(detNum): many_sum = 0 gtRects = [] for gtNum in range(len(recallMat)): if gtNum not in gtDontCarePolsNum and gtExcludeMat[ gtNum] == 0 and detExcludeMat[detNum] == 0: if recallMat[ gtNum, detNum] >= evaluationParams['AREA_RECALL_CONSTRAINT']: many_sum += precisionMat[gtNum, detNum] gtRects.append(gtNum) if many_sum >= evaluationParams['AREA_PRECISION_CONSTRAINT'] and len( gtRects) >= 2: pivots = [] for matchGt in gtRects: pG = gtPols[matchGt] pivots.append([get_midpoints(pG[0][0], pG[0][3]), pG.center()]) for i in range(len(pivots)): for k in range(len(pivots)): if k == i: continue angle = get_angle_3pt(pivots[i][0], pivots[k][1], pivots[i][1]) if angle > 180: angle = 360 - angle if min(angle, 180 - angle) >= 45: return False, [] return True, gtRects else: return False, [] perSampleMetrics = {} methodRecallSum = 0 methodPrecisionSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') gt = rrc_evaluation_funcs.load_zip_file( gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file( submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGlobalCareGt = 0 numGlobalCareDet = 0 arrGlobalConfidences = [] arrGlobalMatches = [] for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 recallAccum = 0. precisionAccum = 0. detMatched = 0 numGtCare = 0 numDetCare = 0 recallMat = np.empty([1, 1]) precisionMat = np.empty([1, 1]) matchMat = np.zeros([1, 1]) gtPols = [] detPols = [] gtPolPoints = [] detPolPoints = [] # pseudo character centers gtCharPoints = [] gtCharCounts = [] # visualization charCounts = np.zeros([1, 1]) recallScore = list() precisionScore = list() #Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [] #Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [] pairs = [] detMatchedNums = [] gtExcludeNums = [] arrSampleConfidences = [] arrSampleMatch = [] sampleAP = 0 evaluationLog = "" pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" if evaluationParams['LTRB']: gtRect = Rectangle(*points) gtPol = rectangle_to_polygon(gtRect) points = polygon_to_points(gtPol) else: gtPol = polygon_from_points(points) gtPols.append(gtPol) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) gtPolPoints.append(points) gtCharPoints.append([]) else: gtCharSize = len(transcription) aspect_ratio = gtPol.aspectRatio() if aspect_ratio > 1.5: points_ver = [ points[6], points[7], points[0], points[1], points[2], points[3], points[4], points[5] ] gtPolPoints.append(points_ver) gtCharPoints.append(gtBoxtoChars(gtCharSize, points_ver)) else: gtCharPoints.append(gtBoxtoChars(gtCharSize, points)) gtPolPoints.append(points) evaluationLog += "GT polygons: " + str(len(gtPols)) + ( " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n") # GT Don't Care overlap for DontCare in gtDontCarePolsNum: for gtNum in list( set(range(len(gtPols))) - set(gtDontCarePolsNum)): if get_intersection(gtPols[gtNum], gtPols[DontCare]) > 0: gtPols[DontCare] -= gtPols[gtNum] if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents( detFile, evaluationParams['CRLF'], evaluationParams['LTRB'], evaluationParams['TRANSCRIPTION'], evaluationParams['CONFIDENCES']) for n in range(len(pointsList)): points = pointsList[n] if evaluationParams['LTRB']: detRect = Rectangle(*points) detPol = rectangle_to_polygon(detRect) points = polygon_to_points(detPol) else: detPol = polygon_from_points(points) detPols.append(detPol) detPolPoints.append(points) evaluationLog += "DET polygons: " + str(len(detPols)) if len(gtPols) > 0 and len(detPols) > 0: #Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] recallMat = np.empty(outputShape) precisionMat = np.empty(outputShape) matchMat = np.zeros(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) gtExcludeMat = np.zeros(len(gtPols), np.int8) detExcludeMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): detCharCounts = [] for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] intersected_area = get_intersection(pD, pG) recallMat[gtNum, detNum] = 0 if pG.area( ) == 0 else intersected_area / pG.area() precisionMat[gtNum, detNum] = 0 if pD.area( ) == 0 else intersected_area / pD.area() detCharCounts.append(np.zeros(len( gtCharPoints[gtNum]))) gtCharCounts.append(detCharCounts) # Find detection Don't Care if len(gtDontCarePolsNum) > 0: for detNum in range(len(detPols)): # many-to-one many_sum = 0 for gtNum in gtDontCarePolsNum: if recallMat[gtNum, detNum] > evaluationParams[ 'AREA_RECALL_CONSTRAINT']: many_sum += precisionMat[gtNum, detNum] if many_sum >= evaluationParams[ 'AREA_PRECISION_CONSTRAINT']: detDontCarePolsNum.append(detNum) else: for gtNum in gtDontCarePolsNum: if precisionMat[ gtNum, detNum] > evaluationParams[ 'AREA_PRECISION_CONSTRAINT']: detDontCarePolsNum.append(detNum) break # many-to-one for mixed DC and non-DC for gtNum in gtDontCarePolsNum: if recallMat[gtNum, detNum] > 0: detPols[detNum] -= gtPols[gtNum] evaluationLog += " (" + str( len(detDontCarePolsNum)) + " don't care)\n" if len( detDontCarePolsNum) > 0 else "\n" # Recalculate matrices for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] intersected_area = get_intersection(pD, pG) recallMat[gtNum, detNum] = 0 if pG.area( ) == 0 else intersected_area / pG.area() precisionMat[gtNum, detNum] = 0 if pD.area( ) == 0 else intersected_area / pD.area() # Find many-to-one matches evaluationLog += "Find many-to-one matches\n" for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: match, matchesGt = many_to_one_match(detNum) if match: pairs.append({ 'gt': matchesGt, 'det': [detNum], 'type': 'MO' }) evaluationLog += "Match GT #" + str( matchesGt) + " with Det #" + str(detNum) + "\n" # Find one-to-one matches evaluationLog += "Find one-to-one matches\n" for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): if gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: match = one_to_one_match(gtNum, detNum) if match: normDist = center_distance( gtPols[gtNum], detPols[detNum]) normDist /= diag(gtPolPoints[gtNum]) + diag( detPolPoints[detNum]) normDist *= 2.0 if normDist < evaluationParams[ 'EV_PARAM_IND_CENTER_DIFF_THR']: pairs.append({ 'gt': [gtNum], 'det': [detNum], 'type': 'OO' }) evaluationLog += "Match GT #" + str( gtNum) + " with Det #" + str( detNum) + "\n" # Find one-to-many matches evaluationLog += "Find one-to-many matches\n" for gtNum in range(len(gtPols)): if gtNum not in gtDontCarePolsNum: match, matchesDet = one_to_many_match(gtNum) if match: pairs.append({ 'gt': [gtNum], 'det': matchesDet, 'type': 'OM' }) evaluationLog += "Match Gt #" + str( gtNum) + " with Det #" + str(matchesDet) + "\n" # Fill match matrix for pair in pairs: matchMat[pair['gt'], pair['det']] = 1 # Fill character matrix char_fill(np.where(matchMat.sum(axis=0) > 0)[0], matchMat) # Recall score for gtNum in range(len(gtRectMat)): if matchMat.sum(axis=1)[gtNum] > 0: recallAccum += len( np.where(sum(gtCharCounts[gtNum]) == 1)[0]) / len( gtCharPoints[gtNum]) if len(np.where(sum(gtCharCounts[gtNum]) == 1) [0]) / len(gtCharPoints[gtNum]) < 1: recallScore.append("<font color=red>" + str( len( np.where( sum(gtCharCounts[gtNum]) == 1)[0])) + "/" + str(len(gtCharPoints[gtNum])) + "</font>") else: recallScore.append( str( len( np.where( sum(gtCharCounts[gtNum]) == 1)[0])) + "/" + str(len(gtCharPoints[gtNum]))) else: recallScore.append("") # Precision score for detNum in range(len(detRectMat)): if matchMat.sum(axis=0)[detNum] > 0: detTotal = 0 detContain = 0 for gtNum in range(len(gtRectMat)): if matchMat[gtNum, detNum] > 0: detTotal += len(gtCharCounts[gtNum][detNum]) detContain += len( np.where( gtCharCounts[gtNum][detNum] == 1)[0]) precisionAccum += detContain / detTotal if detContain / detTotal < 1: precisionScore.append("<font color=red>" + str(detContain) + "/" + str(detTotal) + "</font>") else: precisionScore.append( str(detContain) + "/" + str(detTotal)) else: precisionScore.append("") # Visualization charCounts = np.zeros((len(gtRectMat), len(detRectMat))) for gtNum in range(len(gtRectMat)): for detNum in range(len(detRectMat)): charCounts[gtNum][detNum] = sum( gtCharCounts[gtNum][detNum]) if evaluationParams['CONFIDENCES']: for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: match = detNum == list( filter(lambda p: p['det'] == 10, pairs))[0]['det'] arrSampleConfidences.append(confidencesList[detNum]) arrSampleMatch.append(match) arrGlobalConfidences.append(confidencesList[detNum]) arrGlobalMatches.append(match) numGtCare = (len(gtPols) - len(gtDontCarePolsNum)) numDetCare = (len(detPols) - len(detDontCarePolsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if numDetCare > 0 else float(1) sampleAP = precision else: recall = float(recallAccum) / numGtCare precision = float( 0) if numDetCare == 0 else float(precisionAccum) / numDetCare if evaluationParams['CONFIDENCES'] and evaluationParams[ 'PER_SAMPLE_RESULTS']: sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare) hmean = 0 if ( precision + recall) == 0 else 2.0 * precision * recall / (precision + recall) evaluationLog += "<b>Recall = " + str(round( recallAccum, 2)) + " / " + str(numGtCare) + " = " + str( round(recall, 2)) + "\n</b>" evaluationLog += "<b>Precision = " + str(round( precisionAccum, 2)) + " / " + str(numDetCare) + " = " + str( round(precision, 2)) + "\n</b>" methodRecallSum += recallAccum methodPrecisionSum += precisionAccum numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare if evaluationParams['PER_SAMPLE_RESULTS']: perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'pairs': pairs, 'AP': sampleAP, 'recallMat': [] if len(detPols) > 100 else recallMat.tolist(), 'precisionMat': [] if len(detPols) > 100 else precisionMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtCharPoints': gtCharPoints, 'gtCharCounts': [sum(k).tolist() for k in gtCharCounts], 'charCounts': charCounts.tolist(), 'recallScore': recallScore, 'precisionScore': precisionScore, 'gtDontCare': gtDontCarePolsNum, 'detDontCare': detDontCarePolsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } # Compute MAP and MAR AP = 0 if evaluationParams['CONFIDENCES']: AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt) methodRecall = 0 if numGlobalCareGt == 0 else methodRecallSum / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else methodPrecisionSum / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodMetrics = { 'recall': methodRecall, 'precision': methodPrecision, 'hmean': methodHmean, 'AP': AP } resDict = { 'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics } return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module,alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) def one_to_one_match(row, col): cont = 0 for j in range(len(recallMat[0])): if recallMat[row,j] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row,j] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] : cont = cont +1 if (cont != 1): return False cont = 0 for i in range(len(recallMat)): if recallMat[i,col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[i,col] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] : cont = cont +1 if (cont != 1): return False if recallMat[row,col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row,col] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] : return True return False def num_overlaps_gt(gtNum): cont = 0 for detNum in range(len(detRects)): if detNum not in detDontCareRectsNum: if recallMat[gtNum,detNum] > 0 : cont = cont +1 return cont def num_overlaps_det(detNum): cont = 0 for gtNum in range(len(recallMat)): if gtNum not in gtDontCareRectsNum: if recallMat[gtNum,detNum] > 0 : cont = cont +1 return cont def is_single_overlap(row, col): if num_overlaps_gt(row)==1 and num_overlaps_det(col)==1: return True else: return False def one_to_many_match(gtNum): many_sum = 0 detRects = [] for detNum in range(len(recallMat[0])): if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and detNum not in detDontCareRectsNum: if precisionMat[gtNum,detNum] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] : many_sum += recallMat[gtNum,detNum] detRects.append(detNum) if round(many_sum,4) >=evaluationParams['AREA_RECALL_CONSTRAINT'] : return True,detRects else: return False,[] def many_to_one_match(detNum): many_sum = 0 gtRects = [] for gtNum in range(len(recallMat)): if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCareRectsNum: if recallMat[gtNum,detNum] >= evaluationParams['AREA_RECALL_CONSTRAINT'] : many_sum += precisionMat[gtNum,detNum] gtRects.append(gtNum) if round(many_sum,4) >=evaluationParams['AREA_PRECISION_CONSTRAINT'] : return True,gtRects else: return False,[] def area(a, b): dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin) + 1 dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin) + 1 if (dx>=0) and (dy>=0): return dx*dy else: return 0. def center(r): x = float(r.xmin) + float(r.xmax - r.xmin + 1) / 2.; y = float(r.ymin) + float(r.ymax - r.ymin + 1) / 2.; return Point(x,y) def point_distance(r1, r2): distx = math.fabs(r1.x - r2.x) disty = math.fabs(r1.y - r2.y) return math.sqrt(distx * distx + disty * disty ) def center_distance(r1, r2): return point_distance(center(r1), center(r2)) def diag(r): w = (r.xmax - r.xmin + 1) h = (r.ymax - r.ymin + 1) return math.sqrt(h * h + w * w) def rectangle_to_points(rect): points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin)] return points perSampleMetrics = {} methodRecallSum = 0 methodPrecisionSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') Point = namedtuple('Point', 'x y') gt = rrc_evaluation_funcs.load_zip_file(gtFilePath,evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file(submFilePath,evaluationParams['DET_SAMPLE_NAME_2_ID'],True) numGt = 0; numDet = 0; for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 recallAccum = 0. precisionAccum = 0. gtRects = [] detRects = [] gtPolPoints = [] detPolPoints = [] gtDontCareRectsNum = []#Array of Ground Truth Rectangles' keys marked as don't Care detDontCareRectsNum = []#Array of Detected Rectangles' matched with a don't Care GT pairs = [] evaluationLog = "" recallMat = np.empty([1,1]) precisionMat = np.empty([1,1]) pointsList,_,transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile,evaluationParams['CRLF'],True,True,False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" gtRect = Rectangle(*points) gtRects.append(gtRect) gtPolPoints.append(points) if dontCare: gtDontCareRectsNum.append( len(gtRects)-1 ) evaluationLog += "GT rectangles: " + str(len(gtRects)) + (" (" + str(len(gtDontCareRectsNum)) + " don't care)\n" if len(gtDontCareRectsNum)>0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList,_,_ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile,evaluationParams['CRLF'],True,False,False) for n in range(len(pointsList)): points = pointsList[n] detRect = Rectangle(*points) detRects.append(detRect) detPolPoints.append(points) if len(gtDontCareRectsNum)>0 : for dontCareRectNum in gtDontCareRectsNum: dontCareRect = gtRects[dontCareRectNum] intersected_area = area(dontCareRect,detRect) rdDimensions = ( (detRect.xmax - detRect.xmin+1) * (detRect.ymax - detRect.ymin+1)); if (rdDimensions==0) : precision = 0 else: precision= intersected_area / rdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT'] ): detDontCareRectsNum.append( len(detRects)-1 ) break evaluationLog += "DET rectangles: " + str(len(detRects)) + (" (" + str(len(detDontCareRectsNum)) + " don't care)\n" if len(detDontCareRectsNum)>0 else "\n") if len(gtRects)==0: recall = 1 precision = 0 if len(detRects)>0 else 1 if len(detRects)>0: #Calculate recall and precision matrixs outputShape=[len(gtRects),len(detRects)] recallMat = np.empty(outputShape) precisionMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtRects),np.int8) detRectMat = np.zeros(len(detRects),np.int8) for gtNum in range(len(gtRects)): for detNum in range(len(detRects)): rG = gtRects[gtNum] rD = detRects[detNum] intersected_area = area(rG,rD) rgDimensions = ( (rG.xmax - rG.xmin+1) * (rG.ymax - rG.ymin+1) ); rdDimensions = ( (rD.xmax - rD.xmin+1) * (rD.ymax - rD.ymin+1)); recallMat[gtNum,detNum] = 0 if rgDimensions==0 else intersected_area / rgDimensions precisionMat[gtNum,detNum] = 0 if rdDimensions==0 else intersected_area / rdDimensions # Find one-to-one matches evaluationLog += "Find one-to-one matches\n" for gtNum in range(len(gtRects)): for detNum in range(len(detRects)): if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCareRectsNum and detNum not in detDontCareRectsNum : match = one_to_one_match(gtNum, detNum) if match is True : #in deteval we have to make other validation before mark as one-to-one if is_single_overlap(gtNum, detNum) is True : rG = gtRects[gtNum] rD = detRects[detNum] normDist = center_distance(rG, rD); normDist /= diag(rG) + diag(rD); normDist *= 2.0; if normDist < evaluationParams['EV_PARAM_IND_CENTER_DIFF_THR'] : gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 recallAccum += evaluationParams['MTYPE_OO_O'] precisionAccum += evaluationParams['MTYPE_OO_O'] pairs.append({'gt':gtNum,'det':detNum,'type':'OO'}) evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n" else: evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(detNum) + " normDist: " + str(normDist) + " \n" else: evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(detNum) + " not single overlap\n" # Find one-to-many matches evaluationLog += "Find one-to-many matches\n" for gtNum in range(len(gtRects)): if gtNum not in gtDontCareRectsNum: match,matchesDet = one_to_many_match(gtNum) if match is True : evaluationLog += "num_overlaps_gt=" + str(num_overlaps_gt(gtNum)) #in deteval we have to make other validation before mark as one-to-one if num_overlaps_gt(gtNum)>=2 : gtRectMat[gtNum] = 1 recallAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesDet)==1 else evaluationParams['MTYPE_OM_O']) precisionAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesDet)==1 else evaluationParams['MTYPE_OM_O']*len(matchesDet)) pairs.append({'gt':gtNum,'det':matchesDet,'type': 'OO' if len(matchesDet)==1 else 'OM'}) for detNum in matchesDet : detRectMat[detNum] = 1 evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(matchesDet) + "\n" else: evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(matchesDet) + " not single overlap\n" # Find many-to-one matches evaluationLog += "Find many-to-one matches\n" for detNum in range(len(detRects)): if detNum not in detDontCareRectsNum: match,matchesGt = many_to_one_match(detNum) if match is True : #in deteval we have to make other validation before mark as one-to-one if num_overlaps_det(detNum)>=2 : detRectMat[detNum] = 1 recallAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesGt)==1 else evaluationParams['MTYPE_OM_M']*len(matchesGt)) precisionAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesGt)==1 else evaluationParams['MTYPE_OM_M']) pairs.append({'gt':matchesGt,'det':detNum,'type': 'OO' if len(matchesGt)==1 else 'MO'}) for gtNum in matchesGt : gtRectMat[gtNum] = 1 evaluationLog += "Match GT #" + str(matchesGt) + " with Det #" + str(detNum) + "\n" else: evaluationLog += "Match Discarded GT #" + str(matchesGt) + " with Det #" + str(detNum) + " not single overlap\n" numGtCare = (len(gtRects) - len(gtDontCareRectsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if len(detRects)>0 else float(1) else: recall = float(recallAccum) / numGtCare precision = float(0) if (len(detRects) - len(detDontCareRectsNum))==0 else float(precisionAccum) / (len(detRects) - len(detDontCareRectsNum)) hmean = 0 if (precision + recall)==0 else 2.0 * precision * recall / (precision + recall) methodRecallSum += recallAccum methodPrecisionSum += precisionAccum numGt += len(gtRects) - len(gtDontCareRectsNum) numDet += len(detRects) - len(detDontCareRectsNum) perSampleMetrics[resFile] = { 'precision':precision, 'recall':recall, 'hmean':hmean, 'pairs':pairs, 'recallMat':[] if len(detRects)>100 else recallMat.tolist(), 'precisionMat':[] if len(detRects)>100 else precisionMat.tolist(), 'gtPolPoints':gtPolPoints, 'detPolPoints':detPolPoints, 'gtDontCare':gtDontCareRectsNum, 'detDontCare':detDontCareRectsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } methodRecall = 0 if numGt==0 else methodRecallSum/numGt methodPrecision = 0 if numDet==0 else methodPrecisionSum/numDet methodHmean = 0 if methodRecall + methodPrecision==0 else 2* methodRecall * methodPrecision / (methodRecall + methodPrecision) methodMetrics = {'precision':methodPrecision, 'recall':methodRecall,'hmean': methodHmean } resDict = {'calculated':True,'Message':'','method': methodMetrics,'per_sample': perSampleMetrics} return resDict;
def evaluate_method(gtFilePath, submFilePath, evaluationParams): """ Method evaluate_method: evaluate method and returns the results Results. Dictionary with the following values: - method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 } - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 } """ for module, alias in evaluation_imports().iteritems(): globals()[alias] = importlib.import_module(module) def polygon_from_points(points): """ Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4 """ resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(points[0]) resBoxes[0, 4] = int(points[1]) resBoxes[0, 1] = int(points[2]) resBoxes[0, 5] = int(points[3]) resBoxes[0, 2] = int(points[4]) resBoxes[0, 6] = int(points[5]) resBoxes[0, 3] = int(points[6]) resBoxes[0, 7] = int(points[7]) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_polygon(rect): resBoxes = np.empty([1, 8], dtype='int32') resBoxes[0, 0] = int(rect.xmin) resBoxes[0, 4] = int(rect.ymax) resBoxes[0, 1] = int(rect.xmin) resBoxes[0, 5] = int(rect.ymin) resBoxes[0, 2] = int(rect.xmax) resBoxes[0, 6] = int(rect.ymin) resBoxes[0, 3] = int(rect.xmax) resBoxes[0, 7] = int(rect.ymax) pointMat = resBoxes[0].reshape([2, 4]).T return plg.Polygon(pointMat) def rectangle_to_points(rect): points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin)] return points def get_union(pD, pG): areaA = pD.area(); areaB = pG.area(); return areaA + areaB - get_intersection(pD, pG); def get_intersection_over_union(pD, pG): try: return get_intersection(pD, pG) / get_union(pD, pG); except: return 0 def get_intersection(pD, pG): pInt = pD & pG if len(pInt) == 0: return 0 return pInt.area() def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP perSampleMetrics = {} matchedSum = 0 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') gt = rrc_evaluation_funcs.load_zip_file(gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID']) subm = rrc_evaluation_funcs.load_zip_file(submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True) numGlobalDontCareGt = 0 numGlobalDontCareDet = 0 numGlobalCareGt = 0; numGlobalCareDet = 0; arrGlobalConfidences = []; arrGlobalMatches = []; for resFile in gt: gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 detMatched = 0 iouMat = np.empty([1, 1]) gtPols = [] detPols = [] gtPolPoints = [] detPolPoints = [] # Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [] # Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [] pairs = [] detMatchedNums = [] arrSampleConfidences = []; arrSampleMatch = []; sampleAP = 0; evaluationLog = "" pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile, evaluationParams[ 'CRLF'], evaluationParams[ 'LTRB'], True, False) for n in range(len(pointsList)): points = pointsList[n] transcription = transcriptionsList[n] dontCare = transcription == "###" if evaluationParams['LTRB']: gtRect = Rectangle(*points) gtPol = rectangle_to_polygon(gtRect) else: gtPol = polygon_from_points(points) gtPols.append(gtPol) gtPolPoints.append(points) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) evaluationLog += "GT polygons: " + str(len(gtPols)) + ( " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n") if resFile in subm: detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile, evaluationParams[ 'CRLF'], evaluationParams[ 'LTRB'], False, evaluationParams[ 'CONFIDENCES']) for n in range(len(pointsList)): points = pointsList[n] if evaluationParams['LTRB']: detRect = Rectangle(*points) detPol = rectangle_to_polygon(detRect) else: detPol = polygon_from_points(points) detPols.append(detPol) detPolPoints.append(points) # 过滤掉 don't care 区域 if len(gtDontCarePolsNum) > 0: for dontCarePol in gtDontCarePolsNum: dontCarePol = gtPols[dontCarePol] intersected_area = get_intersection(dontCarePol, detPol) pdDimensions = detPol.area() precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']): detDontCarePolsNum.append(len(detPols) - 1) break evaluationLog += "DET polygons: " + str(len(detPols)) + ( " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n") if len(gtPols) > 0 and len(detPols) > 0: # Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] iouMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum: if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT']: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 detMatched += 1 pairs.append({'gt': gtNum, 'det': detNum}) detMatchedNums.append(detNum) evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n" if evaluationParams['CONFIDENCES']: for detNum in range(len(detPols)): if detNum not in detDontCarePolsNum: # we exclude the don't care detections match = detNum in detMatchedNums arrSampleConfidences.append(confidencesList[detNum]) arrSampleMatch.append(match) arrGlobalConfidences.append(confidencesList[detNum]); arrGlobalMatches.append(match); numGtCare = (len(gtPols) - len(gtDontCarePolsNum)) numDetCare = (len(detPols) - len(detDontCarePolsNum)) if numGtCare == 0: recall = float(1) precision = float(0) if numDetCare > 0 else float(1) sampleAP = precision else: recall = float(detMatched) / numGtCare precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare if evaluationParams['CONFIDENCES'] and evaluationParams['PER_SAMPLE_RESULTS']: sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare) hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / (precision + recall) matchedSum += detMatched numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare numGlobalDontCareGt += len(gtDontCarePolsNum) numGlobalDontCareDet += len(detDontCarePolsNum) if evaluationParams['PER_SAMPLE_RESULTS']: perSampleMetrics[resFile] = { 'precision': precision, 'recall': recall, 'hmean': hmean, 'pairs': pairs, 'AP': sampleAP, 'iouMat': [] if len(detPols) > 100 else iouMat.tolist(), 'gtPolPoints': gtPolPoints, 'detPolPoints': detPolPoints, 'gtDontCare': gtDontCarePolsNum, 'detDontCare': detDontCarePolsNum, 'evaluationParams': evaluationParams, 'evaluationLog': evaluationLog } # Compute MAP and MAR AP = 0 if evaluationParams['CONFIDENCES']: AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt) methodRecall = 0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision) methodMetrics = {'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP} resDict = {'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics} return resDict;