Esempio n. 1
0
def cleval_evaluation(gt_file, submit_file):
    """
    evaluate and returns the results
    Returns with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - per_sample (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 },
                                                         'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    # storing overall result
    overall_result = GlobalResult(PARAMS.E2E)

    # to store per sample evaluation results
    per_sample_metrics = {}

    gt_files = load_zip_file(gt_file, PARAMS.GT_SAMPLE_NAME_2_ID)
    submission_files = load_zip_file(submit_file, PARAMS.DET_SAMPLE_NAME_2_ID,
                                     True)

    # prepare ThreadPool for multi-process
    executor = concurrent.futures.ProcessPoolExecutor(
        max_workers=PARAMS.NUM_WORKERS)
    futures = {}
    bar_len = len(gt_files)

    for file_idx in gt_files:
        gt_file = rrc_evaluation_funcs.decode_utf8(gt_files[file_idx])
        if file_idx in submission_files:
            det_file = decode_utf8(submission_files[file_idx])
            if det_file is None:
                det_file = ""
        else:
            det_file = ""

        future = executor.submit(eval_single_result, gt_file, det_file)
        futures[future] = file_idx

    with tqdm(total=bar_len) as pbar:
        pbar.set_description("Integrating results...")
        for future in concurrent.futures.as_completed(futures):
            file_idx = futures[future]
            result = future.result()
            per_sample_metrics[file_idx] = result
            overall_result.accumulate_stats(result['Rawdata'])
            pbar.update(1)

    executor.shutdown()

    resDict = {
        'calculated': True,
        'Message': '',
        'method': overall_result.to_dict(),
        'per_sample': per_sample_metrics
    }
    return resDict
Esempio n. 2
0
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    for module, alias in evaluation_imports().items():
        globals()[alias] = importlib.import_module(module)

    def polygon_from_points(points):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymax)
        resBoxes[0, 1] = int(rect.xmin)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymin)
        resBoxes[0, 3] = int(rect.xmax)
        resBoxes[0, 7] = int(rect.ymax)

        pointMat = resBoxes[0].reshape([2, 4]).T

        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [
            int(rect.xmin),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymin),
            int(rect.xmin),
            int(rect.ymin)
        ]
        return points

    def get_union(pD, pG):
        areaA = pD.area()
        areaB = pG.area()
        return areaA + areaB - get_intersection(pD, pG)

    def get_intersection_over_union(pD, pG):
        try:
            return get_intersection(pD, pG) / get_union(pD, pG)
        except:
            return 0

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)

            if numGtCare > 0:
                AP /= numGtCare

        return AP

    perSampleMetrics = {}

    matchedSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
    gt = rrc_evaluation_funcs.load_zip_file(
        gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(
        submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalCareGt = 0
    numGlobalCareDet = 0

    arrGlobalConfidences = []
    arrGlobalMatches = []
    for resFile in gt:

        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0

        detMatched = 0

        iouMat = np.empty([1, 1])

        gtPols = []
        detPols = []

        gtPolPoints = []
        detPolPoints = []

        #Array of Ground Truth Polygons' keys marked as don't Care
        gtDontCarePolsNum = []
        #Array of Detected Polygons' matched with a don't Care GT
        detDontCarePolsNum = []

        pairs = []
        detMatchedNums = []

        arrSampleConfidences = []
        arrSampleMatch = []
        sampleAP = 0

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
            gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True,
            False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            gtPolPoints.append(points)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)
        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
            " (" + str(len(gtDontCarePolsNum)) +
            " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        if resFile in subm:

            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
                detFile, evaluationParams['CRLF'], evaluationParams['LTRB'],
                evaluationParams['TRANSCRIPTIONS'],
                evaluationParams['CONFIDENCES'])
            for n in range(len(pointsList)):
                points = pointsList[n]
                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)
                if len(gtDontCarePolsNum) > 0:
                    for dontCarePol in gtDontCarePolsNum:
                        dontCarePol = gtPols[dontCarePol]
                        intersected_area = get_intersection(
                            dontCarePol, detPol)
                        pdDimensions = detPol.area()
                        precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
                        if (precision >
                                evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCarePolsNum.append(len(detPols) - 1)
                            break
            evaluationLog += "DET polygons: " + str(len(detPols)) + (
                " (" + str(len(detDontCarePolsNum)) +
                " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n")

            if len(gtPols) > 0 and len(detPols) > 0:
                #Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                iouMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        iouMat[gtNum,
                               detNum] = get_intersection_over_union(pD, pG)

                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        if gtRectMat[gtNum] == 0 and detRectMat[
                                detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams[
                                    'IOU_CONSTRAINT']:
                                gtRectMat[gtNum] = 1
                                detRectMat[detNum] = 1
                                detMatched += 1
                                pairs.append({'gt': gtNum, 'det': detNum})
                                detMatchedNums.append(detNum)
                                evaluationLog += "Match GT #" + str(
                                    gtNum) + " with Det #" + str(detNum) + "\n"

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        #we exclude the don't care detections
                        match = detNum in detMatchedNums

                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)

                        arrGlobalConfidences.append(confidencesList[detNum])
                        arrGlobalMatches.append(match)
        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
        else:
            recall = float(detMatched) / numGtCare
            precision = 0 if numDetCare == 0 else float(
                detMatched) / numDetCare
            if evaluationParams['CONFIDENCES'] and evaluationParams[
                    'PER_SAMPLE_RESULTS']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch,
                                      numGtCare)

        hmean = 0 if (
            precision +
            recall) == 0 else 2.0 * precision * recall / (precision + recall)

        matchedSum += detMatched
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare

        if evaluationParams['PER_SAMPLE_RESULTS']:
            perSampleMetrics[resFile] = {
                'precision': precision,
                'recall': recall,
                'hmean': hmean,
                'pairs': pairs,
                'AP': sampleAP,
                'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
                'gtPolPoints': gtPolPoints,
                'detPolPoints': detPolPoints,
                'gtDontCare': gtDontCarePolsNum,
                'detDontCare': detDontCarePolsNum,
                'evaluationParams': evaluationParams,
                'evaluationLog': evaluationLog
            }

    # Compute MAP and MAR
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches,
                        numGlobalCareGt)

    methodRecall = 0 if numGlobalCareGt == 0 else float(
        matchedSum) / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else float(
        matchedSum) / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
        methodRecall + methodPrecision)

    methodMetrics = {
        'precision': methodPrecision,
        'recall': methodRecall,
        'hmean': methodHmean,
        'AP': AP
    }

    resDict = {
        'calculated': True,
        'Message': '',
        'method': methodMetrics,
        'per_sample': perSampleMetrics
    }

    return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """
    if evaluationParams['E2E']:
        from hanziconv import HanziConv
        import editdistance

    for module, alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    def polygon_from_points(points):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymax)
        resBoxes[0, 1] = int(rect.xmin)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymin)
        resBoxes[0, 3] = int(rect.xmax)
        resBoxes[0, 7] = int(rect.ymax)

        pointMat = resBoxes[0].reshape([2, 4]).T

        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin),
                  int(rect.xmin), int(rect.ymin)]
        return points

    def get_union(pD, pG):
        areaA = pD.area()
        areaB = pG.area()
        return areaA + areaB - get_intersection(pD, pG)

    def get_intersection_over_union(pD, pG):
        try:
            return get_intersection(pD, pG) / get_union(pD, pG)
        except:
            return 0

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)

            if numGtCare > 0:
                AP /= numGtCare

        return AP

    #from RTWC17
    def normalize_txt(st):
        """
        Normalize Chinese text strings by:
          - remove puncutations and other symbols
          - convert traditional Chinese to simplified
          - convert English characters to lower cases
        """
        st = ''.join(st.split(' '))
        st = re.sub("\"", "", st)
        # remove any this not one of Chinese character, ascii 0-9, and ascii a-z and A-Z
        new_st = re.sub(ur'[^\u4e00-\u9fa5\u0041-\u005a\u0061-\u007a0-9]+', '', st)
        # convert Traditional Chinese to Simplified Chinese
        new_st = HanziConv.toSimplified(new_st)
        # convert uppercase English letters to lowercase
        new_st = new_st.lower()
        return new_st

    def text_distance(str1, str2):
        str1 = normalize_txt(str1)
        str2 = normalize_txt(str2)
        return editdistance.eval(str1, str2)

    perSampleMetrics = {}

    matchedSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')

    gt = rrc_evaluation_funcs.load_zip_file(gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalCareGt = 0
    numGlobalCareDet = 0

    arrGlobalConfidences = []
    arrGlobalMatches = []

    #total edit distance
    total_dist = 0

    for resFile in gt:

        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0

        detMatched = 0

        iouMat = np.empty([1, 1])

        gtPols = []
        detPols = []

        gtTrans = []
        detTrans = []

        gtPolPoints = []
        detPolPoints = []

        # Array of Ground Truth Polygons' keys marked as don't Care
        gtDontCarePolsNum = []
        # Array of Detected Polygons' matched with a don't Care GT
        detDontCarePolsNum = []

        pairs = []
        detMatchedNums = []

        arrSampleConfidences = []
        arrSampleMatch = []
        sampleAP = 0

        example_dist = 0
        match_tuples = []

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile,evaluationParams['CRLF'],evaluationParams['LTRB'],True, False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = (transcription == "###") or (transcription=="?")
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            gtPolPoints.append(points)
            gtTrans.append(transcription)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)

        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
        " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        if resFile in subm:

            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile,evaluationParams['CRLF'],evaluationParams['LTRB'],evaluationParams['E2E'],evaluationParams['CONFIDENCES'])
            for n in range(len(pointsList)):
                points = pointsList[n]

                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)
                if evaluationParams['E2E']:
                    transcription = transcriptionsList[n]
                    detTrans.append(transcription)
                if len(gtDontCarePolsNum) > 0:
                    for dontCarePol in gtDontCarePolsNum:
                        dontCarePol = gtPols[dontCarePol]
                        intersected_area = get_intersection(dontCarePol, detPol)
                        pdDimensions = detPol.area()
                        precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
                        if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCarePolsNum.append(len(detPols) - 1)
                            break

            evaluationLog += "DET polygons: " + str(len(detPols)) + (
            " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n")

            if len(gtPols) > 0 and len(detPols) > 0:
                # Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                iouMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG)

                # match dt index of every gt
                gtMatch = np.empty(len(gtPols), np.int8)
                gtMatch.fill(-1)
                # match gt index of every dt
                dtMatch = np.empty(len(detPols), dtype=np.int8)
                dtMatch.fill(-1)

                for gtNum in range(len(gtPols)):
                    max_iou = 0
                    match_dt_idx = -1

                    for detNum in range(len(detPols)):
                        if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0\
                                and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT']:
                                gtRectMat[gtNum] = 1
                                detRectMat[detNum] = 1
                                detMatched += 1
                                pairs.append({'gt': gtNum, 'det': detNum})
                                detMatchedNums.append(detNum)
                                evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n"

                        if evaluationParams['E2E'] and gtMatch[gtNum] == -1 and dtMatch[detNum] == -1\
                                and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT'] and iouMat[gtNum, detNum] > max_iou:
                                max_iou = iouMat[gtNum, detNum]
                                match_dt_idx = detNum

                    if evaluationParams['E2E'] and match_dt_idx >= 0:
                        gtMatch[gtNum] = match_dt_idx
                        dtMatch[match_dt_idx] = gtNum

                if evaluationParams['E2E']:
                    for gtNum in range(len(gtPols)):
                        if gtNum in gtDontCarePolsNum:
                            continue
                        gt_text = gtTrans[gtNum]
                        if gtMatch[gtNum] >= 0:
                            dt_text = detTrans[gtMatch[gtNum]]
                        else:
                            dt_text = u''
                        dist = text_distance(gt_text, dt_text)
                        example_dist += dist
                        match_tuples.append((gt_text, dt_text, dist))
                    match_tuples.append(("===============","==============", -1))
                    for detNum in range(len(detPols)):
                        if detNum in detDontCarePolsNum:
                            continue
                        if dtMatch[detNum] == -1:
                            gt_text = u''
                            dt_text = detTrans[detNum]
                            dist = text_distance(gt_text, dt_text)
                            example_dist += dist
                            match_tuples.append((gt_text, dt_text, dist))

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        # we exclude the don't care detections
                        match = detNum in detMatchedNums

                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)

                        arrGlobalConfidences.append(confidencesList[detNum])
                        arrGlobalMatches.append(match)
        #avoid when det file don't exist, example_dist=0
        elif evaluationParams['E2E']:
            match_tuples.append(("===============", "==============", -1))
            dt_text = u''
            for gtNum in range(len(gtPols)):
                if gtNum in gtDontCarePolsNum:
                    continue
                gt_text = gtTrans[gtNum]
                dist = text_distance(gt_text, dt_text)
                example_dist += dist
                match_tuples.append((gt_text, dt_text, dist))
        total_dist += example_dist

        if evaluationParams['E2E']:
            logger.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
            logger.debug("file:{}".format(resFile))
            for tp in match_tuples:
                gt_text, dt_text, dist = tp
                logger.debug(u'GT: "{}" matched to DT: "{}", distance = {}'.format(gt_text, dt_text, dist))
            logger.debug('Distance = {:f}'.format(example_dist))
            logger.debug('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')

        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
        else:
            recall = float(detMatched) / numGtCare
            precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
            if evaluationParams['CONFIDENCES'] and evaluationParams['PER_SAMPLE_RESULTS']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare)

        hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / (precision + recall)

        matchedSum += detMatched
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare

        if evaluationParams['PER_SAMPLE_RESULTS']:
            perSampleMetrics[resFile] = {
                'precision': precision,
                'recall': recall,
                'hmean': hmean,
                'pairs': pairs,
                'AP': sampleAP,
                'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
                'gtPolPoints': gtPolPoints,
                'detPolPoints': detPolPoints,
                'gtDontCare': gtDontCarePolsNum,
                'detDontCare': detDontCarePolsNum,
                'evaluationParams': evaluationParams,
                'evaluationLog': evaluationLog
            }
            if evaluationParams['E2E']:
                perSampleMetrics[resFile]['exampleDistance'] = example_dist
                # print("file:{} exampleDistance:{}".format(resFile,example_dist))

    # Compute MAP and MAR
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt)

    methodRecall = 0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
    methodRecall + methodPrecision)
    methodDistance = 0 if len(gt) == 0 else float(total_dist)/len(gt)

    methodMetrics = {'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP, 'distance': methodDistance}

    resDict = {'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics}

    return resDict
Esempio n. 4
0
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """
    for module, alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    def polygon_from_points(points, correctOffset=False):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """

        if correctOffset:  #this will substract 1 from the coordinates that correspond to the xmax and ymax
            points[2] -= 1
            points[4] -= 1
            points[5] -= 1
            points[7] -= 1

        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymax)
        resBoxes[0, 1] = int(rect.xmin)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymin)
        resBoxes[0, 3] = int(rect.xmax)
        resBoxes[0, 7] = int(rect.ymax)

        pointMat = resBoxes[0].reshape([2, 4]).T

        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [
            int(rect.xmin),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymin),
            int(rect.xmin),
            int(rect.ymin)
        ]
        return points

    def get_union(pD, pG):
        areaA = pD.area()
        areaB = pG.area()
        return areaA + areaB - get_intersection(pD, pG)

    def get_intersection_over_union(pD, pG):
        try:
            return get_intersection(pD, pG) / get_union(pD, pG)
        except:
            return 0

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)

            if numGtCare > 0:
                AP /= numGtCare

        return AP

    def transcription_match(transGt,
                            transDet,
                            specialCharacters=unicode('!?.:,*"()·[]/\'',
                                                      'utf-8'),
                            onlyRemoveFirstLastCharacterGT=True):

        if onlyRemoveFirstLastCharacterGT:
            #special characters in GT are allowed only at initial or final position
            if (transGt == transDet):
                return True

            if specialCharacters.find(transGt[0]) > -1:
                if transGt[1:] == transDet:
                    return True

            if specialCharacters.find(transGt[-1]) > -1:
                if transGt[0:len(transGt) - 1] == transDet:
                    return True

            if specialCharacters.find(
                    transGt[0]) > -1 and specialCharacters.find(
                        transGt[-1]) > -1:
                if transGt[1:len(transGt) - 1] == transDet:
                    return True
            return False
        else:
            #Special characters are removed from the begining and the end of both Detection and GroundTruth
            while len(transGt) > 0 and specialCharacters.find(transGt[0]) > -1:
                transGt = transGt[1:]

            while len(transDet) > 0 and specialCharacters.find(
                    transDet[0]) > -1:
                transDet = transDet[1:]

            while len(transGt) > 0 and specialCharacters.find(
                    transGt[-1]) > -1:
                transGt = transGt[0:len(transGt) - 1]

            while len(transDet) > 0 and specialCharacters.find(
                    transDet[-1]) > -1:
                transDet = transDet[0:len(transDet) - 1]

            return transGt == transDet

    def include_in_dictionary(transcription):
        """
        Function used in Word Spotting that finds if the Ground Truth transcription meets the rules to enter into the dictionary. If not, the transcription will be cared as don't care
        """
        #special case 's at final
        if transcription[len(transcription) -
                         2:] == "'s" or transcription[len(transcription) -
                                                      2:] == "'S":
            transcription = transcription[0:len(transcription) - 2]

        #hypens at init or final of the word
        transcription = transcription.strip('-')

        specialCharacters = unicode("'!?.:,*\"()·[]/", "utf-8")
        for character in specialCharacters:
            transcription = transcription.replace(character, ' ')

        transcription = transcription.strip()

        if len(transcription) != len(transcription.replace(" ", "")):
            return False

        if len(transcription) < evaluationParams['MIN_LENGTH_CARE_WORD']:
            return False

        notAllowed = unicode("×÷·", "utf-8")

        range1 = [ord(u'a'), ord(u'z')]
        range2 = [ord(u'A'), ord(u'Z')]
        range3 = [ord(u'À'), ord(u'ƿ')]
        range4 = [ord(u'DŽ'), ord(u'ɿ')]
        range5 = [ord(u'Ά'), ord(u'Ͽ')]
        range6 = [ord(u'-'), ord(u'-')]

        for char in transcription:
            charCode = ord(char)
            if (notAllowed.find(char) != -1):
                return False

            valid = (charCode >= range1[0] and charCode <= range1[1]) or (
                charCode >= range2[0] and charCode <= range2[1]
            ) or (charCode >= range3[0] and charCode <= range3[1]) or (
                charCode >= range4[0] and charCode <= range4[1]) or (
                    charCode >= range5[0]
                    and charCode <= range5[1]) or (charCode >= range6[0]
                                                   and charCode <= range6[1])
            if valid == False:
                return False

        return True

    def include_in_dictionary_transcription(transcription):
        """
        Function applied to the Ground Truth transcriptions used in Word Spotting. It removes special characters or terminations
        """
        #special case 's at final
        if transcription[len(transcription) -
                         2:] == "'s" or transcription[len(transcription) -
                                                      2:] == "'S":
            transcription = transcription[0:len(transcription) - 2]

        #hypens at init or final of the word
        transcription = transcription.strip('-')

        specialCharacters = unicode("'!?.:,*\"()·[]/", "utf-8")
        for character in specialCharacters:
            transcription = transcription.replace(character, ' ')

        transcription = transcription.strip()

        return transcription

    perSampleMetrics = {}

    matchedSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')

    gt = rrc_evaluation_funcs.load_zip_file(
        gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(
        submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalCareGt = 0
    numGlobalCareDet = 0

    arrGlobalConfidences = []
    arrGlobalMatches = []

    for resFile in gt:

        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        if (gtFile is None):
            raise Exception("The file %s is not UTF-8" % resFile)

        recall = 0
        precision = 0
        hmean = 0
        detCorrect = 0
        iouMat = np.empty([1, 1])
        gtPols = []
        detPols = []
        gtTrans = []
        detTrans = []
        gtPolPoints = []
        detPolPoints = []
        gtDontCarePolsNum = [
        ]  #Array of Ground Truth Polygons' keys marked as don't Care
        detDontCarePolsNum = [
        ]  #Array of Detected Polygons' matched with a don't Care GT
        detMatchedNums = []
        pairs = []

        arrSampleConfidences = []
        arrSampleMatch = []
        sampleAP = 0

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
            gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True,
            False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            gtPolPoints.append(points)

            #On word spotting we will filter some transcriptions with special characters
            if evaluationParams['WORD_SPOTTING']:
                if dontCare == False:
                    if include_in_dictionary(transcription) == False:
                        dontCare = True
                    else:
                        transcription = include_in_dictionary_transcription(
                            transcription)

            gtTrans.append(transcription)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)

        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
            " (" + str(len(gtDontCarePolsNum)) +
            " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        if resFile in subm:

            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
                detFile, evaluationParams['CRLF'], evaluationParams['LTRB'],
                True, evaluationParams['CONFIDENCES'])

            for n in range(len(pointsList)):
                points = pointsList[n]
                transcription = transcriptionsList[n]

                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)
                detTrans.append(transcription)

                if len(gtDontCarePolsNum) > 0:
                    for dontCarePol in gtDontCarePolsNum:
                        dontCarePol = gtPols[dontCarePol]
                        intersected_area = get_intersection(
                            dontCarePol, detPol)
                        pdDimensions = detPol.area()
                        precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
                        if (precision >
                                evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCarePolsNum.append(len(detPols) - 1)
                            break

            evaluationLog += "DET polygons: " + str(len(detPols)) + (
                " (" + str(len(detDontCarePolsNum)) +
                " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n")

            if len(gtPols) > 0 and len(detPols) > 0:
                #Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                iouMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        iouMat[gtNum,
                               detNum] = get_intersection_over_union(pD, pG)

                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        if gtRectMat[gtNum] == 0 and detRectMat[
                                detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams[
                                    'IOU_CONSTRAINT']:
                                gtRectMat[gtNum] = 1
                                detRectMat[detNum] = 1
                                #detection matched only if transcription is equal
                                if evaluationParams['WORD_SPOTTING']:
                                    correct = gtTrans[gtNum].upper(
                                    ) == detTrans[detNum].upper()
                                else:
                                    correct = transcription_match(
                                        gtTrans[gtNum].upper(),
                                        detTrans[detNum].upper(),
                                        evaluationParams['SPECIAL_CHARACTERS'],
                                        evaluationParams[
                                            'ONLY_REMOVE_FIRST_LAST_CHARACTER']
                                    ) == True
                                detCorrect += (1 if correct else 0)
                                if correct:
                                    detMatchedNums.append(detNum)
                                pairs.append({
                                    'gt': gtNum,
                                    'det': detNum,
                                    'correct': correct
                                })
                                evaluationLog += "Match GT #" + str(
                                    gtNum) + " with Det #" + str(
                                        detNum) + " trans. correct: " + str(
                                            correct) + "\n"

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        #we exclude the don't care detections
                        match = detNum in detMatchedNums

                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)

                        arrGlobalConfidences.append(confidencesList[detNum])
                        arrGlobalMatches.append(match)

        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
        else:
            recall = float(detCorrect) / numGtCare
            precision = 0 if numDetCare == 0 else float(
                detCorrect) / numDetCare
            if evaluationParams['CONFIDENCES']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch,
                                      numGtCare)

        hmean = 0 if (
            precision +
            recall) == 0 else 2.0 * precision * recall / (precision + recall)

        matchedSum += detCorrect
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare

        perSampleMetrics[resFile] = {
            'precision': precision,
            'recall': recall,
            'hmean': hmean,
            'pairs': pairs,
            'AP': sampleAP,
            'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
            'gtPolPoints': gtPolPoints,
            'detPolPoints': detPolPoints,
            'gtTrans': gtTrans,
            'detTrans': detTrans,
            'gtDontCare': gtDontCarePolsNum,
            'detDontCare': detDontCarePolsNum,
            'evaluationParams': evaluationParams,
            'evaluationLog': evaluationLog
        }

    # Compute AP
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches,
                        numGlobalCareGt)

    methodRecall = 0 if numGlobalCareGt == 0 else float(
        matchedSum) / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else float(
        matchedSum) / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
        methodRecall + methodPrecision)

    methodMetrics = {
        'precision': methodPrecision,
        'recall': methodRecall,
        'hmean': methodHmean,
        'AP': AP
    }

    resDict = {
        'calculated': True,
        'Message': '',
        'method': methodMetrics,
        'per_sample': perSampleMetrics
    }

    return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    for module, alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    def one_to_one_match(row, col):
        cont = 0
        for j in range(len(recallMat[0])):
            if recallMat[row, j] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row, j] >= \
                    evaluationParams['AREA_PRECISION_CONSTRAINT']:
                cont = cont + 1
        if (cont != 1):
            return False
        cont = 0
        for i in range(len(recallMat)):
            if recallMat[i, col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[i, col] >= \
                    evaluationParams['AREA_PRECISION_CONSTRAINT']:
                cont = cont + 1
        if (cont != 1):
            return False

        if recallMat[row, col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row, col] >= \
                evaluationParams['AREA_PRECISION_CONSTRAINT']:
            return True
        return False

    def one_to_many_match(gtNum):
        many_sum = 0
        detRects = []
        for detNum in range(len(recallMat[0])):
            if gtRectMat[gtNum] == 0 and detRectMat[
                    detNum] == 0 and detNum not in detDontCareRectsNum:
                if precisionMat[gtNum, detNum] >= evaluationParams[
                        'AREA_PRECISION_CONSTRAINT']:
                    many_sum += recallMat[gtNum, detNum]
                    detRects.append(detNum)
        if many_sum >= evaluationParams['AREA_RECALL_CONSTRAINT']:
            return True, detRects
        else:
            return False, []

    def many_to_one_match(detNum):
        many_sum = 0
        gtRects = []
        for gtNum in range(len(recallMat)):
            if gtRectMat[gtNum] == 0 and detRectMat[
                    detNum] == 0 and gtNum not in gtDontCareRectsNum:
                if recallMat[
                        gtNum,
                        detNum] >= evaluationParams['AREA_RECALL_CONSTRAINT']:
                    many_sum += precisionMat[gtNum, detNum]
                    gtRects.append(gtNum)
        if many_sum >= evaluationParams['AREA_PRECISION_CONSTRAINT']:
            return True, gtRects
        else:
            return False, []

    def area(a, b):
        dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin) + 1
        dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin) + 1
        if (dx >= 0) and (dy >= 0):
            return dx * dy
        else:
            return 0.

    def center(r):
        x = float(r.xmin) + float(r.xmax - r.xmin + 1) / 2.
        y = float(r.ymin) + float(r.ymax - r.ymin + 1) / 2.
        return Point(x, y)

    def point_distance(r1, r2):
        distx = math.fabs(r1.x - r2.x)
        disty = math.fabs(r1.y - r2.y)
        return math.sqrt(distx * distx + disty * disty)

    def center_distance(r1, r2):
        return point_distance(center(r1), center(r2))

    def diag(r):
        w = (r.xmax - r.xmin + 1)
        h = (r.ymax - r.ymin + 1)
        return math.sqrt(h * h + w * w)

    perSampleMetrics = {}

    methodRecallSum = 0
    methodPrecisionSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
    Point = namedtuple('Point', 'x y')

    gt = rrc_evaluation_funcs.load_zip_file(
        gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(
        submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGt = 0
    numDet = 0

    for resFile in gt:

        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0
        recallAccum = 0.
        precisionAccum = 0.
        gtRects = []
        detRects = []
        gtPolPoints = []
        detPolPoints = []
        gtDontCareRectsNum = [
        ]  # Array of Ground Truth Rectangles' keys marked as don't Care
        detDontCareRectsNum = [
        ]  # Array of Detected Rectangles' matched with a don't Care GT
        pairs = []
        evaluationLog = ""

        recallMat = np.empty([1, 1])
        precisionMat = np.empty([1, 1])

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
            gtFile, evaluationParams['CRLF'], True, True, False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"

            # convert x1,y1,x2,y2,x3,y3,x4,y4 to xmin,ymin,xmax,ymax
            if len(points) == 8:
                points_tmp = np.array(points).reshape(4, 2)
                points_x = points_tmp[:, 0]
                points_y = points_tmp[:, 1]
                xmin = points_x[np.argmin(points_x)]
                xmax = points_x[np.argmax(points_x)]
                ymin = points_y[np.argmin(points_y)]
                ymax = points_y[np.argmax(points_y)]
                points = [xmin, ymin, xmax, ymax]
            gtRect = Rectangle(*points)
            gtRects.append(gtRect)
            gtPolPoints.append(points)
            if dontCare:
                gtDontCareRectsNum.append(len(gtRects) - 1)

        evaluationLog += "GT rectangles: " + str(len(gtRects)) + (
            " (" + str(len(gtDontCareRectsNum)) +
            " don't care)\n" if len(gtDontCareRectsNum) > 0 else "\n")

        if resFile in subm:
            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])
            pointsList, _, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
                detFile, evaluationParams['CRLF'], True, True, False)
            for n in range(len(pointsList)):
                points = pointsList[n]
                # print points
                detRect = Rectangle(*points)
                detRects.append(detRect)
                detPolPoints.append(points)
                if len(gtDontCareRectsNum) > 0:
                    for dontCareRectNum in gtDontCareRectsNum:
                        dontCareRect = gtRects[dontCareRectNum]
                        intersected_area = area(dontCareRect, detRect)
                        rdDimensions = ((detRect.xmax - detRect.xmin + 1) *
                                        (detRect.ymax - detRect.ymin + 1))
                        if (rdDimensions == 0):
                            precision = 0
                        else:
                            precision = intersected_area / rdDimensions
                        if (precision >
                                evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCareRectsNum.append(len(detRects) - 1)
                            break

            evaluationLog += "DET rectangles: " + str(len(detRects)) + (
                " (" + str(len(detDontCareRectsNum)) +
                " don't care)\n" if len(detDontCareRectsNum) > 0 else "\n")

            if len(gtRects) == 0:
                recall = 1
                precision = 0 if len(detRects) > 0 else 1

            if len(detRects) > 0:
                # Calculate recall and precision matrixs
                outputShape = [len(gtRects), len(detRects)]
                recallMat = np.empty(outputShape)
                precisionMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtRects), np.int8)
                detRectMat = np.zeros(len(detRects), np.int8)
                for gtNum in range(len(gtRects)):
                    for detNum in range(len(detRects)):
                        rG = gtRects[gtNum]
                        rD = detRects[detNum]
                        intersected_area = area(rG, rD)
                        rgDimensions = ((rG.xmax - rG.xmin + 1) *
                                        (rG.ymax - rG.ymin + 1))
                        rdDimensions = ((rD.xmax - rD.xmin + 1) *
                                        (rD.ymax - rD.ymin + 1))
                        recallMat[
                            gtNum,
                            detNum] = 0 if rgDimensions == 0 else intersected_area / rgDimensions
                        precisionMat[
                            gtNum,
                            detNum] = 0 if rdDimensions == 0 else intersected_area / rdDimensions

                # Find one-to-one matches
                evaluationLog += "Find one-to-one matches\n"
                for gtNum in range(len(gtRects)):
                    for detNum in range(len(detRects)):
                        if gtRectMat[gtNum] == 0 and detRectMat[
                                detNum] == 0 and gtNum not in gtDontCareRectsNum and detNum not in detDontCareRectsNum:
                            match = one_to_one_match(gtNum, detNum)
                            if match is True:
                                rG = gtRects[gtNum]
                                rD = detRects[detNum]
                                normDist = center_distance(rG, rD)
                                normDist /= diag(rG) + diag(rD)
                                normDist *= 2.0
                                if normDist < evaluationParams[
                                        'EV_PARAM_IND_CENTER_DIFF_THR']:
                                    gtRectMat[gtNum] = 1
                                    detRectMat[detNum] = 1
                                    recallAccum += evaluationParams[
                                        'MTYPE_OO_O']
                                    precisionAccum += evaluationParams[
                                        'MTYPE_OO_O']
                                    pairs.append({
                                        'gt': gtNum,
                                        'det': detNum,
                                        'type': 'OO'
                                    })
                                    evaluationLog += "Match GT #" + str(
                                        gtNum) + " with Det #" + str(
                                            detNum) + "\n"
                                else:
                                    evaluationLog += "Match Discarded GT #" + str(
                                        gtNum) + " with Det #" + str(
                                            detNum) + " normDist: " + str(
                                                normDist) + " \n"
                # Find one-to-many matches
                evaluationLog += "Find one-to-many matches\n"
                for gtNum in range(len(gtRects)):
                    if gtNum not in gtDontCareRectsNum:
                        match, matchesDet = one_to_many_match(gtNum)
                        if match is True:
                            gtRectMat[gtNum] = 1
                            recallAccum += evaluationParams['MTYPE_OM_O']
                            precisionAccum += evaluationParams[
                                'MTYPE_OM_O'] * len(matchesDet)
                            pairs.append({
                                'gt': gtNum,
                                'det': matchesDet,
                                'type': 'OM'
                            })
                            for detNum in matchesDet:
                                detRectMat[detNum] = 1
                            evaluationLog += "Match GT #" + str(
                                gtNum) + " with Det #" + str(matchesDet) + "\n"

                            # Find many-to-one matches
                evaluationLog += "Find many-to-one matches\n"
                for detNum in range(len(detRects)):
                    if detNum not in detDontCareRectsNum:
                        match, matchesGt = many_to_one_match(detNum)
                        if match is True:
                            detRectMat[detNum] = 1
                            recallAccum += evaluationParams[
                                'MTYPE_OM_M'] * len(matchesGt)
                            precisionAccum += evaluationParams['MTYPE_OM_M']
                            pairs.append({
                                'gt': matchesGt,
                                'det': detNum,
                                'type': 'MO'
                            })
                            for gtNum in matchesGt:
                                gtRectMat[gtNum] = 1
                            evaluationLog += "Match GT #" + str(
                                matchesGt) + " with Det #" + str(detNum) + "\n"

                numGtCare = (len(gtRects) - len(gtDontCareRectsNum))
                if numGtCare == 0:
                    recall = float(1)
                    precision = float(0) if len(detRects) > 0 else float(1)
                else:
                    recall = float(recallAccum) / numGtCare
                    precision = float(0) if (
                        len(detRects) - len(detDontCareRectsNum)
                    ) == 0 else float(precisionAccum) / (
                        len(detRects) - len(detDontCareRectsNum))
                hmean = 0 if (precision +
                              recall) == 0 else 2.0 * precision * recall / (
                                  precision + recall)

        evaluationLog += "Recall = " + str(recall) + "\n"
        evaluationLog += "Precision = " + str(precision) + "\n"

        methodRecallSum += recallAccum
        methodPrecisionSum += precisionAccum
        numGt += len(gtRects) - len(gtDontCareRectsNum)
        numDet += len(detRects) - len(detDontCareRectsNum)

        perSampleMetrics[resFile] = {
            'precision': precision,
            'recall': recall,
            'hmean': hmean,
            'pairs': pairs,
            'recallMat': [] if len(detRects) > 100 else recallMat.tolist(),
            'precisionMat':
            [] if len(detRects) > 100 else precisionMat.tolist(),
            'gtPolPoints': gtPolPoints,
            'detPolPoints': detPolPoints,
            'gtDontCare': gtDontCareRectsNum,
            'detDontCare': detDontCareRectsNum,
            'evaluationParams': evaluationParams,
            'evaluationLog': evaluationLog
        }

    methodRecall = 0 if numGt == 0 else methodRecallSum / numGt
    methodPrecision = 0 if numDet == 0 else methodPrecisionSum / numDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
        methodRecall + methodPrecision)

    methodMetrics = {
        'precision': methodPrecision,
        'recall': methodRecall,
        'hmean': methodHmean
    }

    resDict = {
        'calculated': True,
        'Message': '',
        'method': methodMetrics,
        'per_sample': perSampleMetrics
    }

    return resDict
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    for module, alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    def polygon_from_points(points):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymax)
        resBoxes[0, 1] = int(rect.xmin)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymin)
        resBoxes[0, 3] = int(rect.xmax)
        resBoxes[0, 7] = int(rect.ymax)

        pointMat = resBoxes[0].reshape([2, 4]).T

        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [
            int(rect.xmin),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymin),
            int(rect.xmin),
            int(rect.ymin)
        ]
        return points

    def get_union(pD, pG):
        areaA = pD.area()
        areaB = pG.area()
        return areaA + areaB - get_intersection(pD, pG)

    def get_intersection_over_union(pD, pG):
        try:
            return get_intersection(pD, pG) / get_union(pD, pG)
        except:
            return 0

    def funcCt(x):
        if x <= 0.01:
            return 1
        else:
            return 1 - x

    def get_text_intersection_over_union_recall(pD, pG):
        '''
        Ct (cut): Area of ground truth that is not covered by detection bounding box.
        '''
        try:
            Ct = pG.area() - get_intersection(pD, pG)
            assert (Ct >= 0 and Ct <= pG.area()), 'Invalid Ct value'
            assert (pG.area() > 0), 'Invalid Gt'
            return (get_intersection(pD, pG) *
                    funcCt(Ct * 1.0 / pG.area())) / get_union(pD, pG)
        except Exception as e:
            return 0

    def funcOt(x):
        if x <= 0.01:
            return 1
        else:
            return 1 - x

    def get_text_intersection_over_union_precision(pD, pG, gtNum, gtPolys,
                                                   gtDontCarePolsNum):
        '''
        Ot: Outlier gt area
        '''
        Ot = 0
        try:
            inside_pG = pD & pG
            gt_union_inside_pD = None
            gt_union_inside_pD_and_pG = None
            count_initial = 0
            for i in xrange(len(gtPolys)):
                if i != gtNum and gtNum not in gtDontCarePolsNum:  # ignore don't care regions
                    if not get_intersection(pD, gtPolys[i]) == 0:
                        if count_initial == 0:
                            # initial
                            gt_union_inside_pD = gtPolys[i]
                            gt_union_inside_pD_and_pG = inside_pG & gtPolys[i]
                            count_initial = 1
                            continue
                        gt_union_inside_pD = gt_union_inside_pD | gtPolys[i]
                        inside_pG_i = inside_pG & gtPolys[i]
                        gt_union_inside_pD_and_pG = gt_union_inside_pD_and_pG | inside_pG_i

            if not gt_union_inside_pD == None:
                pD_union_with_other_gt = pD & gt_union_inside_pD
                Ot = pD_union_with_other_gt.area(
                ) - gt_union_inside_pD_and_pG.area()
                if Ot <= 1.0e-10:
                    Ot = 0
            else:
                Ot = 0
            assert (Ot >= 0 and Ot <= pD.area() + 1
                    ), ' Invalid Ot value: ' + str(Ot) + ' ' + str(pD.area())
            assert (pD.area() > 0), ' Invalid pD area: ' + str(pD.area())
            return (get_intersection(pD, pG) *
                    funcOt(Ot * 1.0 / pD.area())) / get_union(pD, pG)
        except Exception as e:
            # print(e)
            return 0

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def get_intersection_three(pD, pG, pGi):
        pInt = pD & pG
        pInt_3 = pInt & pGi
        if len(pInt_3) == 0:
            return 0
        return pInt_3.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)

            if numGtCare > 0:
                AP /= numGtCare

        return AP

    perSampleMetrics = {}

    matchedSum = 0
    matchedSum_iou = 0
    matchedSum_tiouGt = 0
    matchedSum_tiouDt = 0
    matchedSum_cutGt = 0
    matchedSum_coverOtherGt = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')

    gt = rrc_evaluation_funcs.load_zip_file(
        gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(
        submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalCareGt = 0
    numGlobalCareDet = 0

    arrGlobalConfidences = []
    arrGlobalMatches = []

    totalNumGtPols = 0
    totalNumDetPols = 0

    # fper_ = open('per_samle_result.txt', 'w')

    for resFile in gt:
        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0

        detMatched = 0
        detMatched_iou = 0
        detMatched_tiouGt = 0
        detMatched_tiouDt = 0
        detMatched_cutGt = 0
        detMatched_coverOtherGt = 0

        iouMat = np.empty([1, 1])

        gtPols = []
        detPols = []

        gtPolPoints = []
        detPolPoints = []

        #Array of Ground Truth Polygons' keys marked as don't Care
        gtDontCarePolsNum = []
        #Array of Detected Polygons' matched with a don't Care GT
        detDontCarePolsNum = []

        pairs = []
        detMatchedNums = []

        arrSampleConfidences = []
        arrSampleMatch = []
        sampleAP = 0

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
            gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True,
            False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            gtPolPoints.append(points)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)

        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
            " (" + str(len(gtDontCarePolsNum)) +
            " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        if resFile in subm:

            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
                detFile, evaluationParams['CRLF'], evaluationParams['LTRB'],
                False, evaluationParams['CONFIDENCES'])
            for n in range(len(pointsList)):
                points = pointsList[n]

                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)
                if len(gtDontCarePolsNum) > 0:
                    for dontCarePol in gtDontCarePolsNum:
                        dontCarePol = gtPols[dontCarePol]
                        intersected_area = get_intersection(
                            dontCarePol, detPol)
                        pdDimensions = detPol.area()
                        precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
                        if (precision >
                                evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCarePolsNum.append(len(detPols) - 1)
                            break

            evaluationLog += "DET polygons: " + str(len(detPols)) + (
                " (" + str(len(detDontCarePolsNum)) +
                " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n")

            if len(gtPols) > 0 and len(detPols) > 0:
                #Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                iouMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                tiouRecallMat = np.empty(outputShape)
                tiouPrecisionMat = np.empty(outputShape)
                tiouGtRectMat = np.zeros(len(gtPols), np.int8)
                tiouDetRectMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        iouMat[gtNum,
                               detNum] = get_intersection_over_union(pD, pG)
                        tiouRecallMat[
                            gtNum,
                            detNum] = get_text_intersection_over_union_recall(
                                pD, pG)
                        tiouPrecisionMat[
                            gtNum,
                            detNum] = get_text_intersection_over_union_precision(
                                pD, pG, gtNum, gtPols, gtDontCarePolsNum)

                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        if gtRectMat[gtNum] == 0 and detRectMat[
                                detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams[
                                    'IOU_CONSTRAINT']:
                                gtRectMat[gtNum] = 1
                                detRectMat[detNum] = 1
                                detMatched += 1
                                detMatched_iou += iouMat[gtNum, detNum]
                                detMatched_tiouGt += tiouRecallMat[gtNum,
                                                                   detNum]
                                detMatched_tiouDt += tiouPrecisionMat[gtNum,
                                                                      detNum]
                                if iouMat[gtNum,
                                          detNum] != tiouRecallMat[gtNum,
                                                                   detNum]:
                                    detMatched_cutGt += 1
                                if iouMat[gtNum,
                                          detNum] != tiouPrecisionMat[gtNum,
                                                                      detNum]:
                                    detMatched_coverOtherGt += 1
                                pairs.append({'gt': gtNum, 'det': detNum})
                                detMatchedNums.append(detNum)
                                evaluationLog += "Match GT #" + str(
                                    gtNum) + " with Det #" + str(detNum) + "\n"

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        #we exclude the don't care detections
                        match = detNum in detMatchedNums

                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)

                        arrGlobalConfidences.append(confidencesList[detNum])
                        arrGlobalMatches.append(match)

        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
            tiouRecall = float(1)
            tiouPrecision = float(0) if numDetCare > 0 else float(1)
        else:
            recall = float(detMatched) / numGtCare
            precision = 0 if numDetCare == 0 else float(
                detMatched) / numDetCare
            iouRecall = float(detMatched_iou) / numGtCare
            iouPrecision = 0 if numDetCare == 0 else float(
                detMatched_iou) / numDetCare
            tiouRecall = float(detMatched_tiouGt) / numGtCare
            tiouPrecision = 0 if numDetCare == 0 else float(
                detMatched_tiouDt) / numDetCare

            if evaluationParams['CONFIDENCES'] and evaluationParams[
                    'PER_SAMPLE_RESULTS']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch,
                                      numGtCare)

        hmean = 0 if (
            precision +
            recall) == 0 else 2.0 * precision * recall / (precision + recall)
        tiouHmean = 0 if (
            tiouPrecision +
            tiouRecall) == 0 else 2.0 * tiouPrecision * tiouRecall / (
                tiouPrecision + tiouRecall)
        iouHmean = 0 if (
            iouPrecision + iouRecall
        ) == 0 else 2.0 * iouPrecision * iouRecall / (iouPrecision + iouRecall)

        matchedSum += detMatched
        matchedSum_iou += detMatched_iou
        matchedSum_tiouGt += detMatched_tiouGt
        matchedSum_tiouDt += detMatched_tiouDt
        matchedSum_cutGt += detMatched_cutGt
        matchedSum_coverOtherGt += detMatched_coverOtherGt
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare

        if evaluationParams['PER_SAMPLE_RESULTS']:
            perSampleMetrics[resFile] = {
                'precision': precision,
                'recall': recall,
                'hmean': hmean,
                'iouPrecision': iouPrecision,
                'iouRecall': iouRecall,
                'iouHmean': iouHmean,
                'tiouPrecision': tiouPrecision,
                'tiouRecall': tiouRecall,
                'tiouHmean': tiouHmean,
                'pairs': pairs,
                'AP': sampleAP,
                'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
                'gtPolPoints': gtPolPoints,
                'detPolPoints': detPolPoints,
                'gtDontCare': gtDontCarePolsNum,
                'detDontCare': detDontCarePolsNum,
                'evaluationParams': evaluationParams,
                'evaluationLog': evaluationLog
            }
        # fper_.writelines(resFile+'\t"IoU: (P: {:.3f}. R: {:.3f}. F: {:.3f})",\t"TIoU: (P: {:.3f}. R: {:.3f}. F: {:.3f})".\n'.format(precision, recall, hmean, tiouPrecision, tiouRecall, tiouHmean))
        try:
            totalNumGtPols += len(gtPols)
            totalNumDetPols += len(detPols)
        except Exception as e:
            raise e
    # fper_.close()

    # Compute MAP and MAR
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches,
                        numGlobalCareGt)

    print('num_gt, num_det: ', numGlobalCareGt, totalNumDetPols)
    methodRecall = 0 if numGlobalCareGt == 0 else float(
        matchedSum) / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else float(
        matchedSum) / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
        methodRecall + methodPrecision)

    methodRecall_iou = 0 if numGlobalCareGt == 0 else float(
        matchedSum_iou) / numGlobalCareGt
    methodPrecision_iou = 0 if numGlobalCareDet == 0 else float(
        matchedSum_iou) / numGlobalCareDet
    iouMethodHmean = 0 if methodRecall_iou + methodPrecision_iou == 0 else 2 * methodRecall_iou * methodPrecision_iou / (
        methodRecall_iou + methodPrecision_iou)

    methodRecall_tiouGt = 0 if numGlobalCareGt == 0 else float(
        matchedSum_tiouGt) / numGlobalCareGt
    methodPrecision_tiouDt = 0 if numGlobalCareDet == 0 else float(
        matchedSum_tiouDt) / numGlobalCareDet
    tiouMethodHmean = 0 if methodRecall_tiouGt + methodPrecision_tiouDt == 0 else 2 * methodRecall_tiouGt * methodPrecision_tiouDt / (
        methodRecall_tiouGt + methodPrecision_tiouDt)

    methodMetrics = {
        'precision': methodPrecision,
        'recall': methodRecall,
        'hmean': methodHmean
    }
    iouMethodMetrics = {
        'iouPrecision': methodPrecision_iou,
        'iouRecall': methodRecall_iou,
        'iouHmean': iouMethodHmean
    }
    tiouMethodMetrics = {
        'tiouPrecision': methodPrecision_tiouDt,
        'tiouRecall': methodRecall_tiouGt,
        'tiouHmean': tiouMethodHmean
    }
    print("recall: ", round(methodRecall, 3), "precision: ",
          round(methodPrecision, 3), "hmean: ", round(methodHmean, 3))
    print("tiouRecall:", round(methodRecall_tiouGt, 3), "tiouPrecision:",
          round(methodPrecision_tiouDt, 3), "tiouHmean:",
          round(tiouMethodHmean, 3))

    resDict = {
        'calculated': True,
        'Message': '',
        'method': methodMetrics,
        'per_sample': perSampleMetrics,
        'iouMethod': iouMethodMetrics,
        'tiouMethod': tiouMethodMetrics
    }

    return resDict
Esempio n. 7
0
def validate_data(gtFilePath, submFilePath, evaluationParams):
    """
    Method validate_data: validates that all files in the results folder are correct (have the correct name contents).
                            Validates also that there are no missing files in the folder.
                            If some error detected, the method raises the error
    """

    gtFile = rrc_evaluation_funcs.decode_utf8(open(gtFilePath, 'rb').read())
    if (gtFile is None):
        raise Exception("The GT file is not UTF-8")

    gtLines = gtFile.split("\r\n" if evaluationParams['CRLF'] else "\n")
    ids = {}
    for line in gtLines:
        line = line.replace("\r", "").replace("\n", "")
        if (line != ""):
            if (evaluationParams['DOUBLE_QUOTES']):
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] +
                    ',\s?\"(.*)\"\s*\t?$', line)
            else:
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.*)$',
                    line)

            if m == None:
                if (evaluationParams['DOUBLE_QUOTES']):
                    raise Exception(
                        ("Line in GT not valid.Found: %s should be: %s" %
                         (line, evaluationParams['SAMPLE_NAME_2_ID'] +
                          ',transcription')).encode('utf-8', 'replace'))
                else:
                    raise Exception(
                        ("Line in GT not valid.Found: %s should be: %s" %
                         (line, evaluationParams['SAMPLE_NAME_2_ID'] +
                          ',"transcription"')).encode('utf-8', 'replace'))
            ids[m.group(1)] = {'gt': m.group(2), 'det': ''}

    submFile = rrc_evaluation_funcs.decode_utf8(
        open(submFilePath, 'rb').read())
    if (submFile is None):
        raise Exception("The Det file is not UTF-8")

    submLines = submFile.split("\r\n" if evaluationParams['CRLF'] else "\n")
    for line in submLines:
        line = line.replace("\r", "").replace("\n", "")
        if (line != ""):
            if (evaluationParams['DOUBLE_QUOTES']):
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] +
                    ',\s?\"(.*)\"\s*\t?$', line)
            else:
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.*)$',
                    line)

            if m == None:
                if (evaluationParams['DOUBLE_QUOTES']):
                    raise Exception(
                        ("Line in results not valid.Found: %s should be: %s" %
                         (line, evaluationParams['SAMPLE_NAME_2_ID'] +
                          ',transcription')).encode('utf-8', 'replace'))
                else:
                    raise Exception(
                        ("Line in results not valid.Found: %s should be: %s" %
                         (line, evaluationParams['SAMPLE_NAME_2_ID'] +
                          ',"transcription"')).encode('utf-8', 'replace'))
            try:
                ids[m.group(1)]['det'] = m.group(2)
            except Exception, e:
                raise Exception((
                    "Line in results not valid. Line: %s Sample item not valid: %s"
                    % (line, m.group(1))).encode('utf-8', 'replace'))
Esempio n. 8
0
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    for module, alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    gtFile = rrc_evaluation_funcs.decode_utf8(open(gtFilePath, 'rb').read())
    gtLines = gtFile.split("\r\n" if evaluationParams['CRLF'] else "\n")
    ids = {}
    for line in gtLines:
        line = line.replace("\r", "").replace("\n", "")
        if (line != ""):
            if (evaluationParams['DOUBLE_QUOTES']):
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] +
                    ',\s?\"(.+)\"$', line)
                ids[m.group(1)] = {
                    "gt": m.group(2).replace("\\\\",
                                             "\\").replace("\\\"", "\""),
                    "det": ""
                }
            else:
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.+)$',
                    line)
                ids[m.group(1)] = {"gt": m.group(2), "det": ""}

    totalDistance = 0.0
    totalLength = 0.0
    totalDistanceUpper = 0.0
    totalLengthUpper = 0.0
    numWords = 0
    correctWords = 0.0
    correctWordsUpper = 0.0

    perSampleMetrics = {}

    submFile = rrc_evaluation_funcs.decode_utf8(
        open(submFilePath, 'rb').read())
    if (submFile is None):
        raise Exception("The file is not UTF-8")

    xls_output = StringIO()
    workbook = xlsxwriter.Workbook(xls_output)
    worksheet = workbook.add_worksheet()
    worksheet.write(1, 1, "sample")
    worksheet.write(1, 2, "gt")
    worksheet.write(1, 3, "E.D.")
    worksheet.write(1, 4, "normalized")
    worksheet.write(1, 5, "E.D. upper")
    worksheet.write(1, 6, "normalized upper")

    submLines = submFile.split("\r\n" if evaluationParams['CRLF'] else "\n")
    for line in submLines:
        line = line.replace("\r", "").replace("\n", "")
        if (line != ""):

            numWords = numWords + 1

            if (evaluationParams['DOUBLE_QUOTES']):
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] +
                    ',\s?\"(.*)\"\s*\t?$', line)
                detected = m.group(2).replace("\\\\",
                                              "\\").replace("\\\"", "\"")
            else:
                m = re.match(
                    r'^' + evaluationParams['SAMPLE_NAME_2_ID'] + ',\s?(.*)$',
                    line)
                detected = m.group(2)

            ids[m.group(1)]['det'] = detected

    row = 1
    for k, v in ids.iteritems():

        gt = v['gt']
        detected = v['det']

        if gt == detected:
            correctWords = correctWords + 1

        if gt.upper() == detected.upper():
            correctWordsUpper = correctWordsUpper + 1

        distance = editdistance.eval(gt, detected)
        length = float(distance) / len(gt)

        distance_up = editdistance.eval(gt.upper(), detected.upper())
        length_up = float(distance_up) / len(gt)

        totalDistance += distance
        totalLength += length

        totalDistanceUpper += distance_up
        totalLengthUpper += length_up

        perSampleMetrics[k] = {
            'gt': gt,
            'det': detected,
            'edist': distance,
            'norm': length,
            'edistUp': distance_up,
            'normUp': length_up
        }
        row = row + 1
        worksheet.write(row, 1, k)
        worksheet.write(row, 2, gt)
        worksheet.write(row, 3, detected)
        worksheet.write(row, 4, distance)
        worksheet.write(row, 5, length)
        worksheet.write(row, 6, distance_up)
        worksheet.write(row, 7, length_up)

    methodMetrics = {
        'totalWords': len(ids),
        'detWords': numWords,
        'crwN': correctWords,
        'crwupN': correctWordsUpper,
        'ted': totalDistance,
        'tedL': totalLength,
        'crw': 0 if numWords == 0 else correctWords / numWords,
        'crwN': correctWords,
        'tedup': totalDistanceUpper,
        'tedupL': totalLengthUpper,
        'crwup': 0 if numWords == 0 else correctWordsUpper / numWords,
        'crwupN': correctWordsUpper
    }

    workbook.close()
    output_items = {'samples.xlsx': xls_output.getvalue()}
    xls_output.close()

    resDict = {
        'calculated': True,
        'Message': '',
        'method': methodMetrics,
        'per_sample': perSampleMetrics,
        'output_items': output_items
    }
    return resDict
Esempio n. 9
0
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    for module, alias in evaluation_imports().items():
        globals()[alias] = importlib.import_module(module)

    def polygon_from_points(points):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymin)
        resBoxes[0, 1] = int(rect.xmax)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymax)
        resBoxes[0, 3] = int(rect.xmin)
        resBoxes[0, 7] = int(rect.ymax)
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [
            int(rect.xmin),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymax),
            int(rect.xmax),
            int(rect.ymin),
            int(rect.xmin),
            int(rect.ymin)
        ]
        return points

    def polygon_to_points(pol):
        pointMat = []
        for p in pol:
            for i in range(len(p)):
                pointMat.extend(p[i])
        return pointMat

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)
            if numGtCare > 0:
                AP /= numGtCare
        return AP

    def point_distance(a, b):
        distx = math.fabs(a[0] - b[0])
        disty = math.fabs(a[1] - b[1])
        return math.sqrt(distx * distx + disty * disty)

    def diag(points):
        diag1 = point_distance((points[0], points[1]), (points[4], points[5]))
        diag2 = point_distance((points[2], points[3]), (points[6], points[7]))
        return (diag1 + diag2) / 2

    def center_distance(p1, p2):
        return point_distance(p1.center(), p2.center())

    def get_midpoints(p1, p2):
        return ((p1[0] + p2[0]) / 2, (p1[1] + p2[1]) / 2)

    def get_angle_3pt(a, b, c):
        """Counterclockwise angle in degrees by turning from a to c around b
            Returns a float between 0.0 and 360.0"""
        ang = math.degrees(
            math.atan2(c[1] - b[1], c[0] - b[0]) -
            math.atan2(a[1] - b[1], a[0] - b[0]))
        return ang + 360 if ang < 0 else ang

    def gtBoxtoChars(num, points):
        chars = []
        assert len(points) == 8
        p1 = get_midpoints([points[0], points[1]], [points[6], points[7]])
        p2 = get_midpoints([points[2], points[3]], [points[4], points[5]])
        unitx = (p2[0] - p1[0]) / num
        unity = (p2[1] - p1[1]) / num
        for i in range(num):
            x = p1[0] + unitx / 2 + unitx * i
            y = p1[1] + unity / 2 + unity * i
            chars.append((x, y))
        return chars

    def char_fill(detNums, matchMat):
        for detNum in detNums:
            detPol = detPols[detNum]
            for gtNum, gtChars in enumerate(gtCharPoints):
                if matchMat[gtNum, detNum] == 1:
                    for gtCharNum, gtChar in enumerate(gtChars):
                        if detPol.isInside(gtChar[0], gtChar[1]):
                            gtCharCounts[gtNum][detNum][gtCharNum] = 1

    def one_to_one_match(row, col):
        cont = 0
        for j in range(len(recallMat[0])):
            if recallMat[row, j] >= evaluationParams[
                    'AREA_RECALL_CONSTRAINT'] and precisionMat[
                        row,
                        j] >= evaluationParams['AREA_PRECISION_CONSTRAINT']:
                cont = cont + 1
        if (cont != 1):
            return False
        cont = 0
        for i in range(len(recallMat)):
            if recallMat[i, col] >= evaluationParams[
                    'AREA_RECALL_CONSTRAINT'] and precisionMat[
                        i,
                        col] >= evaluationParams['AREA_PRECISION_CONSTRAINT']:
                cont = cont + 1
        if (cont != 1):
            return False

        if recallMat[row, col] >= evaluationParams[
                'AREA_RECALL_CONSTRAINT'] and precisionMat[
                    row, col] >= evaluationParams['AREA_PRECISION_CONSTRAINT']:
            return True
        return False

    def one_to_many_match(gtNum):
        many_sum = 0
        detRects = []
        for detNum in range(len(recallMat[0])):
            if detNum not in detDontCarePolsNum and gtExcludeMat[
                    gtNum] == 0 and detExcludeMat[detNum] == 0:
                if precisionMat[gtNum, detNum] >= evaluationParams[
                        'AREA_PRECISION_CONSTRAINT']:
                    many_sum += recallMat[gtNum, detNum]
                    detRects.append(detNum)
        if many_sum >= evaluationParams['AREA_RECALL_CONSTRAINT'] and len(
                detRects) >= 2:
            pivots = []
            for matchDet in detRects:
                pD = polygon_from_points(detPolPoints[matchDet])
                pivots.append([get_midpoints(pD[0][0], pD[0][3]), pD.center()])
            for i in range(len(pivots)):
                for k in range(len(pivots)):
                    if k == i:
                        continue
                    angle = get_angle_3pt(pivots[i][0], pivots[k][1],
                                          pivots[i][1])
                    if angle > 180:
                        angle = 360 - angle
                    if min(angle, 180 - angle) >= 45:
                        return False, []
            return True, detRects
        else:
            return False, []

    def many_to_one_match(detNum):
        many_sum = 0
        gtRects = []
        for gtNum in range(len(recallMat)):
            if gtNum not in gtDontCarePolsNum and gtExcludeMat[
                    gtNum] == 0 and detExcludeMat[detNum] == 0:
                if recallMat[
                        gtNum,
                        detNum] >= evaluationParams['AREA_RECALL_CONSTRAINT']:
                    many_sum += precisionMat[gtNum, detNum]
                    gtRects.append(gtNum)
        if many_sum >= evaluationParams['AREA_PRECISION_CONSTRAINT'] and len(
                gtRects) >= 2:
            pivots = []
            for matchGt in gtRects:
                pG = gtPols[matchGt]
                pivots.append([get_midpoints(pG[0][0], pG[0][3]), pG.center()])
            for i in range(len(pivots)):
                for k in range(len(pivots)):
                    if k == i:
                        continue
                    angle = get_angle_3pt(pivots[i][0], pivots[k][1],
                                          pivots[i][1])
                    if angle > 180:
                        angle = 360 - angle
                    if min(angle, 180 - angle) >= 45:
                        return False, []
            return True, gtRects
        else:
            return False, []

    perSampleMetrics = {}

    methodRecallSum = 0
    methodPrecisionSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')

    gt = rrc_evaluation_funcs.load_zip_file(
        gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(
        submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalCareGt = 0
    numGlobalCareDet = 0

    arrGlobalConfidences = []
    arrGlobalMatches = []

    for resFile in gt:

        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0
        recallAccum = 0.
        precisionAccum = 0.

        detMatched = 0
        numGtCare = 0
        numDetCare = 0

        recallMat = np.empty([1, 1])
        precisionMat = np.empty([1, 1])
        matchMat = np.zeros([1, 1])

        gtPols = []
        detPols = []

        gtPolPoints = []
        detPolPoints = []

        # pseudo character centers
        gtCharPoints = []
        gtCharCounts = []

        # visualization
        charCounts = np.zeros([1, 1])
        recallScore = list()
        precisionScore = list()

        #Array of Ground Truth Polygons' keys marked as don't Care
        gtDontCarePolsNum = []
        #Array of Detected Polygons' matched with a don't Care GT
        detDontCarePolsNum = []

        pairs = []
        detMatchedNums = []
        gtExcludeNums = []

        arrSampleConfidences = []
        arrSampleMatch = []
        sampleAP = 0

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
            gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True,
            False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
                points = polygon_to_points(gtPol)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)
                gtPolPoints.append(points)
                gtCharPoints.append([])
            else:
                gtCharSize = len(transcription)
                aspect_ratio = gtPol.aspectRatio()
                if aspect_ratio > 1.5:
                    points_ver = [
                        points[6], points[7], points[0], points[1], points[2],
                        points[3], points[4], points[5]
                    ]
                    gtPolPoints.append(points_ver)
                    gtCharPoints.append(gtBoxtoChars(gtCharSize, points_ver))
                else:
                    gtCharPoints.append(gtBoxtoChars(gtCharSize, points))
                    gtPolPoints.append(points)
        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
            " (" + str(len(gtDontCarePolsNum)) +
            " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        # GT Don't Care overlap
        for DontCare in gtDontCarePolsNum:
            for gtNum in list(
                    set(range(len(gtPols))) - set(gtDontCarePolsNum)):
                if get_intersection(gtPols[gtNum], gtPols[DontCare]) > 0:
                    gtPols[DontCare] -= gtPols[gtNum]

        if resFile in subm:

            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
                detFile, evaluationParams['CRLF'], evaluationParams['LTRB'],
                evaluationParams['TRANSCRIPTION'],
                evaluationParams['CONFIDENCES'])
            for n in range(len(pointsList)):
                points = pointsList[n]

                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                    points = polygon_to_points(detPol)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)

            evaluationLog += "DET polygons: " + str(len(detPols))

            if len(gtPols) > 0 and len(detPols) > 0:
                #Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                recallMat = np.empty(outputShape)
                precisionMat = np.empty(outputShape)
                matchMat = np.zeros(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                gtExcludeMat = np.zeros(len(gtPols), np.int8)
                detExcludeMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    detCharCounts = []
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        intersected_area = get_intersection(pD, pG)
                        recallMat[gtNum, detNum] = 0 if pG.area(
                        ) == 0 else intersected_area / pG.area()
                        precisionMat[gtNum, detNum] = 0 if pD.area(
                        ) == 0 else intersected_area / pD.area()
                        detCharCounts.append(np.zeros(len(
                            gtCharPoints[gtNum])))
                    gtCharCounts.append(detCharCounts)

                # Find detection Don't Care
                if len(gtDontCarePolsNum) > 0:
                    for detNum in range(len(detPols)):
                        # many-to-one
                        many_sum = 0
                        for gtNum in gtDontCarePolsNum:
                            if recallMat[gtNum, detNum] > evaluationParams[
                                    'AREA_RECALL_CONSTRAINT']:
                                many_sum += precisionMat[gtNum, detNum]
                        if many_sum >= evaluationParams[
                                'AREA_PRECISION_CONSTRAINT']:
                            detDontCarePolsNum.append(detNum)
                        else:
                            for gtNum in gtDontCarePolsNum:
                                if precisionMat[
                                        gtNum, detNum] > evaluationParams[
                                            'AREA_PRECISION_CONSTRAINT']:
                                    detDontCarePolsNum.append(detNum)
                                    break
                        # many-to-one for mixed DC and non-DC
                        for gtNum in gtDontCarePolsNum:
                            if recallMat[gtNum, detNum] > 0:
                                detPols[detNum] -= gtPols[gtNum]

                    evaluationLog += " (" + str(
                        len(detDontCarePolsNum)) + " don't care)\n" if len(
                            detDontCarePolsNum) > 0 else "\n"

                # Recalculate matrices
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        intersected_area = get_intersection(pD, pG)
                        recallMat[gtNum, detNum] = 0 if pG.area(
                        ) == 0 else intersected_area / pG.area()
                        precisionMat[gtNum, detNum] = 0 if pD.area(
                        ) == 0 else intersected_area / pD.area()

                # Find many-to-one matches
                evaluationLog += "Find many-to-one matches\n"
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        match, matchesGt = many_to_one_match(detNum)
                        if match:
                            pairs.append({
                                'gt': matchesGt,
                                'det': [detNum],
                                'type': 'MO'
                            })
                            evaluationLog += "Match GT #" + str(
                                matchesGt) + " with Det #" + str(detNum) + "\n"

                # Find one-to-one matches
                evaluationLog += "Find one-to-one matches\n"
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        if gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            match = one_to_one_match(gtNum, detNum)
                            if match:
                                normDist = center_distance(
                                    gtPols[gtNum], detPols[detNum])
                                normDist /= diag(gtPolPoints[gtNum]) + diag(
                                    detPolPoints[detNum])
                                normDist *= 2.0
                                if normDist < evaluationParams[
                                        'EV_PARAM_IND_CENTER_DIFF_THR']:
                                    pairs.append({
                                        'gt': [gtNum],
                                        'det': [detNum],
                                        'type': 'OO'
                                    })
                                    evaluationLog += "Match GT #" + str(
                                        gtNum) + " with Det #" + str(
                                            detNum) + "\n"

                # Find one-to-many matches
                evaluationLog += "Find one-to-many matches\n"
                for gtNum in range(len(gtPols)):
                    if gtNum not in gtDontCarePolsNum:
                        match, matchesDet = one_to_many_match(gtNum)
                        if match:
                            pairs.append({
                                'gt': [gtNum],
                                'det': matchesDet,
                                'type': 'OM'
                            })
                            evaluationLog += "Match Gt #" + str(
                                gtNum) + " with Det #" + str(matchesDet) + "\n"

                # Fill match matrix
                for pair in pairs:
                    matchMat[pair['gt'], pair['det']] = 1

                # Fill character matrix
                char_fill(np.where(matchMat.sum(axis=0) > 0)[0], matchMat)

                # Recall score
                for gtNum in range(len(gtRectMat)):
                    if matchMat.sum(axis=1)[gtNum] > 0:
                        recallAccum += len(
                            np.where(sum(gtCharCounts[gtNum]) == 1)[0]) / len(
                                gtCharPoints[gtNum])
                        if len(np.where(sum(gtCharCounts[gtNum]) == 1)
                               [0]) / len(gtCharPoints[gtNum]) < 1:
                            recallScore.append("<font color=red>" + str(
                                len(
                                    np.where(
                                        sum(gtCharCounts[gtNum]) == 1)[0])) +
                                               "/" +
                                               str(len(gtCharPoints[gtNum])) +
                                               "</font>")
                        else:
                            recallScore.append(
                                str(
                                    len(
                                        np.where(
                                            sum(gtCharCounts[gtNum]) == 1)[0]))
                                + "/" + str(len(gtCharPoints[gtNum])))
                    else:
                        recallScore.append("")

                # Precision score
                for detNum in range(len(detRectMat)):
                    if matchMat.sum(axis=0)[detNum] > 0:
                        detTotal = 0
                        detContain = 0
                        for gtNum in range(len(gtRectMat)):
                            if matchMat[gtNum, detNum] > 0:
                                detTotal += len(gtCharCounts[gtNum][detNum])
                                detContain += len(
                                    np.where(
                                        gtCharCounts[gtNum][detNum] == 1)[0])
                        precisionAccum += detContain / detTotal
                        if detContain / detTotal < 1:
                            precisionScore.append("<font color=red>" +
                                                  str(detContain) + "/" +
                                                  str(detTotal) + "</font>")
                        else:
                            precisionScore.append(
                                str(detContain) + "/" + str(detTotal))
                    else:
                        precisionScore.append("")

                # Visualization
                charCounts = np.zeros((len(gtRectMat), len(detRectMat)))
                for gtNum in range(len(gtRectMat)):
                    for detNum in range(len(detRectMat)):
                        charCounts[gtNum][detNum] = sum(
                            gtCharCounts[gtNum][detNum])

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        match = detNum == list(
                            filter(lambda p: p['det'] == 10, pairs))[0]['det']
                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)
                        arrGlobalConfidences.append(confidencesList[detNum])
                        arrGlobalMatches.append(match)

        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
        else:
            recall = float(recallAccum) / numGtCare
            precision = float(
                0) if numDetCare == 0 else float(precisionAccum) / numDetCare
            if evaluationParams['CONFIDENCES'] and evaluationParams[
                    'PER_SAMPLE_RESULTS']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch,
                                      numGtCare)

        hmean = 0 if (
            precision +
            recall) == 0 else 2.0 * precision * recall / (precision + recall)

        evaluationLog += "<b>Recall = " + str(round(
            recallAccum, 2)) + " / " + str(numGtCare) + " = " + str(
                round(recall, 2)) + "\n</b>"
        evaluationLog += "<b>Precision = " + str(round(
            precisionAccum, 2)) + " / " + str(numDetCare) + " = " + str(
                round(precision, 2)) + "\n</b>"

        methodRecallSum += recallAccum
        methodPrecisionSum += precisionAccum
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare

        if evaluationParams['PER_SAMPLE_RESULTS']:
            perSampleMetrics[resFile] = {
                'precision': precision,
                'recall': recall,
                'hmean': hmean,
                'pairs': pairs,
                'AP': sampleAP,
                'recallMat': [] if len(detPols) > 100 else recallMat.tolist(),
                'precisionMat':
                [] if len(detPols) > 100 else precisionMat.tolist(),
                'gtPolPoints': gtPolPoints,
                'detPolPoints': detPolPoints,
                'gtCharPoints': gtCharPoints,
                'gtCharCounts': [sum(k).tolist() for k in gtCharCounts],
                'charCounts': charCounts.tolist(),
                'recallScore': recallScore,
                'precisionScore': precisionScore,
                'gtDontCare': gtDontCarePolsNum,
                'detDontCare': detDontCarePolsNum,
                'evaluationParams': evaluationParams,
                'evaluationLog': evaluationLog
            }

    # Compute MAP and MAR
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches,
                        numGlobalCareGt)

    methodRecall = 0 if numGlobalCareGt == 0 else methodRecallSum / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else methodPrecisionSum / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
        methodRecall + methodPrecision)

    methodMetrics = {
        'recall': methodRecall,
        'precision': methodPrecision,
        'hmean': methodHmean,
        'AP': AP
    }

    resDict = {
        'calculated': True,
        'Message': '',
        'method': methodMetrics,
        'per_sample': perSampleMetrics
    }

    return resDict
Esempio n. 10
0
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """    

    for module,alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    def one_to_one_match(row, col):
        cont = 0
        for j in range(len(recallMat[0])):    
            if recallMat[row,j] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row,j] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] :
                cont = cont +1
        if (cont != 1):
            return False
        cont = 0
        for i in range(len(recallMat)):    
            if recallMat[i,col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[i,col] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] :
                cont = cont +1
        if (cont != 1):
            return False
        
        if recallMat[row,col] >= evaluationParams['AREA_RECALL_CONSTRAINT'] and precisionMat[row,col] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] :
            return True
        return False
    
    def num_overlaps_gt(gtNum):
        cont = 0
        for detNum in range(len(detRects)):
            if detNum not in detDontCareRectsNum:
                if recallMat[gtNum,detNum] > 0 :
                    cont = cont +1
        return cont

    def num_overlaps_det(detNum):
        cont = 0
        for gtNum in range(len(recallMat)):    
            if gtNum not in gtDontCareRectsNum:
                if recallMat[gtNum,detNum] > 0 :
                    cont = cont +1
        return cont
    
    def is_single_overlap(row, col):
        if num_overlaps_gt(row)==1 and num_overlaps_det(col)==1:
            return True
        else:
            return False
    
    def one_to_many_match(gtNum):
        many_sum = 0
        detRects = []
        for detNum in range(len(recallMat[0])):    
            if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and detNum not in detDontCareRectsNum:
                if precisionMat[gtNum,detNum] >= evaluationParams['AREA_PRECISION_CONSTRAINT'] :
                    many_sum += recallMat[gtNum,detNum]
                    detRects.append(detNum)
        if round(many_sum,4) >=evaluationParams['AREA_RECALL_CONSTRAINT'] :
            return True,detRects
        else:
            return False,[]         
    
    def many_to_one_match(detNum):
        many_sum = 0
        gtRects = []
        for gtNum in range(len(recallMat)):    
            if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCareRectsNum:
                if recallMat[gtNum,detNum] >= evaluationParams['AREA_RECALL_CONSTRAINT'] :
                    many_sum += precisionMat[gtNum,detNum]
                    gtRects.append(gtNum)
        if round(many_sum,4) >=evaluationParams['AREA_PRECISION_CONSTRAINT'] :
            return True,gtRects
        else:
            return False,[]
    
    def area(a, b):
            dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin) + 1
            dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin) + 1
            if (dx>=0) and (dy>=0):
                    return dx*dy
            else:
                    return 0.
    
    def center(r):
        x = float(r.xmin) + float(r.xmax - r.xmin + 1) / 2.;
        y = float(r.ymin) + float(r.ymax - r.ymin + 1) / 2.;
        return Point(x,y)
        
    def point_distance(r1, r2):
        distx = math.fabs(r1.x - r2.x)
        disty = math.fabs(r1.y - r2.y)
        return math.sqrt(distx * distx + disty * disty )  
        
    
    def center_distance(r1, r2):
        return point_distance(center(r1), center(r2))
    
    def diag(r):
        w = (r.xmax - r.xmin + 1)
        h = (r.ymax - r.ymin + 1)
        return math.sqrt(h * h + w * w)  
    
    def rectangle_to_points(rect):
        points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin)]
        return points    
    
    perSampleMetrics = {}
    
    methodRecallSum = 0
    methodPrecisionSum = 0
    
    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
    Point = namedtuple('Point', 'x y')
    
    gt = rrc_evaluation_funcs.load_zip_file(gtFilePath,evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(submFilePath,evaluationParams['DET_SAMPLE_NAME_2_ID'],True)
   
    numGt = 0;
    numDet = 0;
   
    for resFile in gt:
        
        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0        
        recallAccum = 0.
        precisionAccum = 0.
        gtRects = []
        detRects = []
        gtPolPoints = []
        detPolPoints = []
        gtDontCareRectsNum = []#Array of Ground Truth Rectangles' keys marked as don't Care
        detDontCareRectsNum = []#Array of Detected Rectangles' matched with a don't Care GT
        pairs = []
        evaluationLog = ""
        
        recallMat = np.empty([1,1])
        precisionMat = np.empty([1,1])              
        
        pointsList,_,transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile,evaluationParams['CRLF'],True,True,False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            gtRect = Rectangle(*points)
            gtRects.append(gtRect)
            gtPolPoints.append(points)
            if dontCare:
                gtDontCareRectsNum.append( len(gtRects)-1 )                 
        
        evaluationLog += "GT rectangles: " + str(len(gtRects)) + (" (" + str(len(gtDontCareRectsNum)) + " don't care)\n" if len(gtDontCareRectsNum)>0 else "\n")
        
        if resFile in subm:
            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])
            pointsList,_,_ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile,evaluationParams['CRLF'],True,False,False)
            for n in range(len(pointsList)):
                points = pointsList[n]            
                detRect = Rectangle(*points)
                detRects.append(detRect)
                detPolPoints.append(points)
                if len(gtDontCareRectsNum)>0 :
                    for dontCareRectNum in gtDontCareRectsNum:
                        dontCareRect = gtRects[dontCareRectNum]
                        intersected_area = area(dontCareRect,detRect)
                        rdDimensions = ( (detRect.xmax - detRect.xmin+1) * (detRect.ymax - detRect.ymin+1));
                        if (rdDimensions==0) :
                            precision = 0
                        else:
                            precision= intersected_area / rdDimensions
                        if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT'] ):
                            detDontCareRectsNum.append( len(detRects)-1 )
                            break

            evaluationLog += "DET rectangles: " + str(len(detRects)) + (" (" + str(len(detDontCareRectsNum)) + " don't care)\n" if len(detDontCareRectsNum)>0 else "\n")

            if len(gtRects)==0:
                recall = 1
                precision = 0 if len(detRects)>0 else 1

            if len(detRects)>0:
                #Calculate recall and precision matrixs
                outputShape=[len(gtRects),len(detRects)]
                recallMat = np.empty(outputShape)
                precisionMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtRects),np.int8)
                detRectMat = np.zeros(len(detRects),np.int8)
                for gtNum in range(len(gtRects)):
                    for detNum in range(len(detRects)):
                        rG = gtRects[gtNum]
                        rD = detRects[detNum]
                        intersected_area = area(rG,rD)
                        rgDimensions = ( (rG.xmax - rG.xmin+1) * (rG.ymax - rG.ymin+1) );
                        rdDimensions = ( (rD.xmax - rD.xmin+1) * (rD.ymax - rD.ymin+1));
                        recallMat[gtNum,detNum] = 0 if rgDimensions==0 else  intersected_area / rgDimensions
                        precisionMat[gtNum,detNum] = 0 if rdDimensions==0 else intersected_area / rdDimensions

                # Find one-to-one matches
                evaluationLog += "Find one-to-one matches\n"
                for gtNum in range(len(gtRects)):
                    for detNum in range(len(detRects)):
                        if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCareRectsNum and detNum not in detDontCareRectsNum :
                            match = one_to_one_match(gtNum, detNum)
                            if match is True :
                                #in deteval we have to make other validation before mark as one-to-one
                                if is_single_overlap(gtNum, detNum) is True :
                                    rG = gtRects[gtNum]
                                    rD = detRects[detNum]
                                    normDist = center_distance(rG, rD);
                                    normDist /= diag(rG) + diag(rD);
                                    normDist *= 2.0;
                                    if normDist < evaluationParams['EV_PARAM_IND_CENTER_DIFF_THR'] :
                                        gtRectMat[gtNum] = 1
                                        detRectMat[detNum] = 1
                                        recallAccum += evaluationParams['MTYPE_OO_O']
                                        precisionAccum += evaluationParams['MTYPE_OO_O']
                                        pairs.append({'gt':gtNum,'det':detNum,'type':'OO'})
                                        evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n"
                                    else:
                                        evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(detNum) + " normDist: " + str(normDist) + " \n"
                                else:
                                    evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(detNum) + " not single overlap\n"
                # Find one-to-many matches
                evaluationLog += "Find one-to-many matches\n"
                for gtNum in range(len(gtRects)):
                    if gtNum not in gtDontCareRectsNum:
                        match,matchesDet = one_to_many_match(gtNum)
                        if match is True :
                            evaluationLog += "num_overlaps_gt=" + str(num_overlaps_gt(gtNum))
                            #in deteval we have to make other validation before mark as one-to-one
                            if num_overlaps_gt(gtNum)>=2 :
                                gtRectMat[gtNum] = 1
                                recallAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesDet)==1 else evaluationParams['MTYPE_OM_O'])
                                precisionAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesDet)==1 else evaluationParams['MTYPE_OM_O']*len(matchesDet))
                                pairs.append({'gt':gtNum,'det':matchesDet,'type': 'OO' if len(matchesDet)==1 else 'OM'})
                                for detNum in matchesDet :
                                    detRectMat[detNum] = 1
                                evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(matchesDet) + "\n"
                            else:
                                evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(matchesDet) + " not single overlap\n"    

                # Find many-to-one matches
                evaluationLog += "Find many-to-one matches\n"
                for detNum in range(len(detRects)):
                    if detNum not in detDontCareRectsNum:
                        match,matchesGt = many_to_one_match(detNum)
                        if match is True :
                            #in deteval we have to make other validation before mark as one-to-one
                            if num_overlaps_det(detNum)>=2 :                          
                                detRectMat[detNum] = 1
                                recallAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesGt)==1 else evaluationParams['MTYPE_OM_M']*len(matchesGt))
                                precisionAccum += (evaluationParams['MTYPE_OO_O'] if len(matchesGt)==1 else evaluationParams['MTYPE_OM_M'])
                                pairs.append({'gt':matchesGt,'det':detNum,'type': 'OO' if len(matchesGt)==1 else 'MO'})
                                for gtNum in matchesGt :
                                    gtRectMat[gtNum] = 1
                                evaluationLog += "Match GT #" + str(matchesGt) + " with Det #" + str(detNum) + "\n"
                            else:
                                evaluationLog += "Match Discarded GT #" + str(matchesGt) + " with Det #" + str(detNum) + " not single overlap\n"                                    

                numGtCare = (len(gtRects) - len(gtDontCareRectsNum))
                if numGtCare == 0:
                    recall = float(1)
                    precision = float(0) if len(detRects)>0 else float(1)
                else:
                    recall = float(recallAccum) / numGtCare
                    precision =  float(0) if (len(detRects) - len(detDontCareRectsNum))==0 else float(precisionAccum) / (len(detRects) - len(detDontCareRectsNum))
                hmean = 0 if (precision + recall)==0 else 2.0 * precision * recall / (precision + recall)  

        methodRecallSum += recallAccum
        methodPrecisionSum += precisionAccum
        numGt += len(gtRects) - len(gtDontCareRectsNum)
        numDet += len(detRects) - len(detDontCareRectsNum)

        perSampleMetrics[resFile] = {
                                        'precision':precision,
                                        'recall':recall,
                                        'hmean':hmean,
                                        'pairs':pairs,
                                        'recallMat':[] if len(detRects)>100 else recallMat.tolist(),
                                        'precisionMat':[] if len(detRects)>100 else precisionMat.tolist(),
                                        'gtPolPoints':gtPolPoints,
                                        'detPolPoints':detPolPoints,
                                        'gtDontCare':gtDontCareRectsNum,
                                        'detDontCare':detDontCareRectsNum,
                                        'evaluationParams': evaluationParams,
                                        'evaluationLog': evaluationLog
                                    }
        
    methodRecall = 0 if numGt==0 else methodRecallSum/numGt
    methodPrecision = 0 if numDet==0 else methodPrecisionSum/numDet
    methodHmean = 0 if methodRecall + methodPrecision==0 else 2* methodRecall * methodPrecision / (methodRecall + methodPrecision)
    
    methodMetrics = {'precision':methodPrecision, 'recall':methodRecall,'hmean': methodHmean  }

    resDict = {'calculated':True,'Message':'','method': methodMetrics,'per_sample': perSampleMetrics}
    
    
    return resDict;
Esempio n. 11
0
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    for module, alias in evaluation_imports().iteritems():
        globals()[alias] = importlib.import_module(module)

    def polygon_from_points(points):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymax)
        resBoxes[0, 1] = int(rect.xmin)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymin)
        resBoxes[0, 3] = int(rect.xmax)
        resBoxes[0, 7] = int(rect.ymax)

        pointMat = resBoxes[0].reshape([2, 4]).T

        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin),
                  int(rect.xmin), int(rect.ymin)]
        return points

    def get_union(pD, pG):
        areaA = pD.area();
        areaB = pG.area();
        return areaA + areaB - get_intersection(pD, pG);

    def get_intersection_over_union(pD, pG):
        try:
            return get_intersection(pD, pG) / get_union(pD, pG);
        except:
            return 0

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)

            if numGtCare > 0:
                AP /= numGtCare

        return AP

    perSampleMetrics = {}

    matchedSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')

    gt = rrc_evaluation_funcs.load_zip_file(gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_zip_file(submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalDontCareGt = 0
    numGlobalDontCareDet = 0
    numGlobalCareGt = 0;
    numGlobalCareDet = 0;

    arrGlobalConfidences = [];
    arrGlobalMatches = [];

    for resFile in gt:

        gtFile = rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0

        detMatched = 0

        iouMat = np.empty([1, 1])

        gtPols = []
        detPols = []

        gtPolPoints = []
        detPolPoints = []

        # Array of Ground Truth Polygons' keys marked as don't Care
        gtDontCarePolsNum = []
        # Array of Detected Polygons' matched with a don't Care GT
        detDontCarePolsNum = []

        pairs = []
        detMatchedNums = []

        arrSampleConfidences = [];
        arrSampleMatch = [];
        sampleAP = 0;

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile,
                                                                                                       evaluationParams[
                                                                                                           'CRLF'],
                                                                                                       evaluationParams[
                                                                                                           'LTRB'],
                                                                                                       True, False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            gtPolPoints.append(points)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)

        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
            " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        if resFile in subm:

            detFile = rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile,
                                                                                                        evaluationParams[
                                                                                                            'CRLF'],
                                                                                                        evaluationParams[
                                                                                                            'LTRB'],
                                                                                                        False,
                                                                                                        evaluationParams[
                                                                                                            'CONFIDENCES'])
            for n in range(len(pointsList)):
                points = pointsList[n]

                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)
                # 过滤掉 don't care 区域
                if len(gtDontCarePolsNum) > 0:
                    for dontCarePol in gtDontCarePolsNum:
                        dontCarePol = gtPols[dontCarePol]
                        intersected_area = get_intersection(dontCarePol, detPol)
                        pdDimensions = detPol.area()
                        precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
                        if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCarePolsNum.append(len(detPols) - 1)
                            break

            evaluationLog += "DET polygons: " + str(len(detPols)) + (
                " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n")

            if len(gtPols) > 0 and len(detPols) > 0:
                # Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                iouMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG)

                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT']:
                                gtRectMat[gtNum] = 1
                                detRectMat[detNum] = 1
                                detMatched += 1
                                pairs.append({'gt': gtNum, 'det': detNum})
                                detMatchedNums.append(detNum)
                                evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n"

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        # we exclude the don't care detections
                        match = detNum in detMatchedNums

                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)

                        arrGlobalConfidences.append(confidencesList[detNum]);
                        arrGlobalMatches.append(match);

        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
        else:
            recall = float(detMatched) / numGtCare
            precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
            if evaluationParams['CONFIDENCES'] and evaluationParams['PER_SAMPLE_RESULTS']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare)

        hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / (precision + recall)

        matchedSum += detMatched
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare
        numGlobalDontCareGt += len(gtDontCarePolsNum)
        numGlobalDontCareDet += len(detDontCarePolsNum)

        if evaluationParams['PER_SAMPLE_RESULTS']:
            perSampleMetrics[resFile] = {
                'precision': precision,
                'recall': recall,
                'hmean': hmean,
                'pairs': pairs,
                'AP': sampleAP,
                'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
                'gtPolPoints': gtPolPoints,
                'detPolPoints': detPolPoints,
                'gtDontCare': gtDontCarePolsNum,
                'detDontCare': detDontCarePolsNum,
                'evaluationParams': evaluationParams,
                'evaluationLog': evaluationLog
            }

    # Compute MAP and MAR
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt)

    methodRecall = 0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
                methodRecall + methodPrecision)

    methodMetrics = {'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP}

    resDict = {'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics}

    return resDict;