Beispiel #1
0
def point_score_fun(ann, kp_ann):
    """Matches annotations based on minimum keypoint distance to a reference point on the human"""

    # ignore instances that are much bigger than the image
    image_area = ann["width"] * ann["height"]
    bbox_area = utils.bbox_area(kp_ann["bbox"])
    if bbox_area > 4 * image_area:
        return (0, 0)

    # ignore instances with a low score
    if kp_ann["score"] < 0.1:
        return (0, 0)

    # ignore instances where the point is outside the bbox
    point = np.array(ann["point"])
    px, py = point
    x, y, w, h = kp_ann["bbox"]
    if (px < x) or (px > x + w) or (py < y) or (py >= y + h):
        return (0, 0)

    keypoints = np.array(kp_ann["keypoints"]).reshape(-1, 3)
    keypoint_score = kp_ann["score"]
    xy = keypoints[keypoints[:, 2] > 0, :2]
    distance = np.sqrt(np.sum((xy - point)**2, axis=1))
    if len(distance) > 0:
        # normalize distance by bbox size
        distance /= min(200, max(10, np.sqrt(bbox_area)))
        inv_distance = 1 / min(distance)
    else:
        inv_distance = 0

    result = None

    # no point to match
    if inv_distance == 0:
        result = (0, 0)

    # check if point is on torso
    global _torso_indices
    if (inv_distance > 2) and np.all(keypoints[_torso_indices, 2] > 0):
        torso_keypoints = keypoints[_torso_indices, :2]
        torso = matplotlib.path.Path(torso_keypoints)
        if torso.contains_point(point) and (keypoint_score > 0.1):
            # if there are multiple instances that have the point on the torso then we want to match
            # the one with the highest keypoint score
            # if there are two instances with same score then match the one with the smallest distance
            result = (1e10 * keypoint_score, inv_distance)

    # default case
    if result is None:
        result = (inv_distance, keypoint_score)

    return result
Beispiel #2
0
def score_fun(ann, kp_ann):
    """Scoring function based on bounding box IoU"""
    ann_bbox = ann["bbox"]
    width = ann["width"]
    height = ann["height"]
    kp_ann_bbox = kp_ann["bbox"]

    # ignore instances that are much bigger than the image
    image_area = ann["width"] * ann["height"]
    bbox_area = utils.bbox_area(kp_ann_bbox)
    if bbox_area > 4 * image_area:
        return 0

    # force pose can generate bounding boxes outside the image
    # only match on area inside image
    kp_ann_bbox = utils.bbox_clamp(kp_ann_bbox, width, height)
    return utils.iou(ann_bbox, kp_ann_bbox)
Beispiel #3
0
    def fill_fields(self, anns):
        for ann in anns:
            if "actions" not in ann:
                continue

            keypoint_indices = self.config.meta.keypoint_indices
            centers = np.array(
                utils.keypoint_centers(ann["keypoints"], keypoint_indices))
            centers = centers / self.config.meta.stride

            action_labels = self.config.meta.action_labels(ann["actions"])
            action_mask = np.asarray(action_labels).astype(bool)

            area = utils.bbox_area(ann["bbox"])
            scale = np.sqrt(area) / self.config.meta.stride

            for center in centers:
                self.fill_center(center, action_mask, scale)
Beispiel #4
0
def score_fun(ann, kp_ann):
    ann_bbox = ann["bbox"]
    width = ann["width"]
    height = ann["height"]
    kp_ann_bbox = kp_ann["bbox"]

    # ignore instances that are much bigger than the image
    image_area = ann["width"] * ann["height"]
    bbox_area = utils.bbox_area(kp_ann_bbox)
    if bbox_area > 4 * image_area:
        return 0

    # ignore instances with a low score
    if kp_ann["score"] < 0.1:
        return 0

    # force pose can generate bounding boxes outside the image
    kp_ann_bbox = utils.bbox_clamp(kp_ann_bbox, width, height)
    return utils.iou(ann_bbox, kp_ann_bbox)
Beispiel #5
0
    def __call__(self, fields):
        meta = self.metas[0]
        cifcaf_annotations = self.cifcaf(fields)
        action_probabilities = fields[meta.head_index]
        anns = []

        for cifcaf_ann in cifcaf_annotations:
            bbox = cifcaf_ann.bbox()
            area = utils.bbox_area(bbox)
            scale = np.sqrt(area) / meta.stride
            radius = int(
                np.round(max(self.min_radius, scale * self.scaling_factor)))
            side_length = 2 * radius + 1

            centers = utils.keypoint_centers(cifcaf_ann.data,
                                             meta.keypoint_indices)
            centers = np.array(centers) / meta.stride
            int_centers = np.round(centers - radius).astype(int)

            probability_fields = action_probabilities[:, 0]

            probabilities = []
            for int_center in int_centers:
                i, j = int_center
                box = [j, i, side_length, side_length]
                probabilities.append(utils.read_values(probability_fields,
                                                       box))

            if self.save_raw:
                save_probabilities = np.where(np.isnan(probabilities), None,
                                              probabilities).tolist()
            else:
                save_probabilities = []

            # remove empty arrays
            probabilities = [
                p for p in probabilities
                if (p.size > 0) and not np.isnan(p).all()
            ]

            probabilities = np.array(probabilities)
            if len(probabilities) > 0:
                if self.strategy == "max":
                    probabilities = np.nanmax(probabilities, (0, 2, 3))
                elif self.strategy == "mean":
                    probabilities = np.nanmean(probabilities, (0, 2, 3))
                probabilities = probabilities.tolist()
            else:
                probabilities = [None] * probability_fields.shape[0]

            anns.append(
                annotations.Action(
                    keypoint_ann=cifcaf_ann,
                    keypoint_indices=meta.keypoint_indices,
                    true_actions=None,
                    all_actions=meta.actions,
                    action_probabilities=probabilities,
                    action_probability_fields=save_probabilities,
                ))

        anns.extend(cifcaf_annotations)
        self.visualizer.predicted(action_probabilities)
        return anns