def point_score_fun(ann, kp_ann): """Matches annotations based on minimum keypoint distance to a reference point on the human""" # ignore instances that are much bigger than the image image_area = ann["width"] * ann["height"] bbox_area = utils.bbox_area(kp_ann["bbox"]) if bbox_area > 4 * image_area: return (0, 0) # ignore instances with a low score if kp_ann["score"] < 0.1: return (0, 0) # ignore instances where the point is outside the bbox point = np.array(ann["point"]) px, py = point x, y, w, h = kp_ann["bbox"] if (px < x) or (px > x + w) or (py < y) or (py >= y + h): return (0, 0) keypoints = np.array(kp_ann["keypoints"]).reshape(-1, 3) keypoint_score = kp_ann["score"] xy = keypoints[keypoints[:, 2] > 0, :2] distance = np.sqrt(np.sum((xy - point)**2, axis=1)) if len(distance) > 0: # normalize distance by bbox size distance /= min(200, max(10, np.sqrt(bbox_area))) inv_distance = 1 / min(distance) else: inv_distance = 0 result = None # no point to match if inv_distance == 0: result = (0, 0) # check if point is on torso global _torso_indices if (inv_distance > 2) and np.all(keypoints[_torso_indices, 2] > 0): torso_keypoints = keypoints[_torso_indices, :2] torso = matplotlib.path.Path(torso_keypoints) if torso.contains_point(point) and (keypoint_score > 0.1): # if there are multiple instances that have the point on the torso then we want to match # the one with the highest keypoint score # if there are two instances with same score then match the one with the smallest distance result = (1e10 * keypoint_score, inv_distance) # default case if result is None: result = (inv_distance, keypoint_score) return result
def score_fun(ann, kp_ann): """Scoring function based on bounding box IoU""" ann_bbox = ann["bbox"] width = ann["width"] height = ann["height"] kp_ann_bbox = kp_ann["bbox"] # ignore instances that are much bigger than the image image_area = ann["width"] * ann["height"] bbox_area = utils.bbox_area(kp_ann_bbox) if bbox_area > 4 * image_area: return 0 # force pose can generate bounding boxes outside the image # only match on area inside image kp_ann_bbox = utils.bbox_clamp(kp_ann_bbox, width, height) return utils.iou(ann_bbox, kp_ann_bbox)
def fill_fields(self, anns): for ann in anns: if "actions" not in ann: continue keypoint_indices = self.config.meta.keypoint_indices centers = np.array( utils.keypoint_centers(ann["keypoints"], keypoint_indices)) centers = centers / self.config.meta.stride action_labels = self.config.meta.action_labels(ann["actions"]) action_mask = np.asarray(action_labels).astype(bool) area = utils.bbox_area(ann["bbox"]) scale = np.sqrt(area) / self.config.meta.stride for center in centers: self.fill_center(center, action_mask, scale)
def score_fun(ann, kp_ann): ann_bbox = ann["bbox"] width = ann["width"] height = ann["height"] kp_ann_bbox = kp_ann["bbox"] # ignore instances that are much bigger than the image image_area = ann["width"] * ann["height"] bbox_area = utils.bbox_area(kp_ann_bbox) if bbox_area > 4 * image_area: return 0 # ignore instances with a low score if kp_ann["score"] < 0.1: return 0 # force pose can generate bounding boxes outside the image kp_ann_bbox = utils.bbox_clamp(kp_ann_bbox, width, height) return utils.iou(ann_bbox, kp_ann_bbox)
def __call__(self, fields): meta = self.metas[0] cifcaf_annotations = self.cifcaf(fields) action_probabilities = fields[meta.head_index] anns = [] for cifcaf_ann in cifcaf_annotations: bbox = cifcaf_ann.bbox() area = utils.bbox_area(bbox) scale = np.sqrt(area) / meta.stride radius = int( np.round(max(self.min_radius, scale * self.scaling_factor))) side_length = 2 * radius + 1 centers = utils.keypoint_centers(cifcaf_ann.data, meta.keypoint_indices) centers = np.array(centers) / meta.stride int_centers = np.round(centers - radius).astype(int) probability_fields = action_probabilities[:, 0] probabilities = [] for int_center in int_centers: i, j = int_center box = [j, i, side_length, side_length] probabilities.append(utils.read_values(probability_fields, box)) if self.save_raw: save_probabilities = np.where(np.isnan(probabilities), None, probabilities).tolist() else: save_probabilities = [] # remove empty arrays probabilities = [ p for p in probabilities if (p.size > 0) and not np.isnan(p).all() ] probabilities = np.array(probabilities) if len(probabilities) > 0: if self.strategy == "max": probabilities = np.nanmax(probabilities, (0, 2, 3)) elif self.strategy == "mean": probabilities = np.nanmean(probabilities, (0, 2, 3)) probabilities = probabilities.tolist() else: probabilities = [None] * probability_fields.shape[0] anns.append( annotations.Action( keypoint_ann=cifcaf_ann, keypoint_indices=meta.keypoint_indices, true_actions=None, all_actions=meta.actions, action_probabilities=probabilities, action_probability_fields=save_probabilities, )) anns.extend(cifcaf_annotations) self.visualizer.predicted(action_probabilities) return anns