def point_score_fun(ann, kp_ann):
"""Matches annotations based on minimum keypoint distance to a reference point on the human"""
# ignore instances that are much bigger than the image
image_area = ann["width"] * ann["height"]
bbox_area = utils.bbox_area(kp_ann["bbox"])
if bbox_area > 4 * image_area:
return (0, 0)
# ignore instances with a low score
if kp_ann["score"] < 0.1:
return (0, 0)
# ignore instances where the point is outside the bbox
point = np.array(ann["point"])
px, py = point
x, y, w, h = kp_ann["bbox"]
if (px < x) or (px > x + w) or (py < y) or (py >= y + h):
return (0, 0)
keypoints = np.array(kp_ann["keypoints"]).reshape(-1, 3)
keypoint_score = kp_ann["score"]
xy = keypoints[keypoints[:, 2] > 0, :2]
distance = np.sqrt(np.sum((xy - point)**2, axis=1))
if len(distance) > 0:
# normalize distance by bbox size
distance /= min(200, max(10, np.sqrt(bbox_area)))
inv_distance = 1 / min(distance)
else:
inv_distance = 0
result = None
# no point to match
if inv_distance == 0:
result = (0, 0)
# check if point is on torso
global _torso_indices
if (inv_distance > 2) and np.all(keypoints[_torso_indices, 2] > 0):
torso_keypoints = keypoints[_torso_indices, :2]
torso = matplotlib.path.Path(torso_keypoints)
if torso.contains_point(point) and (keypoint_score > 0.1):
# if there are multiple instances that have the point on the torso then we want to match
# the one with the highest keypoint score
# if there are two instances with same score then match the one with the smallest distance
result = (1e10 * keypoint_score, inv_distance)
# default case
if result is None:
result = (inv_distance, keypoint_score)
return result
def score_fun(ann, kp_ann):
"""Scoring function based on bounding box IoU"""
ann_bbox = ann["bbox"]
width = ann["width"]
height = ann["height"]
kp_ann_bbox = kp_ann["bbox"]
# ignore instances that are much bigger than the image
image_area = ann["width"] * ann["height"]
bbox_area = utils.bbox_area(kp_ann_bbox)
if bbox_area > 4 * image_area:
return 0
# force pose can generate bounding boxes outside the image
# only match on area inside image
kp_ann_bbox = utils.bbox_clamp(kp_ann_bbox, width, height)
return utils.iou(ann_bbox, kp_ann_bbox)
def fill_fields(self, anns):
for ann in anns:
if "actions" not in ann:
continue
keypoint_indices = self.config.meta.keypoint_indices
centers = np.array(
utils.keypoint_centers(ann["keypoints"], keypoint_indices))
centers = centers / self.config.meta.stride
action_labels = self.config.meta.action_labels(ann["actions"])
action_mask = np.asarray(action_labels).astype(bool)
area = utils.bbox_area(ann["bbox"])
scale = np.sqrt(area) / self.config.meta.stride
for center in centers:
self.fill_center(center, action_mask, scale)
def score_fun(ann, kp_ann):
ann_bbox = ann["bbox"]
width = ann["width"]
height = ann["height"]
kp_ann_bbox = kp_ann["bbox"]
# ignore instances that are much bigger than the image
image_area = ann["width"] * ann["height"]
bbox_area = utils.bbox_area(kp_ann_bbox)
if bbox_area > 4 * image_area:
return 0
# ignore instances with a low score
if kp_ann["score"] < 0.1:
return 0
# force pose can generate bounding boxes outside the image
kp_ann_bbox = utils.bbox_clamp(kp_ann_bbox, width, height)
return utils.iou(ann_bbox, kp_ann_bbox)
def __call__(self, fields):
meta = self.metas[0]
cifcaf_annotations = self.cifcaf(fields)
action_probabilities = fields[meta.head_index]
anns = []
for cifcaf_ann in cifcaf_annotations:
bbox = cifcaf_ann.bbox()
area = utils.bbox_area(bbox)
scale = np.sqrt(area) / meta.stride
radius = int(
np.round(max(self.min_radius, scale * self.scaling_factor)))
side_length = 2 * radius + 1
centers = utils.keypoint_centers(cifcaf_ann.data,
meta.keypoint_indices)
centers = np.array(centers) / meta.stride
int_centers = np.round(centers - radius).astype(int)
probability_fields = action_probabilities[:, 0]
probabilities = []
for int_center in int_centers:
i, j = int_center
box = [j, i, side_length, side_length]
probabilities.append(utils.read_values(probability_fields,
box))
if self.save_raw:
save_probabilities = np.where(np.isnan(probabilities), None,
probabilities).tolist()
else:
save_probabilities = []
# remove empty arrays
probabilities = [
p for p in probabilities
if (p.size > 0) and not np.isnan(p).all()
]
probabilities = np.array(probabilities)
if len(probabilities) > 0:
if self.strategy == "max":
probabilities = np.nanmax(probabilities, (0, 2, 3))
elif self.strategy == "mean":
probabilities = np.nanmean(probabilities, (0, 2, 3))
probabilities = probabilities.tolist()
else:
probabilities = [None] * probability_fields.shape[0]
anns.append(
annotations.Action(
keypoint_ann=cifcaf_ann,
keypoint_indices=meta.keypoint_indices,
true_actions=None,
all_actions=meta.actions,
action_probabilities=probabilities,
action_probability_fields=save_probabilities,
))
anns.extend(cifcaf_annotations)
self.visualizer.predicted(action_probabilities)
return anns