def masks_from_df(self, img, df, input_format="RGB", resize_factor=1):
obj_df = df[df.det_type == "object"]
if len(obj_df):
boxes = []
# For input to mask-rcnn
pred_classes = []
pred_boxes = []
for _, row in obj_df.iterrows():
box = boxutils.dfbox_to_norm(row, resize_factor=resize_factor)
boxes.append(box)
for cls in self.thing_idxs:
pred_boxes.append(box)
pred_classes.append(cls)
res = self.masks_from_bboxes(img,
pred_boxes,
pred_classes=pred_classes,
input_format=input_format)
masks = res["masks"]
masks = [
mask.sum(0).detach()
for mask in masks.split(len(self.thing_idxs))
]
res["masks"] = masks
else:
res = {"masks": [], "boxes": [], "scores": [], "classes": []}
return res
def masks_from_df(img, df, input_format="RGB", resize_factor=1):
obj_df = df[df.det_type == "object"]
boxes = []
masks = []
if len(obj_df):
for _, row in obj_df.iterrows():
box = boxutils.dfbox_to_norm(row, resize_factor=resize_factor)
mask, _ = grab_cut(img, box)
masks.append(mask)
boxes.append(box)
return masks, boxes
def links_from_df(hoa_df, resize_factor=1):
link_df = hoa_df[hoa_df.hoa_link == hoa_df.hoa_link]
links = []
for link_idx, link_row in link_df.iterrows():
box_ltrb = boxutils.dfbox_to_norm(link_row,
resize_factor=resize_factor)
box_center = [
box_ltrb[2] + box_ltrb[0],
box_ltrb[3] + box_ltrb[1],
]
link = {
"type": link_row.hoa_link,
"link_source": box_center,
"obj_offset": [link_row.obj_offx, link_row.obj_offy],
"side": link_row.side,
}
links.append(link)
return links
def add_masks_df_viz(ax,
masks_df,
resize_factor=1,
alpha_mask=0.5,
debug=False):
if masks_df.shape[0] > 0:
if debug:
print("Drawing predicted hand and object boxes !")
bboxes_norm = [
boxutils.dfbox_to_norm(box_row)
for _, box_row in masks_df.iterrows()
]
masks = [
coco_mask_decode({
"counts": row[1]["mask"],
"size": [100, 100]
}) for row in masks_df.iterrows()
]
masks = [
resize_mask(mask,
height=int(1080 * resize_factor),
width=int(1920 * resize_factor)) for mask in masks
]
colors = [get_masks_color(obj[1]) for obj in masks_df.iterrows()]
labels = [get_masks_label(obj[1]) for obj in masks_df.iterrows()]
detect2d.visualize_bboxes(
ax,
bboxes_norm,
labels=labels,
label_color="w",
linewidth=2,
color=colors,
)
for label, mask in zip(labels, masks):
base_mask = mask[:, :, np.newaxis].astype(np.float)
# show_mask = np.concatenate(
# [base_mask.repeat(3, 2), base_mask * alpha_mask], 2
# )
show_mask = np.concatenate([
base_mask, base_mask * 0.7, base_mask * 0.8,
base_mask * alpha_mask
], 2)
ax.imshow(show_mask)
def hands_from_df(self, img, hoa_df, resize_factor=1):
hand_df = hoa_df[hoa_df.det_type == "hand"]
hand_boxes = {}
# Keep first right and left hand found in data frame
for hand_idx, row in hand_df.iterrows():
box_ltrb = boxutils.dfbox_to_norm(row, resize_factor=resize_factor)
box_ltwh = [
box_ltrb[0],
box_ltrb[1],
box_ltrb[2] - box_ltrb[0],
box_ltrb[3] - box_ltrb[1],
]
box_side = row.side
hand_key = f"{box_side}_hand"
if hand_key not in hand_boxes:
hand_boxes[hand_key] = np.array(box_ltwh)
for side in ["right", "left"]:
hand_key = f"{side}_hand"
if hand_key not in hand_boxes:
hand_boxes[hand_key] = None
_, pred_hands = self.hand_extractor.regress(img, [hand_boxes],
add_margin=True)
# Retrieve first results
ego_hands = pred_hands[0]
pred_hands = {}
for side in ["right", "left"]:
hand_key = f"{side}_hand"
if hand_key in ego_hands and (ego_hands[hand_key] is not None):
pred_hand = ego_hands[hand_key]
pred_hands[side] = {
"verts": pred_hand["pred_vertices_img"],
"faces": pred_hand["faces"],
"hand_pose": pred_hand["pred_hand_pose"],
}
return pred_hands