def densepose_chart_predictor_output_to_result(
predictor_output: DensePoseChartPredictorOutput,
boxes: Boxes) -> DensePoseChartResult:
"""
Convert densepose chart predictor outputs to results
Args:
predictor_output (DensePoseChartPredictorOutput): DensePose predictor
output to be converted to results, must contain only 1 output
boxes (Boxes): bounding box that corresponds to the predictor output,
must contain only 1 bounding box
Return:
DensePose chart-based result (DensePoseChartResult)
"""
assert len(predictor_output) == 1 and len(boxes) == 1, (
f"Predictor output to result conversion can operate only single outputs"
f", got {len(predictor_output)} predictor outputs and {len(boxes)} boxes"
)
boxes_xyxy_abs = boxes.tensor.clone()
boxes_xywh_abs = BoxMode.convert(boxes_xyxy_abs, BoxMode.XYXY_ABS,
BoxMode.XYWH_ABS)
box_xywh = make_int_box(boxes_xywh_abs[0])
labels = resample_fine_and_coarse_segm_to_bbox(predictor_output,
box_xywh).squeeze(0)
uv = resample_uv_to_bbox(predictor_output, labels, box_xywh)
return DensePoseChartResult(labels=labels, uv=uv)
def get_val_dicts():
data = json.load(open("stars_carla_val.json"))
for im in data:
anno = im['annotations']
for an in anno:
an['bbox_mode'] = BoxMode(an['bbox_mode'])
return data
def getResults(self):
"""
Convert the estimation results to necessary format.
return:
densepose_res_list: it is a list of dict that each element in the list
is corresponding to the reslut of one image. The format of the list is
as follows:
[
# image one
{
'scores': [], # numpy array, a vector contians the confidence of each detected instance
'bboxes': [[x0,y0,x1,y1], ..., [x0,y0,x1,y1]], # numpy array, the coordinates of each
detected bbox.
'pred_densepose': [((3, H, W), iuv_png_data), ..., ((3, H, W), iuv_png_data)]
# H is the height of detected bbox, W is its width. iuv_png_data is the densepose resutls
# encoded into png data.
# Note:
# 1. The fuction of DensePoseResult.decode_png_data in the project DensePose can be used
# to decode iuv_png_data to get iuv data.
# 2. The shape of the decoded iuv data is (3, H, W).
# 3. iuv[0, :, :] is the patch index of image points, indicating which of the 24 surface
# patches the point is on.
# 4. iuv[1, :, :] is the U-coordinate value of image points.
# 5. iuv[2, :, :] is the v-coordinate value of image points.
# 6. More details in https://github.com/dli2016/detectron2/blob/master/projects/DensePose/doc/TOOL_APPLY_NET.md
},
...,
# Image I
{
'scores': [],
'bboxes': [[x0,y0,x1,y1], ..., [x0,y0,x1,y1]],
'pred_densepose': [((3, H, W), iuv_png_data), ..., ((3, H, W), iuv_png_data)]
}
]
"""
densepose_res = self._densepose_res
densepose_res_list = []
# (user custom code START)
# Convert the data format to the necessary one.
for output in densepose_res:
instances = output.to('cpu')
results = {}
if instances.has("scores"):
results['scores'] = instances.scores.to('cpu').numpy()
if instances.has("pred_boxes"):
results['bboxes'] = instances.pred_boxes.tensor.to('cpu').numpy()
if instances.has("pred_densepose"):
boxes_XYWH = BoxMode.convert(
instances.pred_boxes.tensor.to('cpu'),
BoxMode.XYXY_ABS,
BoxMode.XYWH_ABS
)
results["pred_densepose"] = \
instances.get("pred_densepose").to_result(boxes_XYWH).results
densepose_res_list.append(results)
# (user custom code END)
return densepose_res_list
def get_data_dicts(self, json_file):
data = json.load(open(json_file))
for im in data:
anno = im["annotations"]
for an in anno:
an["bbox_mode"] = BoxMode(an["bbox_mode"])
return data
def bbox_wh_to_xy(bbox):
if not isinstance(bbox, (tuple, list)):
original_shape = bbox.shape
bbox = bbox.reshape((-1, 4))
else:
original_shape = None
bbox = BoxMode.convert(
box=bbox, from_mode=BoxMode.XYWH_ABS, to_mode=BoxMode.XYXY_ABS
)
if original_shape is not None:
return bbox.reshape(original_shape)
return bbox
def to_densepose_prediction(instances):
if not instances.has("pred_boxes"):
return None
if not instances.has("pred_densepose"):
return None
pred_boxes = instances.get("pred_boxes").tensor.cpu()
boxes = BoxMode.convert(pred_boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)
pred_densepose = instances.get("pred_densepose").to_result(boxes)
return pred_densepose
def execute_on_outputs(context: Dict[str, Any], entry: Dict[str, Any],
outputs: Instances):
image_fpath = entry["file_name"]
print(f"Processing {image_fpath}")
result = {"file_name": image_fpath}
if outputs.has("scores"):
result["scores"] = outputs.get("scores").cpu()
if outputs.has("pred_boxes"):
result["pred_boxes_XYXY"] = outputs.get("pred_boxes").tensor.cpu()
if outputs.has("pred_densepose"):
boxes_XYWH = BoxMode.convert(result["pred_boxes_XYXY"],
BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)
result["pred_densepose"] = outputs.get("pred_densepose").to_result(
boxes_XYWH)
context["results"].append(result)
def predict_densepose(input_image, predictor):
"""
Predicts densepose output given a cropped and centred input image.
:param input_images: (wh, wh)
:param predictor: instance of detectron2 DefaultPredictor class, created with the
appropriate config file.
"""
orig_h, orig_w = input_image.shape[:2]
outputs = predictor(input_image)["instances"]
bboxes = outputs.pred_boxes.tensor.cpu(
) # Multiple densepose predictions if there are multiple people in the image
bboxes_XYWH = BoxMode.convert(bboxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)
bboxes = bboxes.cpu().detach().numpy()
largest_centred_bbox_index = get_largest_centred_bounding_box(
bboxes, orig_w,
orig_h) # Picks out centred person that is largest in the image.
pred_densepose = outputs.pred_densepose.to_result(bboxes_XYWH)
iuv_arr = DensePoseResult.decode_png_data(
*pred_densepose.results[largest_centred_bbox_index])
# Round bbox to int
largest_bbox = bboxes[largest_centred_bbox_index]
w1 = largest_bbox[0]
w2 = largest_bbox[0] + iuv_arr.shape[2]
h1 = largest_bbox[1]
h2 = largest_bbox[1] + iuv_arr.shape[1]
I_image = np.zeros((orig_h, orig_w))
I_image[int(h1):int(h2), int(w1):int(w2)] = iuv_arr[0, :, :]
# U_image = np.zeros((orig_h, orig_w))
# U_image[int(h1):int(h2), int(w1):int(w2)] = iuv_arr[1, :, :]
# V_image = np.zeros((orig_h, orig_w))
# V_image[int(h1):int(h2), int(w1):int(w2)] = iuv_arr[2, :, :]
vis_I_image = apply_colormap(I_image, vmin=0, vmax=24)
vis_I_image = vis_I_image[:, :, :3]
vis_I_image[I_image == 0, :] = np.zeros(3, dtype=np.uint8)
overlay_vis = cv2.addWeighted(input_image, 0.6, vis_I_image, 0.4, gamma=0)
return I_image, overlay_vis
def load_mosaic(self, image, data_dict):
"""
from https://github.com/ultralytics/yolov3
"""
# loads images in a mosaic
data_dicts = [data_dict]
data_dicts.extend([
copy.deepcopy(self._dataset[random.randint(0,
len(self._dataset) -
1)]) for _ in range(3)
])
images = [image]
labels = []
w, h = 1024, 1024
for i, data_dict in enumerate(data_dicts):
if i > 0:
# load image and boxes
image = self.load_image(data_dict)
# if random.random() < self.mixup_prob:
# image, data_dict = self.load_mixup(image, data_dict)
# data_dicts[i] = data_dict
images.append(image)
if "annotations" in data_dict:
labels.append(
np.array([[obj['category_id']] + obj['bbox']
for obj in data_dict["annotations"]],
dtype=np.float32))
else:
labels.append(np.array([]))
w, h = min(w, data_dict["width"]), min(h, data_dict["height"])
xc, yc = int(random.uniform(w * 0.25, w * 0.75)), int(
random.uniform(h * 0.25, h * 0.75)) # mosaic center x, y
if self.img_format == "BGR":
fill_val = (103, 116, 123)
else: # rgb
fill_val = (123, 116, 103)
mosaic_img = np.full(
(h, w, 3), fill_val,
dtype=np.uint8) # base image with 4 tiles, BGR order
mosaic_labels = []
for i, (image, data_dict) in enumerate(zip(images, data_dicts)):
wi, hi = data_dict['width'], data_dict['height']
# place img in mosaic_img
if i == 0: # top left
x1a, y1a, x2a, y2a = max(xc - wi, 0), max(
yc - hi, 0), xc, yc # xmin, ymin, xmax, ymax (large image)
x1b, y1b, x2b, y2b = wi - (x2a - x1a), hi - (
y2a - y1a), wi, hi # xmin, ymin, xmax, ymax (small image)
elif i == 1: # top right
x1a, y1a, x2a, y2a = xc, max(yc - hi, 0), min(xc + wi, w), yc
x1b, y1b, x2b, y2b = 0, hi - (y2a - y1a), min(wi,
x2a - x1a), hi
elif i == 2: # bottom left
x1a, y1a, x2a, y2a = max(xc - wi, 0), yc, xc, min(h, yc + hi)
x1b, y1b, x2b, y2b = wi - (x2a - x1a), 0, max(xc, wi), min(
y2a - y1a, hi)
elif i == 3: # bottom right
x1a, y1a, x2a, y2a = xc, yc, min(xc + wi, w), min(h, yc + hi)
x1b, y1b, x2b, y2b = 0, 0, min(wi,
x2a - x1a), min(y2a - y1a, hi)
mosaic_img[y1a:y2a,
x1a:x2a] = image[y1b:y2b,
x1b:x2b] # img4[ymin:ymax, xmin:xmax]
padw = x1a - x1b
padh = y1a - y1b
# Labels
if len(labels[i]) > 0:
labels_i = labels[i].copy()
# xywh to xyxy
labels_i[:, 1] += padw
labels_i[:, 2] += padh
labels_i[:, 3] = labels_i[:, 1] + labels_i[:, 3]
labels_i[:, 4] = labels_i[:, 2] + labels_i[:, 4]
mosaic_labels.append(labels_i)
# Concat/clip labels
if len(mosaic_labels):
mosaic_labels = np.concatenate(mosaic_labels, 0)
np.clip(mosaic_labels[:, 1], 0, w, out=mosaic_labels[:, 1])
np.clip(mosaic_labels[:, 3], 0, w, out=mosaic_labels[:, 3])
np.clip(mosaic_labels[:, 2], 0, h, out=mosaic_labels[:, 2])
np.clip(mosaic_labels[:, 4], 0, h, out=mosaic_labels[:, 4])
mosaic_labels.astype(np.int32)
mosaic_labels = mosaic_labels[np.where(
(mosaic_labels[:, 3] - mosaic_labels[:, 1]) *
(mosaic_labels[:, 4] - mosaic_labels[:, 2]) > 0)]
# Augment
# mosaic_img, mosaic_labels = random_affine(mosaic_img,
# mosaic_labels,
# degrees=0.0,
# translate=0.0,
# scale=0.5,
# shear=0.0,
# border=-w // 2) # border to remove
# translate back to detectron2 format
annos = []
for i in range(mosaic_labels.shape[0]):
bbox = [
mosaic_labels[i, 1], mosaic_labels[i, 2],
max(mosaic_labels[i, 3] - mosaic_labels[i, 1], 0),
max(mosaic_labels[i, 4] - mosaic_labels[i, 2], 0)
]
anno = {
'iscrowd': 0,
# xyxy to xywh
'bbox': bbox,
'category_id': int(mosaic_labels[i, 0]),
'bbox_mode': BoxMode(1), # xywh
'area': bbox[2] * bbox[3],
}
annos.append(anno)
data_dict = {
"file_name": data_dicts[0]['file_name'],
"height": h,
"width": w,
"image_id": data_dicts[0]['image_id'],
"annotations": annos
}
return mosaic_img, data_dict
def run(self):
self.beginTaskRun()
# Get input :
input = self.getInput(0)
# Get output :
output = self.getOutput(0)
output_graph = self.getOutput(1)
output_graph.setNewLayer("DensePose")
srcImage = input.getImage()
# Get parameters :
param = self.getParam()
# predictor
if not self.loaded:
print("Chargement du modèle")
if param.cuda == False:
self.cfg.MODEL.DEVICE = "cpu"
self.deviceFrom = "cpu"
else:
self.deviceFrom = "gpu"
self.loaded = True
self.predictor = DefaultPredictor(self.cfg)
# reload model if CUDA check and load without CUDA
elif self.deviceFrom == "cpu" and param.cuda == True:
print("Chargement du modèle")
self.cfg = get_cfg()
add_densepose_config(self.cfg)
self.cfg.merge_from_file(self.folder + "/DensePose_git/configs/"+self.MODEL_NAME_CONFIG+".yaml")
self.cfg.MODEL.WEIGHTS = self.folder + "/models/"+self.MODEL_NAME+".pkl"
self.deviceFrom = "gpu"
self.predictor = DefaultPredictor(self.cfg)
# reload model if CUDA not check and load with CUDA
elif self.deviceFrom == "gpu" and param.cuda == False:
print("Chargement du modèle")
self.cfg = get_cfg()
self.cfg.MODEL.DEVICE = "cpu"
add_densepose_config(self.cfg)
self.cfg.merge_from_file(self.folder + "/DensePose_git/configs/"+self.MODEL_NAME_CONFIG+".yaml")
self.cfg.MODEL.WEIGHTS = self.folder + "/models/"+self.MODEL_NAME+".pkl"
self.deviceFrom = "cpu"
self.predictor = DefaultPredictor(self.cfg)
outputs = self.predictor(srcImage)["instances"]
scores = outputs.get("scores").cpu()
boxes_XYXY = outputs.get("pred_boxes").tensor.cpu()
boxes_XYWH = BoxMode.convert(boxes_XYXY, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)
denseposes = outputs.get("pred_densepose").to_result(boxes_XYWH)
# Number of iso values betwen 0 and 1
self.levels = np.linspace(0, 1, 9)
cmap = cv2.COLORMAP_PARULA
img_colors_bgr = cv2.applyColorMap((self.levels * 255).astype(np.uint8), cmap)
self.level_colors_bgr = [
[int(v) for v in img_color_bgr.ravel()] for img_color_bgr in img_colors_bgr
]
# text and rect graph properties
properties_text = core.GraphicsTextProperty()
properties_text.color = [255,255,255]
properties_text.font_size = 10
properties_rect = core.GraphicsRectProperty()
properties_rect.pen_color = [11,130,41]
properties_line = core.GraphicsPolylineProperty()
properties_line.line_size = 1
self.emitStepProgress()
for i in range(len(denseposes)):
if scores.numpy()[i] > param.proba:
bbox_xywh = boxes_XYWH[i]
bbox_xyxy = boxes_XYXY[i]
result_encoded = denseposes.results[i]
iuv_arr = DensePoseResult.decode_png_data(*result_encoded)
# without indice surface
self.visualize_iuv_arr(srcImage, iuv_arr, bbox_xywh, properties_line, output_graph)
# with indice surface
#self.visualize_iuv_arr_indiceSurface(srcImage, iuv_arr, bbox_xyxy, output_graph)
output_graph.addRectangle(bbox_xyxy[0].item(), bbox_xyxy[1].item(), bbox_xyxy[2].item() - bbox_xyxy[0].item(), bbox_xyxy[3].item() - bbox_xyxy[1].item(),properties_rect)
output_graph.addText(str(scores[i].item())[:5], float(bbox_xyxy[0].item()), float(bbox_xyxy[1].item()), properties_text)
output.setImage(srcImage)
self.emitStepProgress()
self.endTaskRun()