def _visualize_predictions(
self, batched_inputs, predictions, predictions_mode, viz_count, max_predictions, title_suffix=None
):
"""Visualizes images and predictions and sends them to Wandb.
batched_inputs: List[dict], list of inputs, each must contain keys "image"
and "instances"
predictions: List[Instances], list of bbox instances either from RPN or ROI head,
see `_log_visualization_to_wandb` for more details.
predictions_mode: str, must be begin with either 'RPN' or 'ROI', defines whether
`predictions` contain bboxes predicted by RPN or ROI heads.
"""
assert predictions_mode[:3] in ["RPN", "ROI"], "Unsupported proposal visualization mode"
for inp, pred in zip(batched_inputs, predictions):
img = inp["image"]
img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format)
gts = inp["instances"]
suffix = f"_{viz_count}" if title_suffix is None else title_suffix
log_visualization_to_wandb(img, gts, pred, predictions_mode, max_predictions, title_suffix=suffix)
viz_count -= 1 # Visualize up to `viz_count` images only
if viz_count == 0:
break
def test_dataloader():
from slender_det.data import build_detection_test_loader
from slender_det.config import get_cfg
cfg = get_cfg()
cfg.DATASETS.TEST = ("objects365_val", )
data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0])
data_iter = iter(data_loader)
data = next(data_iter)
ipdb.set_trace()
for data in data_iter:
for dataset_dict in data:
img = dataset_dict['image']
img = convert_image_to_rgb(img.permute(1, 2, 0), cfg.INPUT.FORMAT)
v_gt = Visualizer(img, None)
v_gt = v_gt.overlay_instances(
boxes=dataset_dict["instances"].gt_boxes,
masks=dataset_dict["instances"].gt_masks,
)
v_gt.save('/data/tmp/vis_obj365_val_{}.png'.format(
dataset_dict['image_id']))
ipdb.set_trace()
def visualize_training(self, batched_inputs, proposals):
"""
A function used to visualize images and proposals. It shows ground truth
bounding boxes on the original image and up to 20 top-scoring predicted
object proposals on the original image. Users can implement different
visualization functions for different models.
Args:
batched_inputs (list): a list that contains input to the model.
proposals (list): a list that contains predicted proposals. Both
batched_inputs and proposals should have the same length.
"""
from detectron2.utils.visualizer import Visualizer
storage = get_event_storage()
max_vis_prop = 20
for input, prop in zip(batched_inputs, proposals):
img = input["image"]
img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format)
v_gt = Visualizer(img, None)
v_gt = v_gt.overlay_instances(boxes=input["instances"].gt_boxes)
anno_img = v_gt.get_image()
box_size = min(len(prop.proposal_boxes), max_vis_prop)
v_pred = Visualizer(img, None)
v_pred = v_pred.overlay_instances(
boxes=prop.proposal_boxes[0:box_size].tensor.cpu().numpy()
)
prop_img = v_pred.get_image()
vis_img = np.concatenate((anno_img, prop_img), axis=1)
vis_img = vis_img.transpose(2, 0, 1)
vis_name = "Left: GT bounding boxes; Right: Predicted proposals"
storage.put_image(vis_name, vis_img)
break # only visualize one image in a batch
def visualize_training(self, batched_inputs, results):
"""
A function used to visualize ground truth images and final network predictions.
It shows ground truth bounding boxes on the original image and up to 20
predicted object bounding boxes on the original image.
Args:
batched_inputs (list): a list that contains input to the model.
results (List[Instances]): a list of #images elements.
"""
from detectron2.utils.visualizer import Visualizer
assert len(batched_inputs) == len(
results
), "Cannot visualize inputs and results of different sizes"
storage = get_event_storage()
max_boxes = 20
image_index = 0 # only visualize a single image
img = batched_inputs[image_index]["image"]
img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format)
v_gt = Visualizer(img, None)
v_gt = v_gt.overlay_instances(boxes=batched_inputs[image_index]["instances"].gt_boxes)
anno_img = v_gt.get_image()
processed_results = detector_postprocess(results[image_index], img.shape[0], img.shape[1])
predicted_boxes = processed_results.pred_boxes.tensor.detach().cpu().numpy()
v_pred = Visualizer(img, None)
v_pred = v_pred.overlay_instances(boxes=predicted_boxes[0:max_boxes])
prop_img = v_pred.get_image()
vis_img = np.vstack((anno_img, prop_img))
vis_img = vis_img.transpose(2, 0, 1)
vis_name = f"Top: GT bounding boxes; Bottom: {max_boxes} Highest Scoring Results"
storage.put_image(vis_name, vis_img)
def test_rle_to_polygon(res_file):
obj365_anno_file = "datasets/obj365/annotations/objects365_val_20190423.json"
obj_api = COCO(annotation_file=obj365_anno_file)
with open(file=res_file, mode='rb') as f:
predictions = torch.load(f)
anns = list(itertools.chain(*[x["instances"] for x in predictions]))
oss_root = "s3://detection/objects365_raw_data/objects365/val/"
img_format = "BGR"
for ann in anns:
segm = ann['segmentation']
image_dict = obj_api.imgs[ann['image_id']]
file_path = os.path.join(oss_root, image_dict['file_name'])
image = load_image_from_oss(smart_path(file_path), format=img_format)
image = convert_image_to_rgb(image, format=img_format)
v_rle = Visualizer(image, None)
v_rle = v_rle.overlay_instances(masks=[segm], )
v_rle.save('/data/tmp/{}_rle.png'.format(ann['image_id']))
v_polygon = Visualizer(copy.deepcopy(image), None)
v_polygon = v_polygon.overlay_instances(
masks=[_convert_rle_to_polygon(segm)], )
v_polygon.save('/data/tmp/{}_polygon.png'.format(ann['image_id']))
ipdb.set_trace()
def visualize_training(self, batched_inputs, results):
"""
A function used to visualize ground truth images and final network predictions.
It shows ground truth bounding boxes on the original image and up to 20
predicted object bounding boxes on the original image.
Args:
batched_inputs (list): a list that contains input to the model.
results (List[Instances]): a list of #images elements.
"""
assert len(batched_inputs) == len(
results), "Cannot visualize inputs and results of different sizes"
storage = get_event_storage()
max_boxes = 20
image_index = 0 # only visualize a single image
img = batched_inputs[image_index]["image"]
img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format)
anno_img = self.visualizer.draw_instance_groundtruth(
img, batched_inputs[image_index]["instances"])
prop_img = self.visualizer.draw_instance_predictions(
img, results[image_index], max_boxes)
vis_img = np.vstack((anno_img, prop_img))
vis_img = vis_img.transpose(2, 0, 1)
vis_name = f"Top: GT bounding boxes; Bottom: {max_boxes} Highest Scoring Results"
storage.put_image(vis_name, vis_img)
def postprocess_to_image(
image: torch.Tensor,
model: Optional[torch.nn.Module] = None,
denormalize: bool = True,
to_rgb: bool = True,
input_format: Optional[str] = None,
) -> np.ndarray:
"""
Takes the output from Ux-R-CNN and converts it to a displayable RGB image.
"""
if denormalize == True and model is not None and hasattr(
model, "_denormalize"):
output_image = model._denormalize(
image,
mean=model.pixel_mean.clone().to(
image.device), # type: ignore[operator]
std=model.pixel_std.clone().to(
image.device), # type: ignore[operator]
)
else:
output_image = image
output_image = (output_image.detach().to("cpu").permute(
1, 2, 0).numpy().clip(0, 255).astype(np.uint8))
if to_rgb is True:
fmt = input_format if input_format else getattr(
model, "input_format", None)
if fmt is None:
raise ValueError(
"can't convert to rgb (input_format not provided as parameter, nor in model)"
)
output_image = convert_image_to_rgb(output_image, fmt)
return output_image
def visualize_training(self, batched_inputs, results):
"""
A function used to visualize image pairs and flow map.
Args:
batched_inputs (list): a list that contains input to the model.
results (list): a list that contains predicted flows. Both
batched_inputs and proposals should have the same length.
"""
assert len(batched_inputs) == len(results), (
"Cannot visualize inputs and results of different sizes")
storage = get_event_storage()
for input_i, pred_flow_i in zip(batched_inputs, results):
image1 = input_i["image1"]
image1 = convert_image_to_rgb(image1.permute(1, 2, 0),
self.input_format)
image2 = input_i["image2"]
image2 = convert_image_to_rgb(image2.permute(1, 2, 0),
self.input_format)
gt_flow_img = flow2img(input_i["flow_map"].permute(
1, 2, 0).cpu().numpy())
pred_flow_img = flow2img(pred_flow_i["flow"].permute(
1, 2, 0).detach().cpu().numpy())
assert image1.shape == gt_flow_img.shape == pred_flow_img.shape, (
image1.shape, gt_flow_img.shape, pred_flow_img.shape)
h, w, c = image1.shape
vis_img = np.zeros([2 * h, 2 * w, c], dtype=np.uint8)
vis_img[:h, :w] = image1
vis_img[h:, :w] = image2
vis_img[:h, w:] = gt_flow_img
vis_img[h:, w:] = pred_flow_img
vis_img = vis_img.transpose(2, 0, 1)
vis_name = "Left: image pairs; Top-Right: GT flow map; Bottom-Right: pred flow map"
storage.put_image(vis_name, vis_img)
break # only visualize one image in a batch
def visualize_training(self, batched_inputs, results): #image,heatmap):#,
"""
A function used to visualize ground truth images and final network predictions.
It shows ground truth bounding boxes on the original image and up to 20
predicted object bounding boxes on the original image.
Args:
batched_inputs (list): a list that contains input to the model.
results (List[Instances]): a list of #images elements.
"""
from pointscollection.utils import exVisualizer as Visualizer
from detectron2.data.detection_utils import convert_image_to_rgb
assert len(batched_inputs) == len(
results), "Cannot visualize inputs and results of different sizes"
# storage = get_event_storage()
max_boxes = 100
image_index = 0 # only visualize a single image
img = batched_inputs[image_index]["image"]
img = convert_image_to_rgb(img.permute(1, 2, 0), "BGR")
print(batched_inputs[0]['file_name'], batched_inputs[0]['image_id'])
# v_gt = Visualizer(img, None)
# # v_gt = v_gt.overlay_instances(boxes=batched_inputs[image_index]["instances"].gt_boxes)
# anno_img = v_gt.get_image()
processed_results = _postprocess(results[image_index], img.shape[0],
img.shape[1])
predicted_boxes = processed_results.pred_boxes.tensor.detach().cpu(
).numpy()
predicted_mask = processed_results.pred_masks.detach().cpu().numpy()
predicted_points = processed_results.pred_points.detach().cpu().numpy()
v_pred = Visualizer(img, None)
v_pred = v_pred.overlay_instances(boxes=predicted_boxes[0:max_boxes],
masks=predicted_mask[0:max_boxes],
points=predicted_points[0:max_boxes])
prop_img = v_pred.get_image()
vis_img = prop_img # np.vstack((anno_img, prop_img))
# vis_img = vis_img.transpose(2, 0, 1)
# vis_name = f"Top: GT bounding boxes; Bottom: {max_boxes} Highest Scoring Results"
# plt.imshow(vis_img)
# plt.show()
plt.imsave(
'output/result_show/{:0>12}.png'.format(
batched_inputs[0]['image_id']), vis_img)
def test_dataset_dicts():
dataset_dicts = get_detection_dataset_dicts(
['coco_objects365_val_with_masks'],
filter_empty=False,
)
oss_root = "s3://detection/"
img_format = "BGR"
for dataset_dict in dataset_dicts:
file_path = os.path.join(oss_root, dataset_dict['file_name'])
image = load_image_from_oss(smart_path(file_path), format=img_format)
image = convert_image_to_rgb(image, format=img_format)
anns = dataset_dict['annotations']
masks = [ann['segmentation'] for ann in anns]
v_gt = Visualizer(image, None)
v_gt = v_gt.overlay_instances(masks=masks, )
v_gt.save('/data/tmp/test_dd_{}.png'.format(dataset_dict['image_id']))
ipdb.set_trace()
def draw_vis(
input: Dict[str, Any],
metadata: Optional[Metadata],
scale: float = 1.0,
draw_instances: bool = True,
) -> np.ndarray:
"""
input (Dict[str, Any]): a dict containing an "image" key (format
provided by "image_format" key if not RGB) and an optional
"instances" key to draw instances.
note: the "instances" should be on cpu.
"""
img = input["image"]
if input.get("image_format", "RGB") != "RGB":
img = convert_image_to_rgb(img, input["image_format"])
if torch.is_tensor(img):
img = img.numpy().astype("uint8")
if draw_instances and "instances" in input:
vis = visualizer.Visualizer(img, metadata, scale=scale)
visimg = vis.draw_instance_predictions(input["instances"])
img = visimg.get_image()
return img
def visualize_train_input(self, input_dict):
"""
Visulize a single input image of model (also the output from train loader)
used for training, this contains the data augmentation.
"""
per_image = input_dict
cfg = self.cfg
# customization
if hasattr(self._get_meta_arch_class(), "visualize_train_input"):
return self._get_meta_arch_class().visualize_train_input(
self, input_dict)
img = per_image["image"].permute(1, 2, 0).cpu().detach().numpy()
img = utils.convert_image_to_rgb(img, cfg.INPUT.FORMAT)
metadata = MetadataCatalog.get(cfg.DATASETS.TRAIN[0])
scale = 2.0
visualizer = Visualizer(img, metadata=metadata, scale=scale)
if "instances" in per_image:
target_fields = per_image["instances"].get_fields()
labels = [
metadata.thing_classes[i] for i in target_fields["gt_classes"]
]
vis = visualizer.overlay_instances(
labels=labels,
boxes=target_fields.get("gt_boxes", None),
masks=target_fields.get("gt_masks", None),
keypoints=target_fields.get("gt_keypoints", None),
)
if "sem_seg" in per_image:
vis = visualizer.draw_sem_seg(per_image["sem_seg"],
area_threshold=0,
alpha=0.5)
return vis.get_image()
def convery_yuv_bt601_to_rgb(image):
return du.convert_image_to_rgb(image, "YUV-BT.601")
webcv2.imshow(fname, vis.get_image()[:, :, ::-1])
webcv2.waitKey()
else:
filepath = os.path.join(dirname, fname)
print("Saving to {} ...".format(filepath))
vis.save(filepath)
scale = 2.0 if args.show else 1.0
if args.source == "dataloader":
train_data_loader = build_detection_train_loader(cfg)
for batch in train_data_loader:
for per_image in batch:
# Pytorch tensor is in (C, H, W) format
img = per_image["image"].permute(1, 2,
0).cpu().detach().numpy()
img = utils.convert_image_to_rgb(img, cfg.INPUT.FORMAT)
visualizer = Visualizer(img, metadata=metadata, scale=scale)
target_fields = per_image["instances"].get_fields()
labels = [
metadata.thing_classes[i]
for i in target_fields["gt_classes"]
]
vis = visualizer.overlay_instances(
labels=labels,
boxes=target_fields.get("gt_boxes", None),
masks=target_fields.get("gt_masks", None),
keypoints=target_fields.get("gt_keypoints", None),
)
output(vis, str(per_image["image_id"]) + ".jpg")
else:
def forward(self, batched_inputs):
"""
Args:
batched_inputs: a list, batched outputs of :class:`DatasetMapper`.
Each item in the list contains the inputs for one image.
For now, each item in the list is a dict that contains:
* "image": Tensor, image in (C, H, W) format.
* "sem_seg": semantic segmentation ground truth
* "center": center points heatmap ground truth
* "offset": pixel offsets to center points ground truth
* Other information that's included in the original dicts, such as:
"height", "width" (int): the output resolution of the model (may be different
from input resolution), used in inference.
Returns:
list[dict]:
each dict is the results for one image. The dict contains the following keys:
* "panoptic_seg", "sem_seg": see documentation
:doc:`/tutorials/models` for the standard output format
* "instances": available if ``predict_instances is True``. see documentation
:doc:`/tutorials/models` for the standard output format
"""
images = [x["image"].to(self.device) for x in batched_inputs]
images = [(x - self.pixel_mean) / self.pixel_std for x in images]
# To avoid error in ASPP layer when input has different size.
size_divisibility = (
self.size_divisibility
if self.size_divisibility > 0
else self.backbone.size_divisibility
)
images = ImageList.from_tensors(images, size_divisibility)
features = self.backbone(images.tensor)
losses = {}
if "sem_seg" in batched_inputs[0]:
targets = [x["sem_seg"].to(self.device) for x in batched_inputs]
targets = ImageList.from_tensors(
targets, size_divisibility, self.sem_seg_head.ignore_value
).tensor
if "sem_seg_weights" in batched_inputs[0]:
# The default D2 DatasetMapper may not contain "sem_seg_weights"
# Avoid error in testing when default DatasetMapper is used.
weights = [x["sem_seg_weights"].to(self.device) for x in batched_inputs]
weights = ImageList.from_tensors(weights, size_divisibility).tensor
else:
weights = None
else:
targets = None
weights = None
sem_seg_results, sem_seg_losses = self.sem_seg_head(features, targets, weights)
losses.update(sem_seg_losses)
if "center" in batched_inputs[0] and "offset" in batched_inputs[0]:
center_targets = [x["center"].to(self.device) for x in batched_inputs]
center_targets = ImageList.from_tensors(
center_targets, size_divisibility
).tensor.unsqueeze(1)
center_weights = [x["center_weights"].to(self.device) for x in batched_inputs]
center_weights = ImageList.from_tensors(center_weights, size_divisibility).tensor
offset_targets = [x["offset"].to(self.device) for x in batched_inputs]
offset_targets = ImageList.from_tensors(offset_targets, size_divisibility).tensor
offset_weights = [x["offset_weights"].to(self.device) for x in batched_inputs]
offset_weights = ImageList.from_tensors(offset_weights, size_divisibility).tensor
else:
center_targets = None
center_weights = None
offset_targets = None
offset_weights = None
center_results, offset_results, center_losses, offset_losses = self.ins_embed_head(
features, center_targets, center_weights, offset_targets, offset_weights
)
losses.update(center_losses)
losses.update(offset_losses)
if self.training:
return losses
if self.benchmark_network_speed:
return []
processed_results = []
for sem_seg_result, center_result, offset_result, input_per_image, image_size in zip(
sem_seg_results, center_results, offset_results, batched_inputs, images.image_sizes
):
height = input_per_image.get("height")
width = input_per_image.get("width")
r = sem_seg_postprocess(sem_seg_result, image_size, height, width)
c = sem_seg_postprocess(center_result, image_size, height, width)
o = sem_seg_postprocess(offset_result, image_size, height, width)
# Post-processing to get panoptic segmentation.
panoptic_image, _ = get_panoptic_segmentation(
r.argmax(dim=0, keepdim=True),
c,
o,
thing_ids=self.meta.thing_dataset_id_to_contiguous_id.values(),
label_divisor=self.meta.label_divisor,
stuff_area=self.stuff_area,
void_label=-1,
threshold=self.threshold,
nms_kernel=self.nms_kernel,
top_k=self.top_k,
)
# For semantic segmentation evaluation.
processed_results.append({"sem_seg": r})
panoptic_image = panoptic_image.squeeze(0)
semantic_prob = F.softmax(r, dim=0)
# Write results to disk:
img = input_per_image["image"]
from detectron2.utils.visualizer import Visualizer
from detectron2.data.detection_utils import convert_image_to_rgb
from PIL import Image
import os
img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format).astype("uint8")
img = np.array(Image.fromarray(img).resize((width, height)))
v_panoptic = Visualizer(img, self.meta)
v_panoptic = v_panoptic.draw_panoptic_seg_predictions(panoptic_image.cpu(), None)
pan_img = v_panoptic.get_image()
image_path = input_per_image['file_name'].split(os.sep)
image_name = os.path.splitext(image_path[-1])[0]
Image.fromarray(pan_img).save(os.path.join('/home/ahabbas/projects/conseg/affinityNet/output_pdl/coco/eval_vis', image_name + '_panoptic.png'))
# For panoptic segmentation evaluation.
processed_results[-1]["panoptic_seg"] = (panoptic_image, None)
# For instance segmentation evaluation.
if self.predict_instances:
instances = []
panoptic_image_cpu = panoptic_image.cpu().numpy()
for panoptic_label in np.unique(panoptic_image_cpu):
if panoptic_label == -1:
continue
pred_class = panoptic_label // self.meta.label_divisor
isthing = pred_class in list(
self.meta.thing_dataset_id_to_contiguous_id.values()
)
# Get instance segmentation results.
if isthing:
instance = Instances((height, width))
# Evaluation code takes continuous id starting from 0
instance.pred_classes = torch.tensor(
[pred_class], device=panoptic_image.device
)
mask = panoptic_image == panoptic_label
instance.pred_masks = mask.unsqueeze(0)
# Average semantic probability
sem_scores = semantic_prob[pred_class, ...]
sem_scores = torch.mean(sem_scores[mask])
# Center point probability
mask_indices = torch.nonzero(mask).float()
center_y, center_x = (
torch.mean(mask_indices[:, 0]),
torch.mean(mask_indices[:, 1]),
)
center_scores = c[0, int(center_y.item()), int(center_x.item())]
# Confidence score is semantic prob * center prob.
instance.scores = torch.tensor(
[sem_scores * center_scores], device=panoptic_image.device
)
# Get bounding boxes
instance.pred_boxes = BitMasks(instance.pred_masks).get_bounding_boxes()
instances.append(instance)
if len(instances) > 0:
processed_results[-1]["instances"] = Instances.cat(instances)
return processed_results
def convery_yuv_bt601_to_rgb(image: np.ndarray) -> np.ndarray:
return du.convert_image_to_rgb(image, "YUV-BT.601")
def show_iteration(i: int) -> None:
image_from_model = iteration_results[i]
image_rgb = convert_image_to_rgb(image_from_model,
uxrcnn_predictor.input_format)
pil_image = Image.fromarray(image_rgb)
display(pil_image)
iterations_output_path = (pathlib.Path(
"/scratch/rices6/misc/iterations/20201112_coco-detection_scratch_uxrcnn_R50FPN_ae-p5_6x/model-0099999/"
) / f"coco-val-{i_example}")
iterations_output_path.mkdir(parents=True, exist_ok=True)
iteration_results = [None] * (num_iterations + 1)
for i in tqdm.tqdm(range(0, len(iteration_results)), ascii=True, ncols=60):
if i == 0:
iteration_results[i] = example_image
else:
_result = uxrcnn_predictor(iteration_results[i - 1])
_reconstruction = postprocess_to_image(_result["unsupervised"],
model=uxrcnn_predictor.model,
to_rgb=False)
iteration_results[i] = _reconstruction
_image_rgb = convert_image_to_rgb(iteration_results[i],
uxrcnn_predictor.input_format)
_pil_image = Image.fromarray(_image_rgb)
frame_path = iterations_output_path / f"frame-{i:07}.png"
_pil_image.save(frame_path)
# %%
import ipywidgets
@ipywidgets.interact(i=ipywidgets.IntSlider(min=0,
max=len(iteration_results) - 1))
def show_iteration(i: int) -> None:
image_from_model = iteration_results[i]
image_rgb = convert_image_to_rgb(image_from_model,
uxrcnn_predictor.input_format)
pil_image = Image.fromarray(image_rgb)
