def visualize_regression_prediction_i(iou_pred,
i,
input_dir=CONFIG.INPUT_DIR,
seg_dir=CONFIG.IOU_SEG_VIS_DIR,
components_dir=CONFIG.COMPONENTS_DIR):
if os.path.isfile(get_save_path_input_i(i, input_dir=input_dir)):
label_mapping = getattr(
importlib.import_module(CONFIG.DATASET.module_name),
CONFIG.DATASET.class_name)(**CONFIG.DATASET.kwargs, ).label_mapping
pred_mapping = getattr(
importlib.import_module(CONFIG.TRAIN_DATASET.module_name),
CONFIG.TRAIN_DATASET.class_name)(**CONFIG.TRAIN_DATASET.kwargs,
).label_mapping
probs, gt, path = probs_gt_load(i, input_dir=input_dir)
input_image = Image.open(path).convert("RGB")
input_image = np.asarray(input_image.resize(probs.shape[:2][::-1]))
components = components_load(i, components_dir=components_dir)
e = entropy(probs)
pred = np.asarray(np.argmax(probs, axis=-1), dtype='int')
gt[gt == 255] = 0
predc = np.asarray([
pred_mapping[pred[p, q]][1] for p in range(pred.shape[0])
for q in range(pred.shape[1])
])
gtc = np.asarray([
label_mapping[gt[p, q]][1] for p in range(gt.shape[0])
for q in range(gt.shape[1])
])
predc = predc.reshape(input_image.shape)
gtc = gtc.reshape(input_image.shape)
overlay_factor = [1.0, 0.5, 1.0]
img_predc, img_gtc, img_entropy = [
Image.fromarray(
np.uint8(arr * overlay_factor[i] + input_image *
(1 - overlay_factor[i])))
for i, arr in enumerate([predc, gtc,
cm.jet(e)[:, :, :3] * 255.0])
]
img_ioupred = Image.fromarray(visualize_segments(components, iou_pred))
images = [img_gtc, img_predc, img_entropy, img_ioupred]
img_top = np.concatenate(images[2:], axis=1)
img_bottom = np.concatenate(images[:2], axis=1)
img_total = np.concatenate((img_top, img_bottom), axis=0)
image = Image.fromarray(img_total.astype('uint8'), 'RGB')
if not os.path.exists(seg_dir):
os.makedirs(seg_dir)
image.save(join(seg_dir, "image{}.png".format(i)))
plt.close()
print("stored: {}".format(join(seg_dir, "image{}.png".format(i))))
def get_ious_for_image(image_index, iou_pred, thresholds, args):
confusion_matrices_pos = {
t: np.zeros((num_categories, num_categories))
for t in thresholds
}
confusion_matrices_neg = {
t: np.zeros((num_categories, num_categories))
for t in thresholds
}
pred, gt, _ = probs_gt_load(
image_index,
input_dir=join(CONFIG.metaseg_io_path, "input", "deeplabv3plus",
args["dataset"]),
preds=True,
)
# transform a2d2 labels to cityscapes category ids
gt = np.vectorize(label_mappings[args["dataset"]].get)(gt)
# transform predictions to cityscapes category ids
pred = np.vectorize(trainid_to_catid.get)(pred)
# load components for constructing the iou mask based on different IoU thresholds
components = components_load(
image_index,
components_dir=join(CONFIG.metaseg_io_path, "components",
"deeplabv3plus", args["dataset"]),
)
# border of components have been labeled with the negative index of the
# main component itself we want however to include the border of the segment in
# the evaluation which is why we have to make it also positive
components = np.absolute(components)
# -1 because component indices start with 1
components = iou_pred[components - 1]
for t in thresholds:
# confusion_matrices_pos[t] = iou(pred,
# gt,
# n_classes=num_categories,
# update_matrix=confusion_matrices_pos[t],
# ignore_index=0,
# mask=(components >= t))[1]
confusion_matrices_neg[t] = iou(
pred,
gt,
n_classes=num_categories,
update_matrix=confusion_matrices_neg[t],
ignore_index=0,
mask=(components < t),
)[1]
return confusion_matrices_pos, confusion_matrices_neg
def add_heatmap_as_metric_i(heat_dir, key, i):
"""
derive aggregated metrics per image and add to metrics dictionary
:param heat_dir: (str) directory with heatmaps as numpy arrays
:param key: (str) new key to access added metric
:param i: (int) id of the image to be processed
"""
_, _, path = probs_gt_load(i)
heat_name = os.path.basename(path)[:-4] + ".npy"
heatmap = np.load(heat_dir + heat_name)
metrics = metrics_load(i, metrics_dir=CONFIG.METRICS_DIR)
components = components_load(i, components_dir=CONFIG.COMPONENTS_DIR)
keys = [key, key + "_in", key + "_bd", key + "_rel", key + "_rel_in"]
heat_metric = {k: [] for k in keys}
for comp_id in range(1, abs(np.min(components)) + 1):
values = compute_metrics_from_heatmap(heatmap, components, comp_id)
for j, k in enumerate(keys):
heat_metric[k].append(values[j])
metrics.update(heat_metric)
metrics_dump(metrics, i, metrics_dir=CONFIG.METRICS_DIR)
def show_full_image(self, ind, save=False):
"""Displays four panels of the full image belonging to a segment.
Top left: Entropy heatmap of prediction.
Top right: Predicted IoU of each segment.
Bottom left: Source image with ground truth overlay.
Bottom right: Predicted semantic segmentation.
"""
self.log.info(
"{} detailed image...".format("Saving" if save else "Loading"))
box = self.data["box"][ind]
image = np.asarray(
Image.open(self.data["image_path"][self.gi[ind]]).convert("RGB"))
image_index = self.data["image_index"][self.gi[ind]]
iou_pred = self.data["iou_pred"][self.gi[ind]]
dataset = self.data["dataset"][self.gi[ind]]
model_name = self.data["model_name"][self.gi[ind]]
pred, gt, image_path = probs_gt_load(
image_index,
input_dir=join(CONFIG.metaseg_io_path, "input", model_name,
dataset),
)
components = components_load(
image_index,
components_dir=join(CONFIG.metaseg_io_path, "components",
model_name, dataset),
)
e = entropy(pred)
pred = pred.argmax(2)
predc = np.asarray([
self.pred_mapping[pred[ind_i, ind_j]][1]
for ind_i in range(pred.shape[0]) for ind_j in range(pred.shape[1])
]).reshape(image.shape)
overlay_factor = [1.0, 0.5, 1.0]
if self.label_mapping[dataset] is not None:
gtc = np.asarray([
self.label_mapping[dataset][gt[ind_i, ind_j]][1]
for ind_i in range(gt.shape[0]) for ind_j in range(gt.shape[1])
]).reshape(image.shape)
else:
gtc = np.zeros_like(image)
overlay_factor[1] = 0.0
img_predc, img_gtc, img_entropy = [
Image.fromarray(
np.uint8(arr * overlay_factor[i] + image *
(1 - overlay_factor[i])))
for i, arr in enumerate([predc, gtc,
cm.jet(e)[:, :, :3] * 255.0])
]
img_ioupred = Image.fromarray(
self.visualize_segments(components, iou_pred))
images = [img_gtc, img_predc, img_entropy, img_ioupred]
box_line_width = 5
left, upper = max(0, box[0] - box_line_width), max(
0, box[1] - box_line_width)
right, lower = min(pred.shape[1], box[2] + box_line_width), min(
pred.shape[0], box[3] + box_line_width)
for k in images:
draw = ImageDraw.Draw(k)
draw.rectangle([left, upper, right, lower],
outline=(255, 0, 0),
width=box_line_width)
del draw
for k in range(len(images)):
images[k] = np.asarray(images[k]).astype("uint8")
img_top = np.concatenate(images[2:], axis=1)
img_bottom = np.concatenate(images[:2], axis=1)
img_total = np.concatenate((img_top, img_bottom), axis=0)
fig_tmp = plt.figure(max(3, max(plt.get_fignums()) + 1), dpi=self.dpi)
fig_tmp.canvas.set_window_title("Dataset: {}, Image index: {}".format(
dataset, image_index))
ax = fig_tmp.add_subplot(111)
ax.set_axis_off()
ax.imshow(img_total, interpolation="nearest")
if save:
fig_tmp.subplots_adjust(bottom=0,
left=0,
right=1,
top=1,
hspace=0,
wspace=0)
ax.margins(0.05, 0.05)
fig_tmp.gca().xaxis.set_major_locator(plt.NullLocator())
fig_tmp.gca().yaxis.set_major_locator(plt.NullLocator())
fig_tmp.savefig(
join(self.save_dir, "detailed_image_{}.jpg".format(ind)),
bbox_inches="tight",
pad_inches=0.0,
)
self.log.debug("Saved image to {}".format(
join(self.save_dir, "detailed_image_{}.jpg".format(ind))))
else:
fig_tmp.tight_layout(pad=0.0)
fig_tmp.show()
def cutout_components(component_indices,
image_index,
iou_pred,
dataset='a2d2',
min_height=64,
min_width=64,
min_crop_height=128,
min_crop_width=128,
model_name='deeplabv3plus'):
"""Cuts out all components of the image and returns them if they match the minimum size requirements.
Args:
component_indices (sequence): Sequence of local component numbers.
image_index (int): Index of the image to process.
iou_pred (numpy array): Array of iou predictions for each component
dataset (str): Name of the dataset to process.
min_height (int): Minimum height of the component to be processed. Useful if you want to pass the crop to a
neural network.
min_width (int): Minimum height of the component to be processed. Useful if you want to pass the crop to a
neural network .
min_crop_width (int): Minimum width the resulting bounding box should have. If the segment satisfies the
min_width but is smaller then min_crop_width the bounding box is getting enlarged until the min_crop_width
is satisfied.
min_crop_height: Minimum height the resulting bounding box should have. If the segment satisfies the
min_height but is smaller then min_crop_height the bounding box is getting enlarged until the
min_crop_height is satisfied.
model_name (str): Name of the model used.
Returns: Dictionary with
'dataset': Name of the dataset the image belongs to.
'model_name': Name of the model used for the prediction
'data': List of raw image crops containing the matching components
'addresses': List of addresses where to find the crop (path to the image file and corner coordinates of the box
(top, left, bottom, right))
"""
components = components_load(image_index,
components_dir=join(CONFIG.metaseg_io_path,
'components', model_name,
dataset))
crops = {
'dataset': dataset,
'model_name': model_name,
'embeddings': [],
'boxes': [],
'image_index': image_index,
'iou_pred': iou_pred,
'component_indices': [],
'segment_indices': [],
'img_crops': []
}
for cindex in component_indices:
segment_indices = np.argwhere(components == cindex)
if segment_indices.shape[0] > 0:
upper, left = segment_indices.min(0)
lower, right = segment_indices.max(0)
if (lower - upper) < min_height or (right - left) < min_width:
continue
if (right - left) < min_crop_width:
margin = min_crop_width - (right - left)
if left - (margin // 2) < 0:
left = 0
right = left + min_crop_width
elif right + (margin // 2) > components.shape[1]:
right = components.shape[1]
left = right - min_crop_width
if right > components.shape[1] or left < 0:
raise IndexError(
'Image with shape {} is too small for a {} x {} crop'.
format(components.shape, min_crop_height,
min_crop_width))
if (lower - upper) < min_crop_height:
margin = min_crop_height - (lower - upper)
if upper - (margin // 2) < 0:
upper = 0
lower = upper + min_crop_height
elif lower + (margin // 2) > components.shape[0]:
lower = components.shape[0]
upper = lower - min_crop_height
if lower > components.shape[0] or upper < 0:
raise IndexError(
'Image with shape {} is too small for a {} x {} crop'.
format(components.shape, min_crop_height,
min_crop_width))
crops['boxes'].append((left, upper, right, lower))
crops['component_indices'].append(cindex)
crops['segment_indices'].append(segment_indices)
return crops