def _num_instances_per_sid(uids):
# Note: instances in Cityscapes are not always labeled with continuous iids,
# e.g. one image can have instances with iids: 000, 001, 003, 007
# TODO(panos): deprecate this function since it has some bugs,
# use _num_instances_per_sid_v2 instead
uids = np.unique(uids)
_, _, _, sids_iids = decode_uids(uids, experimental_return_sids_iids=True)
sid2Ninstances = collections.defaultdict(lambda: 0)
for sid_iid in sids_iids:
sid, _, _ = decode_uids(sid_iid)
sid2Ninstances[sid] += 1
return sid2Ninstances
def _num_instances_per_sid_v2(uids):
# Note: instances in Cityscapes are not always labeled with continuous iids,
# e.g. one image can have instances with iids: 000, 001, 003, 007
# TODO(panos): move this functionality to utils.format
# np.array is needed since uids are Python ints
# and np.unique implicitly converts them to np.int64
# TODO(panos): remove this need when np.int64 is supported in decode_uids
uids_unique = np.unique(np.array(uids, dtype=np.int32))
_, _, _, sids_iids = decode_uids(uids_unique,
experimental_return_sids_iids=True)
sids_iids_unique = np.unique(sids_iids)
sid2Ninstances = collections.defaultdict(lambda: 0)
for sid_iid in sids_iids_unique:
sid, iid, _ = decode_uids(sid_iid)
if iid >= 0:
sid2Ninstances[sid] += 1
return sid2Ninstances
def _sid2iids(uids):
# a dict mapping a sid to a set of all its iids
# uids: a list of Python int uids
# iids do not need to be consecutive numbers
# TODO(panos): move this functionality to utils.format
sid2iids = collections.defaultdict(set)
for uid in set(uids):
sid, iid, _ = decode_uids(uid)
if iid >= 0:
sid2iids[sid].add(iid)
return sid2iids
def uids_lids2uids_cids(uids_with_lids, lids2cids):
"""
Convert uids with semantic classes encoded as lids to uids with cids.
This function is useful in the Cityscapes context, or other datasets
where the lids and cids separation is used.
"""
uids = uids_with_lids
sids, _, _ = decode_uids(uids)
uids_with_cids = np.where(
uids <= 99, lids2cids[sids],
np.where(uids <= 99_999, lids2cids[sids] * 10**3 + uids % 10**3,
lids2cids[sids] * 10**5 + uids % 10**5))
return uids_with_cids
def experimental_visualize(image_path, label_path, experimental_emphasize_instance_boundary=True):
"""
Visualizes in a pyplot window an image and a label pair from
provided paths. For reading Pillow is used so all paths and formats
must be Pillow-compatible.
Args:
image_path: an image path provided to Pillow.Image.open
label_path: a label path provided to Pillow.Image.open
"""
assert op.exists(image_path)
assert op.exists(label_path)
image = Image.open(image_path)
uids_with_lids = np.array(Image.open(label_path), dtype=np.int32)
# uids according to our hierarchical panoptic format
uids_with_cids = uids_lids2uids_cids(uids_with_lids, LIDS2CIDS)
# We want to visualize on all three levels so we need all the uids levels
# and we do it here for all levels together so we call experimental_uid2color once to have
# same shades across subfigures per plot for easier comparison
sids, iids, _, sids_iids = decode_uids(uids_with_cids, experimental_return_sids_iids=True)
ids_all_levels_unique = np.unique(np.stack([sids, sids_iids, uids_with_cids]))
uid_2_color = experimental_uid2color(list(map(int, ids_all_levels_unique)), CIDS2COLORS)
palette = _sparse_ids_mapping_to_dense_ids_mapping(uid_2_color, (0, 0, 0), dtype=np.uint8)
# using numpy advanced indexing (gathering) a color from the (Ncolors, 3)-shaped palette
# is chosen for each sid, sid_iid, and uid
uids_sem_colored = palette[sids]
uids_sem_inst_colored = palette[sids_iids]
uids_sem_inst_parts_colored = palette[uids_with_cids]
# add boundaries
edge_option = 'sobel' # or 'erosion'
if experimental_emphasize_instance_boundary:
# TODO(panos): simplify this algorithm
# create per-instance binary masks
iids_unique = np.unique(iids)
boundaries = np.full(iids.shape, False)
edges = np.full(iids.shape, False)
for iid in iids_unique:
if 0 <= iid <= 999:
iid_mask = np.equal(iids, iid)
if edge_option == 'sobel':
edge_horizont = ndimage.sobel(iid_mask, 0)
edge_vertical = ndimage.sobel(iid_mask, 1)
edges = np.logical_or(np.hypot(edge_horizont, edge_vertical), edges)
elif edge_option == 'erosion':
boundary = np.logical_xor(iid_mask,
ndimage.binary_erosion(iid_mask, structure=np.ones((4, 4))))
boundaries = np.logical_or(boundaries, boundary)
if edge_option == 'sobel':
boundaries_image = np.uint8(edges)[..., np.newaxis] * np.uint8([[[255, 255, 255]]])
elif edge_option == 'erosion':
boundaries_image = np.uint8(boundaries)[..., np.newaxis] * np.uint8([[[255, 255, 255]]])
uids_sem_inst_colored = np.where(boundaries_image,
boundaries_image,
uids_sem_inst_colored)
uids_sem_inst_parts_colored = np.where(boundaries_image,
boundaries_image,
uids_sem_inst_parts_colored)
# plot
# initialize figure for plotting
_, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)
# for ax in axes:
# ax.set_axis_off()
ax1.imshow(image)
ax1.set_title('image')
ax2.imshow(uids_sem_colored)
ax2.set_title('labels colored on semantic level')
ax3.imshow(uids_sem_inst_colored)
ax3.set_title('labels colored on semantic and instance levels')
ax4.imshow(uids_sem_inst_parts_colored)
ax4.set_title('labels colored on semantic, instance, and parts levels')
plt.show()
def visualize(image_path, label_path):
"""
Visualizes in a pyplot window an image and a label pair from
provided paths. For reading Pillow is used so all paths and formats
must be Pillow-compatible.
Args:
image_path: an image path provided to Pillow.Image.open
label_path: a label path provided to Pillow.Image.open
"""
assert op.exists(image_path)
assert op.exists(label_path)
# Prepare canvases and decode the labels.
image = np.array(Image.open(image_path), dtype=np.uint8)
label = np.array(Image.open(label_path), dtype=np.int32)
uids_unique_org = np.unique(label)
semantic_segmentation = np.zeros((image.shape[0], image.shape[1], 3),
dtype=np.uint8)
instance_segmentation = np.zeros((image.shape[0], image.shape[1], 3),
dtype=np.uint8)
parts_segmentation = np.zeros((image.shape[0], image.shape[1], 3),
dtype=np.uint8)
sids, iids, _ = decode_uids(label)
# Color at the semantic level.
color_generator = IdGenerator(CATEGORIES)
for sid in np.unique(sids):
mask = np.equal(sids, sid)
color = CATEGORIES[sid]['color']
semantic_segmentation[mask] = color
# Color at the semantic and instance level and find the instance-level boundaries.
sids_only = np.where(iids < 0, sids, np.zeros_like(iids))
for sid in np.unique(sids_only):
mask = np.equal(sids_only, sid)
color = color_generator.get_color(sid)
instance_segmentation[mask] = color
sid_iids = np.where(iids >= 0, sids * 10**3 + iids, np.zeros_like(iids))
boundaries = np.full(sid_iids.shape, False)
for sid_iid in np.unique(sid_iids):
if sid_iid != 0:
mask = np.equal(sid_iids, sid_iid)
color = color_generator.get_color(sid_iid // 1000)
instance_segmentation[mask] = color
boundary_horizon = ndimage.sobel(mask, 0)
boundary_vertical = ndimage.sobel(mask, 1)
boundaries = np.logical_or(
np.hypot(boundary_horizon, boundary_vertical), boundaries)
# Color at the part level.
# Conver the original labels into the form for visualization with IdGenerator.
for uid in uids_unique_org:
# If uid is sid or sid_iid, encode them as they are.
if uid <= 99_999:
sid_iid = uid
# If uid is sid_iid_pid, map sid_pid to its corresponding sid and create new label as sid_iid.
else:
sid, iid, pid = decode_uids(uid)
sid_pid = sid * 10**2 + pid
if sid_pid in SID_PID2PARTS_CID:
sid_iid = SID_PID2PARTS_CID[sid_pid] * 10**3 + iid
else:
sid_iid = sid * 10**3 + iid
label[label == uid] = sid_iid
color_generator = IdGenerator(CATEGORIES_PARTS)
for sid_iid in np.unique(label):
# If sid_iid is in the format of sid , use sid for color generation (things and stuff classes differentiated by IdGenerator inherently).
if sid_iid <= 99:
id_ = sid_iid
# If sid_iid is in the format of sid_iid, get sid.
else:
id_ = sid_iid // 1000
mask = label == sid_iid
color = color_generator.get_color(id_)
parts_segmentation[mask] = color
# Depict boundaries.
instance_segmentation[boundaries] = [255, 255, 255]
parts_segmentation[boundaries] = [255, 255, 255]
# plot
# initialize figure for plotting
_, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)
# for ax in axes:
# ax.set_axis_off()
ax1.imshow(image)
ax1.set_title('image')
ax2.imshow(semantic_segmentation)
ax2.set_title('labels colored on semantic level')
ax3.imshow(instance_segmentation)
ax3.set_title('labels colored on semantic and instance levels')
ax4.imshow(parts_segmentation)
ax4.set_title('labels colored on semantic, instance, and parts levels')
plt.show()
def _colorize_uids(uids,
sid2color=None,
experimental_emphasize_instance_boundary=False):
"""
This function colorizes a `uids` array that has values according to the
hierarchical format in the README. For the semantic level it uses the colors
provided in `sid2color` and creates internally colors for the
instance level different instances (random shades of `sid2color`) and
parts level different parts (a blend of parula color map and `sid2color`).
Example usage: ....py
Limitations: only up to 7 parts are supported for now (pids: 0-6)
Args:
uids: np.ndarray, np.int32, 2-D, with values according to hierarchical format in the README
sid2color: Python dict, mapping the semantic class ids (sids) in uids to an RGB color tuple,
with values in [0, 255],
experimental_emphasize_instance_boundaries: Python boolean,
if True will add a white boundary around instances (for now its experimental)
Return:
The following three np.ndarray, np.uint8, 3-D (RGB), with values in [0, 255]:
uids_semantic_colored: `uids` colored on semantic level,
using the colors in `sid2color`
uids_instance_colored: `uids` colored on semantic and instance level,
using shades of colors in `sid2color` to denote different instances
uids_parts_colored: `uids` colored on semantic, instance and part level,
using shades of colors in `sid2color` to denote different instances
and mixing them with parula color palette for pixels with parts
"""
edge_option = 'sobel' # or 'erosion'
if sid2color is None:
raise NotImplementedError(
'Random colors for sid2color will be supported in the future.')
uid_2_colors = _uid2colors(np.unique(uids), sid2color)
# create the palettes so advanced indexing for coloring can be used
# initialize all colors in palettes with white (255, 255, 255) (for easier debugging)
uids_keys = list(uid_2_colors.keys())
# palettes for sids, iids, and pids
palette_sids = np.full((np.max(uids_keys) + 1, 3), 255, dtype=np.uint8)
palette_iids = np.copy(palette_sids)
palette_pids = np.copy(palette_sids)
for uid, colors in uid_2_colors.items():
palette_sids[uid] = colors[0]
palette_iids[uid] = colors[1]
palette_pids[uid] = colors[2]
# create colored images using the palettes
uids_sids_colored = palette_sids[uids]
uids_iids_colored = palette_iids[uids]
uids_pids_colored = palette_pids[uids]
if experimental_emphasize_instance_boundary:
_, iids, _ = decode_uids(uids)
# create per-instance binary masks
iids_unique = np.unique(iids)
boundaries = np.full(iids.shape, False)
edges = np.full(iids.shape, False)
for iid in iids_unique:
if 0 <= iid <= 999:
iid_mask = np.equal(iids, iid)
if edge_option == 'sobel':
edge_horizont = ndimage.sobel(iid_mask, 0)
edge_vertical = ndimage.sobel(iid_mask, 1)
edges = np.logical_or(
np.hypot(edge_horizont, edge_vertical), edges)
elif edge_option == 'erosion':
boundary = np.logical_xor(
iid_mask,
ndimage.binary_erosion(iid_mask,
structure=np.ones((4, 4))))
boundaries = np.logical_or(boundaries, boundary)
if edge_option == 'sobel':
boundaries_image = np.uint8(edges)[..., np.newaxis] * np.uint8(
[[[255, 255, 255]]])
elif edge_option == 'erosion':
boundaries_image = np.uint8(boundaries)[...,
np.newaxis] * np.uint8(
[[[255, 255, 255]]])
uids_iids_colored = np.where(boundaries_image, boundaries_image,
uids_iids_colored)
uids_pids_colored = np.where(boundaries_image, boundaries_image,
uids_pids_colored)
return uids_sids_colored, uids_iids_colored, uids_pids_colored
# generate instance shades
sid2num_instances = _num_instances_per_sid(uids)
sid2shades = {
sid: _generate_shades(id2color[sid], experimental_deltas, Ninstances)
for sid, Ninstances in sid2num_instances.items()
}
# generate the uid to colors mappings
uid_2_colors = dict()
def _add_mapping(uid, sc, ic, pc):
uid_2_colors[uid] = list(map(np.uint8, [sc, ic, pc]))
for uid in np.unique(uids):
sid, iid, pid = decode_uids(uid)
# only semantic labels
if uid <= 99:
scolor = id2color[uid]
_add_mapping(uid, scolor, scolor, scolor)
continue
# from this point onward we have at least semantic labels
scolor = id2color[sid]
icolor = sid2shades[sid][iid]
# only semantic and instance labels
if uid <= 99_999:
_add_mapping(uid, scolor, icolor, icolor)
continue
labels_paths_original = glob.glob(
op.join(BASEPATH_LABELS_ORIGINAL, 'train', '*', '*_instanceIds.png'))
labels_paths_original.extend(
glob.glob(
op.join(BASEPATH_LABELS_ORIGINAL, 'val', '*', '*_instanceIds.png')))
print(len(labels_paths_original))
labels_paths_ours = [
lp.replace('cityscapes',
'cityscapes_panoptic_parts').replace('_instanceIds.png',
'_panopticIds.tif')
for lp in labels_paths_original
]
print(len(labels_paths_ours))
# validate labels
for i, (lp_orig,
lp_ours) in enumerate(zip(labels_paths_original, labels_paths_ours)):
print(f"{i+1}/{len(labels_paths_original)}")
labels_orig = np.array(Image.open(lp_orig), dtype=np.int32)
labels_ours = np.array(Image.open(lp_ours), dtype=np.int32)
_, _, _, sids_iids = decode_uids(labels_ours,
experimental_return_sids_iids=True)
if not np.all(np.equal(labels_orig, sids_iids)):
print(lp_orig, lp_ours, sep='\n')
print(np.unique(labels_orig),
print(np.unique(sids_iids)),
np.unique(labels_ours),
sep='\n')
breakpoint()