def __init__(self, dataroot):
""" """
self.dataroot = dataroot
self.img_dir = f'{self.dataroot}/seg/images'
self.label_dir = f'{self.dataroot}/seg/labels'
self.id_to_classname_map = get_dataloader_id_to_classname_map(dataset_name='bdd')
self.classname_to_id_map = get_classname_to_dataloaderid_map(dataset_name='bdd')
def remap_dataset(dname: str,
remapped_dname: str,
tsv_fpath: str,
old_dataroot: str,
remapped_dataroot: str,
include_ignore_idx_cls: bool = True,
convert_label_from_rgb: bool = False,
num_processes: int = 4):
"""
Given path to a dataset, given names of _names.txt
Remap according to the provided tsv.
(also account for the fact that 255 is always unlabeled)
Args:
- dname: string representing name of taxonomy for original dataset
- remapped_dname: string representing name of taxonomy for new dataset
- tsv_fpath: string representing path to a .tsv file
- old_dataroot: string representing path to original dataset
- remapped_dataroot: string representing path at which to new dataset
- include_ignore_idx_cls: whether to include unlabeled=255 from source
- convert_label_from_rgb: labels of original dataset are stored as RGB
- num_processes: integer representing number of workers to exploit
Returns:
- None
"""
# load colors ordered with class indices, if labels encoded as RGB
dataset_colors = load_dataset_colors_arr(
dname) if convert_label_from_rgb else None
# load up the dictionary from the tsv
classname_remapping_dict = read_label_mapping(filename=tsv_fpath,
label_from=dname,
label_to=remapped_dname,
convert_val_to_int=False)
oldid_to_oldname = get_dataloader_id_to_classname_map(dname)
newname_tonewid_map = get_classname_to_dataloaderid_map(
remapped_dname, include_ignore_idx_cls=include_ignore_idx_cls)
# form one-way mapping between IDs
old_name_to_newid = convert_dictionaries(classname_remapping_dict,
newname_tonewid_map)
class_idx_remapping_dict = convert_dictionaries(oldid_to_oldname,
old_name_to_newid)
label_mapping_arr = form_label_mapping_array(class_idx_remapping_dict)
for split in ['train', 'val']: #'trainval']:# 'val']: #
orig_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths(
dname, split)
remapped_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths(
remapped_dname, split)
send_list_to_workers(
num_processes=num_processes,
list_to_split=orig_relative_img_label_pairs,
worker_func_ptr=relabel_pair_worker,
remapped_relative_img_label_pairs=remapped_relative_img_label_pairs,
label_mapping_arr=label_mapping_arr,
old_dataroot=old_dataroot,
new_dataroot=remapped_dataroot,
dataset_colors=dataset_colors)
def __init__(self, dataroot: str, dname: str) -> None:
"""
"""
self.dataroot = dataroot
self.dname = dname
self.id_to_classname_map = get_dataloader_id_to_classname_map(dname)
self.get_classname_to_id_map = get_classname_to_dataloaderid_map(
dataset_name=dname)
def verify_all_relabeled_dataset_segments(num_processes: int, mld: Any,
dname: str, dataroot: str,
update_records,
require_strict_boundaries: bool):
"""
By using remap.py, we already have converted label img from original taxonomy, to universal taxonomy.
Args:
- num_processes: number of processes to launch; shouldn't exceed number of cores on machine
- mld: Mask Level Dataset
- dname: string representing name of a dataset taxonomy
- dataroot: string representing path to a file directory
- update_records
Returns:
- None
"""
classname_to_id_map = get_classname_to_dataloaderid_map(
dname, include_ignore_idx_cls=True)
# Check for incorrect class names.
for rec in update_records:
valid_orig = rec.orig_class in classname_to_id_map.keys()
valid_relabeled = rec.relabeled_class in classname_to_id_map.keys()
if not (valid_orig and valid_relabeled):
print(rec.__dict__)
print(
f'Invalid universal classname: {rec.orig_class}, {rec.relabeled_class}'
)
quit()
for split in ['train', 'val']:
# Create for each split separately, since SUNRGBD has same file names for different images in train vs. val
parent_fname_to_updatelist_dict = form_fname_to_updatelist_dict(
dname, update_records, split)
split_txt_fpath = f'{_ROOT}/mseg/dataset_lists/{dname}/list/{split}.txt'
# load up the data root and all absolute paths from the txt file
img_label_pairs = generate_all_img_label_pair_fpaths(
data_root=dataroot, split_txt_fpath=split_txt_fpath)
if num_processes > 1:
send_list_to_workers(
num_processes=num_processes,
list_to_split=img_label_pairs,
worker_func_ptr=verify_mask_worker,
parent_fname_to_updatelist_dict=parent_fname_to_updatelist_dict,
mld=mld,
classname_to_id_map=classname_to_id_map,
require_strict_boundaries=require_strict_boundaries,
split=split)
elif num_processes == 1:
# useful for debugging in a single thread
for (img_fpath, label_img_fpath) in img_label_pairs:
verify_label_img_masks(img_fpath, label_img_fpath,
parent_fname_to_updatelist_dict, mld,
classname_to_id_map,
require_strict_boundaries, split)
def __init__(self, dataset_dir):
"""
"""
self.id_to_classname_map = get_dataloader_id_to_classname_map(
dataset_name='sunrgbd-37')
self.classname_to_id_map = get_classname_to_dataloaderid_map(
dataset_name='sunrgbd-37')
self.img_dir = f'{dataset_dir}/image'
self.label_dir = f'{dataset_dir}/semseg-label37'
def __init__(self, semantic_version_dataroot: str,
instance_version_dataroot: str):
"""
Args:
- semantic_version_dataroot: from ADEChallengeData2016
- instance_verson_dataroot: from ADE20K_2016_07_26
"""
self.img_dir = f'{semantic_version_dataroot}/images'
self.ade20k_instance_dataroot = instance_version_dataroot
# following two maps are only used for relabeling original data
self.id_to_classname_map = get_dataloader_id_to_classname_map(
dataset_name='ade20k-151')
self.classname_to_id_map = get_classname_to_dataloaderid_map(
dataset_name='ade20k-151')
self.ade20k_split_nickname_dict = {
'train': 'training',
'val': 'validation'
}
# make it easy to look up an RGB file path, given just the filename stem
self.fname_to_rgbfpath_dict = {}
for split in ['train', 'val']:
ade20k_split_nickname = self.ade20k_split_nickname_dict[split]
rgb_fpaths = glob.glob(
f'{self.img_dir}/{ade20k_split_nickname}/*.jpg')
for rgb_fpath in rgb_fpaths:
fname_stem = Path(rgb_fpath).stem
assert fname_stem not in self.fname_to_rgbfpath_dict
self.fname_to_rgbfpath_dict[fname_stem] = rgb_fpath
# make it easy to look up a SEG RGB file path, given just the filename stem
self.fname_to_segrgbfpath_dict = {}
seg_rgb_fpaths = glob.glob(
f'{self.ade20k_instance_dataroot}/images/**/*seg.png',
recursive=True)
for seg_rgb_fpath in seg_rgb_fpaths:
fname_stem = Path(seg_rgb_fpath).stem
assert '_seg' in fname_stem[-4:]
# remove suffix now
fname_stem = fname_stem.replace('_seg', '')
self.fname_to_segrgbfpath_dict[fname_stem] = seg_rgb_fpath
def test_label_transform_unlabeled(): """ Make sure 255 stays mapped to 255 at each level (to be ignored in cross-entropy loss). """ IGNORE_LABEL = 255 dname = 'mapillary-public65' txt_classnames = load_class_names(dname) name2id = get_classname_to_dataloaderid_map(dname, include_ignore_idx_cls = True) train_idx = name2id['unlabeled'] tc = TaxonomyConverter() # training dataset label traind_label = torch.ones(4,4)*train_idx traind_label = traind_label.type(torch.LongTensor) # Get back the universal label u_label = tc.transform_label(traind_label, dname) u_idx = IGNORE_LABEL gt_u_label = np.ones((4,4)).astype(np.int64) * u_idx assert np.allclose(u_label.numpy(), gt_u_label)
import PIL.Image as Image
import PIL.ImageDraw as ImageDraw
from tqdm import tqdm
from mseg.utils.names_utils import (get_classname_to_dataloaderid_map,
get_dataloader_id_to_classname_map)
"""
AutoNUE labels and classes to store, read, and write annotations.
In their hierarchy, we use 'id-type'='id' (the most fine-grained).
Uses PIllow to raster json polygons.
Ref:
https://github.com/AutoNUE/public-code/blob/master/preperation/createLabels.py
"""
name2labelid = get_classname_to_dataloaderid_map('idd-40',
include_ignore_idx_cls=False)
id2label = get_dataloader_id_to_classname_map('idd-40',
include_ignore_idx_cls=False)
# A point in a polygon
Point = namedtuple('Point', ['x', 'y'])
# Class that contains the information of a single annotated object
class CsObject:
# Constructor
def __init__(self):
# the label
self.label = ""
# the polygon as list of points
self.polygon = []
