def remap_dataset(dname: str, remapped_dname: str, tsv_fpath: str, old_dataroot: str, remapped_dataroot: str, include_ignore_idx_cls: bool = True, convert_label_from_rgb: bool = False, num_processes: int = 4): """ Given path to a dataset, given names of _names.txt Remap according to the provided tsv. (also account for the fact that 255 is always unlabeled) Args: - dname: string representing name of taxonomy for original dataset - remapped_dname: string representing name of taxonomy for new dataset - tsv_fpath: string representing path to a .tsv file - old_dataroot: string representing path to original dataset - remapped_dataroot: string representing path at which to new dataset - include_ignore_idx_cls: whether to include unlabeled=255 from source - convert_label_from_rgb: labels of original dataset are stored as RGB - num_processes: integer representing number of workers to exploit Returns: - None """ # load colors ordered with class indices, if labels encoded as RGB dataset_colors = load_dataset_colors_arr( dname) if convert_label_from_rgb else None # load up the dictionary from the tsv classname_remapping_dict = read_label_mapping(filename=tsv_fpath, label_from=dname, label_to=remapped_dname, convert_val_to_int=False) oldid_to_oldname = get_dataloader_id_to_classname_map(dname) newname_tonewid_map = get_classname_to_dataloaderid_map( remapped_dname, include_ignore_idx_cls=include_ignore_idx_cls) # form one-way mapping between IDs old_name_to_newid = convert_dictionaries(classname_remapping_dict, newname_tonewid_map) class_idx_remapping_dict = convert_dictionaries(oldid_to_oldname, old_name_to_newid) label_mapping_arr = form_label_mapping_array(class_idx_remapping_dict) for split in ['train', 'val']: #'trainval']:# 'val']: # orig_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths( dname, split) remapped_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths( remapped_dname, split) send_list_to_workers( num_processes=num_processes, list_to_split=orig_relative_img_label_pairs, worker_func_ptr=relabel_pair_worker, remapped_relative_img_label_pairs=remapped_relative_img_label_pairs, label_mapping_arr=label_mapping_arr, old_dataroot=old_dataroot, new_dataroot=remapped_dataroot, dataset_colors=dataset_colors)
def test_load_colors(): """ """ tax_data = read_mapillary_config_helper() num_classes = 66 gt_dataset_ordered_colors = np.zeros((66, 3), dtype=np.uint8) for i in range(num_classes): gt_dataset_ordered_colors[i] = np.array(tax_data['labels'][i]['color']) colors = load_dataset_colors_arr('mapillary-public66') assert np.allclose(colors, gt_dataset_ordered_colors)
def __init__(self, dataroot: str): """ Args: - dataroot: string representing path to unzipped Mapillary file Returns: - None """ self.dataroot = dataroot self.dataset_ordered_colors = load_dataset_colors_arr( 'mapillary-public66') self.id_to_classname_map = get_dataloader_id_to_classname_map( dataset_name='mapillary-public66')
def test_load_dataset_colors_arr(): """ """ dataset_name = 'camvid-32' colors_arr = load_dataset_colors_arr(dataset_name) assert np.allclose(colors_arr[0], np.array([64, 128, 64])) assert np.allclose(colors_arr[-1], np.array([64, 192, 0]))
def remap_dataset(dname: str, tsv_fpath: str, old_dataroot: str, remapped_dataroot: str, panoptic_json_path: str, num_processes: int = 4, create_symlink_cpy: bool = False, convert_label_from_rgb: bool = False): """ Given path to a dataset, given names of _names.txt Remap according to the provided tsv. (also account for the fact that 255 is always unlabeled) Args: - dname: string representing name of taxonomy for original dataset - tsv_fpath: string representing path to a .tsv file - old_dataroot: string representing path to original dataset - remapped_dataroot: string representing path at which to new dataset - panoptic_json_path: string representing path to coco-style json file - num_processes: integer representing number of workers to exploit - create_symlink_cpy: adds symbolic links for images in the same folder structure as annotations Returns: - None """ # form one-way mapping between IDs tconv = TaxonomyConverter(train_datasets=[dname], test_datasets=[], tsv_fpath=tsv_fpath) dataset_colors = load_dataset_colors_arr( dname) if convert_label_from_rgb else None for split_idx, split in enumerate(['train', 'val']): panoptic_json_content = None orig_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths( dname, split) if not panoptic_json_path is None: with open( panoptic_json_path.format(split=split, split_idx=str(split_idx)), 'r') as ifile: json_cont = json.load(ifile) panoptic_json_content = { a["file_name"]: a for a in json_cont["annotations"] } if dname[:4] == "coco": #hacky needed for inplace coco support orig_relative_img_label_pairs = [[ fix_path_coco_inplace(p[0]), fix_path_coco_inplace(p[1]) ] for p in orig_relative_img_label_pairs] basedir = 'images/' + split + '/' + dname img_subdirs = list( set([os.path.dirname(p[0]) for p in orig_relative_img_label_pairs])) img_dir_remapping = {} for d in img_subdirs: img_dir_remapping[d] = basedir if len( img_subdirs) == 1 else basedir + '/' + d.replace( '/color', '').replace('/leftImg8bit', '') if create_symlink_cpy: unpriv_symb_link( old_dataroot + '/' + d, remapped_dataroot + '/' + img_dir_remapping[d]) remapped_relative_img_label_pairs = [ (img_dir_remapping[os.path.dirname(p[0])] + '/' + os.path.basename(p[0]), img_dir_remapping[os.path.dirname( p[0])].replace('images', 'annotations') + '/' + os.path.basename(p[0]).replace('.jpg', '.png')) for p in orig_relative_img_label_pairs ] send_list_to_workers( num_processes=num_processes, list_to_split=orig_relative_img_label_pairs, worker_func_ptr=relabel_pair_worker, remapped_relative_img_label_pairs=remapped_relative_img_label_pairs, tax_converter=tconv, panoptic_json_content=panoptic_json_content, old_dataroot=old_dataroot, new_dataroot=remapped_dataroot, dname=dname, dataset_colors=dataset_colors)