def test_map_semantic_img_fast_uint16_stress_test(): """ Test fast method on large example. Map 60,000 classes to a different 60,000 classes with a fast array conversion. Should take less than 1 millisecond. """ semantic_img = np.array(range(60000)).reshape(3000, 20).astype(np.uint16) # will expect uint16 back since 60000 < 65535, which is uint16 max gt_mapped_img1 = np.array(range(60000)).reshape(3000, 20).astype( np.uint16) + 1 label_mapping = {i: i + 1 for i in range(60000)} label_mapping_copy = copy.deepcopy(label_mapping) label_mapping_arr = form_label_mapping_array(label_mapping) dict_is_equal(label_mapping, label_mapping_copy) assert label_mapping == label_mapping_copy start = time.time() mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr) end = time.time() print(f'Took {end-start} sec.') gt_mapped_img2 = map_semantic_img_slow(semantic_img, label_mapping) assert np.allclose(gt_mapped_img1, mapped_img) assert np.allclose(gt_mapped_img2, mapped_img) assert gt_mapped_img1.dtype == mapped_img.dtype assert gt_mapped_img2.dtype == mapped_img.dtype
def remap_dataset(dname: str, remapped_dname: str, tsv_fpath: str, old_dataroot: str, remapped_dataroot: str, include_ignore_idx_cls: bool = True, convert_label_from_rgb: bool = False, num_processes: int = 4): """ Given path to a dataset, given names of _names.txt Remap according to the provided tsv. (also account for the fact that 255 is always unlabeled) Args: - dname: string representing name of taxonomy for original dataset - remapped_dname: string representing name of taxonomy for new dataset - tsv_fpath: string representing path to a .tsv file - old_dataroot: string representing path to original dataset - remapped_dataroot: string representing path at which to new dataset - include_ignore_idx_cls: whether to include unlabeled=255 from source - convert_label_from_rgb: labels of original dataset are stored as RGB - num_processes: integer representing number of workers to exploit Returns: - None """ # load colors ordered with class indices, if labels encoded as RGB dataset_colors = load_dataset_colors_arr( dname) if convert_label_from_rgb else None # load up the dictionary from the tsv classname_remapping_dict = read_label_mapping(filename=tsv_fpath, label_from=dname, label_to=remapped_dname, convert_val_to_int=False) oldid_to_oldname = get_dataloader_id_to_classname_map(dname) newname_tonewid_map = get_classname_to_dataloaderid_map( remapped_dname, include_ignore_idx_cls=include_ignore_idx_cls) # form one-way mapping between IDs old_name_to_newid = convert_dictionaries(classname_remapping_dict, newname_tonewid_map) class_idx_remapping_dict = convert_dictionaries(oldid_to_oldname, old_name_to_newid) label_mapping_arr = form_label_mapping_array(class_idx_remapping_dict) for split in ['train', 'val']: #'trainval']:# 'val']: # orig_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths( dname, split) remapped_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths( remapped_dname, split) send_list_to_workers( num_processes=num_processes, list_to_split=orig_relative_img_label_pairs, worker_func_ptr=relabel_pair_worker, remapped_relative_img_label_pairs=remapped_relative_img_label_pairs, label_mapping_arr=label_mapping_arr, old_dataroot=old_dataroot, new_dataroot=remapped_dataroot, dataset_colors=dataset_colors)
def test_relabel_pair(): """ grayscale -> grayscale label remapping. """ old_dataroot = f'{_TEST_DIR}/test_data' new_dataroot = f'{_TEST_DIR}/test_data' orig_pair = ('rgb_old.jpg', 'remapping_test_data_old/label_old.png') remapped_pair = ('rgb_new.jpg', 'remapping_test_data_new/label_new.png') old_label_fpath = f'{old_dataroot}/{orig_pair[1]}' create_leading_fpath_dirs(old_label_fpath) new_label_fpath = f'{new_dataroot}/{remapped_pair[1]}' semantic_img = np.array([[254, 0, 1], [7, 8, 9]], dtype=np.uint8) imageio.imwrite(old_label_fpath, semantic_img) label_mapping = {254: 253, 0: 255, 1: 0, 7: 6, 8: 7, 9: 8} label_mapping_arr = form_label_mapping_array(label_mapping) relabel_pair(old_dataroot, new_dataroot, orig_pair, remapped_pair, label_mapping_arr, dataset_colors=None) gt_mapped_img = np.array([[253, 255, 0], [6, 7, 8]], dtype=np.uint8) remapped_img = imageio.imread(new_label_fpath) assert np.allclose(gt_mapped_img, remapped_img) os.remove(old_label_fpath) os.remove(new_label_fpath)
def test_form_label_mapping_array_fromzero_uint8(): """ Count from zero, with 8-bit unsigned int data type. """ label_mapping = {0: 1, 1: 2, 3: 4} label_mapping_arr = form_label_mapping_array(label_mapping) # since max class index <= 255, we expect uint8 gt_label_mapping_arr = np.array([1, 2, 0, 4], dtype=np.uint8) assert np.allclose(label_mapping_arr, gt_label_mapping_arr)
def test_form_label_mapping_array_from_nonzero_uint16(): """ """ label_mapping = {655: 300, 654: 255, 653: 100} label_mapping_arr = form_label_mapping_array(label_mapping) # since max class index > 255, we expect uint16 gt_label_mapping_arr = np.zeros((656), dtype=np.uint16) gt_label_mapping_arr[655] = 300 gt_label_mapping_arr[654] = 255 gt_label_mapping_arr[653] = 100 print(label_mapping) assert np.allclose(label_mapping_arr, gt_label_mapping_arr)
def test_map_semantic_img_fast_dontwidenvalue(): """ Test fast method on simple conversion from 2x3 grayscale -> 2x3 grayscale. """ semantic_img = np.array([[300, 301, 302], [300, 301, 302]], dtype=np.uint16) label_mapping = {300: 0, 301: 1, 302: 2} label_mapping_arr = form_label_mapping_array(label_mapping) mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr) # Expect uint8 since max class index in values <= 255, so uint16 unnecessary gt_mapped_img1 = np.array([[0, 1, 2], [0, 1, 2]], dtype=np.uint8) assert np.allclose(gt_mapped_img1, mapped_img) assert mapped_img.dtype == np.uint8
def test_map_semantic_img_fast_widenvalue(): """ Test fast method on simple conversion from 2x3 grayscale -> 2x3 grayscale. """ semantic_img = np.array([[255, 255, 255], [255, 255, 255]], dtype=np.uint8) label_mapping = {255: 256} label_mapping_arr = form_label_mapping_array(label_mapping) mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr) # Expect uint8 since max class index <= 255, so uint16 unnecessary gt_mapped_img1 = np.array([[256, 256, 256], [256, 256, 256]], dtype=np.uint16) assert np.allclose(gt_mapped_img1, mapped_img) assert mapped_img.dtype == np.uint16
def test_map_semantic_img_fast(): """ Test fast method on simple conversion from 2x3 grayscale -> 2x3 grayscale. """ semantic_img = np.array([[254, 0, 1], [7, 8, 9]], dtype=np.uint8) label_mapping = {254: 253, 0: 255, 1: 0, 7: 6, 8: 7, 9: 8} label_mapping_arr = form_label_mapping_array(label_mapping) mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr) # Expect uint8 since max class index <= 255, so uint16 unnecessary gt_mapped_img1 = np.array([[253, 255, 0], [6, 7, 8]], dtype=np.uint8) gt_mapped_img2 = map_semantic_img_slow(semantic_img, label_mapping) assert np.allclose(gt_mapped_img1, mapped_img) assert np.allclose(gt_mapped_img2, mapped_img) assert gt_mapped_img1.dtype == mapped_img.dtype assert gt_mapped_img2.dtype == mapped_img.dtype