def test_map_semantic_img_fast_uint16_stress_test():
"""
Test fast method on large example. Map 60,000 classes to a
different 60,000 classes with a fast array conversion.
Should take less than 1 millisecond.
"""
semantic_img = np.array(range(60000)).reshape(3000, 20).astype(np.uint16)
# will expect uint16 back since 60000 < 65535, which is uint16 max
gt_mapped_img1 = np.array(range(60000)).reshape(3000, 20).astype(
np.uint16) + 1
label_mapping = {i: i + 1 for i in range(60000)}
label_mapping_copy = copy.deepcopy(label_mapping)
label_mapping_arr = form_label_mapping_array(label_mapping)
dict_is_equal(label_mapping, label_mapping_copy)
assert label_mapping == label_mapping_copy
start = time.time()
mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr)
end = time.time()
print(f'Took {end-start} sec.')
gt_mapped_img2 = map_semantic_img_slow(semantic_img, label_mapping)
assert np.allclose(gt_mapped_img1, mapped_img)
assert np.allclose(gt_mapped_img2, mapped_img)
assert gt_mapped_img1.dtype == mapped_img.dtype
assert gt_mapped_img2.dtype == mapped_img.dtype
def remap_dataset(dname: str,
remapped_dname: str,
tsv_fpath: str,
old_dataroot: str,
remapped_dataroot: str,
include_ignore_idx_cls: bool = True,
convert_label_from_rgb: bool = False,
num_processes: int = 4):
"""
Given path to a dataset, given names of _names.txt
Remap according to the provided tsv.
(also account for the fact that 255 is always unlabeled)
Args:
- dname: string representing name of taxonomy for original dataset
- remapped_dname: string representing name of taxonomy for new dataset
- tsv_fpath: string representing path to a .tsv file
- old_dataroot: string representing path to original dataset
- remapped_dataroot: string representing path at which to new dataset
- include_ignore_idx_cls: whether to include unlabeled=255 from source
- convert_label_from_rgb: labels of original dataset are stored as RGB
- num_processes: integer representing number of workers to exploit
Returns:
- None
"""
# load colors ordered with class indices, if labels encoded as RGB
dataset_colors = load_dataset_colors_arr(
dname) if convert_label_from_rgb else None
# load up the dictionary from the tsv
classname_remapping_dict = read_label_mapping(filename=tsv_fpath,
label_from=dname,
label_to=remapped_dname,
convert_val_to_int=False)
oldid_to_oldname = get_dataloader_id_to_classname_map(dname)
newname_tonewid_map = get_classname_to_dataloaderid_map(
remapped_dname, include_ignore_idx_cls=include_ignore_idx_cls)
# form one-way mapping between IDs
old_name_to_newid = convert_dictionaries(classname_remapping_dict,
newname_tonewid_map)
class_idx_remapping_dict = convert_dictionaries(oldid_to_oldname,
old_name_to_newid)
label_mapping_arr = form_label_mapping_array(class_idx_remapping_dict)
for split in ['train', 'val']: #'trainval']:# 'val']: #
orig_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths(
dname, split)
remapped_relative_img_label_pairs = generate_all_img_label_pair_relative_fpaths(
remapped_dname, split)
send_list_to_workers(
num_processes=num_processes,
list_to_split=orig_relative_img_label_pairs,
worker_func_ptr=relabel_pair_worker,
remapped_relative_img_label_pairs=remapped_relative_img_label_pairs,
label_mapping_arr=label_mapping_arr,
old_dataroot=old_dataroot,
new_dataroot=remapped_dataroot,
dataset_colors=dataset_colors)
def test_relabel_pair():
"""
grayscale -> grayscale label remapping.
"""
old_dataroot = f'{_TEST_DIR}/test_data'
new_dataroot = f'{_TEST_DIR}/test_data'
orig_pair = ('rgb_old.jpg', 'remapping_test_data_old/label_old.png')
remapped_pair = ('rgb_new.jpg', 'remapping_test_data_new/label_new.png')
old_label_fpath = f'{old_dataroot}/{orig_pair[1]}'
create_leading_fpath_dirs(old_label_fpath)
new_label_fpath = f'{new_dataroot}/{remapped_pair[1]}'
semantic_img = np.array([[254, 0, 1], [7, 8, 9]], dtype=np.uint8)
imageio.imwrite(old_label_fpath, semantic_img)
label_mapping = {254: 253, 0: 255, 1: 0, 7: 6, 8: 7, 9: 8}
label_mapping_arr = form_label_mapping_array(label_mapping)
relabel_pair(old_dataroot,
new_dataroot,
orig_pair,
remapped_pair,
label_mapping_arr,
dataset_colors=None)
gt_mapped_img = np.array([[253, 255, 0], [6, 7, 8]], dtype=np.uint8)
remapped_img = imageio.imread(new_label_fpath)
assert np.allclose(gt_mapped_img, remapped_img)
os.remove(old_label_fpath)
os.remove(new_label_fpath)
def test_form_label_mapping_array_fromzero_uint8():
"""
Count from zero, with 8-bit unsigned int data type.
"""
label_mapping = {0: 1, 1: 2, 3: 4}
label_mapping_arr = form_label_mapping_array(label_mapping)
# since max class index <= 255, we expect uint8
gt_label_mapping_arr = np.array([1, 2, 0, 4], dtype=np.uint8)
assert np.allclose(label_mapping_arr, gt_label_mapping_arr)
def test_form_label_mapping_array_from_nonzero_uint16():
"""
"""
label_mapping = {655: 300, 654: 255, 653: 100}
label_mapping_arr = form_label_mapping_array(label_mapping)
# since max class index > 255, we expect uint16
gt_label_mapping_arr = np.zeros((656), dtype=np.uint16)
gt_label_mapping_arr[655] = 300
gt_label_mapping_arr[654] = 255
gt_label_mapping_arr[653] = 100
print(label_mapping)
assert np.allclose(label_mapping_arr, gt_label_mapping_arr)
def test_map_semantic_img_fast_dontwidenvalue():
"""
Test fast method on simple conversion from 2x3 grayscale -> 2x3 grayscale.
"""
semantic_img = np.array([[300, 301, 302], [300, 301, 302]],
dtype=np.uint16)
label_mapping = {300: 0, 301: 1, 302: 2}
label_mapping_arr = form_label_mapping_array(label_mapping)
mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr)
# Expect uint8 since max class index in values <= 255, so uint16 unnecessary
gt_mapped_img1 = np.array([[0, 1, 2], [0, 1, 2]], dtype=np.uint8)
assert np.allclose(gt_mapped_img1, mapped_img)
assert mapped_img.dtype == np.uint8
def test_map_semantic_img_fast_widenvalue():
"""
Test fast method on simple conversion from 2x3 grayscale -> 2x3 grayscale.
"""
semantic_img = np.array([[255, 255, 255], [255, 255, 255]], dtype=np.uint8)
label_mapping = {255: 256}
label_mapping_arr = form_label_mapping_array(label_mapping)
mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr)
# Expect uint8 since max class index <= 255, so uint16 unnecessary
gt_mapped_img1 = np.array([[256, 256, 256], [256, 256, 256]],
dtype=np.uint16)
assert np.allclose(gt_mapped_img1, mapped_img)
assert mapped_img.dtype == np.uint16
def test_map_semantic_img_fast():
"""
Test fast method on simple conversion from 2x3 grayscale -> 2x3 grayscale.
"""
semantic_img = np.array([[254, 0, 1], [7, 8, 9]], dtype=np.uint8)
label_mapping = {254: 253, 0: 255, 1: 0, 7: 6, 8: 7, 9: 8}
label_mapping_arr = form_label_mapping_array(label_mapping)
mapped_img = map_semantic_img_fast(semantic_img, label_mapping_arr)
# Expect uint8 since max class index <= 255, so uint16 unnecessary
gt_mapped_img1 = np.array([[253, 255, 0], [6, 7, 8]], dtype=np.uint8)
gt_mapped_img2 = map_semantic_img_slow(semantic_img, label_mapping)
assert np.allclose(gt_mapped_img1, mapped_img)
assert np.allclose(gt_mapped_img2, mapped_img)
assert gt_mapped_img1.dtype == mapped_img.dtype
assert gt_mapped_img2.dtype == mapped_img.dtype
