def test_can_convert_voc_to_yolo(self): target_dataset = Dataset.from_iterable( [ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Bbox(1.0, 2.0, 2.0, 2.0, label=8), Bbox(4.0, 5.0, 2.0, 2.0, label=15), Bbox(5.5, 6, 2, 2, label=22), ]) ], categories=[ label.name for label in VOC.make_voc_categories()[AnnotationType.label] ]) with TestDir() as test_dir: voc_dir = osp.join( __file__[:__file__.rfind(osp.join('tests', ''))], 'tests', 'assets', 'voc_dataset', 'voc_dataset1') yolo_dir = osp.join(test_dir, 'yolo_dir') run(self, 'convert', '-if', 'voc', '-i', voc_dir, '-f', 'yolo', '-o', yolo_dir, '--', '--save-images') parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') compare_datasets(self, target_dataset, parsed_dataset, require_images=True)
def test_can_import_voc_dataset_with_empty_lines_in_subset_lists(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Bbox(1.0, 2.0, 2.0, 2.0, label=8, id=1, group=1, attributes={ 'difficult': False, 'truncated': True, 'occluded': False, 'pose': 'Unspecified' } ) ]) ], categories=VOC.make_voc_categories()) rpath = osp.join('ImageSets', 'Main', 'train.txt') matrix = [ ('voc_detection', '', ''), ('voc_detection', 'train', rpath), ] for format, subset, path in matrix: with self.subTest(format=format, subset=subset, path=path): if subset: expected = expected_dataset.get_subset(subset) else: expected = expected_dataset actual = Dataset.import_from(osp.join(DUMMY_DATASET3_DIR, path), format) compare_datasets(self, expected, actual, require_images=True)
def test_can_save_and_load_voc_segmentation_dataset(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Mask(image=np.ones([10, 20]), label=2, group=1) ]), DatasetItem(id='2007_000002', subset='test', image=np.ones((10, 20, 3))), ], categories=VOC.make_voc_categories()) dataset_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1') rpath = osp.join('ImageSets', 'Segmentation', 'train.txt') matrix = [ ('voc_segmentation', '', ''), ('voc_segmentation', 'train', rpath), ('voc', 'train', rpath), ] for format, subset, path in matrix: with self.subTest(format=format, subset=subset, path=path): if subset: expected = expected_dataset.get_subset(subset) else: expected = expected_dataset with TestDir() as test_dir: self._test_can_save_and_load(test_dir, dataset_dir, expected, format, result_path=path, label_map='voc')
def test_can_import_voc_segmentation_dataset(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Mask(image=np.ones([10, 20]), label=2, group=1) ]), DatasetItem(id='2007_000002', subset='test', image=np.ones((10, 20, 3))), ], categories=VOC.make_voc_categories()) rpath = osp.join('ImageSets', 'Segmentation', 'train.txt') matrix = [ ('voc_segmentation', '', ''), ('voc_segmentation', 'train', rpath), ('voc', 'train', rpath), ] for format, subset, path in matrix: with self.subTest(format=format, subset=subset, path=path): if subset: expected = expected_dataset.get_subset(subset) else: expected = expected_dataset actual = Dataset.import_from(osp.join(DUMMY_DATASET_DIR, path), format) compare_datasets(self, expected, actual, require_images=True)
def test_convert_to_voc_format(self): """ <b>Description:</b> Ensure that the dataset can be converted to VOC format with command `datum convert`. <b>Expected results:</b> A VOC dataset that matches the expected dataset. <b>Steps:</b> 1. Get path to the source dataset from assets. 2. Convert source dataset to VOC format, using the `convert` command. 3. Verify that resulting dataset is equal to the expected dataset. """ label_map = OrderedDict(('label_' + str(i), [None, [], []]) for i in range(10)) label_map['background'] = [None, [], []] label_map.move_to_end('background', last=False) expected_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='default', image=np.ones((16, 16, 3)), annotations=[ Bbox(0.0, 4.0, 4.0, 8.0, attributes={ 'difficult': False, 'truncated': False, 'occluded': False, 'visibility': '1.0', 'ignored': 'False' }, id=1, label=3, group=1 ) ] ) ], categories=VOC.make_voc_categories(label_map)) mot_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], 'tests', 'assets', 'mot_dataset') with TestDir() as test_dir: voc_dir = osp.join(test_dir, 'voc') run(self, 'convert', '-if', 'mot_seq', '-i', mot_dir, '-f', 'voc', '-o', voc_dir, '--', '--save-images') target_dataset = Dataset.import_from(voc_dir, format='voc') compare_datasets(self, expected_dataset, target_dataset, require_images=True)
def test_can_save_and_load_voc_detection_dataset(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Bbox(4.0, 5.0, 2.0, 2.0, label=15, id=2, group=2, attributes={ 'difficult': False, 'truncated': False, 'occluded': False, **{ a.name : a.value % 2 == 1 for a in VOC.VocAction } } ), Bbox(1.0, 2.0, 2.0, 2.0, label=8, id=1, group=1, attributes={ 'difficult': False, 'truncated': True, 'occluded': False, 'pose': 'Unspecified' } ) ]), DatasetItem(id='2007_000002', subset='test', image=np.ones((10, 20, 3))), ], categories=VOC.make_voc_categories()) dataset_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1') rpath = osp.join('ImageSets', 'Main', 'train.txt') matrix = [ ('voc_detection', '', ''), ('voc_detection', 'train', rpath), ] for format, subset, path in matrix: with self.subTest(format=format, subset=subset, path=path): if subset: expected = expected_dataset.get_subset(subset) else: expected = expected_dataset with TestDir() as test_dir: self._test_can_save_and_load(test_dir, dataset_dir, expected, format, result_path=path, label_map='voc')
def test_export_to_voc_format(self): label_map = OrderedDict(('label_%s' % i, [None, [], []]) for i in range(10)) label_map['background'] = [None, [], []] label_map.move_to_end('background', last=False) expected_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', image=np.ones((10, 15, 3)), annotations=[ Bbox(0.0, 2.0, 4.0, 2.0, attributes={ 'difficult': False, 'truncated': False, 'occluded': False }, id=1, label=3, group=1 ), Bbox(3.0, 3.0, 2.0, 3.0, attributes={ 'difficult': False, 'truncated': False, 'occluded': False }, id=2, label=5, group=2 ) ] ) ], categories=VOC.make_voc_categories(label_map)) with TestDir() as test_dir: yolo_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], 'tests', 'assets', 'yolo_dataset') run(self, 'create', '-o', test_dir) run(self, 'import', '-p', test_dir, '-f', 'yolo', yolo_dir) voc_export = osp.join(test_dir, 'voc_export') run(self, 'export', '-p', test_dir, '-f', 'voc', '-o', voc_export, '--', '--save-images') parsed_dataset = Dataset.import_from(voc_export, format='voc') compare_datasets(self, expected_dataset, parsed_dataset, require_images=True)
def test_can_import_voc_layout_dataset(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Bbox(4.0, 5.0, 2.0, 2.0, label=15, id=2, group=2, attributes={ 'difficult': False, 'truncated': False, 'occluded': False, **{ a.name : a.value % 2 == 1 for a in VOC.VocAction } } ), Bbox(5.5, 6.0, 2.0, 2.0, label=22, group=2), ]), DatasetItem(id='2007_000002', subset='test', image=np.ones((10, 20, 3))), ], categories=VOC.make_voc_categories()) rpath = osp.join('ImageSets', 'Layout', 'train.txt') matrix = [ ('voc_layout', '', ''), ('voc_layout', 'train', rpath), ('voc', 'train', rpath), ] for format, subset, path in matrix: with self.subTest(format=format, subset=subset, path=path): if subset: expected = expected_dataset.get_subset(subset) else: expected = expected_dataset actual = Dataset.import_from(osp.join(DUMMY_DATASET_DIR, path), format) compare_datasets(self, expected, actual, require_images=True)
def test_label_projection_with_masks(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[ Bbox(1, 2, 2, 2, label=3, attributes={ 'pose': VOC.VocPose(1).name, 'truncated': True, 'difficult': False, 'occluded': False, }, id=1, group=1, ), ] ), DatasetItem(id='2007_000002', subset='test', image=np.ones((10, 20, 3))), ], categories=VOC.make_voc_categories({ 'background': [(0, 0, 0), [], []], # Added on export 'a': [(128, 0, 0), [], []], # Generated by the transform 'b': [(0, 128, 0), [], []], # Generated by the transform 'cat': [(64, 0, 0), [], []] # Original })) dataset_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1') with TestDir() as test_dir: run(self, 'create', '-o', test_dir) run(self, 'import', '-p', test_dir, '-f', 'voc', dataset_path) run(self, 'transform', '-p', test_dir, '-t', 'project_labels', '--', '-l', 'a', '-l', 'b', '-l', 'cat') parsed_dataset = Dataset.import_from( osp.join(test_dir, 'source-1'), 'voc') compare_datasets(self, expected_dataset, parsed_dataset)
def test_can_save_and_load_voc_dataset(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', image=np.ones((10, 20, 3)), annotations=[Label(i) for i in range(22) if i % 2 == 1] + [ Bbox(4.0, 5.0, 2.0, 2.0, label=15, id=1, group=1, attributes={ 'difficult': False, 'truncated': False, 'occluded': False, **{ a.name : a.value % 2 == 1 for a in VOC.VocAction } }, ), Bbox(1.0, 2.0, 2.0, 2.0, label=8, id=2, group=2, attributes={ 'difficult': False, 'truncated': True, 'occluded': False, 'pose': 'Unspecified' } ), Bbox(5.5, 6.0, 2.0, 2.0, label=22, id=0, group=1), Mask(image=np.ones([10, 20]), label=2, group=1), ]), DatasetItem(id='2007_000002', subset='test', image=np.ones((10, 20, 3))) ], categories=VOC.make_voc_categories()) voc_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1') with TestDir() as test_dir: self._test_can_save_and_load(test_dir, voc_dir, source_dataset, 'voc', label_map='voc')
def categories(self): return VOC.make_voc_categories()
def categories(self): return VOC.make_voc_categories(dst_label_map)
def categories(self): label_map = OrderedDict() label_map['background'] = [(0, 0, 0), [], []] label_map['label_1'] = [(1, 2, 3), [], []] label_map['label_2'] = [(3, 2, 1), [], []] return VOC.make_voc_categories(label_map)
def categories(self): label_map = OrderedDict() label_map['background'] = [None, [], []] label_map['Label_1'] = [None, [], []] label_map['label_2'] = [None, [], []] return VOC.make_voc_categories(label_map)
def test_preparing_dataset_for_train_model(self): """ <b>Description:</b> Testing a particular example of working with VOC dataset. <b>Expected results:</b> A VOC dataset that matches the expected result. <b>Steps:</b> 1. Get path to the source dataset from assets. 2. Create a datumaro project and add source dataset to it. 3. Leave only non-occluded annotations with `filter` command. 4. Split the dataset into subsets with `transform` command. 5. Export the project to a VOC dataset with `export` command. 6. Verify that the resulting dataset is equal to the expected result. """ expected_dataset = Dataset.from_iterable([ DatasetItem(id='c', subset='train', annotations=[ Bbox(3.0, 1.0, 8.0, 5.0, attributes={ 'truncated': False, 'occluded': False, 'difficult': False }, id=1, label=2, group=1 ) ] ), DatasetItem(id='d', subset='test', annotations=[ Bbox(4.0, 4.0, 4.0, 4.0, attributes={ 'truncated': False, 'occluded': False, 'difficult': False }, id=1, label=3, group=1 ) ] ) ], categories=VOC.make_voc_categories()) dataset_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset2') with TestDir() as test_dir: run(self, 'create', '-o', test_dir) run(self, 'import', '-p', test_dir, '-f', 'voc', dataset_path) run(self, 'filter', '-p', test_dir, '-m', 'i+a', '-e', "/item/annotation[occluded='False']") run(self, 'transform', '-p', test_dir, '-t', 'random_split', '--', '-s', 'test:.5', '-s', 'train:.5', '--seed', '1') export_path = osp.join(test_dir, 'dataset') run(self, 'export', '-p', test_dir, '-f', 'voc', '-o', export_path, '--', '--label-map', 'voc') parsed_dataset = Dataset.import_from(export_path, format='voc') compare_datasets(self, expected_dataset, parsed_dataset)
def categories(self): label_map = VOC.make_voc_label_map() label_map['non_voc_label'] = [None, [], []] for label_desc in label_map.values(): label_desc[0] = None # rebuild colormap return VOC.make_voc_categories(label_map)