def test_inplace_save_writes_only_updated_data_with_direct_changes(self): expected = Dataset.from_iterable([ DatasetItem(1, subset='a'), DatasetItem(2, subset='a', image=np.ones((3, 2, 3))), DatasetItem(2, subset='b'), ]) with TestDir() as path: # generate initial dataset dataset = Dataset.from_iterable([ # modified subset DatasetItem(1, subset='a'), # unmodified subset DatasetItem(2, subset='b'), # removed subset DatasetItem(3, subset='c', image=np.ones((2, 2, 3))), ]) dataset.save(path, save_images=True) dataset.put(DatasetItem(2, subset='a', image=np.ones((3, 2, 3)))) dataset.remove(3, 'c') dataset.save(save_images=True) self.assertEqual({'a.json', 'b.json'}, set(os.listdir(osp.join(path, 'annotations')))) self.assertEqual({'2.jpg'}, set(os.listdir(osp.join(path, 'images', 'a')))) compare_datasets_strict(self, expected, Dataset.load(path))
def test_inplace_save_writes_only_updated_data_with_transforms(self): with TestDir() as path: expected = Dataset.from_iterable([ DatasetItem(2, subset='test'), DatasetItem(3, subset='train', image=np.ones((2, 2, 3))), DatasetItem(4, subset='train', image=np.ones((2, 3, 3))), DatasetItem(5, subset='test', point_cloud=osp.join(path, 'point_clouds', 'test', '5.pcd'), related_images=[ Image(data=np.ones((3, 4, 3)), path=osp.join(path, 'test', '5', 'image_0.jpg')), osp.join(path, 'test', '5', 'a', '5.png'), ]), ]) dataset = Dataset.from_iterable([ DatasetItem(1, subset='a'), DatasetItem(2, subset='b'), DatasetItem(3, subset='c', image=np.ones((2, 2, 3))), DatasetItem(4, subset='d', image=np.ones((2, 3, 3))), DatasetItem(5, subset='e', point_cloud='5.pcd', related_images=[ np.ones((3, 4, 3)), 'a/5.png', ]), ]) dataset.save(path, save_images=True) dataset.filter('/item[id >= 2]') dataset.transform('random_split', splits=(('train', 0.5), ('test', 0.5)), seed=42) dataset.save(save_images=True) self.assertEqual( {'images', 'annotations', 'point_clouds', 'related_images'}, set(os.listdir(path))) self.assertEqual({'train.json', 'test.json'}, set(os.listdir(osp.join(path, 'annotations')))) self.assertEqual({'3.jpg', '4.jpg'}, set(os.listdir(osp.join(path, 'images', 'train')))) self.assertEqual({'train', 'c', 'd'}, set(os.listdir(osp.join(path, 'images')))) self.assertEqual(set(), set(os.listdir(osp.join(path, 'images', 'c')))) self.assertEqual(set(), set(os.listdir(osp.join(path, 'images', 'd')))) self.assertEqual( {'image_0.jpg'}, set(os.listdir(osp.join(path, 'related_images', 'test', '5')))) compare_datasets_strict(self, expected, Dataset.load(path))
def test_can_run_self_merge(self): dataset1 = Dataset.from_iterable([ DatasetItem(id=100, subset='train', image=np.ones((10, 6, 3)), annotations=[ Bbox(1, 2, 3, 3, label=0), ]), ], categories=['a', 'b']) dataset2 = Dataset.from_iterable([ DatasetItem(id=100, subset='train', image=np.ones((10, 6, 3)), annotations=[ Bbox(1, 2, 3, 4, label=1), Bbox(5, 6, 2, 3, label=2), ]), ], categories=['a', 'b', 'c']) expected = Dataset.from_iterable( [ DatasetItem(id=100, subset='train', image=np.ones((10, 6, 3)), annotations=[ Bbox(1, 2, 3, 4, label=2, id=1, group=1, attributes={ 'score': 0.5, 'occluded': False, 'difficult': False, 'truncated': False }), Bbox(5, 6, 2, 3, label=3, id=2, group=2, attributes={ 'score': 0.5, 'occluded': False, 'difficult': False, 'truncated': False }), Bbox(1, 2, 3, 3, label=1, id=1, group=1, attributes={ 'score': 0.5, 'is_crowd': False }), ]), ], categories={ AnnotationType.label: LabelCategories.from_iterable(['background', 'a', 'b', 'c']), AnnotationType.mask: MaskCategories(VOC.generate_colormap(4)) }) with TestDir() as test_dir: dataset1_url = osp.join(test_dir, 'dataset1') dataset2_url = osp.join(test_dir, 'dataset2') dataset1.export(dataset1_url, 'coco', save_images=True) dataset2.export(dataset2_url, 'voc', save_images=True) proj_dir = osp.join(test_dir, 'proj') with Project.init(proj_dir) as project: project.import_source('source', dataset2_url, 'voc') result_dir = osp.join(test_dir, 'result') run(self, 'merge', '-o', result_dir, '-p', proj_dir, dataset1_url + ':coco') compare_datasets(self, expected, Dataset.load(result_dir), require_images=True)