def test_inplace_save_writes_only_updated_data_with_direct_changes(self):
expected = Dataset.from_iterable([
DatasetItem(1, subset='a'),
DatasetItem(2, subset='a', image=np.ones((3, 2, 3))),
DatasetItem(2, subset='b'),
])
with TestDir() as path:
# generate initial dataset
dataset = Dataset.from_iterable([
# modified subset
DatasetItem(1, subset='a'),
# unmodified subset
DatasetItem(2, subset='b'),
# removed subset
DatasetItem(3, subset='c', image=np.ones((2, 2, 3))),
])
dataset.save(path, save_images=True)
dataset.put(DatasetItem(2, subset='a', image=np.ones((3, 2, 3))))
dataset.remove(3, 'c')
dataset.save(save_images=True)
self.assertEqual({'a.json', 'b.json'},
set(os.listdir(osp.join(path, 'annotations'))))
self.assertEqual({'2.jpg'},
set(os.listdir(osp.join(path, 'images', 'a'))))
compare_datasets_strict(self, expected, Dataset.load(path))
def test_inplace_save_writes_only_updated_data_with_transforms(self):
with TestDir() as path:
expected = Dataset.from_iterable([
DatasetItem(2, subset='test'),
DatasetItem(3, subset='train', image=np.ones((2, 2, 3))),
DatasetItem(4, subset='train', image=np.ones((2, 3, 3))),
DatasetItem(5,
subset='test',
point_cloud=osp.join(path, 'point_clouds', 'test',
'5.pcd'),
related_images=[
Image(data=np.ones((3, 4, 3)),
path=osp.join(path, 'test', '5',
'image_0.jpg')),
osp.join(path, 'test', '5', 'a', '5.png'),
]),
])
dataset = Dataset.from_iterable([
DatasetItem(1, subset='a'),
DatasetItem(2, subset='b'),
DatasetItem(3, subset='c', image=np.ones((2, 2, 3))),
DatasetItem(4, subset='d', image=np.ones((2, 3, 3))),
DatasetItem(5,
subset='e',
point_cloud='5.pcd',
related_images=[
np.ones((3, 4, 3)),
'a/5.png',
]),
])
dataset.save(path, save_images=True)
dataset.filter('/item[id >= 2]')
dataset.transform('random_split',
splits=(('train', 0.5), ('test', 0.5)),
seed=42)
dataset.save(save_images=True)
self.assertEqual(
{'images', 'annotations', 'point_clouds', 'related_images'},
set(os.listdir(path)))
self.assertEqual({'train.json', 'test.json'},
set(os.listdir(osp.join(path, 'annotations'))))
self.assertEqual({'3.jpg', '4.jpg'},
set(os.listdir(osp.join(path, 'images',
'train'))))
self.assertEqual({'train', 'c', 'd'},
set(os.listdir(osp.join(path, 'images'))))
self.assertEqual(set(),
set(os.listdir(osp.join(path, 'images', 'c'))))
self.assertEqual(set(),
set(os.listdir(osp.join(path, 'images', 'd'))))
self.assertEqual(
{'image_0.jpg'},
set(os.listdir(osp.join(path, 'related_images', 'test', '5'))))
compare_datasets_strict(self, expected, Dataset.load(path))
def test_can_run_self_merge(self):
dataset1 = Dataset.from_iterable([
DatasetItem(id=100,
subset='train',
image=np.ones((10, 6, 3)),
annotations=[
Bbox(1, 2, 3, 3, label=0),
]),
],
categories=['a', 'b'])
dataset2 = Dataset.from_iterable([
DatasetItem(id=100,
subset='train',
image=np.ones((10, 6, 3)),
annotations=[
Bbox(1, 2, 3, 4, label=1),
Bbox(5, 6, 2, 3, label=2),
]),
],
categories=['a', 'b', 'c'])
expected = Dataset.from_iterable(
[
DatasetItem(id=100,
subset='train',
image=np.ones((10, 6, 3)),
annotations=[
Bbox(1,
2,
3,
4,
label=2,
id=1,
group=1,
attributes={
'score': 0.5,
'occluded': False,
'difficult': False,
'truncated': False
}),
Bbox(5,
6,
2,
3,
label=3,
id=2,
group=2,
attributes={
'score': 0.5,
'occluded': False,
'difficult': False,
'truncated': False
}),
Bbox(1,
2,
3,
3,
label=1,
id=1,
group=1,
attributes={
'score': 0.5,
'is_crowd': False
}),
]),
],
categories={
AnnotationType.label:
LabelCategories.from_iterable(['background', 'a', 'b', 'c']),
AnnotationType.mask:
MaskCategories(VOC.generate_colormap(4))
})
with TestDir() as test_dir:
dataset1_url = osp.join(test_dir, 'dataset1')
dataset2_url = osp.join(test_dir, 'dataset2')
dataset1.export(dataset1_url, 'coco', save_images=True)
dataset2.export(dataset2_url, 'voc', save_images=True)
proj_dir = osp.join(test_dir, 'proj')
with Project.init(proj_dir) as project:
project.import_source('source', dataset2_url, 'voc')
result_dir = osp.join(test_dir, 'result')
run(self, 'merge', '-o', result_dir, '-p', proj_dir,
dataset1_url + ':coco')
compare_datasets(self,
expected,
Dataset.load(result_dir),
require_images=True)
