def test_can_catch_pickle_exception(self):
with TestDir() as test_dir:
anno_file = osp.join(test_dir, 'test')
with open(anno_file, 'wb') as file:
pickle.dump(enumerate([1, 2, 3]), file)
with self.assertRaisesRegex(pickle.UnpicklingError, "Global"):
Dataset.import_from(test_dir, 'cifar')
def test_can_convert(self):
source_dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'label_me')
with TestDir() as test_dir:
LabelMeConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'label_me')
compare_datasets(self, source_dataset, parsed_dataset,
require_images=True)
def _import(src_file, task_data):
with TemporaryDirectory() as tmp_dir:
zipfile.ZipFile(src_file).extractall(tmp_dir)
if glob(osp.join(tmp_dir, '*.txt')):
dataset = Dataset.import_from(tmp_dir, 'imagenet_txt', env=dm_env)
else:
dataset = Dataset.import_from(tmp_dir, 'imagenet', env=dm_env)
import_dm_annotations(dataset, task_data)
def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
zipfile.ZipFile(src_file).extractall(tmp_dir)
if glob(osp.join(tmp_dir, '*.txt')):
dataset = Dataset.import_from(tmp_dir, 'imagenet_txt', env=dm_env)
else:
dataset = Dataset.import_from(tmp_dir, 'imagenet', env=dm_env)
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def _import(src_file, task_data):
if zipfile.is_zipfile(src_file):
with TemporaryDirectory() as tmp_dir:
zipfile.ZipFile(src_file).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'kitti_raw', env=dm_env)
import_dm_annotations(dataset, task_data)
else:
dataset = Dataset.import_from(src_file.name, 'kitti_raw', env=dm_env)
import_dm_annotations(dataset, task_data)
def _import(src_file, instance_data):
if zipfile.is_zipfile(src_file):
with TemporaryDirectory() as tmp_dir:
zipfile.ZipFile(src_file).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'coco', env=dm_env)
import_dm_annotations(dataset, instance_data)
else:
dataset = Dataset.import_from(src_file.name,
'coco_instances', env=dm_env)
import_dm_annotations(dataset, instance_data)
def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
if zipfile.is_zipfile(src_file):
zipfile.ZipFile(src_file).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'sly_pointcloud', env=dm_env)
else:
dataset = Dataset.import_from(src_file.name,
'sly_pointcloud', env=dm_env)
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def _import(src_file, instance_data):
with TemporaryDirectory() as tmp_dir:
zipfile.ZipFile(src_file).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'vgg_face2', env=dm_env)
dataset.transform('rename', r"|([^/]+/)?(.+)|\2|")
import_dm_annotations(dataset, instance_data)
def test_can_import_10(self):
expected_dataset = Dataset.from_iterable(
[
DatasetItem(id='image_1',
subset='data_batch_1',
image=np.ones((32, 32, 3)),
annotations=[Label(0)]),
DatasetItem(id='image_2',
subset='test_batch',
image=np.ones((32, 32, 3)),
annotations=[Label(1)]),
DatasetItem(id='image_3',
subset='test_batch',
image=np.ones((32, 32, 3)),
annotations=[Label(3)]),
DatasetItem(id='image_4',
subset='test_batch',
image=np.ones((32, 32, 3)),
annotations=[Label(2)]),
DatasetItem(id='image_5',
subset='test_batch',
image=np.array([[[1, 2, 3], [4, 5, 6]],
[[1, 2, 3], [4, 5, 6]]]),
annotations=[Label(3)])
],
categories=['airplane', 'automobile', 'bird', 'cat'])
dataset = Dataset.import_from(DUMMY_10_DATASET_DIR, 'cifar')
compare_datasets(self, expected_dataset, dataset, require_images=True)
def test_can_save_and_load_with_multiple_labels(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='1',
image=np.ones((8, 8, 3)),
annotations=[Label(0), Label(1)]
),
DatasetItem(id='2',
image=np.ones((8, 8, 3))
),
], categories={
AnnotationType.label: LabelCategories.from_iterable(
'label_' + str(label) for label in range(2)),
})
excepted_dataset = Dataset.from_iterable([
DatasetItem(id='label_0/1',
image=np.ones((8, 8, 3)),
annotations=[Label(0)]
),
DatasetItem(id='label_1/1',
image=np.ones((8, 8, 3)),
annotations=[Label(1)]
),
DatasetItem(id='no_label/2',
image=np.ones((8, 8, 3))
),
], categories=['label_0', 'label_1'])
with TestDir() as test_dir:
ImagenetConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'imagenet')
compare_datasets(self, excepted_dataset, parsed_dataset,
require_images=True)
def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
# put label map from the task if not present
labelmap_file = osp.join(tmp_dir, 'labelmap.txt')
if not osp.isfile(labelmap_file):
labels_meta = instance_data.meta['project']['labels'] \
if isinstance(instance_data, ProjectData) else instance_data.meta['task']['labels']
labels = (label['name'] + ':::' for _, label in labels_meta)
with open(labelmap_file, 'w') as f:
f.write('\n'.join(labels))
# support flat archive layout
anno_dir = osp.join(tmp_dir, 'Annotations')
if not osp.isdir(anno_dir):
anno_files = glob(osp.join(tmp_dir, '**', '*.xml'), recursive=True)
subsets_dir = osp.join(tmp_dir, 'ImageSets', 'Main')
os.makedirs(subsets_dir, exist_ok=True)
with open(osp.join(subsets_dir, 'train.txt'), 'w') as subset_file:
for f in anno_files:
subset_file.write(osp.splitext(osp.basename(f))[0] + '\n')
os.makedirs(anno_dir, exist_ok=True)
for f in anno_files:
shutil.move(f, anno_dir)
dataset = Dataset.import_from(tmp_dir, 'voc', env=dm_env)
dataset.transform('masks_to_polygons')
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def test_inplace_save_writes_only_updated_data(self):
expected = Dataset.from_iterable([
DatasetItem(1, subset='train', image=np.ones((2, 4, 3))),
DatasetItem(2, subset='train', image=np.ones((3, 2, 3))),
],
categories=[])
with TestDir() as path:
dataset = Dataset.from_iterable([
DatasetItem(1, subset='train', image=np.ones((2, 4, 3))),
DatasetItem(
2, subset='train', image=Image(path='2.jpg', size=(3, 2))),
DatasetItem(3, subset='valid', image=np.ones((2, 2, 3))),
],
categories=[])
dataset.export(path, 'wider_face', save_images=True)
dataset.put(
DatasetItem(2, subset='train', image=np.ones((3, 2, 3))))
dataset.remove(3, 'valid')
dataset.save(save_images=True)
self.assertEqual({'1.jpg', '2.jpg'},
set(
os.listdir(
osp.join(path, 'WIDER_train', 'images',
'no_label'))))
self.assertEqual({'wider_face_train_bbx_gt.txt'},
set(os.listdir(osp.join(path,
'wider_face_split'))))
compare_datasets(self,
expected,
Dataset.import_from(path, 'wider_face'),
require_images=True,
ignored_attrs=IGNORE_ALL)
def test_can_save_and_load_with_landmarks(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='name0/name0_0001',
subset='test', image=np.ones((2, 5, 3)),
attributes={
'positive_pairs': ['name0/name0_0002'],
'negative_pairs': []
},
annotations=[
Points([0, 4, 3, 3, 2, 2, 1, 0, 3, 0]),
]
),
DatasetItem(id='name0/name0_0002',
subset='test', image=np.ones((2, 5, 3)),
attributes={
'positive_pairs': [],
'negative_pairs': []
},
annotations=[
Points([0, 5, 3, 5, 2, 2, 1, 0, 3, 0]),
]
),
])
with TestDir() as test_dir:
LfwConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'lfw')
compare_datasets(self, source_dataset, parsed_dataset)
def test_dataset_with_save_dataset_meta_file(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id='a/b/1',
image=np.ones((8, 8, 3)),
subset='train',
annotations=[
Bbox(0, 2, 4, 2, label=2),
Bbox(0,
1,
2,
3,
label=1,
attributes={
'blur': '2',
'expression': '0',
'illumination': '0',
'occluded': '0',
'pose': '2',
'invalid': '0'
}),
]),
],
categories=['face', 'label_0', 'label_1'])
with TestDir() as test_dir:
WiderFaceConverter.convert(source_dataset,
test_dir,
save_images=True,
save_dataset_meta=True)
parsed_dataset = Dataset.import_from(test_dir, 'wider_face')
self.assertTrue(osp.isfile(osp.join(test_dir,
'dataset_meta.json')))
compare_datasets(self, source_dataset, parsed_dataset)
def _import(src_file, instance_data):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'label_me', env=dm_env)
dataset.transform('masks_to_polygons')
import_dm_annotations(dataset, instance_data)
def test_convert_from_voc_format(self):
"""
<b>Description:</b>
Ensure that the dataset can be converted from VOC format with
command `datum convert`.
<b>Expected results:</b>
A ImageNet dataset that matches the expected dataset.
<b>Steps:</b>
1. Get path to the source dataset from assets.
2. Convert source dataset to LabelMe format, using the `convert` command.
3. Verify that resulting dataset is equal to the expected dataset.
"""
labels = sorted([l.name for l in VOC.VocLabel if l.value % 2 == 1])
expected_dataset = Dataset.from_iterable([
DatasetItem(id='/'.join([label, '2007_000001']),
subset='default', annotations=[Label(i)])
for i, label in enumerate(labels)
] + [DatasetItem(id='no_label/2007_000002', subset='default',
image=np.ones((10, 20, 3)))
], categories=labels
)
voc_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1')
with TestDir() as test_dir:
imagenet_dir = osp.join(test_dir, 'imagenet')
run(self, 'convert', '-if', 'voc', '-i', voc_dir,
'-f', 'imagenet', '-o', imagenet_dir, '--', '--save-image')
target_dataset = Dataset.import_from(imagenet_dir, format='imagenet')
compare_datasets(self, expected_dataset, target_dataset,
require_images=True)
def test_can_save_and_load_with_meta_file(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id='image_2',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(0)]),
DatasetItem(
id='image_3', subset='test', image=np.ones((32, 32, 3))),
DatasetItem(id='image_4',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(1)])
],
categories=['label_0', 'label_1'])
with TestDir() as test_dir:
CifarConverter.convert(source_dataset,
test_dir,
save_images=True,
save_dataset_meta=True)
parsed_dataset = Dataset.import_from(test_dir, 'cifar')
self.assertTrue(osp.isfile(osp.join(test_dir,
'dataset_meta.json')))
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def test_can_save_and_load_cifar100(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id='image_2',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(0)]),
DatasetItem(
id='image_3', subset='test', image=np.ones((32, 32, 3))),
DatasetItem(id='image_4',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(1)])
],
categories=[['class_0', 'superclass_0'],
['class_1', 'superclass_0']])
with TestDir() as test_dir:
CifarConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'cifar')
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def test_can_import(self):
expected_dataset = Dataset.from_iterable(
[
DatasetItem(id=1,
image=np.ones((16, 16, 3)),
annotations=[
Bbox(0,
4,
4,
8,
label=2,
attributes={
'occluded': False,
'visibility': 1.0,
'ignored': False,
}),
]),
],
categories={
AnnotationType.label:
LabelCategories.from_iterable('label_' + str(label)
for label in range(10)),
})
dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'mot_seq')
compare_datasets(self, expected_dataset, dataset)
def split_video_command(args):
src_path = osp.abspath(args.src_path)
dst_dir = args.dst_dir
if dst_dir:
if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir):
raise CliException("Directory '%s' already exists "
"(pass --overwrite to overwrite)" % dst_dir)
else:
dst_dir = generate_next_file_name('%s-frames' % osp.basename(src_path))
dst_dir = osp.abspath(dst_dir)
log.info("Exporting frames...")
dataset = Dataset.import_from(src_path,
'video_frames',
name_pattern=args.name_pattern,
step=args.step,
start_frame=args.start_frame,
end_frame=args.end_frame)
dataset.export(format='image_dir',
save_dir=dst_dir,
image_ext=args.image_ext)
log.info("Frames are exported into '%s'" % dst_dir)
return 0
def test_can_save_and_load_with_no_save_images(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='1',
subset='train',
image=np.ones((8, 8, 3)),
annotations=[
Bbox(0, 2, 4, 2, label=1),
Bbox(0,
1,
2,
3,
label=0,
attributes={
'blur': '2',
'expression': '0',
'illumination': '0',
'occluded': '0',
'pose': '2',
'invalid': '0'
}),
Label(1),
])
],
categories=['face', 'label_0'])
with TestDir() as test_dir:
WiderFaceConverter.convert(source_dataset,
test_dir,
save_images=False)
parsed_dataset = Dataset.import_from(test_dir, 'wider_face')
compare_datasets(self, source_dataset, parsed_dataset)
def test_can_import_with_colored_masks(self):
expected_dataset = Dataset.from_iterable(
[
DatasetItem(
id='Stereo_Left/Omni_F/000000',
image=np.ones((1, 5, 3)),
annotations=[
Mask(np.array([[1, 1, 0, 0, 0]]), label=1),
Mask(np.array([[0, 0, 1, 1, 0]]), label=2),
Mask(np.array([[0, 0, 0, 0, 1]]), label=3),
],
),
DatasetItem(
id='Stereo_Left/Omni_F/000001',
image=np.ones((1, 5, 3)),
annotations=[
Mask(np.array([[1, 0, 0, 0, 0]]), label=1),
Mask(np.array([[0, 1, 0, 0, 0]]), label=2),
Mask(np.array([[0, 0, 1, 1, 0]]), label=15),
Mask(np.array([[0, 0, 0, 0, 1]]), label=3),
],
)
],
categories=Synthia.make_categories())
dataset = Dataset.import_from(DUMMY_COLOR_SEGM_DATASET_DIR, 'synthia')
compare_datasets(self, expected_dataset, dataset, require_images=True)
def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='кириллица с пробелом',
image=np.ones((8, 8, 3)),
annotations=[
Bbox(0,
1,
2,
3,
label=0,
attributes={
'blur': '2',
'expression': '0',
'illumination': '0',
'occluded': '0',
'pose': '2',
'invalid': '0'
}),
]),
],
categories=['face'])
with TestDir() as test_dir:
WiderFaceConverter.convert(source_dataset,
test_dir,
save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'wider_face')
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def test_can_save_and_load_without_saving_images(self):
dataset = Dataset.from_iterable([
DatasetItem(id='a',
image=np.ones((5, 5, 3)),
annotations=[
Bbox(1,
2,
3,
4,
label=0,
group=1,
attributes={'score': 1.0}),
Mask(label=1,
group=0,
image=np.ones((5, 5)),
attributes={'box_id': '00000000'})
])
],
categories=['label_0', 'label_1'])
with TestDir() as test_dir:
OpenImagesConverter.convert(dataset, test_dir)
parsed_dataset = Dataset.import_from(test_dir, 'open_images')
compare_datasets(self, dataset, parsed_dataset)
def test_can_delete_labels_from_yolo_dataset(self):
target_dataset = Dataset.from_iterable([
DatasetItem(id='1',
subset='train',
image=np.ones((10, 15, 3)),
annotations=[Bbox(0.0, 2.0, 4.0, 2.0, label=0)])
],
categories=['label_2'])
with TestDir() as test_dir:
yolo_dir = osp.join(
__file__[:__file__.rfind(osp.join('tests', ''))], 'tests',
'assets', 'yolo_dataset')
run(self, 'create', '-o', test_dir)
run(self, 'import', '-p', test_dir, '-f', 'yolo', yolo_dir)
run(self, 'filter', '-p', test_dir, '-m', 'i+a', '-e',
"/item/annotation[label='label_2']")
run(self, 'transform', '-p', test_dir, '-t', 'remap_labels', '--',
'-l', 'label_2:label_2', '--default', 'delete')
export_dir = osp.join(test_dir, 'export')
run(self, 'export', '-p', test_dir, '-o', export_dir, '-f', 'yolo',
'--', '--save-image')
parsed_dataset = Dataset.import_from(export_dir, format='yolo')
compare_datasets(self, target_dataset, parsed_dataset)
def test_can_save_and_load_with_meta_file(self):
dataset = Dataset.from_iterable([
DatasetItem(id='a',
image=np.ones((5, 5, 3)),
annotations=[
Bbox(1,
2,
3,
4,
label=0,
group=1,
attributes={'score': 1.0}),
Mask(label=1,
group=0,
image=np.ones((5, 5)),
attributes={'box_id': '00000000'})
])
],
categories=['label_0', 'label_1'])
with TestDir() as test_dir:
OpenImagesConverter.convert(dataset,
test_dir,
save_images=True,
save_dataset_meta=True)
parsed_dataset = Dataset.import_from(test_dir, 'open_images')
self.assertTrue(osp.isfile(osp.join(test_dir,
'dataset_meta.json')))
compare_datasets(self,
dataset,
parsed_dataset,
require_images=True)
