def test_can_save_and_load_with_multiple_labels(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='1',
image=np.ones((8, 8, 3)),
annotations=[Label(0), Label(1)]
),
DatasetItem(id='2',
image=np.ones((8, 8, 3))
),
], categories={
AnnotationType.label: LabelCategories.from_iterable(
'label_' + str(label) for label in range(2)),
})
excepted_dataset = Dataset.from_iterable([
DatasetItem(id='label_0/1',
image=np.ones((8, 8, 3)),
annotations=[Label(0)]
),
DatasetItem(id='label_1/1',
image=np.ones((8, 8, 3)),
annotations=[Label(1)]
),
DatasetItem(id='no_label/2',
image=np.ones((8, 8, 3))
),
], categories=['label_0', 'label_1'])
with TestDir() as test_dir:
ImagenetConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'imagenet')
compare_datasets(self, excepted_dataset, parsed_dataset,
require_images=True)
def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self):
label_categories = LabelCategories(attributes={'occluded'})
for i in range(10):
label_categories.add(str(i))
label_categories.items[2].attributes.update(['a1', 'a2', 'empty'])
source_dataset = Dataset.from_iterable([
DatasetItem(id='кириллица с пробелом',
subset='s1', image=np.zeros((5, 10, 3)),
annotations=[
Label(1),
]
),
], categories={
AnnotationType.label: label_categories,
})
target_dataset = Dataset.from_iterable([
DatasetItem(id='кириллица с пробелом',
subset='s1', image=np.zeros((5, 10, 3)),
annotations=[
Label(1),
], attributes={'frame': 0}
),
], categories={
AnnotationType.label: label_categories,
})
with TestDir() as test_dir:
self._test_save_and_load(source_dataset,
partial(CvatConverter.convert, save_images=True), test_dir,
target_dataset=target_dataset, require_images=True)
def test_can_save_and_load_with_multiple_labels(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id='1',
subset='train',
annotations=[Label(1),
Label(2),
Label(3)]),
DatasetItem(
id='2',
subset='train',
image=np.zeros((2, 8, 3)),
),
],
categories={
AnnotationType.label:
LabelCategories.from_iterable('label_' + str(label)
for label in range(10)),
})
with TestDir() as test_dir:
ImagenetTxtConverter.convert(source_dataset,
test_dir,
save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'imagenet_txt')
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def append_bbox_voc(annotations, **kwargs):
annotations.append(
Bbox(
1,
1,
2,
2,
label=kwargs["label_id"],
id=kwargs["ann_id"] + 1,
attributes=kwargs["attributes"],
group=kwargs["ann_id"],
)) # obj
annotations.append(
Label(kwargs["label_id"], attributes=kwargs["attributes"]))
annotations.append(
Bbox(
1,
1,
2,
2,
label=kwargs["label_id"] + 3,
group=kwargs["ann_id"],
)) # part
annotations.append(
Label(kwargs["label_id"] + 3, attributes=kwargs["attributes"]))
def test_can_save_and_load_cifar100(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id='image_2',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(0)]),
DatasetItem(
id='image_3', subset='test', image=np.ones((32, 32, 3))),
DatasetItem(id='image_4',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(1)])
],
categories=[['class_0', 'superclass_0'],
['class_1', 'superclass_0']])
with TestDir() as test_dir:
CifarConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'cifar')
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def test_can_save_and_load(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id=0,
subset='test',
image=np.ones((28, 28)),
annotations=[Label(0)]),
DatasetItem(id=1, subset='test', image=np.ones((28, 28))),
DatasetItem(id=2,
subset='test',
image=np.ones((28, 28)),
annotations=[Label(1)])
],
categories={
AnnotationType.label:
LabelCategories.from_iterable(
str(label) for label in range(10)),
})
with TestDir() as test_dir:
MnistConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'mnist')
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def test_remap_labels_ignore_missing_labels_in_secondary_categories(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id=1, annotations=[
Label(0),
])
], categories={
AnnotationType.label: LabelCategories.from_iterable(['a', 'b', 'c']),
AnnotationType.points: PointsCategories.from_iterable([]), # all missing
AnnotationType.mask: MaskCategories.generate(2) # no c color
})
target_dataset = Dataset.from_iterable([
DatasetItem(id=1, annotations=[
Label(0),
]),
], categories={
AnnotationType.label: LabelCategories.from_iterable(['d', 'e', 'f']),
AnnotationType.points: PointsCategories.from_iterable([]),
AnnotationType.mask: MaskCategories.generate(2)
})
actual = transforms.RemapLabels(source_dataset,
mapping={ 'a': 'd', 'b': 'e', 'c': 'f' }, default='delete')
compare_datasets(self, target_dataset, actual)
def test_can_save_and_load_with_no_save_images(self):
source_dataset = Dataset.from_iterable([
DatasetItem(
id='name0_0001',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(0,
attributes={'positive_pairs': ['name0/name0_0002']})
]),
DatasetItem(id='name0_0002',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(0,
attributes={
'positive_pairs': ['name0/name0_0001'],
'negative_pairs': ['name1/name1_0001']
})
]),
DatasetItem(id='name1_0001',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[Label(1, attributes={})]),
],
categories=['name0', 'name1'])
with TestDir() as test_dir:
LfwConverter.convert(source_dataset, test_dir, save_images=False)
parsed_dataset = Dataset.import_from(test_dir, 'lfw')
compare_datasets(self, source_dataset, parsed_dataset)
def test_can_import_with_meta_file(self):
expected_dataset = Dataset.from_iterable(
[
DatasetItem(id='000001',
subset='train',
image=np.ones((3, 4, 3)),
annotations=[Label(1)]),
DatasetItem(id='000002',
subset='train',
image=np.ones((3, 4, 3)),
annotations=[Label(3)]),
DatasetItem(id='000003',
subset='val',
image=np.ones((3, 4, 3)),
annotations=[Label(0)]),
DatasetItem(id='000004',
subset='test',
image=np.ones((3, 4, 3)),
annotations=[Label(2)]),
DatasetItem(id='000005',
subset='test',
image=np.ones((3, 4, 3)),
annotations=[Label(6)])
],
categories=[f'class-{i}' for i in range(7)])
dataset = Dataset.import_from(DUMMY_ALIGN_DATASET_DIR_WITH_META_FILE,
'align_celeba')
compare_datasets(self, expected_dataset, dataset, require_images=True)
def test_can_import(self):
expected_dataset = Dataset.from_iterable(
[
DatasetItem(id=0,
subset='test',
image=np.ones((28, 28)),
annotations=[Label(0)]),
DatasetItem(id=1,
subset='test',
image=np.ones((28, 28)),
annotations=[Label(2)]),
DatasetItem(id=2,
subset='test',
image=np.ones((28, 28)),
annotations=[Label(1)]),
DatasetItem(id=0,
subset='train',
image=np.ones((28, 28)),
annotations=[Label(5)]),
DatasetItem(id=1,
subset='train',
image=np.ones((28, 28)),
annotations=[Label(7)])
],
categories={
AnnotationType.label:
LabelCategories.from_iterable(
str(label) for label in range(10)),
})
dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'mnist')
compare_datasets(self, expected_dataset, dataset)
def test_can_save_and_load_with_no_format_names(self):
source_dataset = Dataset.from_iterable([
DatasetItem(
id='a/1',
image=np.ones((2, 5, 3)),
annotations=[
Label(0,
attributes={
'positive_pairs': ['name0/b/2'],
'negative_pairs': ['d/4']
})
],
),
DatasetItem(
id='b/2', image=np.ones((2, 5, 3)), annotations=[Label(0)]),
DatasetItem(
id='c/3', image=np.ones((2, 5, 3)), annotations=[Label(1)]),
DatasetItem(
id='d/4',
image=np.ones((2, 5, 3)),
),
],
categories=['name0', 'name1'])
with TestDir() as test_dir:
LfwConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'lfw')
compare_datasets(self, source_dataset, parsed_dataset)
def test_can_save_and_load_with_landmarks(self):
source_dataset = Dataset.from_iterable([
DatasetItem(
id='name0_0001',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(0,
attributes={'positive_pairs': ['name0/name0_0002']}),
Points([0, 4, 3, 3, 2, 2, 1, 0, 3, 0], label=0),
]),
DatasetItem(id='name0_0002',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(0),
Points([0, 5, 3, 5, 2, 2, 1, 0, 3, 0], label=0),
]),
],
categories=['name0'])
with TestDir() as test_dir:
LfwConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'lfw')
compare_datasets(self, source_dataset, parsed_dataset)
def test_can_import_10(self):
expected_dataset = Dataset.from_iterable(
[
DatasetItem(id='image_1',
subset='data_batch_1',
image=np.ones((32, 32, 3)),
annotations=[Label(0)]),
DatasetItem(id='image_2',
subset='test_batch',
image=np.ones((32, 32, 3)),
annotations=[Label(1)]),
DatasetItem(id='image_3',
subset='test_batch',
image=np.ones((32, 32, 3)),
annotations=[Label(3)]),
DatasetItem(id='image_4',
subset='test_batch',
image=np.ones((32, 32, 3)),
annotations=[Label(2)]),
DatasetItem(id='image_5',
subset='test_batch',
image=np.array([[[1, 2, 3], [4, 5, 6]],
[[1, 2, 3], [4, 5, 6]]]),
annotations=[Label(3)])
],
categories=['airplane', 'automobile', 'bird', 'cat'])
dataset = Dataset.import_from(DUMMY_10_DATASET_DIR, 'cifar')
compare_datasets(self, expected_dataset, dataset, require_images=True)
def test_can_save_and_load_with_meta_file(self):
source_dataset = Dataset.from_iterable(
[
DatasetItem(id='image_2',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(0)]),
DatasetItem(
id='image_3', subset='test', image=np.ones((32, 32, 3))),
DatasetItem(id='image_4',
subset='test',
image=np.ones((32, 32, 3)),
annotations=[Label(1)])
],
categories=['label_0', 'label_1'])
with TestDir() as test_dir:
CifarConverter.convert(source_dataset,
test_dir,
save_images=True,
save_dataset_meta=True)
parsed_dataset = Dataset.import_from(test_dir, 'cifar')
self.assertTrue(osp.isfile(osp.join(test_dir,
'dataset_meta.json')))
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def setUp(self):
self.source = Dataset.from_iterable([
DatasetItem(id='1', subset='val',
attributes={'qq': 1, 'x': 2},
annotations=[ Label(0, attributes={ 'x': 1, 'y': 2 }) ]),
DatasetItem(id='2', subset='val',
attributes={'qq': 2},
annotations=[ Label(0, attributes={ 'x': 1, 'y': 2 }) ]),
], categories=['a'])
def test_remap_labels(self):
src_dataset = Dataset.from_iterable([
DatasetItem(id=1, annotations=[
# Should be remapped
Label(1),
Bbox(1, 2, 3, 4, label=2),
Mask(image=np.array([1]), label=3),
# Should be deleted
Polygon([1, 1, 2, 2, 3, 4], label=4),
# Should be kept
PolyLine([1, 3, 4, 2, 5, 6]),
Bbox(4, 3, 2, 1, label=5),
])
], categories={
AnnotationType.label: LabelCategories.from_iterable(
f'label{i}' for i in range(6)),
AnnotationType.mask: MaskCategories(
colormap=mask_tools.generate_colormap(6)),
AnnotationType.points: PointsCategories.from_iterable(
[(i, [str(i)]) for i in range(6)])
})
dst_dataset = Dataset.from_iterable([
DatasetItem(id=1, annotations=[
Label(1),
Bbox(1, 2, 3, 4, label=0),
Mask(image=np.array([1]), label=1),
PolyLine([1, 3, 4, 2, 5, 6], label=None),
Bbox(4, 3, 2, 1, label=2),
]),
], categories={
AnnotationType.label: LabelCategories.from_iterable(
['label0', 'label9', 'label5']),
AnnotationType.mask: MaskCategories(colormap={
i: v for i, v in enumerate({
k: v for k, v in mask_tools.generate_colormap(6).items()
if k in { 0, 1, 5 }
}.values())
}),
AnnotationType.points: PointsCategories.from_iterable(
[(0, ['0']), (1, ['1']), (2, ['5'])])
})
actual = transforms.RemapLabels(src_dataset, mapping={
'label1': 'label9', # rename & join with new label9 (from label3)
'label2': 'label0', # rename & join with existing label0
'label3': 'label9', # rename & join with new label9 (from label1)
'label4': '', # delete the label and associated annotations
# 'label5' - unchanged
}, default='keep')
compare_datasets(self, dst_dataset, actual)
def test_can_save_and_load_with_meta_file(self):
source_dataset = Dataset.from_iterable([
DatasetItem(
id='name0_0001',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(0,
attributes={'positive_pairs': ['name0/name0_0002']})
]),
DatasetItem(id='name0_0002',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(0,
attributes={
'positive_pairs': ['name0/name0_0001'],
'negative_pairs': ['name1/name1_0001']
})
]),
DatasetItem(
id='name1_0001',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(1,
attributes={'positive_pairs': ['name1/name1_0002']})
]),
DatasetItem(id='name1_0002',
subset='test',
image=np.ones((2, 5, 3)),
annotations=[
Label(1,
attributes={
'positive_pairs': ['name1/name1_0002'],
'negative_pairs': ['name0/name0_0001']
})
]),
],
categories=['name0', 'name1'])
with TestDir() as test_dir:
LfwConverter.convert(source_dataset,
test_dir,
save_images=True,
save_dataset_meta=True)
parsed_dataset = Dataset.import_from(test_dir, 'lfw')
self.assertTrue(osp.isfile(osp.join(test_dir,
'dataset_meta.json')))
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def __iter__(self):
return iter([
DatasetItem(id='a/0', subset='a', annotations=[
Label(1),
Label(2),
Label(3),
]),
DatasetItem(id=1, subset='b', annotations=[
Label(4),
]),
])
def test_can_remove_all_attrs_by_item_id(self):
expected = Dataset.from_iterable([
DatasetItem(id='1', subset='val',
annotations=[ Label(0) ]),
DatasetItem(id='2', subset='val',
attributes={'qq': 2},
annotations=[ Label(0, attributes={ 'x': 1, 'y': 2 }) ]),
], categories=['a'])
actual = transforms.RemoveAttributes(self.source, ids=[('1', 'val')])
compare_datasets(self, expected, actual)
def test_can_remove_annotations_in_dataset(self):
expected = Dataset.from_iterable([
DatasetItem(id='1', subset='test'),
DatasetItem(id='2', subset='test'),
], categories=['a', 'b'])
dataset = Dataset.from_iterable([
DatasetItem(id='1', subset='test', annotations=[Label(0)]),
DatasetItem(id='2', subset='test', annotations=[Label(1)]),
], categories=['a', 'b'])
actual = transforms.RemoveAnnotations(dataset)
compare_datasets(self, expected, actual)
def test_can_match_items(self):
# items 1 and 3 are unique, item 2 is common and should be merged
source0 = Dataset.from_iterable([
DatasetItem(1, annotations=[ Label(0), ]),
DatasetItem(2, annotations=[ Label(0), ]),
], categories=['a', 'b'])
source1 = Dataset.from_iterable([
DatasetItem(2, annotations=[ Label(1), ]),
DatasetItem(3, annotations=[ Label(0), ]),
], categories=['a', 'b'])
source2 = Dataset.from_iterable([
DatasetItem(2, annotations=[ Label(0), Bbox(1, 2, 3, 4) ]),
], categories=['a', 'b'])
expected = Dataset.from_iterable([
DatasetItem(1, annotations=[
Label(0, attributes={'score': 1/3}),
]),
DatasetItem(2, annotations=[
Label(0, attributes={'score': 2/3}),
Label(1, attributes={'score': 1/3}),
Bbox(1, 2, 3, 4, attributes={'score': 1.0}),
]),
DatasetItem(3, annotations=[
Label(0, attributes={'score': 1/3}),
]),
], categories=['a', 'b'])
merger = IntersectMerge()
merged = merger([source0, source1, source2])
compare_datasets(self, expected, merged)
self.assertEqual(
[
NoMatchingItemError(item_id=('1', DEFAULT_SUBSET_NAME),
sources={1, 2}),
NoMatchingItemError(item_id=('3', DEFAULT_SUBSET_NAME),
sources={0, 2}),
],
sorted((e for e in merger.errors
if isinstance(e, NoMatchingItemError)),
key=lambda e: e.item_id)
)
self.assertEqual(
[
NoMatchingAnnError(item_id=('2', DEFAULT_SUBSET_NAME),
sources={0, 1}, ann=source2.get('2').annotations[1]),
],
sorted((e for e in merger.errors
if isinstance(e, NoMatchingAnnError)),
key=lambda e: e.item_id)
)
def test_can_merge_categories(self):
source0 = Dataset.from_iterable([
DatasetItem(1, annotations=[ Label(0), ]),
], categories={
AnnotationType.label: LabelCategories.from_iterable(['a', 'b']),
AnnotationType.points: PointsCategories.from_iterable([
(0, ['l0', 'l1']),
(1, ['l2', 'l3']),
]),
AnnotationType.mask: MaskCategories({
0: (0, 1, 2),
1: (1, 2, 3),
}),
})
source1 = Dataset.from_iterable([
DatasetItem(1, annotations=[ Label(0), ]),
], categories={
AnnotationType.label: LabelCategories.from_iterable(['c', 'b']),
AnnotationType.points: PointsCategories.from_iterable([
(0, []),
(1, ['l2', 'l3']),
]),
AnnotationType.mask: MaskCategories({
0: (0, 2, 4),
1: (1, 2, 3),
}),
})
expected = Dataset.from_iterable([
DatasetItem(1, annotations=[ Label(0), Label(2), ]),
], categories={
AnnotationType.label: LabelCategories.from_iterable(['a', 'b', 'c']),
AnnotationType.points: PointsCategories.from_iterable([
(0, ['l0', 'l1']),
(1, ['l2', 'l3']),
(2, []),
]),
AnnotationType.mask: MaskCategories({
0: (0, 1, 2),
1: (1, 2, 3),
2: (0, 2, 4),
}),
})
merger = IntersectMerge()
merged = merger([source0, source1])
compare_datasets(self, expected, merged, ignored_attrs={'score'})
def test_can_save_and_load_without_saving_images(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='a', subset='train_1', annotations=[Label(0)]),
DatasetItem(id='b', subset='train_first', annotations=[Label(1)]),
],
categories=['x', 'y'])
with TestDir() as test_dir:
CifarConverter.convert(source_dataset, test_dir, save_images=False)
parsed_dataset = Dataset.import_from(test_dir, 'cifar')
compare_datasets(self,
source_dataset,
parsed_dataset,
require_images=True)
def transform_item(self, item):
annotations = list(item.annotations)
attributes = item.attributes
if item.attributes:
annotations.append(Label(self._label, attributes=item.attributes))
attributes = {}
return item.wrap(annotations=annotations, attributes=attributes)
def test_can_save_and_load_voc_classification_dataset(self):
expected_dataset = Dataset.from_iterable([
DatasetItem(id='2007_000001', subset='train',
image=np.ones((10, 20, 3)),
annotations=[Label(i) for i in range(22) if i % 2 == 1]),
DatasetItem(id='2007_000002', subset='test',
image=np.ones((10, 20, 3))),
], categories=VOC.make_voc_categories())
dataset_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1')
rpath = osp.join('ImageSets', 'Main', 'train.txt')
matrix = [
('voc_classification', '', ''),
('voc_classification', 'train', rpath),
]
for format, subset, path in matrix:
with self.subTest(format=format, subset=subset, path=path):
if subset:
expected = expected_dataset.get_subset(subset)
else:
expected = expected_dataset
with TestDir() as test_dir:
self._test_can_save_and_load(test_dir, dataset_dir,
expected, format, result_path=path, label_map='voc')
def test_can_sample_with_selective_count(self):
source = Dataset.from_iterable([
DatasetItem(i, subset=s, annotations=[Label(l)])
for i, (s, l,
_) in enumerate(product(['a', 'b'], [0, 1, 2], [0, 1, 2]))
],
categories=['a', 'b', 'c'])
actual = LabelRandomSampler(source,
count=2,
label_counts={
'a': 0,
'b': 1
})
counts_a = Counter(a.label for item in actual.get_subset('a')
for a in item.annotations)
counts_b = Counter(a.label for item in actual.get_subset('b')
for a in item.annotations)
self.assertEqual(
counts_a, {
actual.categories()[AnnotationType.label].find('b')[0]: 1,
actual.categories()[AnnotationType.label].find('c')[0]: 2
})
self.assertEqual(
counts_b, {
actual.categories()[AnnotationType.label].find('b')[0]: 1,
actual.categories()[AnnotationType.label].find('c')[0]: 2
})
def test_convert_from_voc_format(self):
"""
<b>Description:</b>
Ensure that the dataset can be converted from VOC format with
command `datum convert`.
<b>Expected results:</b>
A ImageNet dataset that matches the expected dataset.
<b>Steps:</b>
1. Get path to the source dataset from assets.
2. Convert source dataset to LabelMe format, using the `convert` command.
3. Verify that resulting dataset is equal to the expected dataset.
"""
labels = sorted([l.name for l in VOC.VocLabel if l.value % 2 == 1])
expected_dataset = Dataset.from_iterable([
DatasetItem(id='/'.join([label, '2007_000001']),
subset='default', annotations=[Label(i)])
for i, label in enumerate(labels)
] + [DatasetItem(id='no_label/2007_000002', subset='default',
image=np.ones((10, 20, 3)))
], categories=labels
)
voc_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1')
with TestDir() as test_dir:
imagenet_dir = osp.join(test_dir, 'imagenet')
run(self, 'convert', '-if', 'voc', '-i', voc_dir,
'-f', 'imagenet', '-o', imagenet_dir, '--', '--save-image')
target_dataset = Dataset.import_from(imagenet_dir, format='imagenet')
compare_datasets(self, expected_dataset, target_dataset,
require_images=True)
def test_can_extract_mnist(self):
with mock_tfds_data():
tfds_ds, tfds_info = tfds.load('mnist',
split='train',
with_info=True)
tfds_example = next(iter(tfds_ds))
expected_dataset = Dataset.from_iterable(
[
DatasetItem(
id='0',
subset='train',
image=tfds_example['image'].numpy().squeeze(axis=2),
annotations=[Label(tfds_example['label'].numpy())],
),
],
categories=tfds_info.features['label'].names)
extractor = make_tfds_extractor('mnist')
actual_dataset = Dataset(extractor)
compare_datasets(self,
expected_dataset,
actual_dataset,
require_images=True)
def test_can_save_and_load_with_no_save_images(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='1',
subset='train',
image=np.ones((8, 8, 3)),
annotations=[
Bbox(0, 2, 4, 2, label=1),
Bbox(0,
1,
2,
3,
label=0,
attributes={
'blur': '2',
'expression': '0',
'illumination': '0',
'occluded': '0',
'pose': '2',
'invalid': '0'
}),
Label(1),
])
],
categories=['face', 'label_0'])
with TestDir() as test_dir:
WiderFaceConverter.convert(source_dataset,
test_dir,
save_images=False)
parsed_dataset = Dataset.import_from(test_dir, 'wider_face')
compare_datasets(self, source_dataset, parsed_dataset)
def _load_labels(self, items_by_id):
label_categories = self._categories[AnnotationType.label]
# TODO: implement reading of machine-annotated labels
for label_path in self._glob_annotations(
'*' + OpenImagesPath.LABEL_DESCRIPTION_FILE_SUFFIX
):
with self._open_csv_annotation(label_path) as label_reader:
for label_description in label_reader:
image_id = label_description['ImageID']
item = items_by_id.get(image_id)
if item is None:
item = items_by_id.setdefault(
image_id, self._add_item(image_id,
self._get_subset_name(label_path))
)
confidence = float(label_description['Confidence'])
label_name = label_description['LabelName']
label_index, _ = label_categories.find(label_name)
if label_index is None:
raise UndefinedLabel(
item_id=item.id, subset=item.subset,
label_name=label_name, severity=Severity.error)
item.annotations.append(Label(
label=label_index, attributes={'score': confidence}))
