def __init__(self, url):
super().__init__()
assert osp.isdir(url)
items = []
for name in os.listdir(url):
path = osp.join(url, name)
if self._is_image(path):
item_id = osp.splitext(name)[0]
item = DatasetItem(id=item_id, image=path)
items.append((item.id, item))
items = sorted(items, key=lambda e: e[0])
items = OrderedDict(items)
self._items = items
self._subsets = None
def __iter__(self):
yield DatasetItem(
id=1,
annotations=[
# drop non voc label
Bbox(2,
3,
4,
5,
label=self._label('cat'),
id=1,
group=1,
attributes={
'truncated': False,
'difficult': False,
'occluded': False,
}),
])
def test_can_split_and_load(self, fxt_sample_video):
test_dir = scope_add(TestDir())
on_exit_do(MediaManager.get_instance().clear)
expected = Dataset.from_iterable([
DatasetItem('frame_%06d' % i, image=np.ones((4, 6, 3)) * i)
for i in range(4)
])
dataset = Dataset.import_from(fxt_sample_video,
'video_frames',
start_frame=0,
end_frame=4,
name_pattern='frame_%06d')
dataset.export(format='image_dir', save_dir=test_dir, image_ext='.jpg')
actual = Dataset.import_from(test_dir, 'image_dir')
compare_datasets(TestCase(), expected, actual)
def test_can_detect_and_import(self):
env = Environment()
env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]}
env.extractors.items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]}
source_dataset = Dataset.from_iterable([
DatasetItem(id=1, annotations=[Label(2)]),
],
categories=['a', 'b', 'c'])
with TestDir() as test_dir:
source_dataset.save(test_dir)
imported_dataset = Dataset.import_from(test_dir, env=env)
self.assertEqual(imported_dataset.data_path, test_dir)
self.assertEqual(imported_dataset.format, DEFAULT_FORMAT)
compare_datasets(self, source_dataset, imported_dataset)
def test_can_do_lazy_get_on_updated_item(self):
iter_called = False
class TestExtractor(Extractor):
def __iter__(self):
nonlocal iter_called
iter_called = True
return iter([
DatasetItem(1),
DatasetItem(2),
])
dataset = Dataset.from_extractors(TestExtractor())
dataset.put(DatasetItem(2))
self.assertTrue((2, '') in dataset)
self.assertFalse(iter_called)
def test_can_save_dataset_with_no_subsets(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='0001_c2s3_000001_00',
image=np.ones((2, 5, 3)),
attributes={
'camera_id': 1,
'person_id': 1,
'query': True
}),
])
with TestDir() as test_dir:
Market1501Converter.convert(source_dataset,
test_dir,
save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'market1501')
compare_datasets(self, source_dataset, parsed_dataset)
def __iter__(self):
return iter([
DatasetItem(id=1,
image=np.zeros((5, 5, 3)),
annotations=[
Mask(np.array(
[[0, 0, 1, 1, 1], [0, 0, 1, 1, 1],
[1, 1, 0, 1, 1], [1, 1, 1, 0, 0],
[1, 1, 1, 0, 0]], ),
label=2,
id=1,
z_order=0),
Polygon([1, 1, 4, 1, 4, 4, 1, 4],
label=1,
id=2,
z_order=1),
]),
])
def _get(self, item_id, subset_name):
subset = self._subsets[subset_name]
item = subset.items[item_id]
if isinstance(item, str):
image_size = self._image_info.get(item_id)
image = Image(path=osp.join(self._path, item), size=image_size)
anno_path = osp.splitext(image.path)[0] + '.txt'
annotations = self._parse_annotations(anno_path, image)
item = DatasetItem(id=item_id,
subset=subset_name,
image=image,
annotations=annotations)
subset.items[item_id] = item
return item
def test_can_save_dataset_with_no_subsets(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='a/b/c', image=np.zeros((8, 4, 3)),
annotations=[Label(1)]
),
], categories={
AnnotationType.label: LabelCategories.from_iterable(
'label_' + str(label) for label in range(10)),
})
with TestDir() as test_dir:
ImagenetTxtConverter.convert(source_dataset, test_dir,
save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'imagenet_txt')
compare_datasets(self, source_dataset, parsed_dataset,
require_images=True)
def __iter__(self):
return iter([
DatasetItem(
id=1,
image=np.zeros((5, 10, 3)),
annotations=[
Polygon([3.0, 2.5, 1.0, 0.0, 3.5, 0.0, 3.0, 2.5],
label=3,
id=4,
group=4,
attributes={'is_crowd': False}),
Polygon([5.0, 3.5, 4.5, 0.0, 8.0, 0.0, 5.0, 3.5],
label=3,
id=4,
group=4,
attributes={'is_crowd': False}),
]),
])
def test_can_save_and_load_bboxes_with_no_save_images(self):
expected_dataset = Dataset.from_iterable([
DatasetItem(id=3,
subset='train',
image=np.ones((10, 15, 3)),
annotations=[
Polygon([2, 2, 8, 3, 7, 10, 2, 9],
attributes={'text': 'word_2'}),
Bbox(0, 2, 5, 9, attributes={'text': 'word_3'}),
]),
])
with TestDir() as test_dir:
self._test_save_and_load(
expected_dataset,
partial(IcdarTextLocalizationConverter.convert,
save_images=False), test_dir,
'icdar_text_localization')
def test_background_masks_dont_introduce_instances_but_cover_others(self):
dataset = Dataset.from_iterable([
DatasetItem(1, image=np.zeros((4, 1, 1)), annotations=[
Mask([1, 1, 1, 1], label=1, attributes={'z_order': 1}),
Mask([0, 0, 1, 1], label=2, attributes={'z_order': 2}),
Mask([0, 0, 1, 1], label=0, attributes={'z_order': 3}),
])
], categories=['background', 'a', 'b'])
with TestDir() as test_dir:
VocConverter.convert(dataset, test_dir, apply_colormap=False)
cls_mask = load_mask(
osp.join(test_dir, 'SegmentationClass', '1.png'))
inst_mask = load_mask(
osp.join(test_dir, 'SegmentationObject', '1.png'))
self.assertTrue(np.array_equal([0, 1], np.unique(cls_mask)))
self.assertTrue(np.array_equal([0, 1], np.unique(inst_mask)))
def patch(cls, dataset, patch, save_dir, **options):
converter = cls(dataset, save_dir, **options)
annotation_writer = _AnnotationWriter(save_dir)
converter._save(annotation_writer)
annotation_writer.remove_unwritten()
images_dir = osp.join(save_dir, OpenImagesPath.IMAGES_DIR)
for (item_id, subset), status in patch.updated_items.items():
if status != ItemStatus.removed:
continue
item = DatasetItem(item_id, subset=subset)
image_path = osp.join(images_dir,
converter._make_image_filename(item, subdir=subset))
if osp.isfile(image_path):
os.unlink(image_path)
def test_can_save_and_load_images(self):
expected_dataset = Dataset.from_iterable([
DatasetItem(id=1, subset='train', attributes={'id': 1}),
DatasetItem(id=2, subset='train', attributes={'id': 2}),
DatasetItem(id=2, subset='val', attributes={'id': 2}),
DatasetItem(id=3, subset='val', attributes={'id': 3}),
DatasetItem(id=4, subset='val', attributes={'id': 4}),
DatasetItem(id=5, subset='test', attributes={'id': 1}),
])
with TestDir() as test_dir:
self._test_save_and_load(expected_dataset,
CocoImageInfoConverter.convert, test_dir)
def test_can_import_points(self):
expected_dataset = Dataset.from_iterable([
DatasetItem(id=1, subset='train',
image=Image(path='1.jpg', size=(5, 5)),
annotations=[
Points([0, 0, 0, 2, 4, 1], [0, 1, 2],
label=1, group=1, id=1,
attributes={'is_crowd': False}),
Polygon([0, 0, 4, 0, 4, 4],
label=1, group=1, id=1,
attributes={'is_crowd': False}),
Points([1, 2, 3, 4, 2, 3],
group=2, id=2,
attributes={'is_crowd': False}),
Bbox(1, 2, 2, 2,
group=2, id=2,
attributes={'is_crowd': False}),
Points([1, 2, 0, 2, 4, 1],
label=0, group=3, id=3,
attributes={'is_crowd': False}),
Bbox(0, 1, 4, 1,
label=0, group=3, id=3,
attributes={'is_crowd': False}),
Points([0, 0, 1, 2, 3, 4], [0, 1, 2],
group=5, id=5,
attributes={'is_crowd': False}),
Bbox(1, 2, 2, 2,
group=5, id=5,
attributes={'is_crowd': False}),
], attributes={'id': 1}),
], categories={
AnnotationType.label: LabelCategories.from_iterable(['a', 'b']),
AnnotationType.points: PointsCategories.from_iterable(
(i, None, [[0, 1], [1, 2]]) for i in range(2)
),
})
dataset = Dataset.import_from(
osp.join(DUMMY_DATASET_DIR, 'coco_person_keypoints'), 'coco')
compare_datasets(self, expected_dataset, dataset)
def test_dataset(self):
label_categories = LabelCategories()
for i in range(5):
label_categories.add('cat' + str(i))
mask_categories = MaskCategories(
generate_colormap(len(label_categories.items)))
points_categories = PointsCategories()
for index, _ in enumerate(label_categories.items):
points_categories.add(index, ['cat1', 'cat2'], joints=[[0, 1]])
return Dataset.from_iterable([
DatasetItem(id=100, subset='train', image=np.ones((10, 6, 3)),
annotations=[
Caption('hello', id=1),
Caption('world', id=2, group=5),
Label(2, id=3, attributes={
'x': 1,
'y': '2',
}),
Bbox(1, 2, 3, 4, label=4, id=4, z_order=1, attributes={
'score': 1.0,
}),
Bbox(5, 6, 7, 8, id=5, group=5),
Points([1, 2, 2, 0, 1, 1], label=0, id=5, z_order=4),
Mask(label=3, id=5, z_order=2, image=np.ones((2, 3))),
]),
DatasetItem(id=21, subset='train',
annotations=[
Caption('test'),
Label(2),
Bbox(1, 2, 3, 4, label=5, id=42, group=42)
]),
DatasetItem(id=2, subset='val',
annotations=[
PolyLine([1, 2, 3, 4, 5, 6, 7, 8], id=11, z_order=1),
Polygon([1, 2, 3, 4, 5, 6, 7, 8], id=12, z_order=4),
]),
DatasetItem(id=42, subset='test',
attributes={'a1': 5, 'a2': '42'}),
DatasetItem(id=42),
DatasetItem(id=43, image=Image(path='1/b/c.qq', size=(2, 4))),
], categories={
AnnotationType.label: label_categories,
AnnotationType.mask: mask_categories,
AnnotationType.points: points_categories,
})
def test_no_label_diff_with_same_item(self):
detections = 3
anns = [
Label(i, attributes={'score': (1.0 + i) / detections}) \
for i in range(detections)
]
item = DatasetItem(id=1, annotations=anns)
conf_thresh = 0.5
comp = Comparator(conf_threshold=conf_thresh)
result = comp.compare_item_labels(item, item)
matches, a_greater, b_greater = result
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(len([it for it in item.annotations \
if conf_thresh < it.attributes['score']]),
len(matches))
def __iter__(self):
return iter([
DatasetItem(id='a/b/1', subset='a', annotations=[
Bbox(2, 3, 4, 5, label=2,
attributes={
'truncated': True,
VOC.VocAction(1).name: True,
VOC.VocAction(2).name: True,
}
),
Bbox(5, 4, 3, 2, label=self._label('person'),
attributes={
'truncated': True,
VOC.VocAction(1).name: True,
VOC.VocAction(2).name: True,
}
),
]),
])
def _load_items(self, parsed):
for frame_id, item_desc in parsed.items():
filename = item_desc.get('name')
if filename:
filename = self._find_image(filename)
if not filename:
filename = item_desc.get('name')
image_size = (item_desc.get('height'), item_desc.get('width'))
if all(image_size):
image_size = (int(image_size[0]), int(image_size[1]))
else:
image_size = None
image = None
if filename:
image = Image(path=filename, size=image_size)
parsed[frame_id] = DatasetItem(id=frame_id, subset=self._subset,
image=image, annotations=item_desc.get('annotations'))
return parsed
def __iter__(self):
return iter([
DatasetItem(id=1,
image=np.ones((10, 5, 3)),
subset='val',
annotations=[
Polygon([0, 0, 1, 0, 1, 2, 0, 2],
label=0,
id=1,
group=1,
attributes={'is_crowd': False}),
Mask(np.array([[1, 0, 0, 1, 0]] * 5 +
[[1, 1, 1, 1, 0]] * 5),
label=0,
id=2,
group=2,
attributes={'is_crowd': True}),
]),
])
def _generate_classification_dataset(self,
config,
subset=None,
empty_scores=False,
out_range=False,
no_attr=False,
no_img=False):
probs = self._get_probs(out_range)
if subset is None:
self.subset = ["train", "val", "test"]
else:
self.subset = subset
iterable = []
label_cat = LabelCategories()
idx = 0
for label_id, label in enumerate(config.keys()):
num_item = config[label]
label_cat.add(label, attributes=None)
for _ in range(num_item):
scores = probs[idx]
idx += 1
if empty_scores:
scores = []
attr = {"scores": scores}
if no_attr:
attr = {}
img = Image(path=f"test/dataset/{idx}.jpg", size=(90, 90))
if no_img:
img = None
iterable.append(
DatasetItem(
idx,
subset=self.subset[idx % len(self.subset)],
annotations=[Label(
label_id,
attributes=attr,
)],
image=img,
))
categories = {AnnotationType.label: label_cat}
dataset = Dataset.from_iterable(iterable, categories)
return dataset
def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self):
source = Dataset.from_iterable([
DatasetItem(id='кириллица с пробелом',
subset='a',
image=np.ones((5, 1)),
annotations=[
Mask(np.array([[1, 0, 0, 0, 0]]),
label=0,
attributes={'track_id': 2}),
]),
],
categories=['a'])
with TestDir() as test_dir:
self._test_save_and_load(source,
partial(MotsPngConverter.convert,
save_images=True),
test_dir,
require_images=True)
def __iter__(self):
return iter([
DatasetItem(id=1, image=np.zeros((6, 10, 3)),
annotations=[
Mask(np.array([
[0, 1, 1, 1, 0, 1, 1, 1, 1, 0],
[0, 0, 1, 1, 0, 1, 1, 1, 0, 0],
[0, 0, 0, 1, 0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
# only internal fragment (without the border),
# but not everywhere...
),
attributes={ 'is_crowd': True },
label=3, id=4, group=4),
]
),
])
def test_mean_std(self):
expected_mean = [100, 50, 150]
expected_std = [20, 50, 10]
dataset = Dataset.from_iterable([
DatasetItem(id=i, image=np.random.normal(
expected_mean, expected_std, size=(h, w, 3))
)
for i, (w, h) in enumerate([
(3000, 100), (800, 600), (400, 200), (700, 300)
])
])
actual_mean, actual_std = mean_std(dataset)
for em, am in zip(expected_mean, actual_mean):
self.assertAlmostEqual(em, am, places=0)
for estd, astd in zip(expected_std, actual_std):
self.assertAlmostEqual(estd, astd, places=0)
def __iter__(self):
return iter([
DatasetItem(
id='2007_000001',
subset='train',
annotations=[
Bbox(
1,
2,
2,
2,
label=self._label('cat'),
attributes={
'pose': VOC.VocPose(1).name,
'truncated': True,
'difficult': False,
'occluded': False,
},
id=1,
group=1,
),
Bbox(
4,
5,
2,
2,
label=self._label('person'),
attributes={
'truncated': False,
'difficult': False,
'occluded': False,
**{
a.name: a.value % 2 == 1
for a in VOC.VocAction
}
},
id=2,
group=2,
# TODO: Actions and group should be excluded
# as soon as correct merge is implemented
),
]),
])
def test_random_split_gives_error_on_wrong_ratios(self):
source_dataset = Dataset.from_iterable([DatasetItem(id=1)])
with self.assertRaises(Exception):
transforms.RandomSplit(source_dataset,
splits=[
('train', 0.5),
('test', 0.7),
])
with self.assertRaises(Exception):
transforms.RandomSplit(source_dataset, splits=[])
with self.assertRaises(Exception):
transforms.RandomSplit(source_dataset,
splits=[
('train', -0.5),
('test', 1.5),
])
def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self):
source_dataset = Dataset.from_iterable([
DatasetItem(id='кириллица с пробелом', subset='train',
image=np.ones((8, 8, 3)),
annotations=[
Bbox(0, 2, 4, 2, label=2),
Bbox(0, 1, 2, 3, label=4),
]),
], categories={
AnnotationType.label: LabelCategories.from_iterable(
'label_' + str(i) for i in range(10)),
})
with TestDir() as test_dir:
YoloConverter.convert(source_dataset, test_dir, save_images=True)
parsed_dataset = Dataset.import_from(test_dir, 'yolo')
compare_datasets(self, source_dataset, parsed_dataset,
require_images=True)
def __iter__(self):
return iter([
DatasetItem(
id='a/b/1',
subset='a',
annotations=[
Bbox(
2,
3,
4,
5,
label=2,
id=1,
group=1,
attributes={
'truncated': True,
'difficult': False,
'occluded': False,
# no attributes here in the label categories
}),
Bbox(
5,
4,
3,
2,
label=self._label('person'),
id=2,
group=2,
attributes={
'truncated': True,
'difficult': False,
'occluded': False,
VOC.VocAction(1).name: True,
VOC.VocAction(2).name: True,
**{
a.name: False
for a in VOC.VocAction if a.value not in {
1, 2
}
}
}),
]),
])
def __iter__(self):
return iter([
DatasetItem(
id=1,
subset='a',
annotations=[
# overlapping masks, the first should be truncated
# the second and third are different instances
Mask(image=np.array([[0, 0, 0, 1, 0]]),
label=3,
z_order=3),
Mask(image=np.array([[0, 1, 1, 1, 0]]),
label=4,
z_order=1),
Mask(image=np.array([[1, 1, 0, 0, 0]]),
label=3,
z_order=2),
]),
])
def _load_items(self, path):
items = {}
labels = self._categories[AnnotationType.label]._indices
labels = { labels[label_name]: label_name
for label_name in labels }
with open(path, encoding='utf-8') as f:
for line in f:
line = line.strip()
objects = line.split('\"')
if 1 < len(objects):
if len(objects) == 5:
objects[0] = objects[1]
objects[1] = objects[3]
else:
raise Exception("Line %s: unexpected number "
"of quotes in filename" % line)
else:
objects = line.split()
image = objects[0]
item_id = osp.splitext(osp.join(*image.split('/')[2:]))[0]
image_path = osp.join(self._dataset_dir, image.lstrip('/'))
item_annotations = []
if 1 < len(objects):
gt = objects[1]
gt_path = osp.join(self._dataset_dir, gt.lstrip('/'))
mask = lazy_mask(gt_path,
self._categories[AnnotationType.mask].inverse_colormap)
mask = mask() # loading mask through cache
classes = np.unique(mask)
for label_id in classes:
if labels[label_id] in self._labels:
image = self._lazy_extract_mask(mask, label_id)
item_annotations.append(
Mask(image=image, label=label_id))
items[item_id] = DatasetItem(id=item_id, subset=self._subset,
image=image_path, annotations=item_annotations)
return items
