def test_open_images_v4_object_detection():
batch_size = 1
image_size = [256, 256]
dataset = OpenImagesV4BoundingBox(
batch_size=batch_size, pre_processor=ResizeWithGtBoxes(image_size))
dataset = DatasetIterator(dataset)
num_max_boxes = dataset.num_max_boxes
assert dataset.num_max_boxes == OpenImagesV4BoundingBox.count_max_boxes()
for _ in range(5):
images, labels = dataset.feed()
# _show_images_with_boxes(images, labels)
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == num_max_boxes
assert labels.shape[2] == 5
def test_custom_has_validation_open_images_v4_object_detection():
batch_size = 8
image_size = [196, 128]
train_dataset = DummyHasValidation(
subset="train",
batch_size=batch_size,
pre_processor=ResizeWithGtBoxes(image_size))
train_dataset = DatasetIterator(train_dataset)
validation_dataset = DummyHasValidation(
subset="validation",
batch_size=batch_size,
pre_processor=ResizeWithGtBoxes(image_size))
validation_dataset = DatasetIterator(validation_dataset)
num_max_boxes = validation_dataset.num_max_boxes
assert validation_dataset.num_max_boxes == DummyHasValidation.count_max_boxes(
)
assert train_dataset.num_per_epoch == 10
assert validation_dataset.num_per_epoch == 16
assert len(train_dataset.classes) == 44
assert len(validation_dataset.classes) == 44
for _ in range(3):
images, labels = train_dataset.feed()
# _show_images_with_boxes(images, labels)
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == num_max_boxes
assert labels.shape[2] == 5
for _ in range(3):
images, labels = validation_dataset.feed()
# _show_images_with_boxes(images, labels)
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == num_max_boxes
assert labels.shape[2] == 5
def test_filp_top_bottom():
batch_size = 3
image_size = [256, 512]
dataset = Pascalvoc2007(
batch_size=batch_size, pre_processor=ResizeWithGtBoxes(image_size),
augmentor=FlipTopBottom(),
)
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
_show_images_with_boxes(images, labels)
def test_hue():
batch_size = 3
image_size = [256, 512]
dataset = Pascalvoc2007(
batch_size=batch_size, pre_processor=ResizeWithGtBoxes(image_size),
augmentor=Hue((-10, 10)),
)
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
_show_images_with_boxes(images, labels)
def test_ytfaces_facial_landmarks_detection():
batch_size = 1
image_size = [256, 320]
stride = 2
pre_processor = Sequence([
ResizeWithJoints(image_size=image_size),
JointsToGaussianHeatmap(image_size=image_size,
num_joints=68,
stride=stride)
])
dataset = YoutubeFacialLandmarks(subset="train",
batch_size=batch_size,
pre_processor=pre_processor)
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == image_size[0] // stride
assert labels.shape[2] == image_size[1] // stride
assert labels.shape[3] == 68
dataset = YoutubeFacialLandmarks(subset="validation",
batch_size=batch_size,
pre_processor=pre_processor)
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == image_size[0] // stride
assert labels.shape[2] == image_size[1] // stride
assert labels.shape[3] == 68
def test_camvid_custom():
batch_size = 1
train_dataset = DummyCamvidCustom(subset="train", batch_size=batch_size)
assert train_dataset.ignore_class_idx == 3
train_dataset = DatasetIterator(train_dataset)
test_dataset = DummyCamvidCustom(subset="validation",
batch_size=batch_size)
test_dataset = DatasetIterator(test_dataset)
assert train_dataset.num_classes == 10
colors = train_dataset.label_colors
assert len(colors) == 10
_test_camvid_basic(train_dataset, test_dataset)
def test_dataset_iterator_batch_order():
"""Assert that data given by iterator is same whether enabele_prefetch ture or false."""
batch_size = 8
dataset = Dummy(subset="train", batch_size=batch_size)
dataset_iterator = DatasetIterator(dataset, seed=10, enable_prefetch=False)
prefetch_dataset_iterator = DatasetIterator(dataset, seed=10, enable_prefetch=True)
for i in range(0, 30):
images, labels = next(dataset_iterator)
prefetch_images, prefetch_labels = next(prefetch_dataset_iterator)
assert np.all(images == prefetch_images)
assert np.all(labels == prefetch_labels)
def test_ssd_random_crop():
batch_size = 3
image_size = [256, 512]
dataset = Pascalvoc2007(
batch_size=batch_size, pre_processor=ResizeWithGtBoxes(image_size),
augmentor=SSDRandomCrop(),
)
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
_show_images_with_boxes(images, labels)
assert np.all(labels[:, :, 2] <= 512)
assert np.all(labels[:, :, 3] <= 256)
def test_camvid_custom_without_test_dataset():
batch_size = 5
validation_size = 0.2
train_dataset = DummyCamvidCustomWithoutTestDataset(
subset="train", batch_size=batch_size, validation_size=validation_size)
train_dataset = DatasetIterator(train_dataset)
test_dataset = DummyCamvidCustomWithoutTestDataset(
subset="validation",
batch_size=batch_size,
validation_size=validation_size)
test_dataset = DatasetIterator(test_dataset)
assert train_dataset.num_per_epoch == 5 * (1 - validation_size)
assert test_dataset.num_per_epoch == 5 * (validation_size)
def test_cityscapes():
batch_size = 1
train_dataset = Cityscapes(subset="train", batch_size=batch_size)
train_dataset = DatasetIterator(train_dataset)
test_dataset = Cityscapes(subset="validation", batch_size=batch_size)
test_dataset = DatasetIterator(test_dataset)
assert train_dataset.num_classes == 34
colors = train_dataset.label_colors
assert len(colors) == 34
train_image_files, train_label_files = train_dataset.feed()
assert train_image_files.shape[0] == batch_size
assert train_label_files.shape[0] == batch_size
def test_camvid():
batch_size = 1
train_dataset = DummyCamvid(subset="train", batch_size=batch_size)
assert train_dataset.ignore_class_idx is None
train_dataset = DatasetIterator(train_dataset)
test_dataset = DummyCamvid(subset="validation", batch_size=batch_size)
test_dataset = DatasetIterator(test_dataset)
assert train_dataset.num_classes == 11
colors = train_dataset.label_colors
assert len(colors) == 12
train_image_files, train_label_files = train_dataset.feed()
assert train_image_files.shape[0] == 1
assert train_label_files.shape[0] == 1
_test_camvid_basic(train_dataset, test_dataset)
def test_sequence():
batch_size = 3
image_size = [256, 512]
augmentor = Sequence([
FlipLeftRight(),
FlipTopBottom(),
SSDRandomCrop(),
])
dataset = Pascalvoc2007(
batch_size=batch_size, pre_processor=ResizeWithGtBoxes(image_size),
augmentor=augmentor,
)
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
_show_images_with_boxes(images, labels)
def test_dataset_iterator_batch_size():
batch_size = 8
dataset = Dummy(subset="train", batch_size=batch_size)
dataset_iterator = DatasetIterator(dataset)
for i in range(0, 10):
images, labels = next(dataset_iterator)
assert images.shape[0] == batch_size
assert labels.shape[0] == batch_size
batch_size = 32
dataset = Dummy(subset="train", batch_size=batch_size)
dataset_iterator = DatasetIterator(dataset)
for i in range(0, 10):
images, labels = next(dataset_iterator)
assert images.shape[0] == batch_size
assert labels.shape[0] == batch_size
def test_can_iterate(set_test_environment, subset):
batch_size = 1
image_size = (100, 100)
dataset = Div2k(subset, batch_size=batch_size)
iterator = DatasetIterator(dataset)
for _ in range(len(dataset)):
images, labels = iterator.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels[0] is None
def test_open_images_v4_classification():
batch_size = 1
image_size = [256, 256]
dataset = OpenImagesV4Classification(batch_size=batch_size,
pre_processor=Resize(image_size))
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == dataset.num_classes
def test_camvid_custom_without_ignore():
batch_size = 1
train_dataset = DummyCamvidCustomWithoutIgnoreClass(subset="train",
batch_size=batch_size)
assert train_dataset.ignore_class_idx is None
train_dataset = DatasetIterator(train_dataset)
assert train_dataset.num_classes == 12
colors = train_dataset.label_colors
assert len(colors) == 12
def setup_dataset(config, subset, seed):
DatasetClass = config.DATASET_CLASS
dataset_kwargs = {key.lower(): val for key, val in config.DATASET.items()}
# If there is a settings for TFDS, TFDS dataset class will be used.
tfds_kwargs = dataset_kwargs.pop("tfds_kwargs", {})
if tfds_kwargs:
if issubclass(DatasetClass, ObjectDetectionBase):
DatasetClass = TFDSObjectDetection
else:
DatasetClass = TFDSClassification
dataset = DatasetClass(subset=subset, **dataset_kwargs, **tfds_kwargs)
enable_prefetch = dataset_kwargs.pop("enable_prefetch", False)
return DatasetIterator(dataset, seed=seed, enable_prefetch=enable_prefetch)
def test_custom_open_images_v4_object_detection():
validation_size = 0.2
batch_size = 1
image_size = [256, 128]
train_dataset = Dummy(batch_size=batch_size,
validation_size=validation_size,
pre_processor=ResizeWithGtBoxes(image_size))
train_dataset = DatasetIterator(train_dataset)
validation_dataset = Dummy(batch_size=batch_size,
subset="validation",
validation_size=validation_size,
pre_processor=ResizeWithGtBoxes(image_size))
validation_dataset = DatasetIterator(validation_dataset)
num_max_boxes = train_dataset.num_max_boxes
assert train_dataset.num_max_boxes == Dummy.count_max_boxes()
assert train_dataset.num_per_epoch == 10 * (1 - validation_size)
assert validation_dataset.num_per_epoch == 10 * (validation_size)
for _ in range(13):
images, labels = train_dataset.feed()
# _show_images_with_boxes(images, labels)
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == num_max_boxes
assert labels.shape[2] == 5
def setup_dataset(config, subset, rank):
"""helper function from lmnet/train.py to setup the data iterator"""
dataset_class = config.DATASET_CLASS
dataset_kwargs = {key.lower(): val for key, val in config.DATASET.items()}
# If there is a settings for TFDS, TFDS dataset class will be used.
tfds_kwargs = dataset_kwargs.pop("tfds_kwargs", {})
if tfds_kwargs:
if issubclass(dataset_class, ObjectDetectionBase):
dataset_class = TFDSObjectDetection
else:
dataset_class = TFDSClassification
dataset = dataset_class(subset=subset, **dataset_kwargs, **tfds_kwargs)
# TODO (Neil): Enable both train and validation
# For some reasons processes are not terminated cleanly, enable prefetch ONLY for the train dataset.
enable_prefetch = dataset_kwargs.pop("enable_prefetch", False) if subset == 'train' else False
return DatasetIterator(dataset, seed=rank, enable_prefetch=enable_prefetch)
def test_bdd100k_seg():
batch_size = 1
train_dataset = DummyBDD100KSeg(subset="train", batch_size=batch_size)
train_iterator = DatasetIterator(train_dataset)
test_dataset = DummyBDD100KSeg(subset="validation", batch_size=batch_size)
test_iterator = DatasetIterator(test_dataset)
assert train_dataset.num_classes == 41
colors = train_dataset.label_colors
assert len(colors) == 41
train_image_files, train_label_files = train_iterator.feed()
assert train_image_files.shape[0] == batch_size
assert train_label_files.shape[0] == batch_size
test_image_files, test_label_files = test_iterator.feed()
assert test_image_files.shape[0] == batch_size
assert test_label_files.shape[0] == batch_size
def setup_dataset(dataset_class, subset, **kwargs):
dataset = dataset_class(subset=subset, **kwargs)
return DatasetIterator(dataset, seed=0)
