def test_ni_sit_single_dataset_reproducibility_data(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
ni_benchmark_reference = ni_benchmark(mnist_train,
mnist_test,
5,
shuffle=True,
seed=1234)
rep_data = ni_benchmark_reference.get_reproducibility_data()
my_ni_benchmark = ni_benchmark(mnist_train,
mnist_test,
0,
reproducibility_data=rep_data)
self.assertEqual(ni_benchmark_reference.n_experiences,
my_ni_benchmark.n_experiences)
self.assertEqual(
ni_benchmark_reference.train_exps_patterns_assignment,
my_ni_benchmark.train_exps_patterns_assignment,
)
self.assertEqual(ni_benchmark_reference.exp_structure,
my_ni_benchmark.exp_structure)
def test_sit_single_dataset(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
5,
task_labels=False,
shuffle=True,
seed=1234,
)
self.assertEqual(5, my_nc_benchmark.n_experiences)
self.assertEqual(10, my_nc_benchmark.n_classes)
for batch_id in range(my_nc_benchmark.n_experiences):
self.assertEqual(
2, len(my_nc_benchmark.classes_in_experience["train"][batch_id])
)
all_classes = set()
for batch_id in range(5):
all_classes.update(
my_nc_benchmark.classes_in_experience["train"][batch_id]
)
self.assertEqual(10, len(all_classes))
def test_sit_single_dataset_fixed_order(self):
order = [2, 3, 5, 7, 8, 9, 0, 1, 4, 6]
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
5,
task_labels=False,
fixed_class_order=order,
)
all_classes = []
for batch_id in range(5):
all_classes.extend(
my_nc_benchmark.classes_in_experience["train"][batch_id]
)
self.assertEqual(order, all_classes)
def test_sit_single_dataset_fixed_order_subset(self):
order = [2, 3, 5, 8, 9, 1, 4, 6]
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
4,
task_labels=False,
fixed_class_order=order,
)
self.assertEqual(4, len(my_nc_benchmark.classes_in_experience["train"]))
all_classes = set()
for batch_id in range(4):
self.assertEqual(
2, len(my_nc_benchmark.classes_in_experience["train"][batch_id])
)
all_classes.update(
my_nc_benchmark.classes_in_experience["train"][batch_id]
)
self.assertEqual(set(order), all_classes)
def test_sit_single_dataset_remap_indexes_each_exp(self):
order = [2, 3, 5, 8, 9, 1, 4, 6]
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
with self.assertRaises(ValueError):
# class_ids_from_zero_* are mutually exclusive
nc_benchmark(
mnist_train,
mnist_test,
4,
task_labels=False,
fixed_class_order=order,
class_ids_from_zero_from_first_exp=True,
class_ids_from_zero_in_each_exp=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
4,
task_labels=False,
fixed_class_order=order,
class_ids_from_zero_in_each_exp=True,
)
self.assertEqual(4, len(my_nc_benchmark.classes_in_experience["train"]))
all_classes = []
for batch_id in range(4):
self.assertEqual(
2, len(my_nc_benchmark.classes_in_experience["train"][batch_id])
)
all_classes.extend(
my_nc_benchmark.classes_in_experience["train"][batch_id]
)
self.assertEqual(8, len(all_classes))
self.assertListEqual([0, 1], sorted(set(all_classes)))
# Regression test for issue #258
for i, experience in enumerate(my_nc_benchmark.train_stream):
unique_dataset_classes = sorted(set(experience.dataset.targets))
expected_dataset_classes = [0, 1]
self.assertListEqual(
expected_dataset_classes, unique_dataset_classes
)
self.assertListEqual(
sorted(order[2 * i : 2 * (i + 1)]),
sorted(my_nc_benchmark.original_classes_in_exp[i]),
)
def test_ni_sit_single_dataset(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_ni_benchmark = ni_benchmark(
mnist_train,
mnist_test,
5,
shuffle=True,
seed=1234,
balance_experiences=True,
)
self.assertEqual(5, my_ni_benchmark.n_experiences)
self.assertEqual(10, my_ni_benchmark.n_classes)
for batch_id in range(5):
self.assertEqual(
10,
len(my_ni_benchmark.classes_in_experience["train"][batch_id]),
)
_, unique_count = torch.unique(torch.as_tensor(mnist_train.targets),
return_counts=True)
min_batch_size = torch.sum(unique_count //
my_ni_benchmark.n_experiences).item()
max_batch_size = min_batch_size + my_ni_benchmark.n_classes
pattern_count = 0
batch_info: NIExperience
for batch_id, batch_info in enumerate(my_ni_benchmark.train_stream):
cur_train_set = batch_info.dataset
t = batch_info.task_label
self.assertEqual(0, t)
self.assertEqual(batch_id, batch_info.current_experience)
self.assertGreaterEqual(len(cur_train_set), min_batch_size)
self.assertLessEqual(len(cur_train_set), max_batch_size)
pattern_count += len(cur_train_set)
self.assertEqual(len(mnist_train), pattern_count)
self.assertEqual(1, len(my_ni_benchmark.test_stream))
pattern_count = 0
for batch_id, batch_info in enumerate(my_ni_benchmark.test_stream):
cur_test_set = batch_info.dataset
t = batch_info.task_label
self.assertEqual(0, t)
self.assertEqual(batch_id, batch_info.current_experience)
pattern_count += len(cur_test_set)
self.assertEqual(len(mnist_test), pattern_count)
def test_ni_sit_slicing(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_ni_benchmark = ni_benchmark(mnist_train,
mnist_test,
5,
shuffle=True,
seed=1234)
experience: NIExperience
for batch_id, experience in enumerate(my_ni_benchmark.train_stream):
self.assertEqual(batch_id, experience.current_experience)
self.assertIsInstance(experience, NIExperience)
self.assertEqual(1, len(my_ni_benchmark.test_stream))
for batch_id, experience in enumerate(my_ni_benchmark.test_stream):
self.assertEqual(batch_id, experience.current_experience)
self.assertIsInstance(experience, NIExperience)
iterable_slice = [3, 4, 1]
sliced_stream = my_ni_benchmark.train_stream[iterable_slice]
self.assertIsInstance(sliced_stream, ClassificationStream)
self.assertEqual(len(iterable_slice), len(sliced_stream))
self.assertEqual("train", sliced_stream.name)
for batch_id, experience in enumerate(sliced_stream):
self.assertEqual(iterable_slice[batch_id],
experience.current_experience)
self.assertIsInstance(experience, NIExperience)
with self.assertRaises(IndexError):
# The test stream only has one element (the complete test set)
sliced_stream = my_ni_benchmark.test_stream[iterable_slice]
iterable_slice = [0, 0, 0]
sliced_stream = my_ni_benchmark.test_stream[iterable_slice]
self.assertIsInstance(sliced_stream, ClassificationStream)
self.assertEqual(len(iterable_slice), len(sliced_stream))
self.assertEqual("test", sliced_stream.name)
for batch_id, experience in enumerate(sliced_stream):
self.assertEqual(iterable_slice[batch_id],
experience.current_experience)
self.assertIsInstance(experience, NIExperience)
def test_sit_multi_dataset_merge(self):
split_mapping = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
train_part1 = make_nc_transformation_subset(
mnist_train, None, None, range(5)
)
train_part2 = make_nc_transformation_subset(
mnist_train, None, None, range(5, 10)
)
train_part2 = AvalancheSubset(train_part2, class_mapping=split_mapping)
test_part1 = make_nc_transformation_subset(
mnist_test, None, None, range(5)
)
test_part2 = make_nc_transformation_subset(
mnist_test, None, None, range(5, 10)
)
test_part2 = AvalancheSubset(test_part2, class_mapping=split_mapping)
my_nc_benchmark = nc_benchmark(
[train_part1, train_part2],
[test_part1, test_part2],
5,
task_labels=False,
shuffle=True,
seed=1234,
)
self.assertEqual(5, my_nc_benchmark.n_experiences)
self.assertEqual(10, my_nc_benchmark.n_classes)
for batch_id in range(5):
self.assertEqual(
2, len(my_nc_benchmark.classes_in_experience["train"][batch_id])
)
all_classes = set()
for batch_id in range(5):
all_classes.update(
my_nc_benchmark.classes_in_experience["train"][batch_id]
)
self.assertEqual(10, len(all_classes))
def test_nc_sit_slicing(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
5,
task_labels=False,
shuffle=True,
seed=1234,
)
experience: NCExperience
for batch_id, experience in enumerate(my_nc_benchmark.train_stream):
self.assertEqual(batch_id, experience.current_experience)
self.assertIsInstance(experience, NCExperience)
for batch_id, experience in enumerate(my_nc_benchmark.test_stream):
self.assertEqual(batch_id, experience.current_experience)
self.assertIsInstance(experience, NCExperience)
iterable_slice = [3, 4, 1]
sliced_stream = my_nc_benchmark.train_stream[iterable_slice]
self.assertIsInstance(sliced_stream, ClassificationStream)
self.assertEqual(len(iterable_slice), len(sliced_stream))
self.assertEqual("train", sliced_stream.name)
for batch_id, experience in enumerate(sliced_stream):
self.assertEqual(
iterable_slice[batch_id], experience.current_experience
)
self.assertIsInstance(experience, NCExperience)
sliced_stream = my_nc_benchmark.test_stream[iterable_slice]
self.assertIsInstance(sliced_stream, ClassificationStream)
self.assertEqual(len(iterable_slice), len(sliced_stream))
self.assertEqual("test", sliced_stream.name)
for batch_id, experience in enumerate(sliced_stream):
self.assertEqual(
iterable_slice[batch_id], experience.current_experience
)
self.assertIsInstance(experience, NCExperience)
def test_nc_benchmark_classes_in_exp_range(self):
train_set = CIFAR100(
default_dataset_location("cifar100"), train=True, download=True
)
test_set = CIFAR100(
default_dataset_location("cifar100"), train=False, download=True
)
benchmark_instance = nc_benchmark(
train_dataset=train_set,
test_dataset=test_set,
n_experiences=5,
task_labels=False,
seed=1234,
shuffle=False,
)
cie_data = benchmark_instance.classes_in_exp_range(0, None)
self.assertEqual(5, len(cie_data))
for i in range(5):
expected = set(range(i * 20, (i + 1) * 20))
self.assertSetEqual(expected, set(cie_data[i]))
cie_data = benchmark_instance.classes_in_exp_range(1, 4)
self.assertEqual(3, len(cie_data))
for i in range(1, 3):
expected = set(range(i * 20, (i + 1) * 20))
self.assertSetEqual(expected, set(cie_data[i - 1]))
random_class_order = list(range(100))
random.shuffle(random_class_order)
benchmark_instance = nc_benchmark(
train_dataset=train_set,
test_dataset=test_set,
n_experiences=5,
task_labels=False,
seed=1234,
fixed_class_order=random_class_order,
shuffle=False,
)
cie_data = benchmark_instance.classes_in_exp_range(0, None)
self.assertEqual(5, len(cie_data))
for i in range(5):
expected = set(random_class_order[i * 20 : (i + 1) * 20])
self.assertSetEqual(expected, set(cie_data[i]))
def __init__(
self,
root: Union[str, Path] = None,
*,
train=True,
transform=None,
target_transform=None,
loader=default_loader,
download=True,
):
"""
Creates an instance of the OpenLORIS dataset.
:param root: The directory where the dataset can be found or downloaded.
Defaults to None, which means that the default location for
'openloris' will be used.
:param train: If True, the training set will be returned. If False,
the test set will be returned.
:param transform: The transformations to apply to the X values.
:param target_transform: The transformations to apply to the Y values.
:param loader: The image loader to use.
:param download: If True, the dataset will be downloaded if needed.
"""
if root is None:
root = default_dataset_location("openloris")
self.train = train # training set or test set
self.transform = transform
self.target_transform = target_transform
self.loader = loader
super(OpenLORIS, self).__init__(root, download=download, verbose=True)
self._load_dataset()
def _get_fmnist_dataset(dataset_root):
if dataset_root is None:
dataset_root = default_dataset_location('fashionmnist')
train_set = FashionMNIST(dataset_root, train=True, download=True)
test_set = FashionMNIST(dataset_root, train=False, download=True)
return train_set, test_set
def __init__(
self,
root: Union[str, Path] = None,
*,
data_name: str = "clear10",
download: bool = True,
verbose: bool = False,
):
"""
Creates an instance of the CLEAR dataset.
This base class simply download and unzip the CLEAR dataset.
This serves as a base class for _CLEARImage/_CLEARFeature dataset.
:param root: The directory where the dataset can be found or downloaded.
Defaults to None, which means that the default location for
str(data_name) will be used.
:param data_name: Data module name with the google drive url and md5
:param download: If True, the dataset will be downloaded if needed.
"""
if root is None:
root = default_dataset_location(data_name)
assert data_name in _CLEAR_DATA_MODULE
self.data_name = data_name
self.module = _CLEAR_DATA_MODULE[data_name]
super(CLEARDataset, self).__init__(root,
download=download,
verbose=True)
self._load_dataset()
def test_nc_benchmark_transformations_advanced(self):
# Regression for #577
ds = CIFAR100(
root=default_dataset_location("cifar100"),
train=True,
download=True,
)
benchmark = nc_benchmark(
ds,
ds,
n_experiences=10,
shuffle=True,
seed=1234,
task_labels=False,
train_transform=ToTensor(),
eval_transform=None,
)
ds_train_train = benchmark.train_stream[0].dataset
self.assertIsInstance(ds_train_train[0][0], Tensor)
ds_train_eval = benchmark.train_stream[0].dataset.eval()
self.assertIsInstance(ds_train_eval[0][0], Image)
ds_test_eval = benchmark.test_stream[0].dataset
self.assertIsInstance(ds_test_eval[0][0], Image)
ds_test_train = benchmark.test_stream[0].dataset.train()
self.assertIsInstance(ds_test_train[0][0], Tensor)
def _get_omniglot_dataset(dataset_root):
if dataset_root is None:
dataset_root = default_dataset_location('omniglot')
train = Omniglot(root=dataset_root, train=True, download=True)
test = Omniglot(root=dataset_root, train=False, download=True)
return train, test
def test_dataset_benchmark(self):
train_MNIST = MNIST(root=default_dataset_location("mnist"),
train=True,
download=True)
test_MNIST = MNIST(root=default_dataset_location("mnist"),
train=False,
download=True)
train_cifar10 = CIFAR10(root=default_dataset_location("cifar10"),
train=True,
download=True)
test_cifar10 = CIFAR10(root=default_dataset_location("cifar10"),
train=False,
download=True)
generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
def _get_cifar100_dataset(dataset_root):
if dataset_root is None:
dataset_root = default_dataset_location('cifar100')
train_set = CIFAR100(dataset_root, train=True, download=True)
test_set = CIFAR100(dataset_root, train=False, download=True)
return train_set, test_set
def _get_mnist_dataset(dataset_root):
if dataset_root is None:
dataset_root = default_dataset_location("mnist")
train_set = MNIST(root=dataset_root, train=True, download=True)
test_set = MNIST(root=dataset_root, train=False, download=True)
return train_set, test_set
def test_dataset_benchmark_avalanche_dataset(self):
train_MNIST = AvalancheDataset(
MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
),
task_labels=0,
)
test_MNIST = AvalancheDataset(
MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
),
task_labels=0,
)
train_cifar10 = AvalancheDataset(
CIFAR10(
root=default_dataset_location("cifar10"),
train=True,
download=True,
),
task_labels=1,
)
test_cifar10 = AvalancheDataset(
CIFAR10(
root=default_dataset_location("cifar10"),
train=False,
download=True,
),
task_labels=1,
)
generic_benchmark = dataset_benchmark([train_MNIST, train_cifar10],
[test_MNIST, test_cifar10])
self.assertEqual(0, generic_benchmark.train_stream[0].task_label)
self.assertEqual(1, generic_benchmark.train_stream[1].task_label)
self.assertEqual(0, generic_benchmark.test_stream[0].task_label)
self.assertEqual(1, generic_benchmark.test_stream[1].task_label)
def __init__(
self,
root: Union[str, Path] = None,
*,
train=True,
transform=None,
target_transform=None,
loader=default_loader,
download=True,
mini=False,
object_level=True,
):
"""
Creates an instance of the CORe50 dataset.
:param root: root for the datasets data. Defaults to None, which means
that the default location for 'core50' will be used.
:param train: train or test split.
:param transform: eventual transformations to be applied.
:param target_transform: eventual transformation to be applied to the
targets.
:param loader: the procedure to load the instance from the storage.
:param download: boolean to automatically download data. Default to
True.
:param mini: boolean to use the 32x32 version instead of the 128x128.
Default to False.
:param object_level: if the classification is objects based or
category based: 50 or 10 way classification problem. Default to True
(50-way object classification problem)
"""
if root is None:
root = default_dataset_location("core50")
super(CORe50Dataset, self).__init__(root,
download=download,
verbose=True)
self.train = train # training set or test set
self.transform = transform
self.target_transform = target_transform
self.loader = loader
self.object_level = object_level
self.mini = mini
# any scenario and run is good here since we want just to load the
# train images and targets with no particular order
self._scen = "ni"
self._run = 0
self._nbatch = 8
# Download the dataset and initialize metadata
self._load_dataset()
def test_sit_single_dataset_batch_size(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
3,
task_labels=False,
per_exp_classes={0: 5, 2: 2},
)
self.assertEqual(3, my_nc_benchmark.n_experiences)
self.assertEqual(10, my_nc_benchmark.n_classes)
all_classes = set()
for batch_id in range(3):
all_classes.update(
my_nc_benchmark.classes_in_experience["train"][batch_id]
)
self.assertEqual(10, len(all_classes))
self.assertEqual(
5, len(my_nc_benchmark.classes_in_experience["train"][0])
)
self.assertEqual(
3, len(my_nc_benchmark.classes_in_experience["train"][1])
)
self.assertEqual(
2, len(my_nc_benchmark.classes_in_experience["train"][2])
)
def _get_inaturalist_dataset(dataset_root, super_categories, download):
if dataset_root is None:
dataset_root = default_dataset_location("inatuarlist2018")
train_set = INATURALIST2018(dataset_root,
split="train",
supcats=super_categories,
download=download)
test_set = INATURALIST2018(dataset_root,
split="val",
supcats=super_categories,
download=download)
return train_set, test_set
def test_sit_single_dataset_fixed_subset_no_remap_idx(self):
order = [2, 5, 7, 8, 9, 0, 1, 4]
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
2,
task_labels=True,
fixed_class_order=order,
class_ids_from_zero_in_each_exp=False,
)
self.assertEqual(2,
len(my_nc_benchmark.classes_in_experience["train"]))
all_classes = set()
for task_id in range(2):
self.assertEqual(
4,
len(my_nc_benchmark.classes_in_experience["train"][task_id]))
self.assertEqual(
set(order[task_id * 4:(task_id + 1) * 4]),
my_nc_benchmark.original_classes_in_exp[task_id],
)
all_classes.update(
my_nc_benchmark.classes_in_experience["train"][task_id])
self.assertEqual(set(order), all_classes)
def test_sit_single_dataset_fixed_order_subset(self):
order = [2, 5, 7, 8, 9, 0, 1, 4]
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
4,
task_labels=True,
fixed_class_order=order,
class_ids_from_zero_in_each_exp=True,
)
self.assertEqual(4,
len(my_nc_benchmark.classes_in_experience["train"]))
all_classes = []
for task_id in range(4):
self.assertEqual(
2,
len(my_nc_benchmark.classes_in_experience["train"][task_id]))
self.assertEqual(
set(order[task_id * 2:(task_id + 1) * 2]),
my_nc_benchmark.original_classes_in_exp[task_id],
)
all_classes.extend(
my_nc_benchmark.classes_in_experience["train"][task_id])
self.assertEqual([0, 1] * 4, all_classes)
def test_sit_single_dataset_reproducibility_data(self):
mnist_train = MNIST(
root=default_dataset_location("mnist"),
train=True,
download=True,
)
mnist_test = MNIST(
root=default_dataset_location("mnist"),
train=False,
download=True,
)
nc_benchmark_ref = nc_benchmark(
mnist_train,
mnist_test,
5,
task_labels=False,
shuffle=True,
seed=5678,
)
my_nc_benchmark = nc_benchmark(
mnist_train,
mnist_test,
-1,
task_labels=False,
reproducibility_data=nc_benchmark_ref.get_reproducibility_data(),
)
self.assertEqual(
nc_benchmark_ref.train_exps_patterns_assignment,
my_nc_benchmark.train_exps_patterns_assignment,
)
self.assertEqual(
nc_benchmark_ref.test_exps_patterns_assignment,
my_nc_benchmark.test_exps_patterns_assignment,
)
def test_nc_benchmark_transformations_basic(self):
# Regression for #577
ds = CIFAR100(
root=default_dataset_location("cifar100"),
train=True,
download=True,
)
ds = AvalancheDataset(ds, transform=ToTensor())
benchmark = nc_benchmark(
ds, ds, n_experiences=10, shuffle=True, seed=1234, task_labels=False
)
exp_0_dataset = benchmark.train_stream[0].dataset
self.assertIsInstance(exp_0_dataset[0][0], Tensor)
def __init__(
self,
root: Union[str, Path] = None,
*,
train=True,
transform=None,
loader=default_loader,
download=True,
lvis_api=None,
img_ids: List[int] = None,
):
"""
Creates an instance of the LVIS dataset.
:param root: The directory where the dataset can be found or downloaded.
Defaults to None, which means that the default location for
"lvis" will be used.
:param train: If True, the training set will be returned. If False,
the test set will be returned.
:param transform: The transformation to apply to (img, annotations)
values.
:param loader: The image loader to use.
:param download: If True, the dataset will be downloaded if needed.
:param lvis_api: An instance of the LVIS class (from the lvis-api) to
use. Defaults to None, which means that annotations will be loaded
from the annotation json found in the root directory.
:param img_ids: A list representing a subset of images to use. Defaults
to None, which means that the dataset will contain all images
in the LVIS dataset.
"""
if root is None:
root = default_dataset_location("lvis")
self.train = train # training set or test set
self.transform = transform
self.loader = loader
self.bbox_crop = True
self.img_ids = img_ids
self.targets = None
self.lvis_api = lvis_api
super(LvisDataset, self).__init__(root,
download=download,
verbose=True)
self._load_dataset()
def __init__(
self,
root: Union[str, Path] = None,
*,
train=True,
transform=None,
target_transform=None,
loader=default_loader,
download=True
):
"""
:param root: root dir where the dataset can be found or downloaded.
Defaults to None, which means that the default location for
'CUB_200_2011' will be used.
:param train: train or test subset of the original dataset. Default
to True.
:param transform: eventual input data transformations to apply.
Default to None.
:param target_transform: eventual target data transformations to apply.
Default to None.
:param loader: method to load the data from disk. Default to
torchvision default_loader.
:param download: default set to True. If the data is already
downloaded it will skip the download.
"""
if root is None:
root = default_dataset_location("CUB_200_2011")
self.train = train
DownloadableDataset.__init__(
self, root, download=download, verbose=True
)
self._load_dataset()
PathsDataset.__init__(
self,
os.path.join(root, CUB200.images_folder),
self._images,
transform=transform,
target_transform=target_transform,
loader=loader,
)
def __init__(self,
root: Union[str, Path] = None,
*,
transform=None,
loader=default_loader,
mask_loader=default_mask_loader,
download=True):
"""
Creates an instance of the Penn-Fudan dataset.
:param root: The directory where the dataset can be found or downloaded.
Defaults to None, which means that the default location for
"pennfudanped" will be used.
:param transform: The transformation to apply to (img, annotations)
values.
:param loader: The image loader to use.
:param mask_loader: The mask image loader to use.
:param download: If True, the dataset will be downloaded if needed.
"""
if root is None:
root = default_dataset_location("pennfudanped")
self.imgs = None
self.masks = None
self.targets = None
self.transform = transform
self.loader = loader
self.mask_loader = mask_loader
super().__init__(
root,
penn_fudan_data[0],
penn_fudan_data[1],
download=download,
verbose=True
)
self._load_dataset()
def __init__(self,
root: Union[str, Path] = None,
*,
train: bool = True,
transform=None,
target_transform=None,
loader=default_loader,
download=True):
"""
Creates an instance of the Tiny Imagenet dataset.
:param root: folder in which to download dataset. Defaults to None,
which means that the default location for 'tinyimagenet' will be
used.
:param train: True for training set, False for test set.
:param transform: Pytorch transformation function for x.
:param target_transform: Pytorch transformation function for y.
:param loader: the procedure to load the instance from the storage.
:param bool download: If True, the dataset will be downloaded if
needed.
"""
if root is None:
root = default_dataset_location("tinyimagenet")
self.transform = transform
self.target_transform = target_transform
self.train = train
self.loader = loader
super(TinyImagenet, self).__init__(root,
self.filename[1],
self.md5,
download=download,
verbose=True)
self._load_dataset()