def __init__(self, data_dir=None): dataset_builder = tfds.builder("cars196:2.*.*", data_dir=data_dir) dataset_builder.download_and_prepare() # Defines dataset specific train/val/trainval/test splits. tfds_splits = {} tfds_splits["train"] = "train[:{}%]".format(TRAIN_SPLIT_PERCENT) tfds_splits["val"] = "train[{}%:]".format(TRAIN_SPLIT_PERCENT) tfds_splits["trainval"] = "train" tfds_splits["test"] = "test" # Creates a dict with example counts for each split. num_samples_splits = {} trainval_count = dataset_builder.info.splits["train"].num_examples test_count = dataset_builder.info.splits["test"].num_examples num_samples_splits["train"] = (TRAIN_SPLIT_PERCENT * trainval_count) // 100 num_samples_splits[ "val"] = trainval_count - num_samples_splits["train"] num_samples_splits["trainval"] = trainval_count num_samples_splits["test"] = test_count super(CarsData, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn(["image", "label"]), num_classes=dataset_builder.info.features["label"].num_classes)
def __init__(self, config="tfds", data_dir=None): if config == "tfds": dataset_builder = tfds.builder("sun397/tfds:4.*.*", data_dir=data_dir) dataset_builder.download_and_prepare() tfds_splits = { "train": "train", "val": "validation", "test": "test", "trainval": "train+validation", } # Creates a dict with example counts. num_samples_splits = { "test": dataset_builder.info.splits["test"].num_examples, "train": dataset_builder.info.splits["train"].num_examples, "val": dataset_builder.info.splits["validation"].num_examples, } num_samples_splits["trainval"] = (num_samples_splits["train"] + num_samples_splits["val"]) else: raise ValueError("No supported config %r for Sun397Data." % config) super(Sun397Data, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn(["image", "label"]), num_classes=dataset_builder.info.features["label"].num_classes)
def __init__(self, num_classes=10, data_dir=None): dataset_builder = tfds.builder("caltech101:3.*.*", data_dir=data_dir) dataset_builder.download_and_prepare() # Defines dataset specific train/val/trainval/test splits. tfds_splits = {} tfds_splits["train"] = "train[:{}%]".format(_TRAIN_SPLIT_PERCENT) tfds_splits["val"] = "train[{}%:]".format(_TRAIN_SPLIT_PERCENT) tfds_splits["trainval"] = "train" tfds_splits["test"] = "test" # Creates a dict with example counts for each split. trainval_count = dataset_builder.info.splits[tfds.Split.TRAIN].num_examples train_count = (_TRAIN_SPLIT_PERCENT * trainval_count) // 100 test_count = dataset_builder.info.splits[tfds.Split.TEST].num_examples num_samples_splits = dict( train=train_count, val=trainval_count - train_count, trainval=trainval_count, test=test_count) super(Caltech101, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=3000, base_preprocess_fn=base.make_get_tensors_fn(("image", "label")), num_classes=dataset_builder.info.features["label"].num_classes)
def __init__(self, features=("image", "label")): dataset_builder = tfds.builder("imagenet2012:5.*.*") # Defines dataset specific train/val/trainval/test splits. # Note, that the test split for "imagenet2012" dataset is not available. # Thus, we use the val split as test. Moreover, we split the train split # into two parts: new train split and new val split. tfds_splits = {} tfds_splits["train"] = "train[:{}%]".format(TRAIN_SPLIT_PERCENT) tfds_splits["val"] = "train[{}%:]".format(TRAIN_SPLIT_PERCENT) tfds_splits["trainval"] = "train" tfds_splits["test"] = "validation" # Creates a dict with example counts. num_samples_splits = {} trainval_count = dataset_builder.info.splits["train"].num_examples test_count = dataset_builder.info.splits["validation"].num_examples num_samples_splits["train"] = (TRAIN_SPLIT_PERCENT * trainval_count) // 100 num_samples_splits[ "val"] = trainval_count - num_samples_splits["train"] num_samples_splits["trainval"] = trainval_count num_samples_splits["test"] = test_count super(ImageNetData, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn(features), filter_fn=self._get_filter_fn(), num_classes=dataset_builder.info.features["label"].num_classes)
def test_make_get_tensors_fn(self): input_dict = {'tens1': 1, 'tens2': 2, 'tens3': 3} # Normal case. fn = base.make_get_tensors_fn(output_tensors=['tens1', 'tens2']) self.assertTrue(callable(fn)) self.assertEqual(fn(input_dict), {'tens1': 1, 'tens2': 2}) # One output tensor is not specified in the input dict. fn = base.make_get_tensors_fn(output_tensors=['tens1', 'tens2', 'tens4']) self.assertTrue(callable(fn)) with self.assertRaises(KeyError): fn(input_dict) # Empty output. fn = base.make_get_tensors_fn(output_tensors=()) self.assertTrue(callable(fn)) self.assertEqual(fn(input_dict), {})
def __init__(self, config="btgraham-300", heavy_train_augmentation=False, data_dir=None): """Initializer for Diabetic Retinopathy dataset. Args: config: Name of the TFDS config to use for this dataset. heavy_train_augmentation: If True, use heavy data augmentation on the training data. Recommended to achieve SOTA. data_dir: directory for downloading and storing the data. """ config_and_version = config + ":3.*.*" dataset_builder = tfds.builder( "diabetic_retinopathy_detection/{}".format(config_and_version), data_dir=data_dir) self._config = config self._heavy_train_augmentation = heavy_train_augmentation dataset_builder.download_and_prepare() # Defines dataset specific train/val/trainval/test splits. tfds_splits = { "train": "train", "val": "validation", "trainval": "train+validation", "test": "test", "train800": "train[:800]", "val200": "validation[:200]", "train800val200": "train[:800]+validation[:200]", } # Creates a dict with example counts for each split. train_count = dataset_builder.info.splits["train"].num_examples val_count = dataset_builder.info.splits["validation"].num_examples test_count = dataset_builder.info.splits["test"].num_examples num_samples_splits = { "train": train_count, "val": val_count, "trainval": train_count + val_count, "test": test_count, "train800": 800, "val200": 200, "train800val200": 1000, } super(RetinopathyData, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn(["image", "label"]), num_classes=dataset_builder.info.features["label"].num_classes)
def __init__(self, data_dir=None, train_split_percent=None): dataset_builder = tfds.builder("oxford_iiit_pet:3.*.*", data_dir=data_dir) dataset_builder.download_and_prepare() train_split_percent = train_split_percent or TRAIN_SPLIT_PERCENT # Creates a dict with example counts for each split. trainval_count = dataset_builder.info.splits[tfds.Split.TRAIN].num_examples test_count = dataset_builder.info.splits[tfds.Split.TEST].num_examples num_samples_splits = { "train": (train_split_percent * trainval_count) // 100, "val": trainval_count - (train_split_percent * trainval_count) // 100, "trainval": trainval_count, "test": test_count, "train800": 800, "val200": 200, "train800val200": 1000, } # Defines dataset specific train/val/trainval/test splits. tfds_splits = { "train": "train[:{}]".format(num_samples_splits["train"]), "val": "train[{}:]".format(num_samples_splits["train"]), "trainval": tfds.Split.TRAIN, "test": tfds.Split.TEST, "train800": "train[:800]", "val200": "train[{}:{}]".format( num_samples_splits["train"], num_samples_splits["train"]+200), "train800val200": "train[:800]+train[{}:{}]".format( num_samples_splits["train"], num_samples_splits["train"]+200), } super(OxfordIIITPetData, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn(["image", "label"]), num_classes=dataset_builder.info.features["label"].num_classes)
def __init__(self, data_dir=None): dataset_builder = tfds.builder("dtd:3.*.*", data_dir=data_dir) dataset_builder.download_and_prepare() # Defines dataset specific train/val/trainval/test splits. tfds_splits = { "train": "train", "val": "validation", "trainval": "train+validation", "test": "test", "train800": "train[:800]", "val200": "validation[:200]", "train800val200": "train[:800]+validation[:200]", } # Creates a dict with example counts for each split. train_count = dataset_builder.info.splits["train"].num_examples val_count = dataset_builder.info.splits["validation"].num_examples test_count = dataset_builder.info.splits["test"].num_examples num_samples_splits = { "train": train_count, "val": val_count, "trainval": train_count + val_count, "test": test_count, "train800": 800, "val200": 200, "train800val200": 1000, } super(DTDData, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn(["image", "label"]), num_classes=dataset_builder.info.features["label"].num_classes)
def __init__(self, num_classes=10, data_dir=None, train_split_percent=None): if num_classes == 10: dataset_builder = tfds.builder("cifar10:3.*.*", data_dir=data_dir) elif num_classes == 100: dataset_builder = tfds.builder("cifar100:3.*.*", data_dir=data_dir) else: raise ValueError( "Number of classes must be 10 or 100, got {}".format( num_classes)) dataset_builder.download_and_prepare() train_split_percent = train_split_percent or TRAIN_SPLIT_PERCENT # Creates a dict with example counts for each split. trainval_count = dataset_builder.info.splits["train"].num_examples test_count = dataset_builder.info.splits["test"].num_examples num_samples_splits = { "train": (train_split_percent * trainval_count) // 100, "val": trainval_count - (train_split_percent * trainval_count) // 100, "trainval": trainval_count, "test": test_count, "train800": 800, "val200": 200, "train800val200": 1000, } # Defines dataset specific train/val/trainval/test splits. tfds_splits = { "train": "train[:{}]".format(num_samples_splits["train"]), "val": "train[{}:]".format(num_samples_splits["train"]), "trainval": "train", "test": "test", "train800": "train[:800]", "val200": "train[{}:{}]".format(num_samples_splits["train"], num_samples_splits["train"] + 200), "train800val200": "train[:800]+train[{}:{}]".format( num_samples_splits["train"], num_samples_splits["train"] + 200), } super(CifarData, self).__init__( dataset_builder=dataset_builder, tfds_splits=tfds_splits, num_samples_splits=num_samples_splits, num_preprocessing_threads=400, shuffle_buffer_size=10000, # Note: Export only image and label tensors with their original types. base_preprocess_fn=base.make_get_tensors_fn( ["image", "label", "id"]), num_classes=dataset_builder.info.features["label"].num_classes)