def test_default_multi_dir(self):
# No data_dir is passed
# Multiple data_dirs are registered
# -> use default path
file_utils.add_data_dir(self.other_data_dir)
self.assertBuildDataDir(self.builder._build_data_dir(None),
self.default_data_dir)
def test_default_multi_dir_version_exists(self):
# No data_dir is passed
# Multiple data_dirs are registered
# Data found
# -> Re-load existing data
file_utils.add_data_dir(self.other_data_dir)
tf.io.gfile.makedirs(
os.path.join(self.other_data_dir, "dummy_dataset_shared_generator",
"1.0.0"))
self.assertBuildDataDir(self.builder._build_data_dir(None),
self.other_data_dir)
def test_default_multi_dir_duplicate(self):
# If two data dirs contains the dataset, raise an error...
file_utils.add_data_dir(self.other_data_dir)
tf.io.gfile.makedirs(
os.path.join(self.default_data_dir,
"dummy_dataset_shared_generator", "1.0.0"))
tf.io.gfile.makedirs(
os.path.join(self.other_data_dir, "dummy_dataset_shared_generator",
"1.0.0"))
with self.assertRaisesRegex(ValueError,
"found in more than one directory"):
self.builder._build_data_dir(None)
def test_expicit_multi_dir(self):
# If two data dirs contains the same version
# Data dir is explicitly passed
file_utils.add_data_dir(self.other_data_dir)
tf.io.gfile.makedirs(
os.path.join(self.default_data_dir,
"dummy_dataset_shared_generator", "1.0.0"))
tf.io.gfile.makedirs(
os.path.join(self.other_data_dir, "dummy_dataset_shared_generator",
"1.0.0"))
self.assertBuildDataDir(
self.builder._build_data_dir(self.other_data_dir),
self.other_data_dir)
def test_default_multi_dir_old_version_exists(self):
# No data_dir is passed
# Multiple data_dirs are registered
# Data dir contains old versions
# -> use default path
file_utils.add_data_dir(self.other_data_dir)
tf.io.gfile.makedirs(
os.path.join(self.other_data_dir, "dummy_dataset_shared_generator",
"0.1.0"))
tf.io.gfile.makedirs(
os.path.join(self.other_data_dir, "dummy_dataset_shared_generator",
"0.2.0"))
self.assertBuildDataDir(self.builder._build_data_dir(None),
self.default_data_dir)
def test_load_data_dir(self):
"""Ensure that `tfds.load` also supports multiple data_dir."""
file_utils.add_data_dir(self.other_data_dir)
class MultiDirDataset(DummyDatasetSharedGenerator): # pylint: disable=unused-variable
VERSION = utils.Version("1.2.0")
data_dir = os.path.join(self.other_data_dir, "multi_dir_dataset",
"1.2.0")
tf.io.gfile.makedirs(data_dir)
with mock.patch.object(dataset_info.DatasetInfo,
"read_from_directory"):
_, info = load.load("multi_dir_dataset", split=[], with_info=True)
self.assertEqual(info.data_dir, data_dir)
