def test_cannot_add_non_datafiles(self):
"""Ensures that exception will be raised if adding a non-datafile object"""
class NotADatafile:
pass
resource = Dataset()
with self.assertRaises(exceptions.InvalidInputException):
resource.add(NotADatafile())
def test_adding_cloud_datafile_to_local_dataset(self):
"""Test that when a cloud datafile is added to a local dataset, it is downloaded to the root of the dataset."""
with Datafile(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
with tempfile.TemporaryDirectory() as temporary_directory:
dataset = Dataset(path=os.path.join(temporary_directory, "path",
"to", "dataset"))
dataset.add(datafile)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.local_path,
os.path.join(dataset.path, "datafile.dat"))
def test_adding_cloud_datafile_to_cloud_dataset_when_file_is_already_in_dataset_directory(
self):
"""Test that a cloud datafile's path is kept as-is when adding it to a cloud dataset if it is already in the
dataset directory and no `path_in_dataset` is provided.
"""
dataset = Dataset(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "dataset"))
with Datafile(path=storage.path.join(dataset.path, "subfolder",
"datafile.dat"),
mode="w") as (datafile, f):
f.write("hello")
dataset.add(datafile)
self.assertIn(datafile, dataset)
self.assertEqual(
datafile.cloud_path,
storage.path.join(dataset.path, "subfolder", "datafile.dat"))
def test_adding_cloud_datafile_to_cloud_dataset(self):
"""Test that a cloud datafile can be added to a cloud dataset and that it's copied into the dataset root if no
`path_within_dataset` is provided.
"""
dataset = Dataset(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "dataset"))
with Datafile(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
dataset.add(datafile)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.cloud_path,
storage.path.join(dataset.path, "datafile.dat"))
def test_adding_local_datafile_to_local_dataset_when_file_is_already_in_dataset_directory(
self):
"""Test that a local datafile's path is kept as-is when adding it to a local dataset if it is already in the
dataset directory.
"""
with tempfile.TemporaryDirectory() as temporary_directory:
dataset = Dataset(path=os.path.join(temporary_directory, "path",
"to", "dataset"))
with Datafile(path=os.path.join(dataset.path, "subfolder",
"datafile.dat"),
mode="w") as (datafile, f):
f.write("hello")
dataset.add(datafile)
self.assertIn(datafile, dataset)
self.assertEqual(
datafile.local_path,
os.path.join(dataset.path, "subfolder", "datafile.dat"))
def test_adding_local_datafile_to_local_dataset(self):
"""Test that a local datafile can be added to a local dataset and that it is copied to the root of the dataset
if no `path_within_dataset` is provided.
"""
with tempfile.TemporaryDirectory() as temporary_directory:
dataset = Dataset(path=os.path.join(temporary_directory, "path",
"to", "dataset"))
with Datafile(path=os.path.join(temporary_directory, "path", "to",
"datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
dataset.add(datafile)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.local_path,
os.path.join(dataset.path, "datafile.dat"))
def test_providing_path_when_adding_cloud_datafile_to_cloud_dataset_copies_datafile_to_path(
self):
"""Test that providing the `path_within_dataset` parameter when adding a cloud datafile to a cloud dataset
results in the datafile being copied to that location within the dataset.
"""
dataset = Dataset(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "dataset"))
with Datafile(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
path_in_dataset = storage.path.join("another", "path", "datafile.dat")
dataset.add(datafile, path_in_dataset=path_in_dataset)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.cloud_path,
storage.path.join(dataset.path, path_in_dataset))
def test_providing_path_when_adding_local_datafile_to_local_dataset(self):
"""Test that providing the `path_within_dataset` parameter when adding a local datafile to a local dataset
results in the datafile being copied to that location within the dataset.
"""
with tempfile.TemporaryDirectory() as temporary_directory:
dataset = Dataset(path=os.path.join(temporary_directory, "path",
"to", "dataset"))
with Datafile(path=os.path.join(temporary_directory, "path", "to",
"datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
path_in_dataset = os.path.join("another", "path", "datafile.dat")
dataset.add(datafile, path_in_dataset=path_in_dataset)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.local_path,
os.path.join(dataset.path, path_in_dataset))
def test_adding_local_datafile_to_cloud_dataset_uploads_it_to_dataset_root(
self):
"""Test that, when adding a local datafile to a cloud dataset and `path_in_dataset` is not provided, the
datafile is uploaded to the root of the dataset.
"""
dataset = Dataset(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "dataset"))
with tempfile.TemporaryDirectory() as temporary_directory:
with Datafile(path=os.path.join(temporary_directory, "path", "to",
"datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
dataset.add(datafile)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.cloud_path,
storage.path.join(dataset.path, "datafile.dat"))
def test_providing_path_in_dataset_when_adding_cloud_datafile_to_local_dataset(
self):
"""Test that when a cloud datafile is added to a local dataset and the `path_in_dataset` parameter is provided,
it is downloaded to that path within the dataset.
"""
with Datafile(path=storage.path.generate_gs_path(
TEST_BUCKET_NAME, "path", "to", "datafile.dat"),
mode="w") as (
datafile,
f,
):
f.write("hello")
with tempfile.TemporaryDirectory() as temporary_directory:
dataset = Dataset(path=os.path.join(temporary_directory, "path",
"to", "dataset"))
path_in_dataset = os.path.join("another", "path", "datafile.dat")
dataset.add(datafile, path_in_dataset=path_in_dataset)
self.assertIn(datafile, dataset)
self.assertEqual(datafile.local_path,
os.path.join(dataset.path, path_in_dataset))
