def test_cannot_add_non_datafiles(self): """Ensures that exception will be raised if adding a non-datafile object""" class NotADatafile: pass resource = Dataset() with self.assertRaises(exceptions.InvalidInputException): resource.add(NotADatafile())
def test_adding_cloud_datafile_to_local_dataset(self): """Test that when a cloud datafile is added to a local dataset, it is downloaded to the root of the dataset.""" with Datafile(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") with tempfile.TemporaryDirectory() as temporary_directory: dataset = Dataset(path=os.path.join(temporary_directory, "path", "to", "dataset")) dataset.add(datafile) self.assertIn(datafile, dataset) self.assertEqual(datafile.local_path, os.path.join(dataset.path, "datafile.dat"))
def test_adding_cloud_datafile_to_cloud_dataset_when_file_is_already_in_dataset_directory( self): """Test that a cloud datafile's path is kept as-is when adding it to a cloud dataset if it is already in the dataset directory and no `path_in_dataset` is provided. """ dataset = Dataset(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "dataset")) with Datafile(path=storage.path.join(dataset.path, "subfolder", "datafile.dat"), mode="w") as (datafile, f): f.write("hello") dataset.add(datafile) self.assertIn(datafile, dataset) self.assertEqual( datafile.cloud_path, storage.path.join(dataset.path, "subfolder", "datafile.dat"))
def test_adding_cloud_datafile_to_cloud_dataset(self): """Test that a cloud datafile can be added to a cloud dataset and that it's copied into the dataset root if no `path_within_dataset` is provided. """ dataset = Dataset(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "dataset")) with Datafile(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") dataset.add(datafile) self.assertIn(datafile, dataset) self.assertEqual(datafile.cloud_path, storage.path.join(dataset.path, "datafile.dat"))
def test_adding_local_datafile_to_local_dataset_when_file_is_already_in_dataset_directory( self): """Test that a local datafile's path is kept as-is when adding it to a local dataset if it is already in the dataset directory. """ with tempfile.TemporaryDirectory() as temporary_directory: dataset = Dataset(path=os.path.join(temporary_directory, "path", "to", "dataset")) with Datafile(path=os.path.join(dataset.path, "subfolder", "datafile.dat"), mode="w") as (datafile, f): f.write("hello") dataset.add(datafile) self.assertIn(datafile, dataset) self.assertEqual( datafile.local_path, os.path.join(dataset.path, "subfolder", "datafile.dat"))
def test_adding_local_datafile_to_local_dataset(self): """Test that a local datafile can be added to a local dataset and that it is copied to the root of the dataset if no `path_within_dataset` is provided. """ with tempfile.TemporaryDirectory() as temporary_directory: dataset = Dataset(path=os.path.join(temporary_directory, "path", "to", "dataset")) with Datafile(path=os.path.join(temporary_directory, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") dataset.add(datafile) self.assertIn(datafile, dataset) self.assertEqual(datafile.local_path, os.path.join(dataset.path, "datafile.dat"))
def test_providing_path_when_adding_cloud_datafile_to_cloud_dataset_copies_datafile_to_path( self): """Test that providing the `path_within_dataset` parameter when adding a cloud datafile to a cloud dataset results in the datafile being copied to that location within the dataset. """ dataset = Dataset(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "dataset")) with Datafile(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") path_in_dataset = storage.path.join("another", "path", "datafile.dat") dataset.add(datafile, path_in_dataset=path_in_dataset) self.assertIn(datafile, dataset) self.assertEqual(datafile.cloud_path, storage.path.join(dataset.path, path_in_dataset))
def test_providing_path_when_adding_local_datafile_to_local_dataset(self): """Test that providing the `path_within_dataset` parameter when adding a local datafile to a local dataset results in the datafile being copied to that location within the dataset. """ with tempfile.TemporaryDirectory() as temporary_directory: dataset = Dataset(path=os.path.join(temporary_directory, "path", "to", "dataset")) with Datafile(path=os.path.join(temporary_directory, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") path_in_dataset = os.path.join("another", "path", "datafile.dat") dataset.add(datafile, path_in_dataset=path_in_dataset) self.assertIn(datafile, dataset) self.assertEqual(datafile.local_path, os.path.join(dataset.path, path_in_dataset))
def test_adding_local_datafile_to_cloud_dataset_uploads_it_to_dataset_root( self): """Test that, when adding a local datafile to a cloud dataset and `path_in_dataset` is not provided, the datafile is uploaded to the root of the dataset. """ dataset = Dataset(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "dataset")) with tempfile.TemporaryDirectory() as temporary_directory: with Datafile(path=os.path.join(temporary_directory, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") dataset.add(datafile) self.assertIn(datafile, dataset) self.assertEqual(datafile.cloud_path, storage.path.join(dataset.path, "datafile.dat"))
def test_providing_path_in_dataset_when_adding_cloud_datafile_to_local_dataset( self): """Test that when a cloud datafile is added to a local dataset and the `path_in_dataset` parameter is provided, it is downloaded to that path within the dataset. """ with Datafile(path=storage.path.generate_gs_path( TEST_BUCKET_NAME, "path", "to", "datafile.dat"), mode="w") as ( datafile, f, ): f.write("hello") with tempfile.TemporaryDirectory() as temporary_directory: dataset = Dataset(path=os.path.join(temporary_directory, "path", "to", "dataset")) path_in_dataset = os.path.join("another", "path", "datafile.dat") dataset.add(datafile, path_in_dataset=path_in_dataset) self.assertIn(datafile, dataset) self.assertEqual(datafile.local_path, os.path.join(dataset.path, path_in_dataset))