def test_s3path_invalid_fileURL(self): test_url = "an invalid fileURL" with self.assertRaises(ValueError): S3Path(test_url)
def copy(self, source: str, destination: str, recursive: bool = False) -> None: """Move a file or folder between local storage and S3, as well as, S3 and S3 Keyword arguments: source -- source file destination -- destination file recursive -- whether to recursively copy a folder """ if StorageService.path_type(source) == PathType.Local: # from local to S3 if StorageService.path_type(destination) == PathType.Local: raise ValueError("Both source and destination are local files") s3_path = S3Path(destination) if path.isdir(source): if not recursive: raise ValueError( f"Source {source} is a folder. Use --recursive") self.upload_dir(source, s3_path.bucket, s3_path.key) else: self.s3_gateway.upload_file(source, s3_path.bucket, s3_path.key) else: source_s3_path = S3Path(source) if StorageService.path_type(destination) == PathType.S3: # from S3 to S3 dest_s3_path = S3Path(destination) if source_s3_path == dest_s3_path: raise ValueError( f"Source {source} and destination {destination} are the same" ) if source.endswith("/"): if not recursive: raise ValueError( f"Source {source} is a folder. Use --recursive") self.copy_dir( source_s3_path.bucket, source_s3_path.key + "/", dest_s3_path.bucket, dest_s3_path.key, ) else: self.s3_gateway.copy( source_s3_path.bucket, source_s3_path.key, dest_s3_path.bucket, dest_s3_path.key, ) else: # from S3 to local if source.endswith("/"): if not recursive: raise ValueError( f"Source {source} is a folder. Use --recursive") self.download_dir( source_s3_path.bucket, source_s3_path.key + "/", destination, ) else: self.s3_gateway.download_file(source_s3_path.bucket, source_s3_path.key, destination)
def test_s3path_with_subfoler(self): test_s3path = S3Path( "https://bucket-name.s3.Region.amazonaws.com/subfolder/key") self.assertEqual(test_s3path.region, "Region") self.assertEqual(test_s3path.bucket, "bucket-name") self.assertEqual(test_s3path.key, "subfolder/key")
def file_exists(self, filename: str) -> bool: if StorageService.path_type(filename) == PathType.S3: s3_path = S3Path(filename) return self.s3_gateway.object_exists(s3_path.bucket, s3_path.key) else: raise ValueError(f"File {filename} is not an S3 filepath")