def test_download_folder(s3, tmp_directory): Path('dir', 'subdir', 'nested').mkdir(parents=True) Path('dir', 'a').touch() Path('dir', 'b').touch() Path('dir', 'subdir', 'c').touch() Path('dir', 'subdir', 'nested', 'd').touch() for path in chain(iglob('dir/*'), iglob('dir/*/*'), iglob('dir/*/*/*')): path = Path(path) if path.is_file(): s3.upload_file(str(path), 'some-bucket', str(PurePosixPath('my-folder', *path.parts))) shutil.rmtree('dir') client = S3Client('some-bucket', parent='my-folder', path_to_project_root='.') client.download('dir') assert Path('dir', 'a').exists() assert Path('dir', 'b').exists() assert Path('dir', 'subdir', 'c').exists() assert Path('dir', 'subdir', 'nested', 'd').exists()
def test_underscore_download(s3, tmp_directory, arg): Path('some-file-to-upload').touch() s3.upload_file('some-file-to-upload', 'some-bucket', 'my-folder/my-file') client = S3Client('some-bucket', 'my-folder', path_to_project_root='.') client._download(arg, 'my-folder/my-file') assert Path('my-file').exists()
def test_error_when_downloading_non_existing(s3): client = S3Client('some-bucket', parent='my-folder', path_to_project_root='.') with pytest.raises(RemoteFileNotFound) as excinfo: client.download('some-file') assert "Could not download 'some-file' using client" in str(excinfo.value)
def test_underscore_upload(s3, tmp_directory, arg): Path('my-file').touch() client = S3Client('some-bucket', 'my-folder', path_to_project_root='.') client._upload(arg) contents = s3.list_objects(Bucket='some-bucket')['Contents'] assert len(contents) == 1 assert contents[0]['Key'] == 'my-folder/my-file'
def test_upload_file(s3, parent, tmp_directory, file_): Path(file_).parent.mkdir(exist_ok=True) Path(file_).touch() client = S3Client('some-bucket', parent=parent, path_to_project_root='.') client.upload(file_) contents = s3.list_objects(Bucket='some-bucket')['Contents'] assert len(contents) == 1 assert contents[0]['Key'] == str(PurePosixPath(parent, file_))
def test_is_dir(s3, tmp_directory): path = Path('dir', 'a') path.parent.mkdir() path.write_text('hello') s3.upload_file('dir/a', 'some-bucket', str(PurePosixPath('my-folder', 'dir/a'))) client = S3Client('some-bucket', parent='my-folder', path_to_project_root='.') assert client._is_dir('my-folder/dir/')
def test_download(s3, tmp_directory, destination, expected): path = Path('dir', 'a') path.parent.mkdir() path.write_text('hello') s3.upload_file('dir/a', 'some-bucket', str(PurePosixPath('my-folder', 'dir/a'))) path.unlink() client = S3Client('some-bucket', parent='my-folder', path_to_project_root='.') client.download('dir/a', destination=destination) assert Path(expected).read_text() == 'hello'
def test_upload_folder(s3, tmp_directory): Path('dir', 'subdir', 'nested').mkdir(parents=True) Path('dir', 'a').touch() Path('dir', 'b').touch() Path('dir', 'subdir', 'c').touch() Path('dir', 'subdir', 'nested', 'd').touch() client = S3Client('some-bucket', parent='my-folder', path_to_project_root='.') client.upload('dir') contents = s3.list_objects(Bucket='some-bucket')['Contents'] assert set(c['Key'] for c in contents) == { 'my-folder/dir/a', 'my-folder/dir/b', 'my-folder/dir/subdir/c', 'my-folder/dir/subdir/nested/d' }
def test_error_if_missing_project_root(tmp_directory): with pytest.raises(DAGSpecInvalidError) as excinfo: S3Client('some-bucket', 'some-folder') assert 'Cannot initialize' in str(excinfo.value)
def test_can_initialize_if_valid_project_root(tmp_directory): Path('pipeline.yaml').touch() client = S3Client('some-bucket', 'some-folder') assert client._path_to_project_root == Path(tmp_directory).resolve()
def test_close(): S3Client('some-bucket', parent='my-folder', path_to_project_root='.').close()
def test_pickle(): c = S3Client('some-bucket', parent='my-folder', path_to_project_root='.') pickle.loads(pickle.dumps(c))