def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type): from dvc.fs.git import GitFileSystem gen = tmp_dir.scm_gen if fs_type == "git" else tmp_dir.dvc_gen gen({"dir": {"foo": "foo", "bar": "bar"}, "file": "file"}, commit="gen") fs = dvc.dvcfs if fs_type == "dvc" else GitFileSystem(scm=scm, rev="HEAD") callback = fsspec.Callback() fs.download_file( "file", (tmp_dir / "file2").fs_path, callback=callback, ) size = os.path.getsize(tmp_dir / "file") assert (tmp_dir / "file2").read_text() == "file" assert callback.size == size assert callback.value == size callback = fsspec.Callback() fs.download( "dir", (tmp_dir / "dir2").fs_path, callback=callback, ) assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"} assert callback.size == 2 assert callback.value == 2
def test_callback_on_repo_fs(tmp_dir, dvc, scm): tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc") tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git") fs = dvc.repo_fs callback = fsspec.Callback() fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback) assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"} assert callback.size == 2 assert callback.value == 2 callback = fsspec.Callback() fs.download(tmp_dir / "dir" / "foo", tmp_dir / "foo", callback=callback) size = os.path.getsize(tmp_dir / "dir" / "foo") assert (tmp_dir / "foo").read_text() == "foo" assert callback.size == size assert callback.value == size callback = fsspec.Callback() fs.download(tmp_dir / "dir" / "bar", tmp_dir / "bar", callback=callback) size = os.path.getsize(tmp_dir / "dir" / "bar") assert (tmp_dir / "bar").read_text() == "bar" assert callback.size == size assert callback.value == size
def test_download_dir_callback(tmp_dir, dvc, cloud): cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) cloud.gen({"dir": {"foo": "foo", "bar": "bar"}}) callback = fsspec.Callback() fs.download(cloud / "dir", tmp_dir / "dir", callback=callback) assert callback.size == 2 assert callback.value == 2 assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"}
def test_upload_callback(tmp_dir, dvc, cloud): tmp_dir.gen("foo", "foo") cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) expected_size = os.path.getsize(tmp_dir / "foo") callback = fsspec.Callback() fs.upload(tmp_dir / "foo", cloud / "foo", callback=callback) assert callback.size == expected_size assert callback.value == expected_size
def test_download_callback(tmp_dir, dvc, cloud): cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) fs.upload(io.BytesIO(b"foo"), cloud / "foo") expected_size = fs.getsize(cloud / "foo") callback = fsspec.Callback() fs.download_file(cloud / "foo", tmp_dir / "foo", callback=callback) assert callback.size == expected_size assert callback.value == expected_size assert (tmp_dir / "foo").read_text() == "foo"
def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type): gen = tmp_dir.scm_gen if fs_type == "git" else tmp_dir.dvc_gen gen({"dir": {"foo": "foo", "bar": "bar"}, "file": "file"}, commit="gen") fs = dvc.dvcfs if fs_type == "dvc" else scm.get_fs("HEAD") callback = fsspec.Callback() fs.download(tmp_dir / "file", tmp_dir / "file2", callback=callback) size = os.path.getsize(tmp_dir / "file") assert (tmp_dir / "file2").read_text() == "file" assert callback.size == size assert callback.value == size if fs_type == "git": pytest.skip("gitfs does not support download_dir") callback = fsspec.Callback() fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback) assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"} assert callback.size == 2 assert callback.value == 2
def test_get_file_callback(fs, tmpdir, remote_dir): src_file = tmpdir / "a.txt" dest_file = tmpdir / "b.txt" with open(src_file, "wb") as file: file.write(b"data" * 10) fs.put_file(src_file, remote_dir + "/a.txt") callback = fsspec.Callback() fs.get_file( remote_dir + "/a.txt", dest_file, callback=callback, block_size=10 ) assert dest_file.read() == "data" * 10 assert callback.size == 40 assert callback.value == 40
def test_download_callback(tmp_dir, dvc, cloud, local_cloud): cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) (tmp_dir / "to_upload").write_text("foo") fs.upload((tmp_dir / "to_upload").fs_path, (cloud / "foo").fs_path) expected_size = fs.getsize((cloud / "foo").fs_path) callback = fsspec.Callback() fs.download_file( (cloud / "foo").fs_path, (tmp_dir / "foo").fs_path, callback=callback, ) assert callback.size == expected_size assert callback.value == expected_size assert (tmp_dir / "foo").read_text() == "foo"