def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type):
from dvc.fs.git import GitFileSystem
gen = tmp_dir.scm_gen if fs_type == "git" else tmp_dir.dvc_gen
gen({"dir": {"foo": "foo", "bar": "bar"}, "file": "file"}, commit="gen")
fs = dvc.dvcfs if fs_type == "dvc" else GitFileSystem(scm=scm, rev="HEAD")
callback = fsspec.Callback()
fs.download_file(
"file",
(tmp_dir / "file2").fs_path,
callback=callback,
)
size = os.path.getsize(tmp_dir / "file")
assert (tmp_dir / "file2").read_text() == "file"
assert callback.size == size
assert callback.value == size
callback = fsspec.Callback()
fs.download(
"dir",
(tmp_dir / "dir2").fs_path,
callback=callback,
)
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
assert callback.size == 2
assert callback.value == 2
def test_callback_on_repo_fs(tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc")
tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git")
fs = dvc.repo_fs
callback = fsspec.Callback()
fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback)
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
assert callback.size == 2
assert callback.value == 2
callback = fsspec.Callback()
fs.download(tmp_dir / "dir" / "foo", tmp_dir / "foo", callback=callback)
size = os.path.getsize(tmp_dir / "dir" / "foo")
assert (tmp_dir / "foo").read_text() == "foo"
assert callback.size == size
assert callback.value == size
callback = fsspec.Callback()
fs.download(tmp_dir / "dir" / "bar", tmp_dir / "bar", callback=callback)
size = os.path.getsize(tmp_dir / "dir" / "bar")
assert (tmp_dir / "bar").read_text() == "bar"
assert callback.size == size
assert callback.value == size
def test_download_dir_callback(tmp_dir, dvc, cloud):
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
cloud.gen({"dir": {"foo": "foo", "bar": "bar"}})
callback = fsspec.Callback()
fs.download(cloud / "dir", tmp_dir / "dir", callback=callback)
assert callback.size == 2
assert callback.value == 2
assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"}
def test_upload_callback(tmp_dir, dvc, cloud):
tmp_dir.gen("foo", "foo")
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
expected_size = os.path.getsize(tmp_dir / "foo")
callback = fsspec.Callback()
fs.upload(tmp_dir / "foo", cloud / "foo", callback=callback)
assert callback.size == expected_size
assert callback.value == expected_size
def test_download_callback(tmp_dir, dvc, cloud):
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
fs.upload(io.BytesIO(b"foo"), cloud / "foo")
expected_size = fs.getsize(cloud / "foo")
callback = fsspec.Callback()
fs.download_file(cloud / "foo", tmp_dir / "foo", callback=callback)
assert callback.size == expected_size
assert callback.value == expected_size
assert (tmp_dir / "foo").read_text() == "foo"
def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type):
gen = tmp_dir.scm_gen if fs_type == "git" else tmp_dir.dvc_gen
gen({"dir": {"foo": "foo", "bar": "bar"}, "file": "file"}, commit="gen")
fs = dvc.dvcfs if fs_type == "dvc" else scm.get_fs("HEAD")
callback = fsspec.Callback()
fs.download(tmp_dir / "file", tmp_dir / "file2", callback=callback)
size = os.path.getsize(tmp_dir / "file")
assert (tmp_dir / "file2").read_text() == "file"
assert callback.size == size
assert callback.value == size
if fs_type == "git":
pytest.skip("gitfs does not support download_dir")
callback = fsspec.Callback()
fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback)
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
assert callback.size == 2
assert callback.value == 2
def test_get_file_callback(fs, tmpdir, remote_dir):
src_file = tmpdir / "a.txt"
dest_file = tmpdir / "b.txt"
with open(src_file, "wb") as file:
file.write(b"data" * 10)
fs.put_file(src_file, remote_dir + "/a.txt")
callback = fsspec.Callback()
fs.get_file(
remote_dir + "/a.txt", dest_file, callback=callback, block_size=10
)
assert dest_file.read() == "data" * 10
assert callback.size == 40
assert callback.value == 40
def test_download_callback(tmp_dir, dvc, cloud, local_cloud):
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
(tmp_dir / "to_upload").write_text("foo")
fs.upload((tmp_dir / "to_upload").fs_path, (cloud / "foo").fs_path)
expected_size = fs.getsize((cloud / "foo").fs_path)
callback = fsspec.Callback()
fs.download_file(
(cloud / "foo").fs_path,
(tmp_dir / "foo").fs_path,
callback=callback,
)
assert callback.size == expected_size
assert callback.value == expected_size
assert (tmp_dir / "foo").read_text() == "foo"
