async def test_storage_download_regular_file_to_absent_file(
storage_server: Any, make_client: _MakeClient, tmp_path: Path,
storage_path: Path) -> None:
src_file = DATA_FOLDER / "file.txt"
storage_file = storage_path / "file.txt"
storage_file.write_bytes(src_file.read_bytes())
local_dir = tmp_path / "local"
local_dir.mkdir()
local_file = local_dir / "file.txt"
progress = mock.Mock()
async with make_client(storage_server.make_url("/")) as client:
await client.storage.download_file(URL("storage:file.txt"),
URL(local_file.as_uri()),
progress=progress)
expected = src_file.read_bytes()
downloaded = local_file.read_bytes()
assert downloaded == expected
src = URL("storage://user/file.txt")
dst = URL(local_file.as_uri())
file_size = src_file.stat().st_size
progress.start.assert_called_with(StorageProgressStart(
src, dst, file_size))
progress.step.assert_called_with(
StorageProgressStep(src, dst, file_size, file_size))
progress.complete.assert_called_with(
StorageProgressComplete(src, dst, file_size))
async def test_blob_storage_upload_regular_file_new_file(
blob_storage_server: Any,
make_client: _MakeClient,
blob_storage_contents: _ContentsObj,
) -> None:
file_path = DATA_FOLDER / "file.txt"
file_size = file_path.stat().st_size
progress = mock.Mock()
async with make_client(blob_storage_server.make_url("/")) as client:
await client.blob_storage.upload_file(URL(file_path.as_uri()),
URL("blob:foo/file.txt"),
progress=progress)
expected = file_path.read_bytes()
uploaded = blob_storage_contents["file.txt"]
assert uploaded["body"] == expected
src = URL(file_path.as_uri())
dst = URL("blob://default/foo/file.txt")
progress.start.assert_called_with(StorageProgressStart(
src, dst, file_size))
progress.step.assert_called_with(
StorageProgressStep(src, dst, file_size, file_size))
progress.complete.assert_called_with(
StorageProgressComplete(src, dst, file_size))
async def test_storage_upload_regular_file_to_existing_file_target(
storage_server: Any, make_client: _MakeClient,
storage_path: Path) -> None:
file_path = DATA_FOLDER / "file.txt"
file_size = file_path.stat().st_size
target_path = storage_path / "file.txt"
progress = mock.Mock()
async with make_client(storage_server.make_url("/")) as client:
await client.storage.upload_file(URL(file_path.as_uri()),
URL("storage:file.txt"),
progress=progress)
expected = file_path.read_bytes()
uploaded = target_path.read_bytes()
assert uploaded == expected
src = URL(file_path.as_uri())
dst = URL("storage://user/file.txt")
progress.start.assert_called_with(StorageProgressStart(
src, dst, file_size))
progress.step.assert_called_with(
StorageProgressStep(src, dst, file_size, file_size))
progress.complete.assert_called_with(
StorageProgressComplete(src, dst, file_size))
async def test_blob_storage_download_regular_file_to_absent_file(
blob_storage_server: Any,
make_client: _MakeClient,
blob_storage_contents: _ContentsObj,
tmp_path: Path,
) -> None:
local_dir = tmp_path / "local"
local_dir.mkdir()
local_file = local_dir / "file.txt"
progress = mock.Mock()
async with make_client(blob_storage_server.make_url("/")) as client:
await client.blob_storage.download_file(URL("blob:foo/test1.txt"),
URL(local_file.as_uri()),
progress=progress)
expected = blob_storage_contents["test1.txt"]["body"]
downloaded = local_file.read_bytes()
assert downloaded == expected
src = URL("blob://default/foo/test1.txt")
dst = URL(local_file.as_uri())
file_size = len(expected)
progress.start.assert_called_with(StorageProgressStart(
src, dst, file_size))
progress.step.assert_called_with(
StorageProgressStep(src, dst, file_size, file_size))
progress.complete.assert_called_with(
StorageProgressComplete(src, dst, file_size))
def test_quiet_stream_progress(capsys: Any, make_root: _MakeRoot) -> None:
report = create_storage_progress(make_root(False, False, False), False)
src = URL("file:///abc")
dst = URL("storage:xyz")
report.begin(src, dst)
captured = capsys.readouterr()
assert captured.out == f""
report.enter(StorageProgressEnterDir(src, dst))
captured = capsys.readouterr()
assert captured.out == f""
report.start(StorageProgressStart(src, dst, 600))
captured = capsys.readouterr()
assert captured.out == f""
report.step(StorageProgressStep(src, dst, 300, 600))
captured = capsys.readouterr()
assert captured.out == ""
report.step(StorageProgressStep(src, dst, 400, 600))
captured = capsys.readouterr()
assert captured.out == ""
report.complete(StorageProgressComplete(src, dst, 600))
captured = capsys.readouterr()
assert captured.out == f""
report.leave(StorageProgressLeaveDir(src, dst))
captured = capsys.readouterr()
assert captured.out == f""
def test_stream_progress(capsys: Any, make_root: _MakeRoot) -> None:
report = create_storage_progress(make_root(False, False, True), False)
src = URL("file:///abc")
src_str = "/abc" if not sys.platform == "win32" else "\\abc"
dst = URL("storage:xyz")
dst_str = "storage:xyz"
report.begin(src, dst)
captured = capsys.readouterr()
assert captured.out == f"Copy '{src_str}' -> '{dst_str}'\n"
report.enter(StorageProgressEnterDir(src, dst))
captured = capsys.readouterr()
assert captured.out == f"'{src_str}' -> '{dst_str}'\n"
report.start(StorageProgressStart(src, dst, 600))
captured = capsys.readouterr()
assert captured.out == f""
report.step(StorageProgressStep(src, dst, 300, 600))
captured = capsys.readouterr()
assert captured.out == ""
report.step(StorageProgressStep(src, dst, 400, 600))
captured = capsys.readouterr()
assert captured.out == ""
report.complete(StorageProgressComplete(src, dst, 600))
captured = capsys.readouterr()
assert captured.out == f"'{src_str}' -> '{dst_str}'\n"
report.leave(StorageProgressLeaveDir(src, dst))
captured = capsys.readouterr()
assert captured.out == f""
async def test_storage_upload_not_a_file(storage_server: Any,
make_client: _MakeClient,
storage_path: Path) -> None:
file_path = Path(os.devnull).absolute()
target_path = storage_path / "file.txt"
progress = mock.Mock()
async with make_client(storage_server.make_url("/")) as client:
await client.storage.upload_file(URL(file_path.as_uri()),
URL("storage:file.txt"),
progress=progress)
uploaded = target_path.read_bytes()
assert uploaded == b""
src = URL(file_path.as_uri())
dst = URL("storage://user/file.txt")
progress.start.assert_called_with(StorageProgressStart(src, dst, 0))
progress.step.assert_not_called()
progress.complete.assert_called_with(StorageProgressComplete(src, dst, 0))
async def test_blob_storage_upload_not_a_file(
blob_storage_server: Any,
make_client: _MakeClient,
blob_storage_contents: _ContentsObj,
) -> None:
file_path = Path(os.devnull).absolute()
progress = mock.Mock()
async with make_client(blob_storage_server.make_url("/")) as client:
await client.blob_storage.upload_file(URL(file_path.as_uri()),
URL("blob:foo/file.txt"),
progress=progress)
uploaded = blob_storage_contents["file.txt"]
assert uploaded["body"] == b""
src = URL(file_path.as_uri())
dst = URL("blob://default/foo/file.txt")
progress.start.assert_called_with(StorageProgressStart(src, dst, 0))
progress.step.assert_not_called()
progress.complete.assert_called_with(StorageProgressComplete(src, dst, 0))
async def test_blob_storage_download_large_file(
blob_storage_server: Any,
make_client: _MakeClient,
blob_storage_contents: _ContentsObj,
tmp_path: Path,
) -> None:
local_dir = tmp_path / "local"
local_dir.mkdir()
local_file = local_dir / "file.txt"
progress = mock.Mock()
# 16 * 4 = 64 MB of data
large_payload = b"yncuNRzU0xhKSqIh" * (4 * 1024 * 1024)
blob_storage_contents["folder2/big_file"] = {
"key": "folder2/big_file",
"size": len(large_payload),
"last_modified": datetime(2019, 1, 3).timestamp(),
"body": large_payload,
}
async with make_client(blob_storage_server.make_url("/")) as client:
await client.blob_storage.download_file(
URL("blob:foo/folder2/big_file"),
URL(local_file.as_uri()),
progress=progress,
)
downloaded = local_file.read_bytes()
assert downloaded == large_payload
src = URL("blob://default/foo/folder2/big_file")
dst = URL(local_file.as_uri())
file_size = len(large_payload)
progress.start.assert_called_with(StorageProgressStart(
src, dst, file_size))
progress.step.assert_called_with(
StorageProgressStep(src, dst, file_size, file_size))
progress.complete.assert_called_with(
StorageProgressComplete(src, dst, file_size))
def test_tty_progress(capsys: Any, make_root: _MakeRoot) -> None:
report = create_storage_progress(make_root(False, True, False), True)
src = URL("file:///abc")
dst = URL("storage:xyz")
src_f = URL("file:///abc/file.txt")
dst_f = URL("storage:xyz/file.txt")
report.begin(src, dst)
captured = capsys.readouterr()
assert captured.out == f"Copy 'file:///abc' => 'storage:xyz'\n"
report.enter(StorageProgressEnterDir(src, dst))
assert unstyle(report) == ["'file:///abc' ..."]
report.start(StorageProgressStart(src_f, dst_f, 600))
assert unstyle(report) == [
"'file:///abc' ...", "'file.txt' [0.00%] 0B of 600B"
]
report.step(StorageProgressStep(src_f, dst_f, 300, 600))
assert unstyle(report) == [
"'file:///abc' ...", "'file.txt' [50.00%] 300B of 600B"
]
report.step(StorageProgressStep(src_f, dst_f, 400, 600))
assert unstyle(report) == [
"'file:///abc' ...", "'file.txt' [66.67%] 400B of 600B"
]
report.complete(StorageProgressComplete(src_f, dst_f, 600))
assert unstyle(report) == ["'file:///abc' ...", "'file.txt' 600B"]
report.leave(StorageProgressLeaveDir(src, dst))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc' DONE",
]
async def test_blob_storage_upload_large_file(
blob_storage_server: Any,
make_client: _MakeClient,
blob_storage_contents: _ContentsObj,
tmp_path: Path,
) -> None:
local_dir = tmp_path / "local"
local_dir.mkdir(exist_ok=True)
local_file = local_dir / "file.txt"
with local_file.open("wb") as f:
for i in range(1024):
f.write(b"yncuNRzU0xhKSqIh" * (4 * 1024))
progress = mock.Mock()
async with make_client(blob_storage_server.make_url("/")) as client:
await client.blob_storage.upload_file(
URL(local_file.as_uri()),
URL("blob:foo/folder2/big_file"),
progress=progress,
)
expected = local_file.read_bytes()
uploaded = blob_storage_contents["folder2/big_file"]
assert uploaded["body"] == expected
src = URL(local_file.as_uri())
dst = URL("blob://default/foo/folder2/big_file")
file_size = len(expected)
progress.start.assert_called_with(StorageProgressStart(
src, dst, file_size))
progress.step.assert_called_with(
StorageProgressStep(src, dst, file_size, file_size))
progress.complete.assert_called_with(
StorageProgressComplete(src, dst, file_size))
def test_tty_nested(make_root: _MakeRoot) -> None:
report = create_storage_progress(make_root(False, True, False), True)
src = URL("file:///abc")
dst = URL("storage:xyz")
src_f = URL("file:///abc/file.txt")
dst_f = URL("storage:xyz/file.txt")
src2 = URL("file:///abc/cde")
dst2 = URL("storage:xyz/cde")
src2_f = URL("file:///abc/cde/file.txt")
dst2_f = URL("storage:xyz/cde/file.txt")
report.enter(StorageProgressEnterDir(src, dst))
assert unstyle(report) == ["'file:///abc' ..."]
report.start(StorageProgressStart(src_f, dst_f, 600))
assert unstyle(report) == [
"'file:///abc' ...", "'file.txt' [0.00%] 0B of 600B"
]
report.step(StorageProgressStep(src_f, dst_f, 300, 600))
assert unstyle(report) == [
"'file:///abc' ...", "'file.txt' [50.00%] 300B of 600B"
]
report.step(StorageProgressStep(src_f, dst_f, 400, 600))
assert unstyle(report) == [
"'file:///abc' ...", "'file.txt' [66.67%] 400B of 600B"
]
report.complete(StorageProgressComplete(src_f, dst_f, 600))
assert unstyle(report) == ["'file:///abc' ...", "'file.txt' 600B"]
report.enter(StorageProgressEnterDir(src2, dst2))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc/cde' ...",
]
report.start(StorageProgressStart(src2_f, dst2_f, 800))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc/cde' ...",
"'file.txt' [0.00%] 0B of 800B",
]
report.step(StorageProgressStep(src2_f, dst2_f, 300, 800))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc/cde' ...",
"'file.txt' [37.50%] 300B of 800B",
]
report.complete(StorageProgressComplete(src2_f, dst_f, 800))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc/cde' ...",
"'file.txt' 800B",
]
report.leave(StorageProgressLeaveDir(src2, dst2))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc/cde' ...",
"'file.txt' 800B",
"'file:///abc/cde' DONE",
]
report.leave(StorageProgressLeaveDir(src, dst))
assert unstyle(report) == [
"'file:///abc' ...",
"'file.txt' 600B",
"'file:///abc/cde' ...",
"'file.txt' 800B",
"'file:///abc/cde' DONE",
"'file:///abc' DONE",
]
