def download(self,
partial_download=None,
force_overwrite=False,
cleanup=False):
"""Download data to `save_dir` and optionally print a message.
Args:
partial_download (list or None):
A list of keys of remotes to partially download.
If None, all data is downloaded
force_overwrite (bool):
If True, existing files are overwritten by the downloaded files.
cleanup (bool):
Whether to delete any zip/tar files after extracting.
Raises:
ValueError: if invalid keys are passed to partial_download
IOError: if a downloaded file's checksum is different from expected
"""
download_utils.downloader(
self.data_home,
remotes=self.remotes,
partial_download=partial_download,
info_message=self._download_info,
force_overwrite=force_overwrite,
cleanup=cleanup,
)
def test_download_multipart_zip(mocker, mock_download_from_remote, mock_unzip):
Path("tests/resources/foo.zip").touch()
Path("tests/resources/foo.z01").touch()
multipart_zip_remote = {
"foo": [
download_utils.RemoteFileMetadata(
filename="foo.zip", url="a", checksum=("1234")
),
download_utils.RemoteFileMetadata(
filename="foo.z01", url="b", checksum=("2345")
),
]
}
download_utils.downloader(
"tests/resources", multipart_zip_remote, force_overwrite=False, cleanup=True
)
mock_download_from_remote.assert_has_calls(
[
mocker.call(multipart_zip_remote["foo"][0], "tests/resources", False),
mocker.call(multipart_zip_remote["foo"][1], "tests/resources", False),
]
)
mock_unzip.assert_called_once_with("tests/resources/foo_single.zip", cleanup=True)
def test_downloader(mocker, mock_path):
mock_zip = mocker.patch.object(download_utils, "download_zip_file")
mock_tar = mocker.patch.object(download_utils, "download_tar_file")
mock_download_from_remote = mocker.patch.object(download_utils,
"download_from_remote")
zip_remote = download_utils.RemoteFileMetadata(filename="remote.zip",
url="a",
checksum=("1234"))
tar_remote = download_utils.RemoteFileMetadata(filename="remote.tar.gz",
url="a",
checksum=("1234"))
file_remote = download_utils.RemoteFileMetadata(filename="remote.txt",
url="a",
checksum=("1234"))
# Zip only
download_utils.downloader("a", remotes={"b": zip_remote})
mock_zip.assert_called_once_with(zip_remote, "a", False, False)
mocker.resetall()
# tar only
download_utils.downloader("a", remotes={"b": tar_remote})
mock_tar.assert_called_once_with(tar_remote, "a", False, False)
mocker.resetall()
# file only
download_utils.downloader("a", remotes={"b": file_remote})
mock_download_from_remote.assert_called_once_with(file_remote, "a", False)
mocker.resetall()
# zip and tar
download_utils.downloader("a", remotes={"b": zip_remote, "c": tar_remote})
mock_zip.assert_called_once_with(zip_remote, "a", False, False)
mock_tar.assert_called_once_with(tar_remote, "a", False, False)
mocker.resetall()
# zip and file
download_utils.downloader("a", remotes={"b": zip_remote, "c": file_remote})
mock_zip.assert_called_once_with(zip_remote, "a", False, False)
mock_download_from_remote.assert_called_once_with(file_remote, "a", False)
mocker.resetall()
# tar and file
download_utils.downloader("a", remotes={"b": tar_remote, "c": file_remote})
mock_tar.assert_called_once_with(tar_remote, "a", False, False)
mock_download_from_remote.assert_called_once_with(file_remote, "a", False)
mocker.resetall()
# zip and tar and file
download_utils.downloader("a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
})
mock_zip.assert_called_once_with(zip_remote, "a", False, False)
mock_download_from_remote.assert_called_once_with(file_remote, "a", False)
mock_tar.assert_called_once_with(tar_remote, "a", False, False)
mocker.resetall()
# test partial download
download_utils.downloader(
"a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
},
partial_download=["b", "d"],
)
mock_zip.assert_called_once_with(zip_remote, "a", False, False)
mock_download_from_remote.assert_called_once_with(file_remote, "a", False)
mocker.resetall()
# test bad type partial download
with pytest.raises(ValueError):
download_utils.downloader(
"a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
},
partial_download="b",
)
with pytest.raises(ValueError):
download_utils.downloader(
"a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
},
partial_download=["d", "e"],
)
# test info message
download_utils.downloader("a", info_message="I am a message!")
mocker.resetall()
# test download twice - defaults
download_utils.downloader("a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
})
download_utils.downloader("a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
})
# test download twice - cleanup=True
download_utils.downloader("a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
},
cleanup=True)
download_utils.downloader("a",
remotes={
"b": zip_remote,
"c": tar_remote,
"d": file_remote
})
def test_downloader_with_server_tar(httpserver):
httpserver.serve_content(
open("tests/resources/remote.tar.gz", "rb").read())
TEST_REMOTE = download_utils.RemoteFileMetadata(
filename="remote.tar.gz",
url=httpserver.url,
checksum=("9042f5eebdcd0b94aa7a3c9bf12dc51d"),
)
save_dir = "tests/resources/_tmp_test_download_utils"
_clean(save_dir)
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
# test downloading twice
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
_clean(save_dir)
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
cleanup=True)
# test downloading twice
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
_clean(save_dir)
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
# test downloading twice
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
force_overwrite=True)
_clean(save_dir)
def test_downloader_with_server_zip(httpserver):
httpserver.serve_content(open("tests/resources/remote.zip", "rb").read())
TEST_REMOTE = download_utils.RemoteFileMetadata(
filename="remote.zip",
url=httpserver.url,
checksum=("7a31ccfa28bfa3fb112d16c96e9d9a89"),
)
save_dir = "tests/resources/_tmp_test_download_utils"
_clean(save_dir)
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
# test downloading twice
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
_clean(save_dir)
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
cleanup=True)
# test downloading twice
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
_clean(save_dir)
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
# test downloading twice
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
force_overwrite=True)
_clean(save_dir)
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
cleanup=True)
# test downloading twice
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
force_overwrite=True)
_clean(save_dir)
def test_downloader_with_server_file(httpserver):
httpserver.serve_content(open("tests/resources/remote.wav").read())
TEST_REMOTE = download_utils.RemoteFileMetadata(
filename="remote.wav",
url=httpserver.url,
checksum=("3f77d0d69dc41b3696f074ad6bf2852f"),
)
save_dir = "tests/resources/tmp_download_test"
_clean(save_dir)
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
# test downloading twice
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
_clean(save_dir)
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
cleanup=True)
# test downloading twice
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
_clean(save_dir)
download_utils.downloader(save_dir, remotes={"b": TEST_REMOTE})
# test downloading twice
download_utils.downloader(save_dir,
remotes={"b": TEST_REMOTE},
force_overwrite=True)
_clean(save_dir)