def test_download_overwrite(event_loop, httpserver, tmpdir):
httpserver.serve_content('SIMPLE = T')
fname = "testing123"
filename = str(tmpdir.join(fname))
with open(filename, "w") as fh:
fh.write("Hello world")
dl = Downloader(loop=event_loop, overwrite=True)
dl.enqueue_file(httpserver.url, filename=filename, chunksize=200)
f = dl.download()
assert isinstance(f, Results)
assert len(f) == 1
assert f[0] == filename
with open(filename) as fh:
assert fh.read() == "SIMPLE = T"
def test_proxy_passed_as_kwargs_to_get(tmpdir, url, proxy):
with mock.patch("aiohttp.client.ClientSession._request",
new_callable=mock.MagicMock) as patched:
dl = Downloader()
dl.enqueue_file(url, path=Path(tmpdir), max_splits=None)
assert dl.queued_downloads == 1
dl.download()
assert patched.called, "`ClientSession._request` not called"
assert list(patched.call_args) == [('GET', url), {
'allow_redirects':
True,
'timeout':
ClientTimeout(total=0, connect=None, sock_read=90, sock_connect=None),
'proxy':
proxy
}]
def test_download_no_overwrite(httpserver, tmpdir):
httpserver.serve_content('SIMPLE = T')
fname = "testing123"
filename = str(tmpdir.join(fname))
with open(filename, "w") as fh:
fh.write("Hello world")
dl = Downloader()
dl.enqueue_file(httpserver.url, filename=filename, chunksize=200)
f = dl.download()
assert isinstance(f, Results)
assert len(f) == 1
assert f[0] == filename
with open(filename) as fh:
# If the contents is the same as when we wrote it, it hasn't been
# overwritten
assert fh.read() == "Hello world"
def test_ftp(tmpdir):
tmpdir = str(tmpdir)
dl = Downloader()
dl.enqueue_file(
"ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz",
path=tmpdir)
dl.enqueue_file(
"ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz",
path=tmpdir)
dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir)
dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir)
f = dl.download()
assert len(f) == 1
assert len(f.errors) == 3
def test_ftp_http(tmpdir, httpserver):
tmpdir = str(tmpdir)
httpserver.serve_content('SIMPLE = T')
dl = Downloader()
dl.enqueue_file(
"ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz",
path=tmpdir)
dl.enqueue_file(
"ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz",
path=tmpdir)
dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir)
dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir)
dl.enqueue_file(httpserver.url, path=tmpdir)
dl.enqueue_file("http://noaurl.notadomain/noafile", path=tmpdir)
assert dl.queued_downloads == 6
f = dl.download()
assert len(f) == 2
assert len(f.errors) == 4
