def test_download_overwrite(event_loop, httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("Hello world") dl = Downloader(loop=event_loop, overwrite=True) dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename with open(filename) as fh: assert fh.read() == "SIMPLE = T"
def test_proxy_passed_as_kwargs_to_get(tmpdir, url, proxy): with mock.patch("aiohttp.client.ClientSession._request", new_callable=mock.MagicMock) as patched: dl = Downloader() dl.enqueue_file(url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert patched.called, "`ClientSession._request` not called" assert list(patched.call_args) == [('GET', url), { 'allow_redirects': True, 'timeout': ClientTimeout(total=0, connect=None, sock_read=90, sock_connect=None), 'proxy': proxy }]
def test_download_no_overwrite(httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("Hello world") dl = Downloader() dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename with open(filename) as fh: # If the contents is the same as when we wrote it, it hasn't been # overwritten assert fh.read() == "Hello world"
def test_ftp(tmpdir): tmpdir = str(tmpdir) dl = Downloader() dl.enqueue_file( "ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz", path=tmpdir) dl.enqueue_file( "ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir) f = dl.download() assert len(f) == 1 assert len(f.errors) == 3
def test_ftp_http(tmpdir, httpserver): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader() dl.enqueue_file( "ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz", path=tmpdir) dl.enqueue_file( "ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir) dl.enqueue_file(httpserver.url, path=tmpdir) dl.enqueue_file("http://noaurl.notadomain/noafile", path=tmpdir) assert dl.queued_downloads == 6 f = dl.download() assert len(f) == 2 assert len(f.errors) == 4