def test_raises_other_exception(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader() dl.enqueue_file(httpserver.url, path=tmpdir) with pytest.raises(ValueError): dl.download()
def test_no_progress(httpserver, tmpdir, capsys): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader(progress=False) dl.enqueue_file(httpserver.url, path=tmpdir) dl.download() # Check that there was not stdout captured = capsys.readouterr().out assert not captured
def test_default_user_agent(event_loop, httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T', headers={'Content-Disposition': "attachment; filename=testfile.fits"}) dl = Downloader(loop=event_loop) dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert 'User-Agent' in httpserver.requests[0].headers assert httpserver.requests[0].headers['User-Agent'] == f"parfive/{parfive.__version__} aiohttp/{aiohttp.__version__} python/{sys.version[:5]}"
def test_custom_user_agent(event_loop, httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T', headers={'Content-Disposition': "attachment; filename=testfile.fits"}) dl = Downloader(loop=event_loop, headers={'User-Agent': 'test value 299792458'}) dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert 'User-Agent' in httpserver.requests[0].headers assert httpserver.requests[0].headers['User-Agent'] == "test value 299792458"
def test_download_unique(httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) filenames = [filename, filename + '.fits', filename + '.fits.gz'] dl = Downloader(overwrite='unique') # Write files to both the target filenames. for fn in filenames: with open(fn, "w") as fh: fh.write("Hello world") dl.enqueue_file(httpserver.url, filename=fn, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == len(filenames) for fn in f: assert fn not in filenames assert f"{fname}.1" in fn
def test_raises_other_exception(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader() dl.enqueue_file(httpserver.url, path=tmpdir) res = dl.download() assert isinstance(res.errors[0].exception, ValueError)
def test_notaurl(tmpdir): tmpdir = str(tmpdir) dl = Downloader(progress=False) dl.enqueue_file("http://notaurl.wibble/file", path=tmpdir) f = dl.download() assert len(f.errors) == 1 assert isinstance(f.errors[0].exception, aiohttp.ClientConnectionError)
def test_download_partial(event_loop, httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader(loop=event_loop) dl.enqueue_file(httpserver.url, filename=lambda resp, url: Path(tmpdir)/"filename") f = dl.download() assert len(f) == 1 # strip the http:// assert "filename" in f[0]
def test_download(event_loop, httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader(loop=event_loop) dl.enqueue_file(httpserver.url, path=Path(tmpdir)) f = dl.download() assert len(f) == 1 # strip the http:// assert httpserver.url[7:] in f[0]
def test_ftp_pasv_command(tmpdir): tmpdir = str(tmpdir) dl = Downloader() dl.enqueue_file( "ftp://ftp.ngdc.noaa.gov/STP/swpc_products/daily_reports/solar_region_summaries/2002/04/20020414SRS.txt", path=tmpdir, passive_commands=["pasv"]) assert dl.queued_downloads == 1 f = dl.download() assert len(f) == 1 assert len(f.errors) == 0
def test_proxy_passed_as_kwargs_to_get(event_loop, tmpdir, url, proxy): with mock.patch( "aiohttp.client.ClientSession._request", new_callable=mock.MagicMock ) as patched: dl = Downloader(loop=event_loop) dl.enqueue_file(url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert patched.called, "`ClientSession._request` not called" assert list(patched.call_args) == [('GET', url), {'allow_redirects': True, 'timeout': ClientTimeout(total=300, connect=None, sock_read=90, sock_connect=None), 'proxy': proxy }]
def test_ftp(tmpdir): tmpdir = str(tmpdir) dl = Downloader() dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir) f = dl.download() assert len(f) == 1 assert len(f.errors) == 3
def test_changed_max_conn(httpserver, tmpdir): # Check that changing max_conn works after creating Downloader tmpdir = str(tmpdir) httpserver.serve_content( 'SIMPLE = T', headers={'Content-Disposition': "attachment; filename=testfile.fits"}) dl = Downloader(max_conn=4) dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) dl.max_conn = 3 f = dl.download() validate_test_file(f)
def test_download_ranged_http(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( 'SIMPLE = T', headers={'Content-Disposition': "attachment; filename=testfile.fits"}) dl = Downloader() dl.enqueue_file(httpserver.url, path=Path(tmpdir)) assert dl.queued_downloads == 1 f = dl.download() validate_test_file(f)
def test_ftp_http(tmpdir, httpserver): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T') dl = Downloader() dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir) dl.enqueue_file(httpserver.url, path=tmpdir) dl.enqueue_file("http://noaurl.notadomain/noafile", path=tmpdir) f = dl.download() assert len(f) == 2 assert len(f.errors) == 4
def test_download_ranged_http(event_loop, httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('SIMPLE = T', headers={'Content-Disposition': "attachment; filename=testfile.fits"}) dl = Downloader(loop=event_loop) dl.enqueue_file(httpserver.url, path=Path(tmpdir)) assert dl.queued_downloads == 1 f = dl.download() assert len(f) == 1 assert Path(f[0]).name == "testfile.fits" assert sha256sum(f[0]) == "a1c58cd340e3bd33f94524076f1fa5cf9a7f13c59d5272a9d4bc0b5bc436d9b3"
def test_download_filename(event_loop, httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) dl = Downloader(loop=event_loop) dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename
def test_retrieve_some_content(testserver, tmpdir): """ Test that the downloader handles errors properly. """ tmpdir = str(tmpdir) dl = Downloader() nn = 5 for i in range(nn): dl.enqueue_file(testserver.url, path=tmpdir) f = dl.download() assert len(f) == nn - 1 assert len(f.errors) == 1
def test_regression_download_ranged_http(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content('S', headers={ 'Content-Disposition': "attachment; filename=testfile.fits", 'Accept-Ranges': "bytes" }) dl = Downloader() dl.enqueue_file(httpserver.url, path=Path(tmpdir)) assert dl.queued_downloads == 1 f = dl.download() assert len(f.errors) == 0, f.errors
def test_retry(tmpdir, testserver): tmpdir = str(tmpdir) dl = Downloader() nn = 5 for i in range(nn): dl.enqueue_file(testserver.url, path=tmpdir) f = dl.download() assert len(f) == nn - 1 assert len(f.errors) == 1 f2 = dl.retry(f) assert len(f2) == nn assert len(f2.errors) == 0
def test_download_filename(httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("SIMPLE = T") dl = Downloader() dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename
def test_download_overwrite(event_loop, httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("Hello world") dl = Downloader(loop=event_loop, overwrite=True) dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename with open(filename) as fh: assert fh.read() == "SIMPLE = T"
def test_download_no_overwrite(httpserver, tmpdir): httpserver.serve_content('SIMPLE = T') fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("Hello world") dl = Downloader() dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename with open(filename) as fh: # If the contents is the same as when we wrote it, it hasn't been # overwritten assert fh.read() == "Hello world"
def test_empty_download(event_loop, tmpdir): dl = Downloader(loop=event_loop) f = dl.download() assert len(f) == 0
def test_empty_download(tmpdir): dl = Downloader() f = dl.download() assert len(f) == 0