async def test_fetch_model_http_redirect(mock_aioresponses) -> None: url = get_data_url('subdir/redirect.alias') # aioresponses doesn't properly handle relative URLs in redirects, so we # need to use an absolute URL. new_url = get_data_url('direct.alias') mock_aioresponses.get(url, headers={'Location': new_url}, status=307) with await fetch_aiohttp.fetch_model(url, DummyModel) as model: assert len(model.ranges) == 2
def test_fetcher_caching(mock_responses) -> None: with fetch_requests.Fetcher() as fetcher: model1 = fetcher.get(get_data_url('rfi_mask_ranges.h5'), DummyModel) model2 = fetcher.get(get_data_url('indirect.alias'), DummyModel) model3 = fetcher.get(get_data_url('direct.alias'), DummyModel) assert model1 is model2 assert model1 is model3 assert not model1.is_closed assert len(mock_responses.calls) == 3 assert model1.is_closed
async def test_fetcher_caching(mock_aioresponses) -> None: async with fetch_aiohttp.Fetcher() as fetcher: model1 = await fetcher.get(get_data_url('rfi_mask_ranges.h5'), DummyModel) model2 = await fetcher.get(get_data_url('indirect.alias'), DummyModel) model3 = await fetcher.get(get_data_url('direct.alias'), DummyModel) assert model1 is model2 assert model1 is model3 assert not model1.is_closed # Not supported by aioresponses # assert len(mock_aioresponses.calls) == 3 assert model1.is_closed
def test_http_file_content_encoding_get(mock_responses): url = get_data_url('rfi_mask_ranges.h5') mock_responses.replace(responses.HEAD, url, headers={ 'Content-Length': str(len(get_data('rfi_mask_ranges.h5'))), 'Accept-Ranges': 'bytes' }) mock_responses.replace(responses.GET, url, headers={ 'Content-Length': str(len(get_data('rfi_mask_ranges.h5'))), 'Accept-Ranges': 'bytes', 'Content-Encoding': 'gzip' }) with requests.Session() as session: with fetch_requests.HttpFile(session, url) as file: with pytest.raises( OSError, match='Server provided Content-Encoding header'): file.read()
def test_bad_shape(mock_responses) -> None: url = get_data_url('band_mask_ranges_2d.h5') with pytest.raises( models.DataError, match='ranges dataset should be 1-dimensional, but is 2-dimensional' ): fetch_requests.fetch_model(url, band_mask.BandMask)
async def test_fetch_model_checksum_ok(mock_aioresponses) -> None: data = get_data('rfi_mask_ranges.h5') digest = hashlib.sha256(data).hexdigest() url = get_data_url(f'sha256_{digest}.h5') mock_aioresponses.get(url, content_type='application/x-hdf5', body=data) with await fetch_aiohttp.fetch_model(url, DummyModel) as model: assert model.checksum == digest
def test_http_file_forbidden(mock_responses): url = get_data_url('does_not_exist') mock_responses.add(responses.HEAD, url, status=403) with requests.Session() as session: with pytest.raises(PermissionError) as exc_info: fetch_requests.HttpFile(session, url) assert exc_info.value.filename == url
def test_fetch_model_http_redirect(mock_responses) -> None: url = get_data_url('subdir/redirect.alias') mock_responses.add(responses.GET, url, headers={'Location': '../direct.alias'}, status=307) with fetch_requests.fetch_model(url, DummyModel) as model: assert len(model.ranges) == 2
def test_http_file_ranges_not_accepted(mock_responses): url = get_data_url('rfi_mask_ranges.h5') with requests.Session() as session: with pytest.raises( OSError, match='Server does not accept byte ranges') as exc_info: fetch_requests.HttpFile(session, url) assert exc_info.value.filename == url
async def test_fetch_model_checksum_bad(mock_aioresponses) -> None: data = get_data('rfi_mask_ranges.h5') digest = hashlib.sha256(data).hexdigest() url = get_data_url(f'sha256_{digest}.h5') # Now invalidate the digest data += b'blahblahblah' mock_aioresponses.get(url, content_type='application/x-hdf5', body=data) with pytest.raises(models.ChecksumError) as exc_info: await fetch_aiohttp.fetch_model(url, DummyModel) assert exc_info.value.url == url
def test_http_file_url_attr(mock_responses, web_server): url = get_data_url('subdir/redirect.h5') new_url = web_server('rfi_mask_ranges.h5') mock_responses.add(responses.HEAD, url, headers={'Location': new_url}, status=307) mock_responses.add_passthru(web_server('')) with requests.Session() as session: with fetch_requests.HttpFile(session, url) as file: assert file.url == new_url
async def test_fetch_model_bad_content_type(mock_aioresponses) -> None: data = get_data('rfi_mask_ranges.h5') url = get_data_url('bad_content_type.h5') mock_aioresponses.get(url, content_type='image/png', body=data) with pytest.raises( models.FileTypeError, match='Expected application/x-hdf5, not image/png') as exc_info: async with fetch_aiohttp.Fetcher() as fetcher: await fetcher.get(url, DummyModel) assert exc_info.value.url == url assert exc_info.value.original_url == url
def test_http_file_no_content_length(mock_responses): url = get_data_url('rfi_mask_ranges.h5') mock_responses.replace(responses.HEAD, url, content_type='application/x-hdf5', headers={'Accept-Ranges': 'bytes'}) with requests.Session() as session: with pytest.raises(OSError, match='Server did not provide Content-Length header' ) as exc_info: fetch_requests.HttpFile(session, url) assert exc_info.value.filename == url
def test_fetch_model_bad_content_type(mock_responses, lazy) -> None: data = get_data('rfi_mask_ranges.h5') url = get_data_url('bad_content_type.h5') mock_responses.add(responses.HEAD, url, content_type='image/png', headers={ 'Accept-Ranges': 'bytes', 'Content-Length': str(len(data)) }) mock_responses.add(responses.GET, url, content_type='image/png', body=data) with pytest.raises( models.FileTypeError, match='Expected application/x-hdf5, not image/png') as exc_info: with fetch_requests.Fetcher() as fetcher: fetcher.get(url, DummyModel, lazy=lazy) assert exc_info.value.url == url assert exc_info.value.original_url == url
def test_http_file_range_ignored(mock_responses): url = get_data_url('rfi_mask_ranges.h5') data = get_data('rfi_mask_ranges.h5') mock_responses.replace(responses.HEAD, url, headers={ 'Accept-Ranges': 'bytes', 'Content-Length': str(len(data)) }) mock_responses.replace(responses.GET, url, body=data, stream=True) with requests.Session() as session: with fetch_requests.HttpFile(session, url) as file: with pytest.raises(OSError, match='Did not receive expected byte range'): file.read(10) # Reading the whole file should work even if the server doesn't send # back partial content. file.seek(0) test_data = file.read(len(data)) assert test_data == data
def test_missing_dataset(filename: str, mock_responses) -> None: url = get_data_url(filename) with pytest.raises(models.DataError, match='Model is missing ranges dataset'): fetch_requests.fetch_model(url, rfi_mask.RFIMask)
def test_bad_model_format(mock_responses) -> None: url = get_data_url('band_mask_bad_format.h5') with pytest.raises(models.ModelFormatError) as exc_info: fetch_requests.fetch_model(url, band_mask.BandMask) assert str( exc_info.value) == "Unknown model_format 'not_ranges' for band_mask"
async def test_fetch_model_connection_error(mock_aioresponses) -> None: # aioresponses raises ConnectionError for any unregistered URL with pytest.raises(aiohttp.ClientConnectionError): await fetch_aiohttp.fetch_model(get_data_url('does_not_exist.h5'), DummyModel)
def test_fetch_model_connection_error(mock_responses) -> None: # responses raises ConnectionError for any unregistered URL with pytest.raises(requests.ConnectionError): fetch_requests.fetch_model(get_data_url('does_not_exist.h5'), DummyModel)