async def test_fetch_model_http_redirect(mock_aioresponses) -> None:
url = get_data_url('subdir/redirect.alias')
# aioresponses doesn't properly handle relative URLs in redirects, so we
# need to use an absolute URL.
new_url = get_data_url('direct.alias')
mock_aioresponses.get(url, headers={'Location': new_url}, status=307)
with await fetch_aiohttp.fetch_model(url, DummyModel) as model:
assert len(model.ranges) == 2
def test_fetcher_caching(mock_responses) -> None:
with fetch_requests.Fetcher() as fetcher:
model1 = fetcher.get(get_data_url('rfi_mask_ranges.h5'), DummyModel)
model2 = fetcher.get(get_data_url('indirect.alias'), DummyModel)
model3 = fetcher.get(get_data_url('direct.alias'), DummyModel)
assert model1 is model2
assert model1 is model3
assert not model1.is_closed
assert len(mock_responses.calls) == 3
assert model1.is_closed
async def test_fetcher_caching(mock_aioresponses) -> None:
async with fetch_aiohttp.Fetcher() as fetcher:
model1 = await fetcher.get(get_data_url('rfi_mask_ranges.h5'),
DummyModel)
model2 = await fetcher.get(get_data_url('indirect.alias'), DummyModel)
model3 = await fetcher.get(get_data_url('direct.alias'), DummyModel)
assert model1 is model2
assert model1 is model3
assert not model1.is_closed
# Not supported by aioresponses
# assert len(mock_aioresponses.calls) == 3
assert model1.is_closed
def test_http_file_content_encoding_get(mock_responses):
url = get_data_url('rfi_mask_ranges.h5')
mock_responses.replace(responses.HEAD,
url,
headers={
'Content-Length':
str(len(get_data('rfi_mask_ranges.h5'))),
'Accept-Ranges':
'bytes'
})
mock_responses.replace(responses.GET,
url,
headers={
'Content-Length':
str(len(get_data('rfi_mask_ranges.h5'))),
'Accept-Ranges':
'bytes',
'Content-Encoding':
'gzip'
})
with requests.Session() as session:
with fetch_requests.HttpFile(session, url) as file:
with pytest.raises(
OSError, match='Server provided Content-Encoding header'):
file.read()
def test_bad_shape(mock_responses) -> None:
url = get_data_url('band_mask_ranges_2d.h5')
with pytest.raises(
models.DataError,
match='ranges dataset should be 1-dimensional, but is 2-dimensional'
):
fetch_requests.fetch_model(url, band_mask.BandMask)
async def test_fetch_model_checksum_ok(mock_aioresponses) -> None:
data = get_data('rfi_mask_ranges.h5')
digest = hashlib.sha256(data).hexdigest()
url = get_data_url(f'sha256_{digest}.h5')
mock_aioresponses.get(url, content_type='application/x-hdf5', body=data)
with await fetch_aiohttp.fetch_model(url, DummyModel) as model:
assert model.checksum == digest
def test_http_file_forbidden(mock_responses):
url = get_data_url('does_not_exist')
mock_responses.add(responses.HEAD, url, status=403)
with requests.Session() as session:
with pytest.raises(PermissionError) as exc_info:
fetch_requests.HttpFile(session, url)
assert exc_info.value.filename == url
def test_fetch_model_http_redirect(mock_responses) -> None:
url = get_data_url('subdir/redirect.alias')
mock_responses.add(responses.GET,
url,
headers={'Location': '../direct.alias'},
status=307)
with fetch_requests.fetch_model(url, DummyModel) as model:
assert len(model.ranges) == 2
def test_http_file_ranges_not_accepted(mock_responses):
url = get_data_url('rfi_mask_ranges.h5')
with requests.Session() as session:
with pytest.raises(
OSError,
match='Server does not accept byte ranges') as exc_info:
fetch_requests.HttpFile(session, url)
assert exc_info.value.filename == url
async def test_fetch_model_checksum_bad(mock_aioresponses) -> None:
data = get_data('rfi_mask_ranges.h5')
digest = hashlib.sha256(data).hexdigest()
url = get_data_url(f'sha256_{digest}.h5')
# Now invalidate the digest
data += b'blahblahblah'
mock_aioresponses.get(url, content_type='application/x-hdf5', body=data)
with pytest.raises(models.ChecksumError) as exc_info:
await fetch_aiohttp.fetch_model(url, DummyModel)
assert exc_info.value.url == url
def test_http_file_url_attr(mock_responses, web_server):
url = get_data_url('subdir/redirect.h5')
new_url = web_server('rfi_mask_ranges.h5')
mock_responses.add(responses.HEAD,
url,
headers={'Location': new_url},
status=307)
mock_responses.add_passthru(web_server(''))
with requests.Session() as session:
with fetch_requests.HttpFile(session, url) as file:
assert file.url == new_url
async def test_fetch_model_bad_content_type(mock_aioresponses) -> None:
data = get_data('rfi_mask_ranges.h5')
url = get_data_url('bad_content_type.h5')
mock_aioresponses.get(url, content_type='image/png', body=data)
with pytest.raises(
models.FileTypeError,
match='Expected application/x-hdf5, not image/png') as exc_info:
async with fetch_aiohttp.Fetcher() as fetcher:
await fetcher.get(url, DummyModel)
assert exc_info.value.url == url
assert exc_info.value.original_url == url
def test_http_file_no_content_length(mock_responses):
url = get_data_url('rfi_mask_ranges.h5')
mock_responses.replace(responses.HEAD,
url,
content_type='application/x-hdf5',
headers={'Accept-Ranges': 'bytes'})
with requests.Session() as session:
with pytest.raises(OSError,
match='Server did not provide Content-Length header'
) as exc_info:
fetch_requests.HttpFile(session, url)
assert exc_info.value.filename == url
def test_fetch_model_bad_content_type(mock_responses, lazy) -> None:
data = get_data('rfi_mask_ranges.h5')
url = get_data_url('bad_content_type.h5')
mock_responses.add(responses.HEAD,
url,
content_type='image/png',
headers={
'Accept-Ranges': 'bytes',
'Content-Length': str(len(data))
})
mock_responses.add(responses.GET, url, content_type='image/png', body=data)
with pytest.raises(
models.FileTypeError,
match='Expected application/x-hdf5, not image/png') as exc_info:
with fetch_requests.Fetcher() as fetcher:
fetcher.get(url, DummyModel, lazy=lazy)
assert exc_info.value.url == url
assert exc_info.value.original_url == url
def test_http_file_range_ignored(mock_responses):
url = get_data_url('rfi_mask_ranges.h5')
data = get_data('rfi_mask_ranges.h5')
mock_responses.replace(responses.HEAD,
url,
headers={
'Accept-Ranges': 'bytes',
'Content-Length': str(len(data))
})
mock_responses.replace(responses.GET, url, body=data, stream=True)
with requests.Session() as session:
with fetch_requests.HttpFile(session, url) as file:
with pytest.raises(OSError,
match='Did not receive expected byte range'):
file.read(10)
# Reading the whole file should work even if the server doesn't send
# back partial content.
file.seek(0)
test_data = file.read(len(data))
assert test_data == data
def test_missing_dataset(filename: str, mock_responses) -> None:
url = get_data_url(filename)
with pytest.raises(models.DataError, match='Model is missing ranges dataset'):
fetch_requests.fetch_model(url, rfi_mask.RFIMask)
def test_bad_model_format(mock_responses) -> None:
url = get_data_url('band_mask_bad_format.h5')
with pytest.raises(models.ModelFormatError) as exc_info:
fetch_requests.fetch_model(url, band_mask.BandMask)
assert str(
exc_info.value) == "Unknown model_format 'not_ranges' for band_mask"
async def test_fetch_model_connection_error(mock_aioresponses) -> None:
# aioresponses raises ConnectionError for any unregistered URL
with pytest.raises(aiohttp.ClientConnectionError):
await fetch_aiohttp.fetch_model(get_data_url('does_not_exist.h5'),
DummyModel)
def test_fetch_model_connection_error(mock_responses) -> None:
# responses raises ConnectionError for any unregistered URL
with pytest.raises(requests.ConnectionError):
fetch_requests.fetch_model(get_data_url('does_not_exist.h5'),
DummyModel)