def test_retrieve_weather(monkeypatch, mock_dataset_arome: xr.Dataset, _get_mock_repository_dir):
five_days_ago = datetime.utcnow() - timedelta(days=5)
one_month_ago = five_days_ago - relativedelta(months=1)
# In stead of returning the regular data
def mock_fill_dataset_with_data(self, arome_coordinates, begin, end):
return mock_dataset_arome
monkeypatch.setattr(AromeRepository, 'gather_period', mock_fill_dataset_with_data)
arome_model = AromeModel()
# The coordinates requested are those of Amsterdam and Arnhem
ds = arome_model.get_weather(coords=[GeoPosition(52.3667, 4.8945), GeoPosition(51.9851, 5.8987)],
begin=one_month_ago,
end=five_days_ago)
# TODO: len doesn't match expectations for the field. Find out why
# TODO: The mock-format needs to be enhanced to properly pass the formatting round. This already happens when not
# mocking, but needs to be supported by the mock-result as well, to get this to properly work offline..
assert ds is not None
print("DS == ", ds)
assert "fake_factor_1" in ds
assert len(ds["fake_factor_1"]) == 48 # 49 prediction moments per time in the mock dataset
assert len(ds["fake_factor_1"][0]) == 95 # 96 periods in the mock dataset
assert isinstance(ds, xr.Dataset)
def mock_dataset_era5(mock_coordinates, mock_factors):
"""
returns a mock Xarray Dataset for
Args:
mock_coordinates:
mock_factors:
Returns:
"""
timeline = pd.date_range(end=(datetime.utcnow() - relativedelta(days=6)),
periods=96,
freq="1H",
closed="left")
coord_indices = coords_to_pd_index(
[GeoPosition(51.873419, 5.705929),
GeoPosition(53.2194, 6.5665)])
weather_factors = mock_factors
data_dict = {
weather_factor: (
["time", "coord"],
np.zeros(shape=(len(timeline), len(coord_indices)),
dtype=np.float64),
)
for weather_factor in weather_factors
}
ds = xr.Dataset(data_vars=data_dict,
coords={
"time": timeline,
"coord": coord_indices
})
ds = ds.unstack("coord")
return ds
def mock_download_era5sl_file(self, weather_factors, years, months, days,
area_box, target_location):
timeline = pd.date_range(end=datetime.utcnow(),
periods=96,
freq="1H",
closed="left")
coord_indices = coords_to_pd_index([
GeoPosition(coordinate[0], coordinate[1])
for coordinate in mock_coordinates
])
coord = coord_indices[0]
data_dict = {
weather_factor: (
["time", "longitude", "latitude"],
np.zeros(shape=(len(timeline), 1, 1), dtype=np.float64),
)
for weather_factor in weather_factors
}
ds = xr.Dataset(data_vars=data_dict,
coords={
"time": timeline,
"longitude": [coord[1]],
"latitude": [coord[0]]
})
ds.to_netcdf(path=target_location, format="NETCDF4", engine="netcdf4")
def _tuples_to_geo_positions(
coords: List[Tuple[float, float]]) -> List[GeoPosition]:
# Convert the Tuples in a list to a list of Geo Positions
coords = [
GeoPosition(coordinate[0], coordinate[1]) for coordinate in coords
]
return coords
def mock_single_value_dataset(mock_coordinates):
mock_geoposition_coordinates = [
GeoPosition(coordinate[0], coordinate[1])
for coordinate in mock_coordinates
]
mock_factor = ['temperature', 'precipitation', 'mock_unknown_field']
timeline = [datetime.now()]
coord_indices = coords_to_pd_index(mock_geoposition_coordinates)
data_dict = {
weather_factor: (
["time", "coord"],
np.zeros(shape=(len(timeline), len(coord_indices)),
dtype=np.float64),
)
for weather_factor in mock_factor
}
ds = xr.Dataset(data_vars=data_dict,
coords={
"time": timeline,
"coord": coord_indices
})
# Fill the single temperature value with 25 degrees Celsius
ds['temperature'].data = [[np.float64(25), np.float64(25)]]
# Fill the single precipitation value with 32 millimeters Celsius
ds['precipitation'].data = [[np.float64(32), np.float64(32)]]
# Fill the field to test the handling of unknown values with 66
ds['mock_unknown_field'].data = [[np.float64(66), np.float64(66)]]
return ds
def test_retrieve_weather(monkeypatch, mock_coordinates, start, end):
mock_geoposition_coordinates = [
GeoPosition(coordinate[0], coordinate[1])
for coordinate in mock_coordinates
]
# TODO: Monkeypatch the download call to test without connection
# TEST 1: Regular usage, with a non-existing factor
pluim_model = PluimModel()
# Factors contain both existing and non-existing factors. Non-existing factors should just be ignored..
mock_factors = [
'fake_factor_1', 'wind_speed', 'wind_direction',
'short_time_wind_speed', 'temperature', 'precipitation', 'cape'
]
ds = pluim_model.get_weather(coords=mock_geoposition_coordinates,
begin=start,
end=end,
weather_factors=mock_factors)
assert ds is not None
assert "wind_speed" in ds
assert "precipitation_sum" not in ds
assert "fake_factor_1" not in ds
assert isinstance(ds, xr.Dataset)
# TEST 2: Empty list of weather factors should get the full set
mock_factors = None
ds = pluim_model.get_weather(coords=mock_geoposition_coordinates,
begin=start,
end=end,
weather_factors=mock_factors)
assert ds is not None
assert "wind_speed" in ds
assert "precipitation_sum" in ds
assert isinstance(ds, xr.Dataset)
# TEST 3: Test for HTTPError handling of non-200 status codes
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
def json(self):
return self.json_data
def mock_request_get(self, *args, **kwargs):
return MockResponse({"dummy": "value"}, 404)
monkeypatch.setattr(requests, "get", mock_request_get)
with pytest.raises(requests.exceptions.HTTPError) as e:
pluim_model.get_weather(coords=mock_geoposition_coordinates,
begin=start,
end=end,
weather_factors=mock_factors)
assert str(e.value.args[0]
)[:53] == "Failed to retrieve data from the KNMI website"
def test_retrieve_weather(mock_coordinates, start, end, inseason):
mock_geoposition_coordinates = [
GeoPosition(coordinate[0], coordinate[1])
for coordinate in mock_coordinates
]
# TODO: Monkeypatch the download call to test without connection
daggegevens_model = DagGegevensModel()
ds = daggegevens_model.get_weather(coords=mock_geoposition_coordinates,
begin=start,
end=end,
inseason=inseason)
assert ds is not None
assert "TN" in ds
assert len(ds["TN"]) == 31
assert isinstance(ds, xr.Dataset)
def test_get_weather(mock_coordinates, start, end):
mock_geo_coordinates = [
GeoPosition(coordinate[0], coordinate[1])
for coordinate in mock_coordinates
]
aw_model = ActueleWaarnemingenModel()
# TODO: Monkeypatch the download call to test without connection
ds = aw_model.get_weather(coords=mock_geo_coordinates,
begin=start,
end=end)
assert ds is not None
assert "temperature" in ds
assert len(ds["temperature"]) == 1
assert isinstance(ds, xr.Dataset)
def _find_closest_stn_single(stn_stations: pd.DataFrame, coord: GeoPosition):
"""
A function that finds the closest station to a single GeoPosition
Args:
stn_stations: A Pandas Dataframe containing all of the station data for the KNMI stations
coord: A Geoposition to find the nearest station to.
Returns:
A station number indicating its index in the supplied dataframe.
"""
stn_stations["distance"] = stn_stations.apply(lambda x: great_circle(
(x["lat"], x["lon"]), coord.get_WGS84()).km,
axis=1)
# Find the stn with the lowest distance to the location
min_ind = np.argmin(stn_stations["distance"].values)
# Return the found stn
return stn_stations.loc[min_ind, "STN"]
def test_location_type_detection():
# Tests to verify proper type detection for regular and extreme coordinate values
# Test 1: Out of bounds 'coordinate' for both WGS84 and RD, using auto-detection
with pytest.raises(ValueError) as e:
assert GeoPosition(52.2, 182)
assert str(
e.value.args[0]
) == "No valid coordinate system could be determined from the coordinates given.."
# Test 2: Value in bounds as WSG84, but specified as RD and out of bounds for RD.
with pytest.raises(ValueError) as e:
assert GeoPosition(52.2, 90, 'RD')
assert str(e.value.args[0]) == "Invalid coordinates for type were used"
# Test 3: Value in bounds as RD, but specified as WGS84 and out of bounds for WGS84.
with pytest.raises(ValueError) as e:
assert GeoPosition(155000, 463000, 'WGS84')
assert str(e.value.args[0]) == "Invalid coordinates for type were used"
# Test 4: Highest possible and lowest possible values for WGS84, autodetect.
assert GeoPosition(180, 90)
assert GeoPosition(-180, -90)
# Test 5: Highest possible and lowest possible values for RD, autodetect.
assert GeoPosition(7000, 289000)
assert GeoPosition(300000, 629000)
# Test 6: Unknown coordinate system is passed for a coordinate. Coordinate resolves (to RD) though.
assert GeoPosition(155000, 463000, 'MARSHMALLOW').system == "RD", \
'GeoPosition should determine RD to be the correctformat'
# Test 7: Unknown coordinate system is passed for a coordinate. Coordinate does not resolve.
with pytest.raises(ValueError) as e:
GeoPosition(-1000, -2000, 'MARSHMALLOW')
assert str(
e.value.args[0]
) == "No valid coordinate system could be determined from the coordinates given.."
def test_retrieve_weather(monkeypatch, mock_coordinates, start, end):
mock_geoposition_coordinates = [
GeoPosition(coordinate[0], coordinate[1])
for coordinate in mock_coordinates
]
# TODO: Monkeypatch the download call to test without connection
uurgegevens_model = UurgegevensModel()
ds = uurgegevens_model.get_weather(coords=mock_geoposition_coordinates,
begin=start,
end=end)
assert ds is not None
assert "TD" in ds
assert isinstance(ds, xr.Dataset)
# TEST 3: Test for HTTPError handling of non-200 status codes
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
def json(self):
return self.json_data
def mock_request_post(*args, **kwargs):
return MockResponse({"dummy": "value"}, 404)
monkeypatch.setattr(requests, "post", mock_request_post)
with pytest.raises(requests.exceptions.HTTPError) as e:
uurgegevens_model.get_weather(coords=mock_geoposition_coordinates,
begin=start,
end=end,
weather_factors=None)
assert str(
e.value.args[0]) == "Failed to retrieve data from the KNMI website"
def get_grid_coordinates(
self, coordinates: List[GeoPosition]) -> List[GeoPosition]:
print("This method is abstract and should be overridden.")
return [GeoPosition(0, 0)]
def test_coordinate_live_locations():
"""
Tests using existing locations in the Netherlands to verify proper conversion between RD and WGS84
Location positions were verified using reliable online registers containing RD coordinates for streets, matching
them to their Google WGS84 locations, and using a known reliable tertiary conversion tool to confirm everything.
"""
# Test 1: "Onze Lieve Vrouwe Toren" - Amersfoort (building and the center point of the RD coordinate system)
geo_pos = GeoPosition(155000, 463000, "RD")
assert geo_pos.get_RD() == (
155000, 463000), "Error - Value not properly saved as RD-value"
assert geo_pos.get_WGS84() == (
52.15517440,
5.38720621), "Error - WGS84 value not within allowed constraints"
# Test 2: Wijnbergseweg - Braamt (street on the middle east side of the country)
geo_pos = GeoPosition(215803, 438150)
assert geo_pos.get_RD() == (
215803, 438150), "Error - Value not properly identified or saved as RD"
assert geo_pos.get_WGS84() == pytest.approx((51.92849584, 6.27121733), rel=1e-5), \
"Error - WGS84 value not within allowed constraints"
# Test 3: Admiralengracht - Amsterdam (street on the middle west side of the country)
geo_pos = GeoPosition(52.36954423, 4.85667541)
assert geo_pos.get_RD() == (
118868, 486984), "Error - RD value not within allowed constraints"
assert geo_pos.get_WGS84() == (
52.36954423,
4.85667541), "Error - Value not properly identified or saved as WGS84"
# Test 4: Gasthuisstraat - Dokkum (street on the middle north side of the country)
geo_pos = GeoPosition(195703, 593452)
assert geo_pos.get_RD() == (
195703, 593452), "Error - Value not properly identified or saved as RD"
assert geo_pos.get_WGS84() == pytest.approx((53.3259597, 5.9980788), rel=1e-5), \
"Error - WGS84 value not within allowed constraints"
# Test 5: Burgemeester de Grauwstraat - Baarle Nassau (street on the middle south side of the country)
geo_pos = GeoPosition(51.4450399, 4.9284643)
assert geo_pos.get_RD() == (
123108, 384094), "Error - RD value not within allowed constraints"
assert geo_pos.get_WGS84() == (
51.4450399,
4.9284643), "Error - Value not properly identified or saved as WGS84"