def adjust_datetimes(self):
"""
Adjust ``start_date`` and ``end_date`` attributes to match
minute marks for respective RadarParameter.
- RADOLAN_CDC is always published at HH:50.
https://opendata.dwd.de/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/ # noqa:E501,B950
- RQ_REFLECTIVITY is published each 15 minutes.
https://opendata.dwd.de/weather/radar/radvor/rq/
- All other radar formats are published in intervals of 5 minutes.
https://opendata.dwd.de/weather/radar/composit/fx/
https://opendata.dwd.de/weather/radar/sites/dx/boo/
"""
if (
self.parameter == DWDRadarParameter.RADOLAN_CDC
or self.parameter in RADAR_PARAMETERS_RADOLAN
):
# Align "start_date" to the most recent 50 minute mark available.
self.start_date = raster_minutes(self.start_date, 50)
# When "end_date" is given as timedelta, resolve it.
if isinstance(self.end_date, timedelta):
self.end_date = self.start_date + self.end_date
# Use "end_date = start_date" to make the machinery
# pick a single file from the fileindex.
if self.end_date is None:
self.end_date = self.start_date + timedelta(microseconds=1)
elif self.parameter == DWDRadarParameter.RQ_REFLECTIVITY:
# Align "start_date" to the 15 minute mark before tm.
self.start_date = round_minutes(self.start_date, 15)
# When "end_date" is given as timedelta, resolve it.
if isinstance(self.end_date, timedelta):
self.end_date = self.start_date + self.end_date
# Expand "end_date" to the end of the 5 minute mark.
if self.end_date is None:
self.end_date = self.start_date + timedelta(minutes=15)
else:
# Align "start_date" to the 5 minute mark before tm.
self.start_date = round_minutes(self.start_date, 5)
# When "end_date" is given as timedelta, resolve it.
if isinstance(self.end_date, timedelta):
self.end_date = self.start_date + self.end_date
# Expand "end_date" to the end of the 5 minute mark.
if self.end_date is None:
self.end_date = self.start_date + timedelta(minutes=5)
def test_radar_request_site_historic_sweep_vol_v_hdf5_yesterday():
"""
Example for testing radar/site sweep-precipitation for a specific date,
this time in HDF5 format.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DWDRadarData(
parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H,
start_date=timestamp,
site=DWDRadarSite.BOO,
fmt=DWDRadarDataFormat.HDF5,
subset=DWDRadarDataSubset.SIMPLE,
)
results = list(request.collect_data())
# Verify number of elements.
assert len(results) == 10
# Get payload from first file.
buffer = results[0].data
payload = buffer.getvalue()
# Verify data.
assert payload.startswith(b"\x89HDF\r\n")
# Verify more details.
# h5dump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950
hdf = h5py.File(buffer, "r")
assert hdf["/how/radar_system"] is not None
assert hdf["/how"].attrs.get("task") == b"Sc_Vol-5Min-NG-01_BOO"
assert hdf["/what"].attrs.get("source") == b"WMO:10132,NOD:deboo"
assert hdf["/how"].attrs.get("scan_count") == 10
assert hdf["/dataset1/how"].attrs.get("scan_index") == 1
assert hdf["/dataset1/data1/data"].shape == (360, 180)
timestamp = round_minutes(request.start_date, 5)
assert hdf["/what"].attrs.get("date") == bytes(
timestamp.strftime("%Y%m%d"), encoding="ascii")
assert (hdf["/what"].attrs.get("time").startswith(
bytes(timestamp.strftime("%H%M"), encoding="ascii")))
# Verify that the second file is the second scan / elevation level.
buffer = results[1].data
hdf = h5py.File(buffer, "r")
assert hdf["/how"].attrs.get("scan_count") == 10
assert hdf["/dataset1/how"].attrs.get("scan_index") == 2
timestamp = round_minutes(request.start_date, 5)
assert hdf["/what"].attrs.get("date") == bytes(
timestamp.strftime("%Y%m%d"), encoding="ascii")
assert (hdf["/what"].attrs.get("time").startswith(
bytes(timestamp.strftime("%H%M"), encoding="ascii")))
def test_radar_request_site_historic_sweep_vol_v_bufr_yesterday():
"""
Example for testing radar/site sweep_vol_v for a specific date,
this time in BUFR format.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DWDRadarData(
parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H,
start_date=timestamp,
site=DWDRadarSite.ASB,
fmt=DWDRadarDataFormat.BUFR,
)
buffer = next(request.collect_data())[1]
payload = buffer.getvalue()
# Read BUFR file.
decoder = pybufrkit.decoder.Decoder()
bufr = decoder.process(payload, info_only=True)
# Verify timestamp in BUFR metadata.
timestamp_aligned = round_minutes(timestamp, 5)
bufr_timestamp = datetime(
bufr.year.value + 2000,
bufr.month.value,
bufr.day.value,
bufr.hour.value,
bufr.minute.value,
)
assert timestamp_aligned == bufr_timestamp
def test_radar_request_site_historic_px250_bufr_yesterday():
"""
Example for testing radar/site PX250 for a specific date.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DWDRadarData(
parameter=DWDRadarParameter.PX250_REFLECTIVITY,
start_date=timestamp,
site=DWDRadarSite.BOO,
)
buffer = next(request.collect_data())[1]
payload = buffer.getvalue()
# Verify data.
header = b"\x00\x00\x00\x00\x00...BUFR"
assert re.match(header, payload), payload[:20]
# Read BUFR file.
decoder = pybufrkit.decoder.Decoder()
bufr = decoder.process(payload, info_only=True)
# Verify timestamp in BUFR metadata.
timestamp_aligned = round_minutes(timestamp, 5)
bufr_timestamp = datetime(
bufr.year.value,
bufr.month.value,
bufr.day.value,
bufr.hour.value,
bufr.minute.value,
)
assert timestamp_aligned == bufr_timestamp
def test_radar_request_site_historic_dx_yesterday():
"""
Verify acquisition of radar/site/DX data works
when using a specific date.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DWDRadarData(
parameter=DWDRadarParameter.DX_REFLECTIVITY,
start_date=timestamp,
site=DWDRadarSite.BOO,
)
buffer = next(request.collect_data())[1]
payload = buffer.getvalue()
# Verify data.
# TODO: Use wradlib to parse binary format.
# https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html
timestamp_aligned = round_minutes(timestamp, 5)
date_time = timestamp_aligned.strftime("%d%H%M")
month_year = timestamp_aligned.strftime("%m%y")
header = f"DX{date_time}10132{month_year}BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS" # noqa:E501,B950
assert re.match(bytes(header, encoding="ascii"), payload)
def test_radar_request_site_latest_dx_reflectivity():
"""
Example for testing radar SITES latest.
"""
request = DwdRadarValues(
parameter=DwdRadarParameter.DX_REFLECTIVITY,
start_date=DwdRadarDate.LATEST,
site=DwdRadarSite.BOO,
)
buffer = next(request.query())[1]
requested_header = wrl.io.read_radolan_header(buffer)
requested_attrs = wrl.io.radolan.parse_dx_header(requested_header)
# Verify data.
timestamp_aligned = round_minutes(datetime.utcnow(), 5)
assert timestamp_aligned.strftime(
"%m%y") == requested_attrs["datetime"].strftime("%m%y")
attrs = {
"producttype": "DX",
"version": " 2",
"cluttermap": 0,
"dopplerfilter": 4,
"statfilter": 0,
"elevprofile": [0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8],
"message": "",
}
skip_attrs = ["radarid", "datetime", "bytes"]
for attr in skip_attrs:
requested_attrs.pop(attr, None)
assert requested_attrs == attrs
def test_radar_request_site_historic_sweep_vol_v_hdf5_timerange():
"""
Example for testing radar/site sweep-precipitation for a specific date,
this time in HDF5 format, with timerange.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DwdRadarValues(
parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H,
start_date=timestamp,
end_date=timedelta(hours=0.5),
site=DwdRadarSite.BOO,
fmt=DwdRadarDataFormat.HDF5,
subset=DwdRadarDataSubset.SIMPLE,
)
# Verify number of elements.
results = list(request.query())
if len(results) == 0:
raise pytest.skip("Data currently not available")
assert len(results) == 60
hdf = h5py.File(results[0].data, "r")
assert hdf["/how"].attrs.get("scan_count") == 10
assert hdf["/dataset1/how"].attrs.get("scan_index") == 1
timestamp = round_minutes(request.start_date, 5)
assert hdf["/what"].attrs.get("date") == bytes(
timestamp.strftime("%Y%m%d"), encoding="ascii")
assert hdf["/what"].attrs.get("time").startswith(
bytes(timestamp.strftime("%H%M"), encoding="ascii"))
def test_radar_request_site_historic_sweep_pcp_v_hdf5_yesterday():
"""
Example for testing radar/site sweep-precipitation for a specific date,
this time in HDF5 format.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DwdRadarValues(
parameter=DwdRadarParameter.SWEEP_PCP_VELOCITY_H,
start_date=timestamp,
site=DwdRadarSite.BOO,
fmt=DwdRadarDataFormat.HDF5,
subset=DwdRadarDataSubset.SIMPLE,
)
results = list(request.query())
if len(results) == 0:
raise pytest.skip("Data currently not available")
# Verify number of elements.
assert len(results) == 1
# Get payload.
buffer = results[0][1]
payload = buffer.getvalue()
# Verify data.
assert payload.startswith(b"\x89HDF\r\n")
# Verify more details.
# h5dump ras07-stqual-pcpng01_sweeph5onem_vradh_00-2020093000403400-boo-10132-hd5
hdf = h5py.File(buffer, "r")
assert hdf["/how/radar_system"] is not None
assert hdf["/how"].attrs.get("task") == b"Sc_Pcp-NG-01_BOO"
assert hdf["/what"].attrs.get("source") == b"WMO:10132,NOD:deboo"
assert hdf["/how"].attrs.get("scan_count") == 1
assert hdf["/dataset1/how"].attrs.get("scan_index") == 1
assert hdf["/dataset1/data1/data"].shape == (360, 600)
timestamp = round_minutes(request.start_date, 5)
assert hdf["/what"].attrs.get("date") == bytes(
timestamp.strftime("%Y%m%d"), encoding="ascii")
assert hdf["/what"].attrs.get("time").startswith(
bytes(timestamp.strftime("%H%M"), encoding="ascii"))
def test_radar_request_site_historic_dx_timerange():
"""
Verify acquisition of radar/site/DX data works
when using a specific date, with timerange.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DwdRadarValues(
parameter=DwdRadarParameter.DX_REFLECTIVITY,
start_date=timestamp,
end_date=timedelta(hours=0.5),
site=DwdRadarSite.BOO,
)
# Verify number of elements.
results = list(request.query())
if len(results) == 0:
raise pytest.skip("Data currently not available")
assert len(results) == 6
buffer = results[0].data
# Verify data.
requested_header = wrl.io.read_radolan_header(buffer)
requested_attrs = wrl.io.radolan.parse_dx_header(requested_header)
timestamp_aligned = round_minutes(timestamp, 5)
assert timestamp_aligned.strftime(
"%m%y") == requested_attrs["datetime"].strftime("%m%y")
assert timestamp_aligned.strftime(
"%d%H%M") == requested_attrs["datetime"].strftime("%d%H%M")
attrs = {
"producttype": "DX",
"version": " 2",
"cluttermap": 0,
"dopplerfilter": 4,
"statfilter": 0,
"elevprofile": [0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8],
"message": "",
}
skip_attrs = ["bytes", "radarid", "datetime"]
for attr in skip_attrs:
requested_attrs.pop(attr, None)
assert requested_attrs == attrs
def test_radar_request_site_latest_dx_reflectivity():
"""
Example for testing radar SITES latest.
"""
request = DwdRadarValues(
parameter=DwdRadarParameter.DX_REFLECTIVITY,
start_date=DwdRadarDate.LATEST,
site=DwdRadarSite.BOO,
)
buffer = next(request.query())[1]
payload = buffer.getvalue()
timestamp_aligned = round_minutes(datetime.utcnow(), 5)
month_year = timestamp_aligned.strftime("%m%y")
header = f"DX......10132{month_year}BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS" # noqa:E501,B950
assert re.match(bytes(header, encoding="ascii"), payload[:160])
def test_radar_request_site_historic_px250_bufr_yesterday():
"""
Example for testing radar/site PX250 for a specific date.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DwdRadarValues(
parameter=DwdRadarParameter.PX250_REFLECTIVITY,
start_date=timestamp,
site=DwdRadarSite.BOO,
)
results = list(request.query())
if len(results) == 0:
raise pytest.skip("Data currently not available")
buffer = results[0].data
payload = buffer.getvalue()
# Verify data.
header = b"\x00\x00\x00\x00\x00...BUFR"
assert re.match(header, payload), payload[:20]
# Read BUFR file.
decoder = pybufrkit.decoder.Decoder()
bufr = decoder.process(payload, info_only=True)
# Verify timestamp in BUFR metadata.
timestamp_aligned = round_minutes(timestamp, 5)
bufr_timestamp = datetime(
bufr.year.value,
bufr.month.value,
bufr.day.value,
bufr.hour.value,
bufr.minute.value,
)
assert timestamp_aligned == bufr_timestamp
def test_radar_request_site_historic_sweep_pcp_v_bufr_yesterday():
"""
Example for testing radar/site sweep-precipitation for a specific date,
this time in BUFR format.
"""
timestamp = datetime.utcnow() - timedelta(days=1)
request = DwdRadarValues(
parameter=DwdRadarParameter.SWEEP_PCP_VELOCITY_H,
start_date=timestamp,
site=DwdRadarSite.ASB,
fmt=DwdRadarDataFormat.BUFR,
)
results = list(request.query())
if len(results) == 0:
raise pytest.skip("Data currently not available")
buffer = results[1]
payload = buffer.getvalue()
# Read BUFR file.
decoder = pybufrkit.decoder.Decoder()
bufr = decoder.process(payload, info_only=True)
# Verify timestamp in BUFR metadata.
timestamp_aligned = round_minutes(timestamp, 5)
bufr_timestamp = datetime(
bufr.year.value + 2000,
bufr.month.value,
bufr.day.value,
bufr.hour.value,
bufr.minute.value,
)
assert timestamp_aligned == bufr_timestamp
def test_round_5min():
tm = datetime(2010, 1, 1, 0, 4, 42)
tm_aligned = round_minutes(tm, 5)
assert tm_aligned == datetime(2010, 1, 1, 0, 0)
