def adjust_datetimes(self): """ Adjust ``start_date`` and ``end_date`` attributes to match minute marks for respective RadarParameter. - RADOLAN_CDC is always published at HH:50. https://opendata.dwd.de/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/ # noqa:E501,B950 - RQ_REFLECTIVITY is published each 15 minutes. https://opendata.dwd.de/weather/radar/radvor/rq/ - All other radar formats are published in intervals of 5 minutes. https://opendata.dwd.de/weather/radar/composit/fx/ https://opendata.dwd.de/weather/radar/sites/dx/boo/ """ if ( self.parameter == DWDRadarParameter.RADOLAN_CDC or self.parameter in RADAR_PARAMETERS_RADOLAN ): # Align "start_date" to the most recent 50 minute mark available. self.start_date = raster_minutes(self.start_date, 50) # When "end_date" is given as timedelta, resolve it. if isinstance(self.end_date, timedelta): self.end_date = self.start_date + self.end_date # Use "end_date = start_date" to make the machinery # pick a single file from the fileindex. if self.end_date is None: self.end_date = self.start_date + timedelta(microseconds=1) elif self.parameter == DWDRadarParameter.RQ_REFLECTIVITY: # Align "start_date" to the 15 minute mark before tm. self.start_date = round_minutes(self.start_date, 15) # When "end_date" is given as timedelta, resolve it. if isinstance(self.end_date, timedelta): self.end_date = self.start_date + self.end_date # Expand "end_date" to the end of the 5 minute mark. if self.end_date is None: self.end_date = self.start_date + timedelta(minutes=15) else: # Align "start_date" to the 5 minute mark before tm. self.start_date = round_minutes(self.start_date, 5) # When "end_date" is given as timedelta, resolve it. if isinstance(self.end_date, timedelta): self.end_date = self.start_date + self.end_date # Expand "end_date" to the end of the 5 minute mark. if self.end_date is None: self.end_date = self.start_date + timedelta(minutes=5)
def test_radar_request_site_historic_sweep_vol_v_hdf5_yesterday(): """ Example for testing radar/site sweep-precipitation for a specific date, this time in HDF5 format. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=timestamp, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.SIMPLE, ) results = list(request.collect_data()) # Verify number of elements. assert len(results) == 10 # Get payload from first file. buffer = results[0].data payload = buffer.getvalue() # Verify data. assert payload.startswith(b"\x89HDF\r\n") # Verify more details. # h5dump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 hdf = h5py.File(buffer, "r") assert hdf["/how/radar_system"] is not None assert hdf["/how"].attrs.get("task") == b"Sc_Vol-5Min-NG-01_BOO" assert hdf["/what"].attrs.get("source") == b"WMO:10132,NOD:deboo" assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 1 assert hdf["/dataset1/data1/data"].shape == (360, 180) timestamp = round_minutes(request.start_date, 5) assert hdf["/what"].attrs.get("date") == bytes( timestamp.strftime("%Y%m%d"), encoding="ascii") assert (hdf["/what"].attrs.get("time").startswith( bytes(timestamp.strftime("%H%M"), encoding="ascii"))) # Verify that the second file is the second scan / elevation level. buffer = results[1].data hdf = h5py.File(buffer, "r") assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 2 timestamp = round_minutes(request.start_date, 5) assert hdf["/what"].attrs.get("date") == bytes( timestamp.strftime("%Y%m%d"), encoding="ascii") assert (hdf["/what"].attrs.get("time").startswith( bytes(timestamp.strftime("%H%M"), encoding="ascii")))
def test_radar_request_site_historic_sweep_vol_v_bufr_yesterday(): """ Example for testing radar/site sweep_vol_v for a specific date, this time in BUFR format. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=timestamp, site=DWDRadarSite.ASB, fmt=DWDRadarDataFormat.BUFR, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Read BUFR file. decoder = pybufrkit.decoder.Decoder() bufr = decoder.process(payload, info_only=True) # Verify timestamp in BUFR metadata. timestamp_aligned = round_minutes(timestamp, 5) bufr_timestamp = datetime( bufr.year.value + 2000, bufr.month.value, bufr.day.value, bufr.hour.value, bufr.minute.value, ) assert timestamp_aligned == bufr_timestamp
def test_radar_request_site_historic_px250_bufr_yesterday(): """ Example for testing radar/site PX250 for a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.PX250_REFLECTIVITY, start_date=timestamp, site=DWDRadarSite.BOO, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. header = b"\x00\x00\x00\x00\x00...BUFR" assert re.match(header, payload), payload[:20] # Read BUFR file. decoder = pybufrkit.decoder.Decoder() bufr = decoder.process(payload, info_only=True) # Verify timestamp in BUFR metadata. timestamp_aligned = round_minutes(timestamp, 5) bufr_timestamp = datetime( bufr.year.value, bufr.month.value, bufr.day.value, bufr.hour.value, bufr.minute.value, ) assert timestamp_aligned == bufr_timestamp
def test_radar_request_site_historic_dx_yesterday(): """ Verify acquisition of radar/site/DX data works when using a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.DX_REFLECTIVITY, start_date=timestamp, site=DWDRadarSite.BOO, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html timestamp_aligned = round_minutes(timestamp, 5) date_time = timestamp_aligned.strftime("%d%H%M") month_year = timestamp_aligned.strftime("%m%y") header = f"DX{date_time}10132{month_year}BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS" # noqa:E501,B950 assert re.match(bytes(header, encoding="ascii"), payload)
def test_radar_request_site_latest_dx_reflectivity(): """ Example for testing radar SITES latest. """ request = DwdRadarValues( parameter=DwdRadarParameter.DX_REFLECTIVITY, start_date=DwdRadarDate.LATEST, site=DwdRadarSite.BOO, ) buffer = next(request.query())[1] requested_header = wrl.io.read_radolan_header(buffer) requested_attrs = wrl.io.radolan.parse_dx_header(requested_header) # Verify data. timestamp_aligned = round_minutes(datetime.utcnow(), 5) assert timestamp_aligned.strftime( "%m%y") == requested_attrs["datetime"].strftime("%m%y") attrs = { "producttype": "DX", "version": " 2", "cluttermap": 0, "dopplerfilter": 4, "statfilter": 0, "elevprofile": [0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], "message": "", } skip_attrs = ["radarid", "datetime", "bytes"] for attr in skip_attrs: requested_attrs.pop(attr, None) assert requested_attrs == attrs
def test_radar_request_site_historic_sweep_vol_v_hdf5_timerange(): """ Example for testing radar/site sweep-precipitation for a specific date, this time in HDF5 format, with timerange. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DwdRadarValues( parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=timestamp, end_date=timedelta(hours=0.5), site=DwdRadarSite.BOO, fmt=DwdRadarDataFormat.HDF5, subset=DwdRadarDataSubset.SIMPLE, ) # Verify number of elements. results = list(request.query()) if len(results) == 0: raise pytest.skip("Data currently not available") assert len(results) == 60 hdf = h5py.File(results[0].data, "r") assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 1 timestamp = round_minutes(request.start_date, 5) assert hdf["/what"].attrs.get("date") == bytes( timestamp.strftime("%Y%m%d"), encoding="ascii") assert hdf["/what"].attrs.get("time").startswith( bytes(timestamp.strftime("%H%M"), encoding="ascii"))
def test_radar_request_site_historic_sweep_pcp_v_hdf5_yesterday(): """ Example for testing radar/site sweep-precipitation for a specific date, this time in HDF5 format. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DwdRadarValues( parameter=DwdRadarParameter.SWEEP_PCP_VELOCITY_H, start_date=timestamp, site=DwdRadarSite.BOO, fmt=DwdRadarDataFormat.HDF5, subset=DwdRadarDataSubset.SIMPLE, ) results = list(request.query()) if len(results) == 0: raise pytest.skip("Data currently not available") # Verify number of elements. assert len(results) == 1 # Get payload. buffer = results[0][1] payload = buffer.getvalue() # Verify data. assert payload.startswith(b"\x89HDF\r\n") # Verify more details. # h5dump ras07-stqual-pcpng01_sweeph5onem_vradh_00-2020093000403400-boo-10132-hd5 hdf = h5py.File(buffer, "r") assert hdf["/how/radar_system"] is not None assert hdf["/how"].attrs.get("task") == b"Sc_Pcp-NG-01_BOO" assert hdf["/what"].attrs.get("source") == b"WMO:10132,NOD:deboo" assert hdf["/how"].attrs.get("scan_count") == 1 assert hdf["/dataset1/how"].attrs.get("scan_index") == 1 assert hdf["/dataset1/data1/data"].shape == (360, 600) timestamp = round_minutes(request.start_date, 5) assert hdf["/what"].attrs.get("date") == bytes( timestamp.strftime("%Y%m%d"), encoding="ascii") assert hdf["/what"].attrs.get("time").startswith( bytes(timestamp.strftime("%H%M"), encoding="ascii"))
def test_radar_request_site_historic_dx_timerange(): """ Verify acquisition of radar/site/DX data works when using a specific date, with timerange. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DwdRadarValues( parameter=DwdRadarParameter.DX_REFLECTIVITY, start_date=timestamp, end_date=timedelta(hours=0.5), site=DwdRadarSite.BOO, ) # Verify number of elements. results = list(request.query()) if len(results) == 0: raise pytest.skip("Data currently not available") assert len(results) == 6 buffer = results[0].data # Verify data. requested_header = wrl.io.read_radolan_header(buffer) requested_attrs = wrl.io.radolan.parse_dx_header(requested_header) timestamp_aligned = round_minutes(timestamp, 5) assert timestamp_aligned.strftime( "%m%y") == requested_attrs["datetime"].strftime("%m%y") assert timestamp_aligned.strftime( "%d%H%M") == requested_attrs["datetime"].strftime("%d%H%M") attrs = { "producttype": "DX", "version": " 2", "cluttermap": 0, "dopplerfilter": 4, "statfilter": 0, "elevprofile": [0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], "message": "", } skip_attrs = ["bytes", "radarid", "datetime"] for attr in skip_attrs: requested_attrs.pop(attr, None) assert requested_attrs == attrs
def test_radar_request_site_latest_dx_reflectivity(): """ Example for testing radar SITES latest. """ request = DwdRadarValues( parameter=DwdRadarParameter.DX_REFLECTIVITY, start_date=DwdRadarDate.LATEST, site=DwdRadarSite.BOO, ) buffer = next(request.query())[1] payload = buffer.getvalue() timestamp_aligned = round_minutes(datetime.utcnow(), 5) month_year = timestamp_aligned.strftime("%m%y") header = f"DX......10132{month_year}BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS" # noqa:E501,B950 assert re.match(bytes(header, encoding="ascii"), payload[:160])
def test_radar_request_site_historic_px250_bufr_yesterday(): """ Example for testing radar/site PX250 for a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DwdRadarValues( parameter=DwdRadarParameter.PX250_REFLECTIVITY, start_date=timestamp, site=DwdRadarSite.BOO, ) results = list(request.query()) if len(results) == 0: raise pytest.skip("Data currently not available") buffer = results[0].data payload = buffer.getvalue() # Verify data. header = b"\x00\x00\x00\x00\x00...BUFR" assert re.match(header, payload), payload[:20] # Read BUFR file. decoder = pybufrkit.decoder.Decoder() bufr = decoder.process(payload, info_only=True) # Verify timestamp in BUFR metadata. timestamp_aligned = round_minutes(timestamp, 5) bufr_timestamp = datetime( bufr.year.value, bufr.month.value, bufr.day.value, bufr.hour.value, bufr.minute.value, ) assert timestamp_aligned == bufr_timestamp
def test_radar_request_site_historic_sweep_pcp_v_bufr_yesterday(): """ Example for testing radar/site sweep-precipitation for a specific date, this time in BUFR format. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DwdRadarValues( parameter=DwdRadarParameter.SWEEP_PCP_VELOCITY_H, start_date=timestamp, site=DwdRadarSite.ASB, fmt=DwdRadarDataFormat.BUFR, ) results = list(request.query()) if len(results) == 0: raise pytest.skip("Data currently not available") buffer = results[1] payload = buffer.getvalue() # Read BUFR file. decoder = pybufrkit.decoder.Decoder() bufr = decoder.process(payload, info_only=True) # Verify timestamp in BUFR metadata. timestamp_aligned = round_minutes(timestamp, 5) bufr_timestamp = datetime( bufr.year.value + 2000, bufr.month.value, bufr.day.value, bufr.hour.value, bufr.minute.value, ) assert timestamp_aligned == bufr_timestamp
def test_round_5min(): tm = datetime(2010, 1, 1, 0, 4, 42) tm_aligned = round_minutes(tm, 5) assert tm_aligned == datetime(2010, 1, 1, 0, 0)