def test_radar_request_composite_historic_radolan_rw_yesterday(): """ Verify acquisition of radar/composite/radolan_rw data works when using a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.RW_REFLECTIVITY, start_date=timestamp, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html date_time = request.start_date.strftime("%d%H%M") month_year = request.start_date.strftime("%m%y") header = ( f"RW{date_time}10000{month_year}BY.......VS 3SW 2.28.1PR E-01INT 60GP 900x 900MF 00000001MS " # noqa:E501,B950 f"..<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,(nhb,)?oft,eis,(tur,)?(isn,)?fbg,mem>" # noqa:E501,B950 ) assert re.match(bytes(header, encoding="ascii"), payload[:160])
def test_radar_request_radvor_rq_yesterday(): """ Verify acquisition of radar/radvor/rq data works when using a specific date. Querying one point in time should yield 3 results for a single 15 minute time step. https://opendata.dwd.de/weather/radar/radvor/rq/ """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.RQ_REFLECTIVITY, start_date=timestamp, ) results = list(request.collect_data()) assert len(results) == 3 payload = results[0].data.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html date_time = request.start_date.strftime("%d%H%M") month_year = request.start_date.strftime("%m%y") header = ( f"RQ{date_time}10000{month_year}BY.......VS 3SW 2.28.1PR E-01INT 60GP 900x 900VV 0MF 00000008QN ...MS " # noqa:E501,B950 f"..<asb,boo,drs,eis,ess,(fbg,)?fld,hnr,(isn,)?mem(,neu,nhb,oft,pro,ros(,tur)?(,umd)?)?" # noqa:E501,B950 ) assert re.match(bytes(header, encoding="ascii"), payload[:180])
def test_radar_request_site_historic_pe_binary_yesterday(): """ Verify acquisition of radar/site/PE_ECHO_TOP data works when using a specific date. This time, we will use the BINARY data format. """ # Acquire data from yesterday at this time. timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.PE_ECHO_TOP, start_date=timestamp, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.BINARY, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html date_time = request.start_date.strftime("%d%H") month_year = request.start_date.strftime("%m%y") header = ( f"PE{date_time}..10132{month_year}BY ....?VS 1LV12 1.0 2.0 3.0 4.0 5.0 " # noqa:E501,B950 f"6.0 7.0 8.0 9.0 10.0 11.0 12.0CO0CD0CS0ET 5.0FL....MS") assert re.match(bytes(header, encoding="ascii"), payload[:160])
def test_radar_request_site_historic_pe_bufr(): """ Verify acquisition of radar/site/PE_ECHO_TOP data works when using a specific date. This time, we will use the BUFR data format. """ # Acquire data from yesterday at this time. timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.PE_ECHO_TOP, start_date=timestamp, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.BUFR, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. header = b"\x00\x00\x00\x00\x00...BUFR" assert re.match(header, payload), payload[:20] # Read BUFR file. decoder = pybufrkit.decoder.Decoder() decoder.process(payload, info_only=True)
def test_radar_request_composite_historic_fx_yesterday(): """ Example for testing radar/composite FX for a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.FX_REFLECTIVITY, start_date=timestamp, ) results = list(request.collect_data()) # Verify number of results. assert len(results) == 25 # Verify data. payload = results[0].data.getvalue() # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html date_time = request.start_date.strftime("%d%H%M") month_year = request.start_date.strftime("%m%y") header = ( f"FX{date_time}10000{month_year}BY.......VS 3SW 2.12.0PR E-01INT 5GP 900x 900VV 000MF 00000002MS " # noqa:E501,B950 f"..<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,(nhb,)?oft,eis,tur,(isn,)?fbg,mem>" ) assert re.match(bytes(header, encoding="ascii"), payload[:160])
def test_radar_request_site_historic_px250_bufr_yesterday(): """ Example for testing radar/site PX250 for a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.PX250_REFLECTIVITY, start_date=timestamp, site=DWDRadarSite.BOO, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. header = b"\x00\x00\x00\x00\x00...BUFR" assert re.match(header, payload), payload[:20] # Read BUFR file. decoder = pybufrkit.decoder.Decoder() bufr = decoder.process(payload, info_only=True) # Verify timestamp in BUFR metadata. timestamp_aligned = round_minutes(timestamp, 5) bufr_timestamp = datetime( bufr.year.value, bufr.month.value, bufr.day.value, bufr.hour.value, bufr.minute.value, ) assert timestamp_aligned == bufr_timestamp
def test_radar_request_radolan_cdc_most_recent(): """ Example for testing radar sites most recent RADOLAN_CDC. """ request = DWDRadarData( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.DAILY, start_date=DWDRadarDate.MOST_RECENT, ) results = list(request.collect_data()) assert len(results) == 1 payload = results[0].data.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html date_time = request.start_date.strftime("%d%H%M") month_year = request.start_date.strftime("%m%y") header = ( f"SF{date_time}10000{month_year}BY.......VS 3SW 2.28.1PR E-01INT1440GP 900x 900MS " # noqa:E501,B950 f"..<asb,boo,ros,hnr,umd,pro,ess,fld,drs,neu,(nhb,)?oft,eis,tur,(isn,)?fbg,mem>" ) assert re.match(bytes(header, encoding="ascii"), payload[:180])
def test_radar_request_site_historic_pe_timerange(format): """ Verify acquisition of radar/site/PE_ECHO_TOP data works when using date ranges. The proof will use these parameters to acquire data: - start_date: Yesterday at this time - end_date: start_date + 1 hour This time, we will test both the BINARY and BUFR data format. """ start_date = datetime.utcnow() - timedelta(days=1) end_date = timedelta(hours=1) request = DWDRadarData( parameter=DWDRadarParameter.PE_ECHO_TOP, start_date=start_date, end_date=end_date, site=DWDRadarSite.BOO, fmt=format, ) assert request.start_date.minute % 5 == 0 # Verify number of elements. results = list(request.collect_data()) assert len(results) == 12
def test_radar_request_radolan_cdc_historic_hourly_data(): """ Verify data acquisition for RADOLAN_CDC/hourly/historical. """ request = DWDRadarData( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.HOURLY, period=DWDRadarPeriod.HISTORICAL, start_date="2019-08-08 00:50:00", ) assert request == DWDRadarData( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.HOURLY, period=DWDRadarPeriod.HISTORICAL, start_date=datetime(year=2019, month=8, day=8, hour=0, minute=50, second=0), ) with Path(HERE, "radolan_hourly_201908080050").open("rb") as f: radolan_hourly = BytesIO(f.read()) radolan_hourly_test = next(request.collect_data()).data assert radolan_hourly.getvalue() == radolan_hourly_test.getvalue()
def test_radar_request_site_historic_sweep_vol_v_bufr_yesterday(): """ Example for testing radar/site sweep_vol_v for a specific date, this time in BUFR format. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=timestamp, site=DWDRadarSite.ASB, fmt=DWDRadarDataFormat.BUFR, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Read BUFR file. decoder = pybufrkit.decoder.Decoder() bufr = decoder.process(payload, info_only=True) # Verify timestamp in BUFR metadata. timestamp_aligned = round_minutes(timestamp, 5) bufr_timestamp = datetime( bufr.year.value + 2000, bufr.month.value, bufr.day.value, bufr.hour.value, bufr.minute.value, ) assert timestamp_aligned == bufr_timestamp
def test_radar_request_radvor_re_yesterday(): """ Verify acquisition of radar/radvor/re data works when using a specific date. Querying one point in time should yield 25 results for a single 5 minute time step. https://opendata.dwd.de/weather/radar/radvor/re/ """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.RE_REFLECTIVITY, start_date=timestamp, ) results = list(request.collect_data()) assert len(results) == 25 payload = results[0].data.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html date_time = request.start_date.strftime("%d%H%M") month_year = request.start_date.strftime("%m%y") header = ( f"RE{date_time}10000{month_year}BY.......VS 3SW P10000.HPR E-03INT 60GP 900x 900VV 000MF 00000008QN " # noqa:E501,B950 f"016MS...<deasb,deboo,dedrs,deeis,deess,(defbg,)?defld,dehnr,(deisn,)?demem(,deneu,denhb,deoft,depro,deros(,detur)?(,deumd)?)?" # noqa:E501,B950 ) assert re.match(bytes(header, encoding="ascii"), payload[:200])
def test_radar_request_site_current_sweep_vol_v_hdf5_single(): """ Example for testing radar sites single current SWEEP_VOL, this time in OPERA HDF5 (ODIM_H5) format. """ request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=DWDRadarDate.CURRENT, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.SIMPLE, elevation=1, ) results = list(request.collect_data()) assert len(results) <= 1 if results: assert "vradh_01" in results[0].url buffer = results[0].data hdf = h5py.File(buffer, "r") assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 2
def test_radar_request_site_historic_dx_yesterday(): """ Verify acquisition of radar/site/DX data works when using a specific date. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.DX_REFLECTIVITY, start_date=timestamp, site=DWDRadarSite.BOO, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() # Verify data. # TODO: Use wradlib to parse binary format. # https://docs.wradlib.org/en/stable/notebooks/radolan/radolan_format.html timestamp_aligned = round_minutes(timestamp, 5) date_time = timestamp_aligned.strftime("%d%H%M") month_year = timestamp_aligned.strftime("%m%y") header = f"DX{date_time}10132{month_year}BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS" # noqa:E501,B950 assert re.match(bytes(header, encoding="ascii"), payload)
def test_radar_request_site_historic_sweep_vol_v_hdf5_yesterday(): """ Example for testing radar/site sweep-precipitation for a specific date, this time in HDF5 format. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=timestamp, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.SIMPLE, ) results = list(request.collect_data()) # Verify number of elements. assert len(results) == 10 # Get payload from first file. buffer = results[0].data payload = buffer.getvalue() # Verify data. assert payload.startswith(b"\x89HDF\r\n") # Verify more details. # h5dump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 hdf = h5py.File(buffer, "r") assert hdf["/how/radar_system"] is not None assert hdf["/how"].attrs.get("task") == b"Sc_Vol-5Min-NG-01_BOO" assert hdf["/what"].attrs.get("source") == b"WMO:10132,NOD:deboo" assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 1 assert hdf["/dataset1/data1/data"].shape == (360, 180) timestamp = round_minutes(request.start_date, 5) assert hdf["/what"].attrs.get("date") == bytes( timestamp.strftime("%Y%m%d"), encoding="ascii") assert (hdf["/what"].attrs.get("time").startswith( bytes(timestamp.strftime("%H%M"), encoding="ascii"))) # Verify that the second file is the second scan / elevation level. buffer = results[1].data hdf = h5py.File(buffer, "r") assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 2 timestamp = round_minutes(request.start_date, 5) assert hdf["/what"].attrs.get("date") == bytes( timestamp.strftime("%Y%m%d"), encoding="ascii") assert (hdf["/what"].attrs.get("time").startswith( bytes(timestamp.strftime("%H%M"), encoding="ascii")))
def radar_scan_volume(): request_velocity = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=DWDRadarDate.MOST_RECENT, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.POLARIMETRIC, ) request_reflectivity = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_REFLECTIVITY_H, start_date=DWDRadarDate.MOST_RECENT, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.POLARIMETRIC, ) log.info( f"Acquiring radar SWEEP_VOL data for {DWDRadarSite.BOO} at " f"{request_velocity.start_date}" ) # Submit requests. results = chain( request_velocity.collect_data(), request_reflectivity.collect_data() ) # Collect list of buffers. files = list(map(lambda item: item.data, results)) # Decode data using wradlib. data = wrl.io.open_odim(files) # Output debug information. radar_info(data) # Plot and display data. plot(data) pl.show() # Remove temporary files. for tmpfile in files: os.unlink(tmpfile)
def test_radar_request_radolan_cdc_current_5min(): """ Verify data acquisition for RADOLAN_CDC/5 minutes. FIXME: Does not work as expected yet. """ request = DWDRadarData( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.MINUTE_5, start_date=DWDRadarDate.CURRENT, ) list(request.collect_data())
def test_radar_request_site_without_site(): """ Verify requesting site data without site croaks. """ with pytest.raises(ValueError) as excinfo: request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_PCP_VELOCITY_H, start_date=DWDRadarDate.LATEST, ) list(request.collect_data()) assert excinfo.typename == "ValueError" assert str(excinfo.value).startswith("Argument 'site' is missing")
def test_radar_request_radolan_cdc_current(time_resolution): """ Verify data acquisition for current RADOLAN_CDC/daily+hourly. Remark: More often than not, this data is not available when looking at CURRENT. """ request = DWDRadarData( parameter=DWDRadarParameter.RADOLAN_CDC, start_date=DWDRadarDate.CURRENT, resolution=time_resolution, ) list(request.collect_data())
def test_radar_request_site_latest_sweep_pcp_v_hdf5_wrong_parameters(): """ Verify requesting HDF5 data without RadarDataFormat croaks. """ with pytest.raises(ValueError) as excinfo: request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_PCP_VELOCITY_H, site=DWDRadarSite.BOO, start_date=DWDRadarDate.CURRENT, ) list(request.collect_data()) assert excinfo.typename == "ValueError" assert str(excinfo.value).startswith("Argument 'format' is missing")
def test_radar_request_site_historic_pe_wrong_parameters(): """ Verify acquisition of radar/site/PE_ECHO_TOP data croaks when omitting RadarDataFormat. """ with pytest.raises(ValueError) as excinfo: request = DWDRadarData( parameter=DWDRadarParameter.PE_ECHO_TOP, site=DWDRadarSite.BOO, start_date=datetime.utcnow(), ) next(request.collect_data()) assert excinfo.typename == "ValueError" assert str(excinfo.value).startswith("Argument 'format' is missing")
def test_radar_request_composite_historic_radolan_rw_timerange(): """ Verify acquisition of radar/composite/radolan_rw data works when using a specific date, with timerange. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.RW_REFLECTIVITY, start_date=timestamp, end_date=timedelta(hours=3), ) results = list(request.collect_data()) # Verify number of results. assert len(results) == 3
def test_radar_request_hdf5_without_subset(): """ Verify requesting HDF5 data without "subset" croaks. """ with pytest.raises(ValueError) as excinfo: request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_PCP_VELOCITY_H, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, start_date=DWDRadarDate.MOST_RECENT, ) list(request.collect_data()) assert excinfo.typename == "ValueError" assert str(excinfo.value).startswith("Argument 'subset' is missing")
def test_radar_request_radolan_cdc_latest(time_resolution): """ Verify requesting latest RADOLAN_CDC croaks. """ with pytest.raises(ValueError) as excinfo: request = DWDRadarData( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=time_resolution, start_date=DWDRadarDate.LATEST, ) list(request.collect_data()) assert excinfo.typename == "ValueError" assert str( excinfo.value).startswith("RADOLAN_CDC data has no '-latest-' files")
def test_radar_request_site_historic_px250_bufr_timerange(): """ Example for testing radar/site PX250 for a specific date, with timerange. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.PX250_REFLECTIVITY, start_date=timestamp, end_date=timedelta(hours=1), site=DWDRadarSite.BOO, ) # Verify number of elements. results = list(request.collect_data()) assert len(results) == 12
def test_radar_request_site_latest_sweep_pcp_v_hdf5(): """ Verify requesting latest HDF5 data croaks. """ with pytest.raises(ValueError) as excinfo: request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_PCP_VELOCITY_H, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, start_date=DWDRadarDate.LATEST, ) list(request.collect_data()) assert excinfo.typename == "ValueError" assert str(excinfo.value).startswith("HDF5 data has no '-latest-' files")
def test_radar_request_site_historic_dx_timerange(): """ Verify acquisition of radar/site/DX data works when using a specific date, with timerange. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.DX_REFLECTIVITY, start_date=timestamp, end_date=timedelta(hours=0.5), site=DWDRadarSite.BOO, ) # Verify number of elements. results = list(request.collect_data()) assert len(results) == 6
def test_radar_request_site_most_recent_sweep_vol_v_hdf5(): """ Example for testing radar sites most recent full SWEEP_VOL, this time in OPERA HDF5 (ODIM_H5) format. """ request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=DWDRadarDate.MOST_RECENT, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.SIMPLE, ) results = list(request.collect_data()) # Verify number of results. assert len(results) == 10 buffer = results[0].data payload = buffer.getvalue() # Verify data. assert payload.startswith(b"\x89HDF\r\n") # Verify more details. # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 hdf = h5py.File(buffer, "r") assert hdf["/how/radar_system"] is not None assert hdf["/how"].attrs.get("task") == b"Sc_Vol-5Min-NG-01_BOO" assert hdf["/what"].attrs.get("source") == b"WMO:10132,NOD:deboo" assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 1 assert hdf["/dataset1/data1/data"].shape == (360, 180) # Verify that the second file is the second scan / elevation level. buffer = results[1].data hdf = h5py.File(buffer, "r") assert hdf["/how"].attrs.get("scan_count") == 10 assert hdf["/dataset1/how"].attrs.get("scan_index") == 2
def test_radar_request_site_historic_sweep_vol_v_bufr_timerange(): """ Example for testing radar/site sweep_vol_v for a specific date, this time in BUFR format, with timerange. """ timestamp = datetime.utcnow() - timedelta(days=1) request = DWDRadarData( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, start_date=timestamp, end_date=timedelta(hours=0.5), site=DWDRadarSite.ASB, fmt=DWDRadarDataFormat.BUFR, ) # Verify number of elements. results = list(request.collect_data()) assert len(results) == 60
def test_radar_request_site_historic_pe_future(caplog): """ Verify that ``DWDRadarRequest`` will properly emit log messages when hitting empty results. This time for PE_ECHO_TOP data. """ request = DWDRadarData( parameter=DWDRadarParameter.PE_ECHO_TOP, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.BUFR, start_date="2099-01-01 00:00:00", ) results = list(request.collect_data()) assert results == [] assert "WARNING" in caplog.text assert "No radar file found" in caplog.text
def test_radar_request_site_latest_dx_reflectivity(): """ Example for testing radar SITES latest. """ request = DWDRadarData( parameter=DWDRadarParameter.DX_REFLECTIVITY, start_date=DWDRadarDate.LATEST, site=DWDRadarSite.BOO, ) buffer = next(request.collect_data())[1] payload = buffer.getvalue() timestamp_aligned = round_minutes(datetime.utcnow(), 5) day = timestamp_aligned.strftime("%d") month_year = timestamp_aligned.strftime("%m%y") header = f"DX{day}....10132{month_year}BY.....VS 2CO0CD4CS0EP0.80.80.80.80.80.80.80.8MS" # noqa:E501,B950 assert re.match(bytes(header, encoding="ascii"), payload[:160])