def test_radar_fileindex_radolan_reflectivity_bin(parameter): file_index = create_fileindex_radar(parameter=parameter, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match( f"*/weather/radar/radolan/{parameter.value}/*---bin") for url in urls)
def test_radar_fileindex_sites_px250_reflectivity_bufr(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.PX250_REFLECTIVITY, site=DwdRadarSite.BOO, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all("/weather/radar/sites/px250/boo" in url for url in urls)
def test_radar_fileindex_composite_rx_reflectivity_bin(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.RX_REFLECTIVITY, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match("*/weather/radar/composit/rx/*---bin") for url in urls)
def test_radar_fileindex_composite_pg_reflectivity_bufr(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.PG_REFLECTIVITY, fmt=DwdRadarDataFormat.BUFR, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match("*/weather/radar/composit/pg/*---bufr") for url in urls)
def test_radar_fileindex_sites_px_reflectivity_bufr(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.PX_REFLECTIVITY, site=DwdRadarSite.BOO, fmt=DwdRadarDataFormat.BUFR, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match("*/weather/radar/sites/px/boo/*---buf") for url in urls)
def test_radar_fileindex_sites_sweep_bufr(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H, site=DwdRadarSite.BOO, fmt=DwdRadarDataFormat.BUFR, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match("*/weather/radar/sites/sweep_vol_v/boo/*--buf.bz2") for url in urls)
def test_radar_fileindex_radolan_cdc_5minutes(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.RADOLAN_CDC, resolution=DwdRadarResolution.MINUTE_5, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match( "*/climate_environment/CDC/grids_germany/5_minutes/radolan/reproc/2017_002/bin/*/YW2017*.tar" ) for url in urls if not url.endswith(".tar.gz"))
def test_radar_fileindex_radolan_cdc_hourly_historical(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.RADOLAN_CDC, resolution=DwdRadarResolution.HOURLY, period=DwdRadarPeriod.HISTORICAL, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match( "*/climate_environment/CDC/grids_germany/hourly/radolan/historical/bin/*/RW*.tar.gz" ) for url in urls if not url.endswith(".pdf"))
def test_radar_fileindex_radolan_cdc_daily_recent(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.RADOLAN_CDC, resolution=DwdRadarResolution.DAILY, period=DwdRadarPeriod.RECENT, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( PurePath(url).match( "*/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/*---bin.gz" ) for url in urls if not url.endswith(".pdf"))
def test_radar_fileindex_sites_sweep_vol_v_hdf5_simple(): file_index = create_fileindex_radar( parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H, site=DwdRadarSite.BOO, fmt=DwdRadarDataFormat.HDF5, subset=DwdRadarDataSubset.SIMPLE, ) urls = file_index[DwdColumns.FILENAME.value].tolist() assert all("/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_simple" in url for url in urls)
def collect_radar_data( parameter: Optional[DwdRadarParameter], resolution: Optional[Resolution] = None, period: Optional[Period] = None, site: Optional[DwdRadarSite] = None, fmt: Optional[DwdRadarDataFormat] = None, subset: Optional[DwdRadarDataSubset] = None, elevation: Optional[int] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, verify: Optional[bool] = True, ) -> RadarResult: """ Collect radar data for given parameters. :param parameter: The radar moment to request :param resolution: Time resolution for RadarParameter.RADOLAN_CDC, either daily or hourly or 5 minutes. :param period: Period type for RadarParameter.RADOLAN_CDC :param site: Site/station if parameter is one of RADAR_PARAMETERS_SITES :param fmt: Data format (BINARY, BUFR, HDF5) :param subset: The subset (simple or polarimetric) for HDF5 data. :param elevation: :param start_date: Start date :param end_date: End date :param verify: Whether to verify the response :return: ``RadarResult`` item """ # Find latest file. if start_date == DwdRadarDate.LATEST: file_index = create_fileindex_radar( parameter=parameter, site=site, fmt=fmt, parse_datetime=False, ) # Find "-latest-" file. filenames = file_index[DwdColumns.FILENAME.value].tolist() latest_file = list(filter(lambda x: "-latest-" in x, filenames))[0] # Yield single "RadarResult" item. result = next(_download_generic_data(url=latest_file)) yield result else: if parameter == DwdRadarParameter.RADOLAN_CDC: if period: period_types = [period] else: period_types = [ Period.RECENT, Period.HISTORICAL, ] results = [] for period in period_types: file_index = create_fileindex_radolan_cdc( resolution=resolution, period=period) # Filter for dates range if start_date and end_date are defined. if period == Period.RECENT: file_index = file_index[ (file_index[DwdColumns.DATETIME.value] >= start_date) & (file_index[DwdColumns.DATETIME.value] < end_date)] # This is for matching historical data, e.g. "RW-200509.tar.gz". else: file_index = file_index[ (file_index[DwdColumns.DATETIME.value].dt.year == start_date.year) & (file_index[DwdColumns.DATETIME.value].dt.month == start_date.month)] results.append(file_index) file_index = pd.concat(results) if file_index.empty: # TODO: Extend this log message. log.warning( f"No radar file found for {parameter}, {site}, {fmt}") return # Iterate list of files and yield "RadarResult" items. for _, row in file_index.iterrows(): url = row[DwdColumns.FILENAME.value] try: yield from download_radolan_data(url, start_date, end_date) except FailedDownload as e: log.exception(e) else: file_index = create_fileindex_radar( parameter=parameter, site=site, fmt=fmt, subset=subset, parse_datetime=True, ) # Filter for dates range if start_date and end_date are defined. file_index = file_index[ (file_index[DwdColumns.DATETIME.value] >= start_date) & (file_index[DwdColumns.DATETIME.value] < end_date)] # Filter SWEEP_VOL_VELOCITY_H and SWEEP_VOL_REFLECTIVITY_H by elevation. if elevation is not None: filename = file_index[DwdColumns.FILENAME.value] file_index = file_index[ (filename.str.contains(f"vradh_{elevation:02d}")) | (filename.str.contains(f"sweep_vol_v_{elevation}")) | (filename.str.contains(f"dbzh_{elevation:02d}")) | (filename.str.contains(f"sweep_vol_z_{elevation}"))] if file_index.empty: log.warning( f"No radar file found for {parameter}, {site}, {fmt}") return # Iterate list of files and yield "RadarResult" items. for _, row in file_index.iterrows(): date_time = row[DwdColumns.DATETIME.value] url = row[DwdColumns.FILENAME.value] try: for result in _download_generic_data(url=url): if result.timestamp is None: result.timestamp = date_time if verify: if fmt == DwdRadarDataFormat.HDF5: verify_hdf5(result.data) yield result except Exception: # pragma: no cover log.exception("Unable to read HDF5 file")