def test_radar_fileindex_sites_px250_reflectivity_bufr(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.PX250_REFLECTIVITY, site=DWDRadarSite.BOO, ) urls = file_index["FILENAME"].tolist() assert all("/weather/radar/sites/px250/boo" in url for url in urls)
def test_radar_fileindex_composite_rx_reflectivity_bin(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.RX_REFLECTIVITY, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match("*/weather/radar/composit/rx/*---bin") for url in urls )
def test_radar_fileindex_radolan_reflectivity_bin(parameter): file_index = create_fileindex_radar( parameter=parameter, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match(f"*/weather/radar/radolan/{parameter.value}/*---bin") for url in urls )
def test_radar_fileindex_composite_pg_reflectivity_bufr(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.PG_REFLECTIVITY, fmt=DWDRadarDataFormat.BUFR, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match("*/weather/radar/composit/pg/*---bufr") for url in urls )
def test_radar_fileindex_sites_px_reflectivity_bufr(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.PX_REFLECTIVITY, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.BUFR, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match("*/weather/radar/sites/px/boo/*---buf") for url in urls )
def test_radar_fileindex_sites_sweep_bufr(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.BUFR, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match("*/weather/radar/sites/sweep_vol_v/boo/*--buf.bz2") for url in urls )
def test_radar_fileindex_sites_sweep_vol_v_hdf5_simple(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.SWEEP_VOL_VELOCITY_H, site=DWDRadarSite.BOO, fmt=DWDRadarDataFormat.HDF5, subset=DWDRadarDataSubset.SIMPLE, ) urls = file_index["FILENAME"].tolist() assert all( "/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_simple" in url for url in urls )
def test_radar_fileindex_radolan_cdc_hourly_historical(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.HOURLY, period=DWDRadarPeriod.HISTORICAL, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match( "*/climate_environment/CDC/grids_germany/hourly/radolan/historical/bin/*/RW*.tar.gz" ) for url in urls if not url.endswith(".pdf") )
def test_radar_fileindex_radolan_cdc_daily_recent(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.DAILY, period=DWDRadarPeriod.RECENT, ) urls = file_index["FILENAME"].tolist() assert all( PurePath(url).match( "*/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/*---bin.gz" ) for url in urls if not url.endswith(".pdf") )
def test_radar_fileindex_radolan_cdc_5minutes(): file_index = create_fileindex_radar( parameter=DWDRadarParameter.RADOLAN_CDC, resolution=DWDRadarResolution.MINUTE_5, ) urls = file_index["FILENAME"].tolist() for url in urls: print(url) assert all( PurePath(url).match( "*/climate_environment/CDC/grids_germany/5_minutes/radolan/reproc/2017_002/bin/*/YW2017*.tar" ) for url in urls if not url.endswith(".tar.gz") )
def collect_radar_data( parameter: DWDRadarParameter, resolution: Optional[DWDRadarResolution] = None, period: Optional[DWDRadarPeriod] = None, site: Optional[DWDRadarSite] = None, fmt: Optional[DWDRadarDataFormat] = None, subset: Optional[DWDRadarDataSubset] = None, elevation: Optional[int] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, ) -> RadarResult: """ Collect radar data for given parameters. :param parameter: The radar moment to request :param site: Site/station if parameter is one of RADAR_PARAMETERS_SITES :param fmt: Data format (BINARY, BUFR, HDF5) :param subset: The subset (simple or polarimetric) for HDF5 data. :param elevation: :param start_date: Start date :param end_date: End date :param resolution: Time resolution for RadarParameter.RADOLAN_CDC, either daily or hourly or 5 minutes. :param period: Period type for RadarParameter.RADOLAN_CDC :return: ``RadarResult`` item """ # Find latest file. if start_date == DWDRadarDate.LATEST: file_index = create_fileindex_radar( parameter=parameter, site=site, fmt=fmt, parse_datetime=False, ) # Find "-latest-" file. filenames = file_index["FILENAME"].tolist() latest_file = list(filter(lambda x: "-latest-" in x, filenames))[0] # Yield single "RadarResult" item. result = next(_download_generic_data(url=latest_file)) yield result else: if parameter == DWDRadarParameter.RADOLAN_CDC: if period: period_types = [period] else: period_types = [ DWDRadarPeriod.RECENT, DWDRadarPeriod.HISTORICAL, ] results = [] for period in period_types: file_index = create_fileindex_radolan_cdc( resolution=resolution, period=period) # Filter for dates range if start_date and end_date are defined. if period == DWDRadarPeriod.RECENT: file_index = file_index[ (file_index[DWDMetaColumns.DATETIME.value] >= start_date) & (file_index[DWDMetaColumns.DATETIME.value] < end_date)] # This is for matching historical data, e.g. "RW-200509.tar.gz". else: file_index = file_index[ (file_index[DWDMetaColumns.DATETIME.value].dt.year == start_date.year) & (file_index[DWDMetaColumns.DATETIME.value].dt.month == start_date.month)] results.append(file_index) file_index = pd.concat(results) if file_index.empty: # TODO: Extend this log message. log.warning( f"No radar file found for {parameter}, {site}, {fmt}") return # Iterate list of files and yield "RadarResult" items. for _, row in file_index.iterrows(): url = row[DWDMetaColumns.FILENAME.value] yield download_radolan_data(start_date, url) else: file_index = create_fileindex_radar( parameter=parameter, site=site, fmt=fmt, subset=subset, parse_datetime=True, ) # Filter for dates range if start_date and end_date are defined. file_index = file_index[ (file_index[DWDMetaColumns.DATETIME.value] >= start_date) & (file_index[DWDMetaColumns.DATETIME.value] < end_date)] # Filter SWEEP_VOL_VELOCITY_H and SWEEP_VOL_REFLECTIVITY_H by elevation. if elevation is not None: filename = file_index[DWDMetaColumns.FILENAME.value] file_index = file_index[ (filename.str.contains(f"vradh_{elevation:02d}")) | (filename.str.contains(f"sweep_vol_v_{elevation}")) | (filename.str.contains(f"dbzh_{elevation:02d}")) | (filename.str.contains(f"sweep_vol_z_{elevation}"))] if file_index.empty: log.warning( f"No radar file found for {parameter}, {site}, {fmt}") return # Iterate list of files and yield "RadarResult" items. for _, row in file_index.iterrows(): date_time = row[DWDMetaColumns.DATETIME.value] url = row[DWDMetaColumns.FILENAME.value] for result in _download_generic_data(url=url): if result.timestamp is None: result.timestamp = date_time yield result