def test_radar_fileindex_radolan_reflectivity_bin(parameter):
file_index = create_fileindex_radar(parameter=parameter, )
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match(
f"*/weather/radar/radolan/{parameter.value}/*---bin")
for url in urls)
def test_radar_fileindex_sites_px250_reflectivity_bufr():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.PX250_REFLECTIVITY,
site=DwdRadarSite.BOO,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all("/weather/radar/sites/px250/boo" in url for url in urls)
def test_radar_fileindex_composite_rx_reflectivity_bin():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.RX_REFLECTIVITY, )
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match("*/weather/radar/composit/rx/*---bin")
for url in urls)
def test_radar_fileindex_composite_pg_reflectivity_bufr():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.PG_REFLECTIVITY,
fmt=DwdRadarDataFormat.BUFR,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match("*/weather/radar/composit/pg/*---bufr")
for url in urls)
def test_radar_fileindex_sites_px_reflectivity_bufr():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.PX_REFLECTIVITY,
site=DwdRadarSite.BOO,
fmt=DwdRadarDataFormat.BUFR,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match("*/weather/radar/sites/px/boo/*---buf")
for url in urls)
def test_radar_fileindex_sites_sweep_bufr():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H,
site=DwdRadarSite.BOO,
fmt=DwdRadarDataFormat.BUFR,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match("*/weather/radar/sites/sweep_vol_v/boo/*--buf.bz2")
for url in urls)
def test_radar_fileindex_radolan_cdc_5minutes():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.RADOLAN_CDC,
resolution=DwdRadarResolution.MINUTE_5,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match(
"*/climate_environment/CDC/grids_germany/5_minutes/radolan/reproc/2017_002/bin/*/YW2017*.tar"
) for url in urls if not url.endswith(".tar.gz"))
def test_radar_fileindex_radolan_cdc_hourly_historical():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.RADOLAN_CDC,
resolution=DwdRadarResolution.HOURLY,
period=DwdRadarPeriod.HISTORICAL,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match(
"*/climate_environment/CDC/grids_germany/hourly/radolan/historical/bin/*/RW*.tar.gz"
) for url in urls if not url.endswith(".pdf"))
def test_radar_fileindex_radolan_cdc_daily_recent():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.RADOLAN_CDC,
resolution=DwdRadarResolution.DAILY,
period=DwdRadarPeriod.RECENT,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all(
PurePath(url).match(
"*/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/*---bin.gz"
) for url in urls if not url.endswith(".pdf"))
def test_radar_fileindex_sites_sweep_vol_v_hdf5_simple():
file_index = create_fileindex_radar(
parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H,
site=DwdRadarSite.BOO,
fmt=DwdRadarDataFormat.HDF5,
subset=DwdRadarDataSubset.SIMPLE,
)
urls = file_index[DwdColumns.FILENAME.value].tolist()
assert all("/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_simple" in url
for url in urls)
def collect_radar_data(
parameter: Optional[DwdRadarParameter],
resolution: Optional[Resolution] = None,
period: Optional[Period] = None,
site: Optional[DwdRadarSite] = None,
fmt: Optional[DwdRadarDataFormat] = None,
subset: Optional[DwdRadarDataSubset] = None,
elevation: Optional[int] = None,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
verify: Optional[bool] = True,
) -> RadarResult:
"""
Collect radar data for given parameters.
:param parameter: The radar moment to request
:param resolution: Time resolution for RadarParameter.RADOLAN_CDC,
either daily or hourly or 5 minutes.
:param period: Period type for RadarParameter.RADOLAN_CDC
:param site: Site/station if parameter is one of
RADAR_PARAMETERS_SITES
:param fmt: Data format (BINARY, BUFR, HDF5)
:param subset: The subset (simple or polarimetric) for HDF5 data.
:param elevation:
:param start_date: Start date
:param end_date: End date
:param verify: Whether to verify the response
:return: ``RadarResult`` item
"""
# Find latest file.
if start_date == DwdRadarDate.LATEST:
file_index = create_fileindex_radar(
parameter=parameter,
site=site,
fmt=fmt,
parse_datetime=False,
)
# Find "-latest-" file.
filenames = file_index[DwdColumns.FILENAME.value].tolist()
latest_file = list(filter(lambda x: "-latest-" in x, filenames))[0]
# Yield single "RadarResult" item.
result = next(_download_generic_data(url=latest_file))
yield result
else:
if parameter == DwdRadarParameter.RADOLAN_CDC:
if period:
period_types = [period]
else:
period_types = [
Period.RECENT,
Period.HISTORICAL,
]
results = []
for period in period_types:
file_index = create_fileindex_radolan_cdc(
resolution=resolution, period=period)
# Filter for dates range if start_date and end_date are defined.
if period == Period.RECENT:
file_index = file_index[
(file_index[DwdColumns.DATETIME.value] >= start_date)
& (file_index[DwdColumns.DATETIME.value] < end_date)]
# This is for matching historical data, e.g. "RW-200509.tar.gz".
else:
file_index = file_index[
(file_index[DwdColumns.DATETIME.value].dt.year ==
start_date.year)
& (file_index[DwdColumns.DATETIME.value].dt.month ==
start_date.month)]
results.append(file_index)
file_index = pd.concat(results)
if file_index.empty:
# TODO: Extend this log message.
log.warning(
f"No radar file found for {parameter}, {site}, {fmt}")
return
# Iterate list of files and yield "RadarResult" items.
for _, row in file_index.iterrows():
url = row[DwdColumns.FILENAME.value]
try:
yield from download_radolan_data(url, start_date, end_date)
except FailedDownload as e:
log.exception(e)
else:
file_index = create_fileindex_radar(
parameter=parameter,
site=site,
fmt=fmt,
subset=subset,
parse_datetime=True,
)
# Filter for dates range if start_date and end_date are defined.
file_index = file_index[
(file_index[DwdColumns.DATETIME.value] >= start_date)
& (file_index[DwdColumns.DATETIME.value] < end_date)]
# Filter SWEEP_VOL_VELOCITY_H and SWEEP_VOL_REFLECTIVITY_H by elevation.
if elevation is not None:
filename = file_index[DwdColumns.FILENAME.value]
file_index = file_index[
(filename.str.contains(f"vradh_{elevation:02d}"))
| (filename.str.contains(f"sweep_vol_v_{elevation}"))
| (filename.str.contains(f"dbzh_{elevation:02d}"))
| (filename.str.contains(f"sweep_vol_z_{elevation}"))]
if file_index.empty:
log.warning(
f"No radar file found for {parameter}, {site}, {fmt}")
return
# Iterate list of files and yield "RadarResult" items.
for _, row in file_index.iterrows():
date_time = row[DwdColumns.DATETIME.value]
url = row[DwdColumns.FILENAME.value]
try:
for result in _download_generic_data(url=url):
if result.timestamp is None:
result.timestamp = date_time
if verify:
if fmt == DwdRadarDataFormat.HDF5:
verify_hdf5(result.data)
yield result
except Exception: # pragma: no cover
log.exception("Unable to read HDF5 file")
