def test_dwd_observations_stations_geojson():
# Existing combination of parameters
request = DwdObservationRequest(
DwdObservationDataset.CLIMATE_SUMMARY,
DwdObservationResolution.DAILY,
DwdObservationPeriod.HISTORICAL,
)
results = request.filter_by_station_id(station_id=("00001", ))
assert not results.df.empty
geojson = results.to_ogc_feature_collection()
properties = geojson["features"][0]["properties"]
geometry = geojson["features"][0]["geometry"]
assert properties["name"] == "Aach"
assert properties["state"] == "Baden-Württemberg"
assert geometry == {
"type": "Point",
"coordinates": [8.8493, 47.8413, 478.0],
}
def test_dwd_observations_stations_filter_empty():
# Existing combination of parameters
request = DwdObservationRequest(
DwdObservationDataset.CLIMATE_SUMMARY,
DwdObservationResolution.DAILY,
DwdObservationPeriod.HISTORICAL,
)
df = request.filter_by_station_id(station_id=("FizzBuzz", )).df
assert df.empty
def test_dwd_observations_stations_filter():
# Existing combination of parameters
request = DwdObservationRequest(
DwdObservationDataset.CLIMATE_SUMMARY,
DwdObservationResolution.DAILY,
DwdObservationPeriod.HISTORICAL,
)
df = request.filter_by_station_id(station_id=("00001", )).df
assert not df.empty
assert_frame_equal(df, EXPECTED_DF)
def test_dwd_observation_data_dataset():
""" Request a parameter set"""
request = DwdObservationRequest(
parameter=["kl"],
resolution="daily",
period=["recent", "historical"],
)
stations = request.filter_by_station_id(station_id=(1, ))
assert stations == DwdObservationRequest(
parameter=[DwdObservationDataset.CLIMATE_SUMMARY],
resolution=DwdObservationResolution.DAILY,
period=[DwdObservationPeriod.HISTORICAL, DwdObservationPeriod.RECENT],
start_date=None,
end_date=None,
).filter_by_station_id(station_id=(1, ), )
request = DwdObservationRequest(
parameter=[DwdObservationDataset.CLIMATE_SUMMARY],
resolution=DwdObservationResolution.DAILY,
period=[DwdObservationPeriod.HISTORICAL, DwdObservationPeriod.RECENT],
).filter_by_station_id(station_id=(1, ), )
assert request == DwdObservationRequest(
parameter=[DwdObservationDataset.CLIMATE_SUMMARY],
resolution=DwdObservationResolution.DAILY,
period=[DwdObservationPeriod.HISTORICAL, DwdObservationPeriod.RECENT],
start_date=None,
end_date=None,
).filter_by_station_id(station_id=(1, ), )
assert request.parameter == [(
DwdObservationDataset.CLIMATE_SUMMARY,
DwdObservationDataset.CLIMATE_SUMMARY,
)]
def dwd_values(
kind: str,
stations: str = Query(default=None),
parameter: str = Query(default=None),
resolution: str = Query(default=None),
period: str = Query(default=None),
mosmix_type: str = Query(default=None),
date: str = Query(default=None),
sql: str = Query(default=None),
tidy: bool = Query(default=True),
):
"""
Acquire data from DWD.
# TODO: Obtain lat/lon distance/number information.
:param provider:
:param kind: string for product, either observation or forecast
:param stations: Comma-separated list of station identifiers.
:param parameter: Observation measure
:param resolution: Frequency/granularity of measurement interval
:param period: Recent or historical files
:param mosmix_type: MOSMIX type. Either "small" or "large".
:param date: Date or date range
:param sql: SQL expression
:param tidy: Whether to return data in tidy format. Default: True.
:return:
"""
if kind not in ["observation", "mosmix"]:
return HTTPException(
status_code=404,
detail=f"Unknown value for query argument 'kind={kind}' {kind}",
)
if stations is None:
raise HTTPException(
status_code=400, detail="Query argument 'stations' is required"
)
station_ids = map(str, read_list(stations))
if kind == "observation":
if parameter is None or resolution is None or period is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'parameter', 'resolution' "
"and 'period' are required",
)
# Data acquisition.
request = DwdObservationRequest(
parameter=parameter,
resolution=resolution,
period=period,
tidy=tidy,
si_units=False,
)
else:
if parameter is None or mosmix_type is None:
raise HTTPException(
status_code=400, detail="Query argument 'mosmix_type' is required"
)
request = DwdMosmixRequest(
parameter=parameter, mosmix_type=mosmix_type, si_units=False
)
# Postprocessing.
results = request.filter_by_station_id(station_id=station_ids).values.all()
if date is not None:
results.filter_by_date(date)
if sql is not None:
results.filter_by_sql(sql)
data = json.loads(results.to_json())
return make_json_response(data)
def run():
"""
Usage:
wetterdienst dwd observation stations --parameter=<parameter> --resolution=<resolution> --period=<period> [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> [--station=<station>] [--period=<period>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> --latitude=<latitude> --longitude=<longitude> [--period=<period>] [--rank=<rank>] [--distance=<distance>] [--tidy] [--date=<date>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd forecast stations [--parameter=<parameter>] [--mosmix-type=<mosmix-type>] [--date=<date>] [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd forecast values --parameter=<parameter> [--mosmix-type=<mosmix-type>] --station=<station> [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd about [parameters] [resolutions] [periods]
wetterdienst dwd about coverage [--parameter=<parameter>] [--resolution=<resolution>] [--period=<period>]
wetterdienst dwd about fields --parameter=<parameter> --resolution=<resolution> --period=<period> [--language=<language>]
wetterdienst radar stations [--odim-code=<odim-code>] [--wmo-code=<wmo-code>] [--country-name=<country-name>]
wetterdienst dwd radar stations
wetterdienst restapi [--listen=<listen>] [--reload]
wetterdienst explorer [--listen=<listen>] [--reload]
wetterdienst --version
wetterdienst (-h | --help)
Options:
--parameter=<parameter> Parameter Set/Parameter, e.g. "kl" or "precipitation_height", etc.
--resolution=<resolution> Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1"
--period=<period> Dataset period: "historical", "recent", "now"
--station=<station> Comma-separated list of station identifiers
--latitude=<latitude> Latitude for filtering by geoposition.
--longitude=<longitude> Longitude for filtering by geoposition.
--rank=<rank> Rank of nearby stations when filtering by geoposition.
--distance=<distance> Maximum distance in km when filtering by geoposition.
--date=<date> Date for filtering data. Can be either a single date(time) or
an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals.
--mosmix-type=<mosmix-type> type of mosmix, either 'small' or 'large'
--sql=<sql> SQL query to apply to DataFrame.
--format=<format> Output format. [Default: json]
--target=<target> Output target for storing data into different data sinks.
--language=<language> Output language. [Default: en]
--version Show version information
--debug Enable debug messages
--listen=<listen> HTTP server listen address.
--reload Run service and dynamically reload changed files
-h --help Show this screen
Examples requesting observation stations:
# Get list of all stations for daily climate summary data in JSON format
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent
# Get list of all stations in CSV format
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --format=csv
# Get list of specific stations
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411
# Get list of specific stations in GeoJSON format
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson
Examples requesting observation values:
# Get daily climate summary data for specific stations
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent
# Get daily climate summary data for specific stations in CSV format
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent
# Get daily climate summary data for specific stations in tidy format
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --tidy
# Limit output to specific date
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01
# Limit output to specified date range in ISO-8601 time interval format
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05
# The real power horse: Acquire data across historical+recent data sets
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --date=1969-01-01/2020-06-11
# Acquire monthly data for 2020-05
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2020-05
# Acquire monthly data from 2017-01 to 2019-12
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2017-01/2019-12
# Acquire annual data for 2019
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2019
# Acquire annual data from 2010 to 2020
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2010/2020
# Acquire hourly data
wetterdienst dwd observation values --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12
Examples requesting forecast stations:
wetterdienst dwd forecast stations
Examples requesting forecast values:
wetterdienst dwd forecast values --parameter=ttt,ff --station=65510
Examples using geospatial features:
# Acquire stations and readings by geoposition, request specific number of nearby stations.
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5
wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5 --date=2020-06-30
# Acquire stations and readings by geoposition, request stations within specific distance.
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25
wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30
Examples using SQL filtering:
# Find stations by state.
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'"
# Find stations by name (LIKE query).
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')"
# Find stations by name (regexp query).
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))"
# Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees celsius.
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE temperature_air_max_200 < 2.0;"
# Filter measurements: Same as above, but use tidy format.
# FIXME: Currently, this does not work, see https://github.com/earthobservations/wetterdienst/issues/377.
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE parameter='temperature_air_max_200' AND value < 2.0;" --tidy
Examples for inquiring metadata:
# Display list of available parameters (air_temperature, precipitation, pressure, ...)
wetterdienst dwd about parameters
# Display list of available resolutions (10_minutes, hourly, daily, ...)
wetterdienst dwd about resolutions
# Display list of available periods (historical, recent, now)
wetterdienst dwd about periods
# Display coverage/correlation between parameters, resolutions and periods.
# This can answer questions like ...
wetterdienst dwd about coverage
# Tell me all periods and resolutions available for 'air_temperature'.
wetterdienst dwd about coverage --parameter=air_temperature
# Tell me all parameters available for 'daily' resolution.
wetterdienst dwd about coverage --resolution=daily
Examples for exporting data to files:
# Export list of stations into spreadsheet
wetterdienst dwd observations stations --parameter=kl --resolution=daily --period=recent --target=file://stations.xlsx
# Shortcut command for fetching readings
alias fetch="wetterdienst dwd observations values --station=1048,4411 --parameter=kl --resolution=daily --period=recent"
# Export readings into spreadsheet (Excel-compatible)
fetch --target="file://observations.xlsx"
# Export readings into Parquet format and display head of Parquet file
fetch --target="file://observations.parquet"
# Check Parquet file
parquet-tools schema observations.parquet
parquet-tools head observations.parquet
# Export readings into Zarr format
fetch --target="file://observations.zarr"
Examples for exporting data to databases:
# Shortcut command for fetching readings
alias fetch="wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent"
# Store readings to DuckDB
fetch --target="duckdb:///dwd.duckdb?table=weather"
# Store readings to InfluxDB
fetch --target="influxdb://localhost/?database=dwd&table=weather"
# Store readings to CrateDB
fetch --target="crate://localhost/?database=dwd&table=weather"
Invoke the HTTP REST API service:
# Start service on standard port, listening on http://localhost:7890.
wetterdienst restapi
# Start service on standard port and watch filesystem changes.
# This is suitable for development.
wetterdienst restapi --reload
# Start service on public interface and specific port.
wetterdienst restapi --listen=0.0.0.0:8890
Invoke the Wetterdienst Explorer UI service:
# Start service on standard port, listening on http://localhost:7891.
wetterdienst explorer
# Start service on standard port and watch filesystem changes.
# This is suitable for development.
wetterdienst explorer --reload
# Start service on public interface and specific port.
wetterdienst explorer --listen=0.0.0.0:8891
"""
appname = f"{__appname__} {__version__}"
# Read command line options.
options = normalize_options(docopt(run.__doc__, version=appname))
# Setup logging.
debug = options.get("debug")
log_level = logging.INFO
if debug: # pragma: no cover
log_level = logging.DEBUG
setup_logging(log_level)
# Run HTTP service.
if options.restapi: # pragma: no cover
listen_address = options.listen
log.info(f"Starting {appname}")
log.info(f"Starting HTTP web service on http://{listen_address}")
from wetterdienst.ui.restapi import start_service
start_service(listen_address, reload=options.reload)
return
# Run UI service.
if options.explorer: # pragma: no cover
listen_address = options.listen
log.info(f"Starting {appname}")
log.info(f"Starting UI web service on http://{listen_address}")
from wetterdienst.ui.explorer.app import start_service
start_service(listen_address, reload=options.reload)
return
# Handle radar data inquiry. Currently, "stations only".
if options.radar:
if options.dwd:
data = DwdRadarSites().all()
else:
if options.odim_code:
data = OperaRadarSites().by_odimcode(options.odim_code)
elif options.wmo_code:
data = OperaRadarSites().by_wmocode(options.wmo_code)
elif options.country_name:
data = OperaRadarSites().by_countryname(options.country_name)
else:
data = OperaRadarSites().all()
output = json.dumps(data, indent=4)
print(output)
return
# Output domain information.
if options.about:
about(options)
return
# Sanity checks.
if (options["values"] or options.forecast) and options.format == "geojson":
raise KeyError("GeoJSON format only available for stations output")
# Acquire station list, also used for readings if required.
# Filtering applied for distance (a.k.a. nearby) and pre-selected stations
stations = None
if options.observation:
stations = DwdObservationRequest(
parameter=read_list(options.parameter),
resolution=options.resolution,
period=options.period,
tidy=options.tidy,
si_units=False,
)
elif options.forecast:
stations = DwdMosmixRequest(
parameter=read_list(options.parameter),
mosmix_type=DwdMosmixType.LARGE,
tidy=options.tidy,
si_units=False,
)
if options.latitude and options.longitude:
if options.rank:
stations = stations.filter_by_rank(
latitude=float(options.latitude),
longitude=float(options.longitude),
rank=int(options.rank),
)
elif options.distance:
stations = stations.filter_by_distance(
latitude=float(options.latitude),
longitude=float(options.longitude),
distance=int(options.distance),
)
else:
raise DocoptExit(
"Geospatial queries need either --rank or --distance")
results = stations
elif options.station:
results = stations.filter_by_station_id(read_list(options.station))
else:
results = stations.all()
df = pd.DataFrame()
if options.stations:
pass
elif options["values"]:
try:
# TODO: Add stream-based processing here.
results = results.values.all()
except ValueError as ex:
log.exception(ex)
sys.exit(1)
df = results.df
if df.empty:
log.error("No data available for given constraints")
sys.exit(1)
# Filter readings by datetime expression.
if options["values"] and options.date:
results.filter_by_date(options.date)
# Apply filtering by SQL.
if options.sql:
if options.tidy:
log.error("Combining SQL filtering with tidy format not possible")
sys.exit(1)
log.info(f"Filtering with SQL: {options.sql}")
results.filter_by_sql(options.sql)
# Emit to data sink, e.g. write to database.
if options.target:
results.to_target(options.target)
return
# Render to output format.
try:
if options.format == "json":
output = results.to_json()
elif options.format == "csv":
output = results.to_csv()
elif options.format == "geojson":
output = results.to_geojson()
else:
raise KeyError("Unknown output format")
except KeyError as ex:
log.error(
f'{ex}. Output format must be one of "json", "geojson", "csv".')
sys.exit(1)
print(output)
