def about(options: Munch):
"""
Output possible arguments for command line options
"--parameter", "--resolution" and "--period".
:param options: Normalized docopt command line options.
"""
def output(thing):
for item in thing:
if item:
if hasattr(item, "value"):
value = item.value
else:
value = item
print("-", value)
if options.parameters:
output(DwdObservationDataset)
elif options.resolutions:
output(DwdObservationResolution)
elif options.periods:
output(DwdObservationPeriod)
elif options.coverage:
metadata = DwdObservationRequest.discover(
filter_=options.resolution,
dataset=read_list(options.parameter),
flatten=False,
)
output = json.dumps(metadata, indent=4)
print(output)
elif options.fields:
metadata = DwdObservationRequest.describe_fields(
dataset=read_list(options.parameter),
resolution=options.resolution,
period=read_list(options.period),
language=options.language,
)
output = pformat(dict(metadata))
print(output)
else:
log.error(
'Please invoke "wetterdienst dwd about" with one of these subcommands:'
)
output(["parameters", "resolutions", "periods", "coverage"])
sys.exit(1)
def about(options: Munch):
"""
Output possible arguments for command line options
"--parameter", "--resolution" and "--period".
:param options: Normalized docopt command line options.
"""
def output(thing):
for item in thing:
if item:
if hasattr(item, "value"):
value = item.value
else:
value = item
print("-", value)
if options.parameters:
output(DWDObservationParameterSet)
elif options.resolutions:
output(DWDObservationResolution)
elif options.periods:
output(DWDObservationPeriod)
elif options.coverage:
output = json.dumps(
DWDObservationMetadata(
resolution=options.resolution,
parameter_set=read_list(options.parameter),
period=read_list(options.period),
).discover_parameter_sets(),
indent=4,
)
print(output)
else:
log.error(
'Please invoke "wetterdienst dwd about" with one of these subcommands:'
)
output(["parameters", "resolutions", "periods", "coverage"])
sys.exit(1)
def about(options: Munch):
"""
Output possible arguments for command line options
"--parameter", "--resolution" and "--period".
:param options: Normalized docopt command line options.
"""
def output(thing):
for item in thing:
if item:
if hasattr(item, "value"):
value = item.value
else:
value = item
print("-", value)
if options.parameters:
output(Parameter)
elif options.resolutions:
output(TimeResolution)
elif options.periods:
output(PeriodType)
elif options.coverage:
print(
discover_climate_observations(
time_resolution=options.resolution,
parameter=read_list(options.parameter),
period_type=read_list(options.period),
)
)
else:
log.error(
'Please invoke "wetterdienst dwd about" with one of these subcommands:'
)
output(["parameters", "resolutions", "periods", "coverage"])
sys.exit(1)
def dwd_values(
kind: str,
stations: str = Query(default=None),
parameter: str = Query(default=None),
resolution: str = Query(default=None),
period: str = Query(default=None),
mosmix_type: str = Query(default=None),
date: str = Query(default=None),
sql: str = Query(default=None),
tidy: bool = Query(default=True),
):
"""
Acquire data from DWD.
# TODO: Obtain lat/lon distance/number information.
:param provider:
:param kind: string for product, either observation or forecast
:param stations: Comma-separated list of station identifiers.
:param parameter: Observation measure
:param resolution: Frequency/granularity of measurement interval
:param period: Recent or historical files
:param mosmix_type: MOSMIX type. Either "small" or "large".
:param date: Date or date range
:param sql: SQL expression
:param tidy: Whether to return data in tidy format. Default: True.
:return:
"""
if kind not in ["observation", "mosmix"]:
return HTTPException(
status_code=404,
detail=f"Unknown value for query argument 'kind={kind}' {kind}",
)
if stations is None:
raise HTTPException(
status_code=400, detail="Query argument 'stations' is required"
)
station_ids = map(str, read_list(stations))
if kind == "observation":
if parameter is None or resolution is None or period is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'parameter', 'resolution' "
"and 'period' are required",
)
# Data acquisition.
request = DwdObservationRequest(
parameter=parameter,
resolution=resolution,
period=period,
tidy=tidy,
si_units=False,
)
else:
if parameter is None or mosmix_type is None:
raise HTTPException(
status_code=400, detail="Query argument 'mosmix_type' is required"
)
request = DwdMosmixRequest(
parameter=parameter, mosmix_type=mosmix_type, si_units=False
)
# Postprocessing.
results = request.filter_by_station_id(station_id=station_ids).values.all()
if date is not None:
results.filter_by_date(date)
if sql is not None:
results.filter_by_sql(sql)
data = json.loads(results.to_json())
return make_json_response(data)
def dwd_readings(
product: str,
station: str = Query(default=None),
parameter: str = Query(default=None),
resolution: str = Query(default=None),
period: str = Query(default=None),
mosmix_type: str = Query(default=None, alias="mosmix-type"),
date: str = Query(default=None),
sql: str = Query(default=None),
):
"""
Acquire data from DWD.
# TODO: Obtain lat/lon distance/number information.
:param product: string for product, either observations or mosmix
:param station: Comma-separated list of station identifiers.
:param parameter: Observation measure
:param resolution: Frequency/granularity of measurement interval
:param period: Recent or historical files
:param mosmix_type type of mosmix, either small or large
:param date: Date or date range
:param sql: SQL expression
:return:
"""
if product not in ["observations", "mosmix"]:
return HTTPException(status_code=404, detail=f"product {product} not found")
if station is None:
raise HTTPException(
status_code=400, detail="Query argument 'station' is required"
)
station_ids = map(str, read_list(station))
if product == "observations":
if parameter is None or resolution is None or period is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'parameter', 'resolution' "
"and 'period' are required",
)
parameter = parse_enumeration_from_template(
parameter, DWDObservationParameterSet
)
resolution = parse_enumeration_from_template(
resolution, DWDObservationResolution
)
period = parse_enumeration_from_template(period, DWDObservationPeriod)
# Data acquisition.
readings = DWDObservationData(
station_ids=station_ids,
parameters=parameter,
resolution=resolution,
periods=period,
tidy_data=True,
humanize_parameters=True,
)
else:
if mosmix_type is None:
raise HTTPException(
status_code=400, detail="Query argument 'mosmix_type' is required"
)
mosmix_type = parse_enumeration_from_template(mosmix_type, DWDMosmixType)
readings = DWDMosmixData(station_ids=station_ids, mosmix_type=mosmix_type)
# Postprocessing.
df = readings.all()
if date is not None:
df = df.dwd.filter_by_date(date, resolution)
df = df.dwd.lower()
if sql is not None:
df = df.io.sql(sql)
data = json.loads(df.to_json(orient="records", date_format="iso"))
return make_json_response(data)
def dwd_readings(
station: str = Query(default=None),
parameter: str = Query(default=None),
resolution: str = Query(default=None),
period: str = Query(default=None),
date: str = Query(default=None),
sql: str = Query(default=None),
):
"""
Acquire data from DWD.
# TODO: Obtain lat/lon distance/number information.
:param station: Comma-separated list of station identifiers.
:param parameter: Observation measure
:param resolution: Frequency/granularity of measurement interval
:param period: Recent or historical files
:param date: Date or date range
:param sql: SQL expression
:return:
"""
if station is None:
raise HTTPException(
status_code=400, detail="Query argument 'station' is required"
)
if parameter is None or resolution is None or period is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'parameter', 'resolution' "
"and 'period' are required",
)
station_ids = map(int, read_list(station))
parameter = parse_enumeration_from_template(parameter, DWDObservationParameterSet)
resolution = parse_enumeration_from_template(resolution, DWDObservationResolution)
period = parse_enumeration_from_template(period, DWDObservationPeriod)
# Data acquisition.
observations = DWDObservationData(
station_ids=station_ids,
parameters=parameter,
resolution=resolution,
periods=period,
tidy_data=True,
humanize_column_names=True,
)
# Postprocessing.
df = observations.collect_safe()
if date is not None:
df = df.dwd.filter_by_date(date, resolution)
df = df.dwd.lower()
if sql is not None:
df = df.io.sql(sql)
data = json.loads(df.to_json(orient="records", date_format="iso"))
return make_json_response(data)
def values(
provider: str = Query(default=None),
network: str = Query(default=None),
parameter: str = Query(default=None),
resolution: str = Query(default=None),
period: str = Query(default=None),
date: str = Query(default=None),
issue: str = Query(default="latest"),
all_: str = Query(alias="all", default=False),
station: str = Query(default=None),
name: str = Query(default=None),
coordinates: str = Query(default=None),
rank: int = Query(default=None),
distance: float = Query(default=None),
bbox: str = Query(default=None),
sql: str = Query(default=None),
sql_values: str = Query(alias="sql-values", default=None),
humanize: bool = Query(default=True),
tidy: bool = Query(default=True),
si_units: bool = Query(alias="si-units", default=True),
skip_empty: bool = Query(alias="skip-empty", default=False),
skip_threshold: float = Query(
alias="skip-threshold", default=0.95, gt=0, le=1),
dropna: bool = Query(alias="dropna", default=False),
pretty: bool = Query(default=False),
debug: bool = Query(default=False),
):
"""
Acquire data from DWD.
:param provider:
:param network: string for network of provider
:param parameter: Observation measure
:param resolution: Frequency/granularity of measurement interval
:param period: Recent or historical files
:param date: Date or date range
:param issue:
:param all_:
:param station:
:param name:
:param coordinates:
:param rank:
:param distance:
:param bbox:
:param sql: SQL expression
:param sql_values:
:param fmt:
:param humanize:
:param tidy: Whether to return data in tidy format. Default: True.
:param si_units:
:param pretty:
:param debug:
:return:
"""
# TODO: Add geojson support
fmt = "json"
if provider is None or network is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'provider' and 'network' are required",
)
if parameter is None or resolution is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'parameter', 'resolution' "
"and 'date' are required",
)
if fmt not in ("json", "geojson"):
raise HTTPException(
status_code=400,
detail="format argument must be one of json, geojson",
)
set_logging_level(debug)
try:
api: ScalarRequestCore = Wetterdienst(provider, network)
except ProviderError:
return HTTPException(
status_code=404,
detail=f"Given combination of provider and network not available. "
f"Choose provider and network from {Wetterdienst.discover()}",
)
parameter = read_list(parameter)
if period:
period = read_list(period)
if station:
station = read_list(station)
try:
values_ = get_values(
api=api,
parameter=parameter,
resolution=resolution,
date=date,
issue=issue,
period=period,
all_=all_,
station_id=station,
name=name,
coordinates=coordinates,
rank=rank,
distance=distance,
bbox=bbox,
sql=sql,
sql_values=sql_values,
si_units=si_units,
skip_empty=skip_empty,
skip_threshold=skip_threshold,
dropna=dropna,
tidy=tidy,
humanize=humanize,
)
except Exception as e:
log.exception(e)
return HTTPException(status_code=404, detail=str(e))
indent = None
if pretty:
indent = 4
output = values_.df
output[Columns.DATE.value] = output[Columns.DATE.value].apply(
lambda ts: ts.isoformat())
output = output.replace({np.NaN: None, pd.NA: None})
output = output.to_dict(orient="records")
output = make_json_response(output, api.provider)
output = json.dumps(output, indent=indent, ensure_ascii=False)
return Response(content=output, media_type="application/json")
def stations(
provider: str = Query(default=None),
network: str = Query(default=None),
parameter: str = Query(default=None),
resolution: str = Query(default=None),
period: str = Query(default=None),
all_: str = Query(alias="all", default=False),
station_id: str = Query(default=None),
name: str = Query(default=None),
coordinates: str = Query(default=None),
rank: int = Query(default=None),
distance: float = Query(default=None),
bbox: str = Query(default=None),
sql: str = Query(default=None),
fmt: str = Query(alias="format", default="json"),
debug: bool = Query(default=False),
pretty: bool = Query(default=False),
):
if provider is None or network is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'provider' and 'network' are required",
)
if parameter is None or resolution is None:
raise HTTPException(
status_code=400,
detail="Query arguments 'parameter', 'resolution' "
"and 'period' are required",
)
if fmt not in ("json", "geojson"):
raise HTTPException(
status_code=400,
detail="format argument must be one of json, geojson",
)
set_logging_level(debug)
try:
api = Wetterdienst(provider, network)
except ProviderError:
return HTTPException(
status_code=404,
detail=
f"Choose provider and network from {app.url_path_for('coverage')}",
)
parameter = read_list(parameter)
if period:
period = read_list(period)
if station_id:
station_id = read_list(station_id)
try:
stations_ = get_stations(
api=api,
parameter=parameter,
resolution=resolution,
period=period,
date=None,
issue=None,
all_=all_,
station_id=station_id,
name=name,
coordinates=coordinates,
rank=rank,
distance=distance,
bbox=bbox,
sql=sql,
tidy=False,
si_units=False,
humanize=False,
skip_empty=False,
skip_threshold=0.95,
dropna=False,
)
except (KeyError, ValueError) as e:
return HTTPException(status_code=404, detail=str(e))
if not stations_.parameter or not stations_.resolution:
return HTTPException(
status_code=404,
detail=
f"No parameter found for provider {provider}, network {network}, "
f"parameter(s) {parameter} and resolution {resolution}.",
)
stations_.df = stations_.df.replace({np.nan: None, pd.NA: None})
indent = None
if pretty:
indent = 4
if fmt == "json":
output = stations_.to_dict()
elif fmt == "geojson":
output = stations_.to_ogc_feature_collection()
output = make_json_response(output, api.provider)
output = json.dumps(output, indent=indent, ensure_ascii=False)
return Response(content=output, media_type="application/json")
def run():
"""
Usage:
wetterdienst dwd observation stations --parameter=<parameter> --resolution=<resolution> --period=<period> [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> [--station=<station>] [--period=<period>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> --latitude=<latitude> --longitude=<longitude> [--period=<period>] [--rank=<rank>] [--distance=<distance>] [--tidy] [--date=<date>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd forecast stations [--parameter=<parameter>] [--mosmix-type=<mosmix-type>] [--date=<date>] [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd forecast values --parameter=<parameter> [--mosmix-type=<mosmix-type>] --station=<station> [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd about [parameters] [resolutions] [periods]
wetterdienst dwd about coverage [--parameter=<parameter>] [--resolution=<resolution>] [--period=<period>]
wetterdienst dwd about fields --parameter=<parameter> --resolution=<resolution> --period=<period> [--language=<language>]
wetterdienst radar stations [--odim-code=<odim-code>] [--wmo-code=<wmo-code>] [--country-name=<country-name>]
wetterdienst dwd radar stations
wetterdienst restapi [--listen=<listen>] [--reload]
wetterdienst explorer [--listen=<listen>] [--reload]
wetterdienst --version
wetterdienst (-h | --help)
Options:
--parameter=<parameter> Parameter Set/Parameter, e.g. "kl" or "precipitation_height", etc.
--resolution=<resolution> Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1"
--period=<period> Dataset period: "historical", "recent", "now"
--station=<station> Comma-separated list of station identifiers
--latitude=<latitude> Latitude for filtering by geoposition.
--longitude=<longitude> Longitude for filtering by geoposition.
--rank=<rank> Rank of nearby stations when filtering by geoposition.
--distance=<distance> Maximum distance in km when filtering by geoposition.
--date=<date> Date for filtering data. Can be either a single date(time) or
an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals.
--mosmix-type=<mosmix-type> type of mosmix, either 'small' or 'large'
--sql=<sql> SQL query to apply to DataFrame.
--format=<format> Output format. [Default: json]
--target=<target> Output target for storing data into different data sinks.
--language=<language> Output language. [Default: en]
--version Show version information
--debug Enable debug messages
--listen=<listen> HTTP server listen address.
--reload Run service and dynamically reload changed files
-h --help Show this screen
Examples requesting observation stations:
# Get list of all stations for daily climate summary data in JSON format
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent
# Get list of all stations in CSV format
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --format=csv
# Get list of specific stations
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411
# Get list of specific stations in GeoJSON format
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson
Examples requesting observation values:
# Get daily climate summary data for specific stations
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent
# Get daily climate summary data for specific stations in CSV format
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent
# Get daily climate summary data for specific stations in tidy format
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --tidy
# Limit output to specific date
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01
# Limit output to specified date range in ISO-8601 time interval format
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05
# The real power horse: Acquire data across historical+recent data sets
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --date=1969-01-01/2020-06-11
# Acquire monthly data for 2020-05
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2020-05
# Acquire monthly data from 2017-01 to 2019-12
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2017-01/2019-12
# Acquire annual data for 2019
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2019
# Acquire annual data from 2010 to 2020
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2010/2020
# Acquire hourly data
wetterdienst dwd observation values --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12
Examples requesting forecast stations:
wetterdienst dwd forecast stations
Examples requesting forecast values:
wetterdienst dwd forecast values --parameter=ttt,ff --station=65510
Examples using geospatial features:
# Acquire stations and readings by geoposition, request specific number of nearby stations.
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5
wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5 --date=2020-06-30
# Acquire stations and readings by geoposition, request stations within specific distance.
wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25
wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30
Examples using SQL filtering:
# Find stations by state.
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'"
# Find stations by name (LIKE query).
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')"
# Find stations by name (regexp query).
wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))"
# Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees celsius.
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE temperature_air_max_200 < 2.0;"
# Filter measurements: Same as above, but use tidy format.
# FIXME: Currently, this does not work, see https://github.com/earthobservations/wetterdienst/issues/377.
wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE parameter='temperature_air_max_200' AND value < 2.0;" --tidy
Examples for inquiring metadata:
# Display list of available parameters (air_temperature, precipitation, pressure, ...)
wetterdienst dwd about parameters
# Display list of available resolutions (10_minutes, hourly, daily, ...)
wetterdienst dwd about resolutions
# Display list of available periods (historical, recent, now)
wetterdienst dwd about periods
# Display coverage/correlation between parameters, resolutions and periods.
# This can answer questions like ...
wetterdienst dwd about coverage
# Tell me all periods and resolutions available for 'air_temperature'.
wetterdienst dwd about coverage --parameter=air_temperature
# Tell me all parameters available for 'daily' resolution.
wetterdienst dwd about coverage --resolution=daily
Examples for exporting data to files:
# Export list of stations into spreadsheet
wetterdienst dwd observations stations --parameter=kl --resolution=daily --period=recent --target=file://stations.xlsx
# Shortcut command for fetching readings
alias fetch="wetterdienst dwd observations values --station=1048,4411 --parameter=kl --resolution=daily --period=recent"
# Export readings into spreadsheet (Excel-compatible)
fetch --target="file://observations.xlsx"
# Export readings into Parquet format and display head of Parquet file
fetch --target="file://observations.parquet"
# Check Parquet file
parquet-tools schema observations.parquet
parquet-tools head observations.parquet
# Export readings into Zarr format
fetch --target="file://observations.zarr"
Examples for exporting data to databases:
# Shortcut command for fetching readings
alias fetch="wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent"
# Store readings to DuckDB
fetch --target="duckdb:///dwd.duckdb?table=weather"
# Store readings to InfluxDB
fetch --target="influxdb://localhost/?database=dwd&table=weather"
# Store readings to CrateDB
fetch --target="crate://localhost/?database=dwd&table=weather"
Invoke the HTTP REST API service:
# Start service on standard port, listening on http://localhost:7890.
wetterdienst restapi
# Start service on standard port and watch filesystem changes.
# This is suitable for development.
wetterdienst restapi --reload
# Start service on public interface and specific port.
wetterdienst restapi --listen=0.0.0.0:8890
Invoke the Wetterdienst Explorer UI service:
# Start service on standard port, listening on http://localhost:7891.
wetterdienst explorer
# Start service on standard port and watch filesystem changes.
# This is suitable for development.
wetterdienst explorer --reload
# Start service on public interface and specific port.
wetterdienst explorer --listen=0.0.0.0:8891
"""
appname = f"{__appname__} {__version__}"
# Read command line options.
options = normalize_options(docopt(run.__doc__, version=appname))
# Setup logging.
debug = options.get("debug")
log_level = logging.INFO
if debug: # pragma: no cover
log_level = logging.DEBUG
setup_logging(log_level)
# Run HTTP service.
if options.restapi: # pragma: no cover
listen_address = options.listen
log.info(f"Starting {appname}")
log.info(f"Starting HTTP web service on http://{listen_address}")
from wetterdienst.ui.restapi import start_service
start_service(listen_address, reload=options.reload)
return
# Run UI service.
if options.explorer: # pragma: no cover
listen_address = options.listen
log.info(f"Starting {appname}")
log.info(f"Starting UI web service on http://{listen_address}")
from wetterdienst.ui.explorer.app import start_service
start_service(listen_address, reload=options.reload)
return
# Handle radar data inquiry. Currently, "stations only".
if options.radar:
if options.dwd:
data = DwdRadarSites().all()
else:
if options.odim_code:
data = OperaRadarSites().by_odimcode(options.odim_code)
elif options.wmo_code:
data = OperaRadarSites().by_wmocode(options.wmo_code)
elif options.country_name:
data = OperaRadarSites().by_countryname(options.country_name)
else:
data = OperaRadarSites().all()
output = json.dumps(data, indent=4)
print(output)
return
# Output domain information.
if options.about:
about(options)
return
# Sanity checks.
if (options["values"] or options.forecast) and options.format == "geojson":
raise KeyError("GeoJSON format only available for stations output")
# Acquire station list, also used for readings if required.
# Filtering applied for distance (a.k.a. nearby) and pre-selected stations
stations = None
if options.observation:
stations = DwdObservationRequest(
parameter=read_list(options.parameter),
resolution=options.resolution,
period=options.period,
tidy=options.tidy,
si_units=False,
)
elif options.forecast:
stations = DwdMosmixRequest(
parameter=read_list(options.parameter),
mosmix_type=DwdMosmixType.LARGE,
tidy=options.tidy,
si_units=False,
)
if options.latitude and options.longitude:
if options.rank:
stations = stations.filter_by_rank(
latitude=float(options.latitude),
longitude=float(options.longitude),
rank=int(options.rank),
)
elif options.distance:
stations = stations.filter_by_distance(
latitude=float(options.latitude),
longitude=float(options.longitude),
distance=int(options.distance),
)
else:
raise DocoptExit(
"Geospatial queries need either --rank or --distance")
results = stations
elif options.station:
results = stations.filter_by_station_id(read_list(options.station))
else:
results = stations.all()
df = pd.DataFrame()
if options.stations:
pass
elif options["values"]:
try:
# TODO: Add stream-based processing here.
results = results.values.all()
except ValueError as ex:
log.exception(ex)
sys.exit(1)
df = results.df
if df.empty:
log.error("No data available for given constraints")
sys.exit(1)
# Filter readings by datetime expression.
if options["values"] and options.date:
results.filter_by_date(options.date)
# Apply filtering by SQL.
if options.sql:
if options.tidy:
log.error("Combining SQL filtering with tidy format not possible")
sys.exit(1)
log.info(f"Filtering with SQL: {options.sql}")
results.filter_by_sql(options.sql)
# Emit to data sink, e.g. write to database.
if options.target:
results.to_target(options.target)
return
# Render to output format.
try:
if options.format == "json":
output = results.to_json()
elif options.format == "csv":
output = results.to_csv()
elif options.format == "geojson":
output = results.to_geojson()
else:
raise KeyError("Unknown output format")
except KeyError as ex:
log.error(
f'{ex}. Output format must be one of "json", "geojson", "csv".')
sys.exit(1)
print(output)
def get_stations(options: Munch) -> pd.DataFrame:
"""
Convenience utility function to dispatch command
line options related to geospatial requests.
It will get used from both acquisition requests for station-
and measurement data.
It is able to handle both "number of stations nearby"
and "maximum distance in kilometers" query flavors.
:param options: Normalized docopt command line options.
:return: nearby_stations
"""
# Obtain DWIM date range for station liveness.
# TODO: Obtain from user.
# However, some dates will not work and Pandas will croak with:
# KeyError: "None of [Int64Index([0, 0, 0, 0, 0], dtype='int64')] are in the [index]"
# See also https://github.com/earthobservations/wetterdienst/pull/145#discussion_r487698588.
days500 = datetime.utcnow() + timedelta(days=-500)
now = datetime.utcnow() + timedelta(days=-2)
start_date = datetime(days500.year, days500.month, days500.day)
end_date = datetime(now.year, now.month, now.day)
stations = None
if options.stations or (options.latitude and options.longitude):
if options.observations:
stations = DWDObservationStations(
parameter_set=options.parameter,
resolution=options.resolution,
period=options.period,
start_date=start_date,
end_date=end_date,
)
elif options.forecasts:
stations = DWDMosmixStations()
if options.latitude and options.longitude:
if options.number:
stations = stations.nearby_number(
latitude=float(options.latitude),
longitude=float(options.longitude),
num_stations_nearby=int(options.number),
)
elif options.distance:
stations = stations.nearby_radius(
latitude=float(options.latitude),
longitude=float(options.longitude),
max_distance_in_km=int(options.distance),
)
else:
stations = stations.all()
if options.station:
station_ids = read_list(options.station)
if options.observations:
stations = stations[stations.STATION_ID.isin(station_ids)]
else:
stations = stations[stations.WMO_ID.isin(station_ids)]
return stations
return pd.DataFrame()
def run():
"""
Usage:
wetterdienst dwd observations stations --parameter=<parameter> --resolution=<resolution> --period=<period> [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--number=<number>] [--distance=<distance>] [--sql=<sql>] [--format=<format>]
wetterdienst dwd observations readings --parameter=<parameter> --resolution=<resolution> --station=<station> [--period=<period>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd observations readings --parameter=<parameter> --resolution=<resolution> --latitude=<latitude> --longitude=<longitude> [--period=<period>] [--number=<number>] [--distance=<distance>] [--tidy] [--date=<date>] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd forecasts stations [--date=<date>] [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--number=<number>] [--distance=<distance>] [--sql=<sql>] [--format=<format>]
wetterdienst dwd forecasts readings --mosmix-type=<mosmix-type> --station=<station> [--parameter=<parameter>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>]
wetterdienst dwd about [parameters] [resolutions] [periods]
wetterdienst dwd about coverage [--parameter=<parameter>] [--resolution=<resolution>] [--period=<period>]
wetterdienst dwd about fields --parameter=<parameter> --resolution=<resolution> --period=<period> [--language=<language>]
wetterdienst service [--listen=<listen>]
wetterdienst --version
wetterdienst (-h | --help)
Options:
--parameter=<parameter> Parameter Set/Parameter, e.g. "kl" or "precipitation_height", etc.
--resolution=<resolution> Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1"
--period=<period> Dataset period: "historical", "recent", "now"
--station=<station> Comma-separated list of station identifiers
--latitude=<latitude> Latitude for filtering by geoposition.
--longitude=<longitude> Longitude for filtering by geoposition.
--number=<number> Number of nearby stations when filtering by geoposition.
--distance=<distance> Maximum distance in km when filtering by geoposition.
--date=<date> Date for filtering data. Can be either a single date(time) or
an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals.
--mosmix-type=<mosmix-type> type of mosmix, either 'small' or 'large'
--sql=<sql> SQL query to apply to DataFrame.
--format=<format> Output format. [Default: json]
--target=<target> Output target for storing data into different data sinks.
--language=<language> Output language. [Default: en]
--version Show version information
--debug Enable debug messages
--listen=<listen> HTTP server listen address. [Default: localhost:7890]
-h --help Show this screen
Examples requesting stations:
# Get list of all stations for daily climate summary data in JSON format
wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent
# Get list of all stations in CSV format
wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --format=csv
# Get list of specific stations
wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411
# Get list of specific stations in GeoJSON format
wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson
Examples requesting readings:
# Get daily climate summary data for specific stations
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent
# Optionally save/restore to/from disk in order to avoid asking upstream servers each time
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent
# Limit output to specific date
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01
# Limit output to specified date range in ISO-8601 time interval format
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05
# The real power horse: Acquire data across historical+recent data sets
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=historical,recent --date=1969-01-01/2020-06-11
# Acquire monthly data for 2020-05
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=monthly --period=recent,historical --date=2020-05
# Acquire monthly data from 2017-01 to 2019-12
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=monthly --period=recent,historical --date=2017-01/2019-12
# Acquire annual data for 2019
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=annual --period=recent,historical --date=2019
# Acquire annual data from 2010 to 2020
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=annual --period=recent,historical --date=2010/2020
# Acquire hourly data
wetterdienst dwd readings --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12
Examples using geospatial features:
# Acquire stations and readings by geoposition, request specific number of nearby stations.
wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5
wetterdienst dwd readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 --date=2020-06-30
# Acquire stations and readings by geoposition, request stations within specific radius.
wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25
wetterdienst dwd readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30
Examples using SQL filtering:
# Find stations by state.
wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'"
# Find stations by name (LIKE query).
wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')"
# Find stations by name (regexp query).
wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))"
# Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees.
wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE element='temperature_air_max_200' AND value < 2.0;"
Examples for inquiring metadata:
# Display list of available parameters (air_temperature, precipitation, pressure, ...)
wetterdienst dwd about parameters
# Display list of available resolutions (10_minutes, hourly, daily, ...)
wetterdienst dwd about resolutions
# Display list of available periods (historical, recent, now)
wetterdienst dwd about periods
# Display coverage/correlation between parameters, resolutions and periods.
# This can answer questions like ...
wetterdienst dwd about coverage
# Tell me all periods and resolutions available for 'air_temperature'.
wetterdienst dwd about coverage --parameter=air_temperature
# Tell me all parameters available for 'daily' resolution.
wetterdienst dwd about coverage --resolution=daily
Examples for exporting data to databases:
# Shortcut command for fetching readings from DWD
alias fetch="wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent"
# Store readings to DuckDB
fetch --target="duckdb://database=dwd.duckdb&table=weather"
# Store readings to InfluxDB
fetch --target="influxdb://localhost/?database=dwd&table=weather"
# Store readings to CrateDB
fetch --target="crate://localhost/?database=dwd&table=weather"
Run as HTTP service:
wetterdienst dwd service
wetterdienst dwd service --listen=0.0.0.0:9999
"""
appname = f"{__appname__} {__version__}"
# Read command line options.
options = normalize_options(docopt(run.__doc__, version=appname))
# Setup logging.
debug = options.get("debug")
log_level = logging.INFO
if debug: # pragma: no cover
log_level = logging.DEBUG
setup_logging(log_level)
# Run service.
if options.service: # pragma: no cover
listen_address = options.listen
log.info(f"Starting {appname}")
log.info(f"Starting web service on {listen_address}")
from wetterdienst.service import start_service
start_service(listen_address)
return
# Output domain information.
if options.about:
about(options)
return
# Sanity checks.
if (options.readings or options.forecasts) and options.format == "geojson":
raise KeyError("GeoJSON format only available for stations output")
# Acquire station list, also used for readings if required.
# Filtering applied for distance (a.k.a. nearby) and pre-selected stations
df = get_stations(options)
if options.stations and df.empty:
log.error("No data available for given constraints")
sys.exit(1)
# Acquire observations.
if options.readings:
# Use list of station identifiers.
if options.station:
station_ids = read_list(options.station)
elif options.latitude and options.longitude:
try:
station_ids = df.STATION_ID.unique()
except AttributeError:
station_ids = df.WMO_ID.unique()
else:
raise KeyError(
"Either --station or --latitude, --longitude required")
# Funnel all parameters to the workhorse.
if options.observations:
readings = DWDObservationData(
station_ids=station_ids,
parameters=read_list(options.parameter),
resolution=options.resolution,
periods=read_list(options.period),
humanize_parameters=True,
tidy_data=options.tidy,
)
elif options.forecasts:
readings = DWDMosmixData(
station_ids=station_ids,
parameters=read_list(options.parameter),
mosmix_type=options.mosmix_type,
humanize_parameters=True,
tidy_data=options.tidy,
)
# Collect data and merge together.
try:
df = readings.all()
except ValueError as ex:
log.exception(ex)
sys.exit(1)
# Sanity checks.
if df.empty:
log.error("No data available")
sys.exit(1)
# Filter readings by datetime expression.
if options.readings and options.date:
resolution = None
if options.observations:
resolution = readings.resolution
df = df.dwd.filter_by_date(options.date, resolution)
# Make column names lowercase.
df = df.dwd.lower()
# Apply filtering by SQL.
if options.sql:
log.info(f"Filtering with SQL: {options.sql}")
df = df.io.sql(options.sql)
# Emit to data sink, e.g. write to database.
if options.target:
log.info(f"Writing data to target {options.target}")
df.io.export(options.target)
return
# Render to output format.
try:
output = df.dwd.format(options.format)
except KeyError as ex:
log.error(
f'{ex}. Output format must be one of "json", "geojson", "csv", "excel".'
)
sys.exit(1)
print(output)