def dwd_stations( kind: str, parameter: str = Query(default=None), resolution: str = Query(default=None), period: str = Query(default=None), mosmix_type: str = Query(default=None), longitude: float = Query(default=None), latitude: float = Query(default=None), rank: int = Query(default=None), distance: int = Query(default=None), sql: str = Query(default=None), ): if kind not in ["observation", "forecast"]: return HTTPException(status_code=404, detail=f"product {kind} not found") # Data acquisition. if kind == "observation": if parameter is None or resolution is None or period is None: raise HTTPException( status_code=400, detail="Query arguments 'parameter', 'resolution' " "and 'period' are required", ) stations = DwdObservationRequest( parameter=parameter, resolution=resolution, period=period, si_units=False ) else: stations = DwdMosmixRequest( parameter=parameter, mosmix_type=mosmix_type, si_units=False ) if longitude and latitude and (rank or distance): if rank: results = stations.filter_by_rank( latitude=latitude, longitude=longitude, rank=rank ) else: results = stations.filter_by_distance( latitude=latitude, longitude=longitude, distance=distance, unit="km" ) else: results = stations.all() # Postprocessing. if sql is not None: results.filter_by_sql(sql) results.fill_gaps() return make_json_response(results.to_dict())
def test_dwd_observation_weather_phenomena(): """Test for DWD weather phenomena data, thanks saschnet (https://github.com/saschnet) for providing the sample, see also https://github.com/earthobservations/wetterdienst/issues/647 """ Settings.tidy = True Settings.humanize = False Settings.si_units = False request = DwdObservationRequest( resolution=DwdObservationResolution.HOURLY, parameter=[DwdObservationParameter.HOURLY.WEATHER_PHENOMENA.WEATHER], start_date=datetime(year=2022, month=3, day=1, tzinfo=timezone.utc), end_date=datetime(year=2022, month=3, day=31, tzinfo=timezone.utc), ) res = request.all().df.dropna() assert len(res) > 0
def run(): """ Usage: wetterdienst dwd observation stations --parameter=<parameter> --resolution=<resolution> --period=<period> [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> [--station=<station>] [--period=<period>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> --latitude=<latitude> --longitude=<longitude> [--period=<period>] [--rank=<rank>] [--distance=<distance>] [--tidy] [--date=<date>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd forecast stations [--parameter=<parameter>] [--mosmix-type=<mosmix-type>] [--date=<date>] [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd forecast values --parameter=<parameter> [--mosmix-type=<mosmix-type>] --station=<station> [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd about [parameters] [resolutions] [periods] wetterdienst dwd about coverage [--parameter=<parameter>] [--resolution=<resolution>] [--period=<period>] wetterdienst dwd about fields --parameter=<parameter> --resolution=<resolution> --period=<period> [--language=<language>] wetterdienst radar stations [--odim-code=<odim-code>] [--wmo-code=<wmo-code>] [--country-name=<country-name>] wetterdienst dwd radar stations wetterdienst restapi [--listen=<listen>] [--reload] wetterdienst explorer [--listen=<listen>] [--reload] wetterdienst --version wetterdienst (-h | --help) Options: --parameter=<parameter> Parameter Set/Parameter, e.g. "kl" or "precipitation_height", etc. --resolution=<resolution> Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1" --period=<period> Dataset period: "historical", "recent", "now" --station=<station> Comma-separated list of station identifiers --latitude=<latitude> Latitude for filtering by geoposition. --longitude=<longitude> Longitude for filtering by geoposition. --rank=<rank> Rank of nearby stations when filtering by geoposition. --distance=<distance> Maximum distance in km when filtering by geoposition. --date=<date> Date for filtering data. Can be either a single date(time) or an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. --mosmix-type=<mosmix-type> type of mosmix, either 'small' or 'large' --sql=<sql> SQL query to apply to DataFrame. --format=<format> Output format. [Default: json] --target=<target> Output target for storing data into different data sinks. --language=<language> Output language. [Default: en] --version Show version information --debug Enable debug messages --listen=<listen> HTTP server listen address. --reload Run service and dynamically reload changed files -h --help Show this screen Examples requesting observation stations: # Get list of all stations for daily climate summary data in JSON format wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent # Get list of all stations in CSV format wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --format=csv # Get list of specific stations wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 # Get list of specific stations in GeoJSON format wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson Examples requesting observation values: # Get daily climate summary data for specific stations wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent # Get daily climate summary data for specific stations in CSV format wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent # Get daily climate summary data for specific stations in tidy format wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --tidy # Limit output to specific date wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01 # Limit output to specified date range in ISO-8601 time interval format wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05 # The real power horse: Acquire data across historical+recent data sets wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --date=1969-01-01/2020-06-11 # Acquire monthly data for 2020-05 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2020-05 # Acquire monthly data from 2017-01 to 2019-12 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2017-01/2019-12 # Acquire annual data for 2019 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2019 # Acquire annual data from 2010 to 2020 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2010/2020 # Acquire hourly data wetterdienst dwd observation values --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12 Examples requesting forecast stations: wetterdienst dwd forecast stations Examples requesting forecast values: wetterdienst dwd forecast values --parameter=ttt,ff --station=65510 Examples using geospatial features: # Acquire stations and readings by geoposition, request specific number of nearby stations. wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5 wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5 --date=2020-06-30 # Acquire stations and readings by geoposition, request stations within specific distance. wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30 Examples using SQL filtering: # Find stations by state. wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'" # Find stations by name (LIKE query). wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')" # Find stations by name (regexp query). wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))" # Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees celsius. wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE temperature_air_max_200 < 2.0;" # Filter measurements: Same as above, but use tidy format. # FIXME: Currently, this does not work, see https://github.com/earthobservations/wetterdienst/issues/377. wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE parameter='temperature_air_max_200' AND value < 2.0;" --tidy Examples for inquiring metadata: # Display list of available parameters (air_temperature, precipitation, pressure, ...) wetterdienst dwd about parameters # Display list of available resolutions (10_minutes, hourly, daily, ...) wetterdienst dwd about resolutions # Display list of available periods (historical, recent, now) wetterdienst dwd about periods # Display coverage/correlation between parameters, resolutions and periods. # This can answer questions like ... wetterdienst dwd about coverage # Tell me all periods and resolutions available for 'air_temperature'. wetterdienst dwd about coverage --parameter=air_temperature # Tell me all parameters available for 'daily' resolution. wetterdienst dwd about coverage --resolution=daily Examples for exporting data to files: # Export list of stations into spreadsheet wetterdienst dwd observations stations --parameter=kl --resolution=daily --period=recent --target=file://stations.xlsx # Shortcut command for fetching readings alias fetch="wetterdienst dwd observations values --station=1048,4411 --parameter=kl --resolution=daily --period=recent" # Export readings into spreadsheet (Excel-compatible) fetch --target="file://observations.xlsx" # Export readings into Parquet format and display head of Parquet file fetch --target="file://observations.parquet" # Check Parquet file parquet-tools schema observations.parquet parquet-tools head observations.parquet # Export readings into Zarr format fetch --target="file://observations.zarr" Examples for exporting data to databases: # Shortcut command for fetching readings alias fetch="wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent" # Store readings to DuckDB fetch --target="duckdb:///dwd.duckdb?table=weather" # Store readings to InfluxDB fetch --target="influxdb://localhost/?database=dwd&table=weather" # Store readings to CrateDB fetch --target="crate://localhost/?database=dwd&table=weather" Invoke the HTTP REST API service: # Start service on standard port, listening on http://localhost:7890. wetterdienst restapi # Start service on standard port and watch filesystem changes. # This is suitable for development. wetterdienst restapi --reload # Start service on public interface and specific port. wetterdienst restapi --listen=0.0.0.0:8890 Invoke the Wetterdienst Explorer UI service: # Start service on standard port, listening on http://localhost:7891. wetterdienst explorer # Start service on standard port and watch filesystem changes. # This is suitable for development. wetterdienst explorer --reload # Start service on public interface and specific port. wetterdienst explorer --listen=0.0.0.0:8891 """ appname = f"{__appname__} {__version__}" # Read command line options. options = normalize_options(docopt(run.__doc__, version=appname)) # Setup logging. debug = options.get("debug") log_level = logging.INFO if debug: # pragma: no cover log_level = logging.DEBUG setup_logging(log_level) # Run HTTP service. if options.restapi: # pragma: no cover listen_address = options.listen log.info(f"Starting {appname}") log.info(f"Starting HTTP web service on http://{listen_address}") from wetterdienst.ui.restapi import start_service start_service(listen_address, reload=options.reload) return # Run UI service. if options.explorer: # pragma: no cover listen_address = options.listen log.info(f"Starting {appname}") log.info(f"Starting UI web service on http://{listen_address}") from wetterdienst.ui.explorer.app import start_service start_service(listen_address, reload=options.reload) return # Handle radar data inquiry. Currently, "stations only". if options.radar: if options.dwd: data = DwdRadarSites().all() else: if options.odim_code: data = OperaRadarSites().by_odimcode(options.odim_code) elif options.wmo_code: data = OperaRadarSites().by_wmocode(options.wmo_code) elif options.country_name: data = OperaRadarSites().by_countryname(options.country_name) else: data = OperaRadarSites().all() output = json.dumps(data, indent=4) print(output) return # Output domain information. if options.about: about(options) return # Sanity checks. if (options["values"] or options.forecast) and options.format == "geojson": raise KeyError("GeoJSON format only available for stations output") # Acquire station list, also used for readings if required. # Filtering applied for distance (a.k.a. nearby) and pre-selected stations stations = None if options.observation: stations = DwdObservationRequest( parameter=read_list(options.parameter), resolution=options.resolution, period=options.period, tidy=options.tidy, si_units=False, ) elif options.forecast: stations = DwdMosmixRequest( parameter=read_list(options.parameter), mosmix_type=DwdMosmixType.LARGE, tidy=options.tidy, si_units=False, ) if options.latitude and options.longitude: if options.rank: stations = stations.filter_by_rank( latitude=float(options.latitude), longitude=float(options.longitude), rank=int(options.rank), ) elif options.distance: stations = stations.filter_by_distance( latitude=float(options.latitude), longitude=float(options.longitude), distance=int(options.distance), ) else: raise DocoptExit( "Geospatial queries need either --rank or --distance") results = stations elif options.station: results = stations.filter_by_station_id(read_list(options.station)) else: results = stations.all() df = pd.DataFrame() if options.stations: pass elif options["values"]: try: # TODO: Add stream-based processing here. results = results.values.all() except ValueError as ex: log.exception(ex) sys.exit(1) df = results.df if df.empty: log.error("No data available for given constraints") sys.exit(1) # Filter readings by datetime expression. if options["values"] and options.date: results.filter_by_date(options.date) # Apply filtering by SQL. if options.sql: if options.tidy: log.error("Combining SQL filtering with tidy format not possible") sys.exit(1) log.info(f"Filtering with SQL: {options.sql}") results.filter_by_sql(options.sql) # Emit to data sink, e.g. write to database. if options.target: results.to_target(options.target) return # Render to output format. try: if options.format == "json": output = results.to_json() elif options.format == "csv": output = results.to_csv() elif options.format == "geojson": output = results.to_geojson() else: raise KeyError("Unknown output format") except KeyError as ex: log.error( f'{ex}. Output format must be one of "json", "geojson", "csv".') sys.exit(1) print(output)