def start_service(listen_address: Optional[str] = None, reload: Optional[bool] = False): # pragma: no cover from uvicorn.main import run setup_logging() if listen_address is None: listen_address = "127.0.0.1:7890" host, port = listen_address.split(":") port = int(port) run(app="wetterdienst.ui.restapi:app", host=host, port=port, reload=reload)
def start_service( listen_address: Optional[str] = None, reload: Optional[bool] = False ): # pragma: no cover """ This entrypoint will be used by `wetterdienst.cli`. """ setup_logging() if listen_address is None: listen_address = "127.0.0.1:7891" host, port = listen_address.split(":") port = int(port) app.server.run(host=host, port=port, debug=reload)
def main(): """Run example.""" setup_logging() mosmix_example()
def main(): setup_logging() mosmix_example()
def run(): """ Usage: wetterdienst dwd observation stations --parameter=<parameter> --resolution=<resolution> --period=<period> [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> [--station=<station>] [--period=<period>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd observation values --parameter=<parameter> --resolution=<resolution> --latitude=<latitude> --longitude=<longitude> [--period=<period>] [--rank=<rank>] [--distance=<distance>] [--tidy] [--date=<date>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd forecast stations [--parameter=<parameter>] [--mosmix-type=<mosmix-type>] [--date=<date>] [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--rank=<rank>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd forecast values --parameter=<parameter> [--mosmix-type=<mosmix-type>] --station=<station> [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd about [parameters] [resolutions] [periods] wetterdienst dwd about coverage [--parameter=<parameter>] [--resolution=<resolution>] [--period=<period>] wetterdienst dwd about fields --parameter=<parameter> --resolution=<resolution> --period=<period> [--language=<language>] wetterdienst radar stations [--odim-code=<odim-code>] [--wmo-code=<wmo-code>] [--country-name=<country-name>] wetterdienst dwd radar stations wetterdienst restapi [--listen=<listen>] [--reload] wetterdienst explorer [--listen=<listen>] [--reload] wetterdienst --version wetterdienst (-h | --help) Options: --parameter=<parameter> Parameter Set/Parameter, e.g. "kl" or "precipitation_height", etc. --resolution=<resolution> Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1" --period=<period> Dataset period: "historical", "recent", "now" --station=<station> Comma-separated list of station identifiers --latitude=<latitude> Latitude for filtering by geoposition. --longitude=<longitude> Longitude for filtering by geoposition. --rank=<rank> Rank of nearby stations when filtering by geoposition. --distance=<distance> Maximum distance in km when filtering by geoposition. --date=<date> Date for filtering data. Can be either a single date(time) or an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. --mosmix-type=<mosmix-type> type of mosmix, either 'small' or 'large' --sql=<sql> SQL query to apply to DataFrame. --format=<format> Output format. [Default: json] --target=<target> Output target for storing data into different data sinks. --language=<language> Output language. [Default: en] --version Show version information --debug Enable debug messages --listen=<listen> HTTP server listen address. --reload Run service and dynamically reload changed files -h --help Show this screen Examples requesting observation stations: # Get list of all stations for daily climate summary data in JSON format wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent # Get list of all stations in CSV format wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --format=csv # Get list of specific stations wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 # Get list of specific stations in GeoJSON format wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson Examples requesting observation values: # Get daily climate summary data for specific stations wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent # Get daily climate summary data for specific stations in CSV format wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent # Get daily climate summary data for specific stations in tidy format wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --tidy # Limit output to specific date wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01 # Limit output to specified date range in ISO-8601 time interval format wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05 # The real power horse: Acquire data across historical+recent data sets wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --date=1969-01-01/2020-06-11 # Acquire monthly data for 2020-05 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2020-05 # Acquire monthly data from 2017-01 to 2019-12 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=monthly --date=2017-01/2019-12 # Acquire annual data for 2019 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2019 # Acquire annual data from 2010 to 2020 wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=annual --date=2010/2020 # Acquire hourly data wetterdienst dwd observation values --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12 Examples requesting forecast stations: wetterdienst dwd forecast stations Examples requesting forecast values: wetterdienst dwd forecast values --parameter=ttt,ff --station=65510 Examples using geospatial features: # Acquire stations and readings by geoposition, request specific number of nearby stations. wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5 wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --rank=5 --date=2020-06-30 # Acquire stations and readings by geoposition, request stations within specific distance. wetterdienst dwd observation stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 wetterdienst dwd observation values --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30 Examples using SQL filtering: # Find stations by state. wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'" # Find stations by name (LIKE query). wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')" # Find stations by name (regexp query). wetterdienst dwd observation stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))" # Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees celsius. wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE temperature_air_max_200 < 2.0;" # Filter measurements: Same as above, but use tidy format. # FIXME: Currently, this does not work, see https://github.com/earthobservations/wetterdienst/issues/377. wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE parameter='temperature_air_max_200' AND value < 2.0;" --tidy Examples for inquiring metadata: # Display list of available parameters (air_temperature, precipitation, pressure, ...) wetterdienst dwd about parameters # Display list of available resolutions (10_minutes, hourly, daily, ...) wetterdienst dwd about resolutions # Display list of available periods (historical, recent, now) wetterdienst dwd about periods # Display coverage/correlation between parameters, resolutions and periods. # This can answer questions like ... wetterdienst dwd about coverage # Tell me all periods and resolutions available for 'air_temperature'. wetterdienst dwd about coverage --parameter=air_temperature # Tell me all parameters available for 'daily' resolution. wetterdienst dwd about coverage --resolution=daily Examples for exporting data to files: # Export list of stations into spreadsheet wetterdienst dwd observations stations --parameter=kl --resolution=daily --period=recent --target=file://stations.xlsx # Shortcut command for fetching readings alias fetch="wetterdienst dwd observations values --station=1048,4411 --parameter=kl --resolution=daily --period=recent" # Export readings into spreadsheet (Excel-compatible) fetch --target="file://observations.xlsx" # Export readings into Parquet format and display head of Parquet file fetch --target="file://observations.parquet" # Check Parquet file parquet-tools schema observations.parquet parquet-tools head observations.parquet # Export readings into Zarr format fetch --target="file://observations.zarr" Examples for exporting data to databases: # Shortcut command for fetching readings alias fetch="wetterdienst dwd observation values --station=1048,4411 --parameter=kl --resolution=daily --period=recent" # Store readings to DuckDB fetch --target="duckdb:///dwd.duckdb?table=weather" # Store readings to InfluxDB fetch --target="influxdb://localhost/?database=dwd&table=weather" # Store readings to CrateDB fetch --target="crate://localhost/?database=dwd&table=weather" Invoke the HTTP REST API service: # Start service on standard port, listening on http://localhost:7890. wetterdienst restapi # Start service on standard port and watch filesystem changes. # This is suitable for development. wetterdienst restapi --reload # Start service on public interface and specific port. wetterdienst restapi --listen=0.0.0.0:8890 Invoke the Wetterdienst Explorer UI service: # Start service on standard port, listening on http://localhost:7891. wetterdienst explorer # Start service on standard port and watch filesystem changes. # This is suitable for development. wetterdienst explorer --reload # Start service on public interface and specific port. wetterdienst explorer --listen=0.0.0.0:8891 """ appname = f"{__appname__} {__version__}" # Read command line options. options = normalize_options(docopt(run.__doc__, version=appname)) # Setup logging. debug = options.get("debug") log_level = logging.INFO if debug: # pragma: no cover log_level = logging.DEBUG setup_logging(log_level) # Run HTTP service. if options.restapi: # pragma: no cover listen_address = options.listen log.info(f"Starting {appname}") log.info(f"Starting HTTP web service on http://{listen_address}") from wetterdienst.ui.restapi import start_service start_service(listen_address, reload=options.reload) return # Run UI service. if options.explorer: # pragma: no cover listen_address = options.listen log.info(f"Starting {appname}") log.info(f"Starting UI web service on http://{listen_address}") from wetterdienst.ui.explorer.app import start_service start_service(listen_address, reload=options.reload) return # Handle radar data inquiry. Currently, "stations only". if options.radar: if options.dwd: data = DwdRadarSites().all() else: if options.odim_code: data = OperaRadarSites().by_odimcode(options.odim_code) elif options.wmo_code: data = OperaRadarSites().by_wmocode(options.wmo_code) elif options.country_name: data = OperaRadarSites().by_countryname(options.country_name) else: data = OperaRadarSites().all() output = json.dumps(data, indent=4) print(output) return # Output domain information. if options.about: about(options) return # Sanity checks. if (options["values"] or options.forecast) and options.format == "geojson": raise KeyError("GeoJSON format only available for stations output") # Acquire station list, also used for readings if required. # Filtering applied for distance (a.k.a. nearby) and pre-selected stations stations = None if options.observation: stations = DwdObservationRequest( parameter=read_list(options.parameter), resolution=options.resolution, period=options.period, tidy=options.tidy, si_units=False, ) elif options.forecast: stations = DwdMosmixRequest( parameter=read_list(options.parameter), mosmix_type=DwdMosmixType.LARGE, tidy=options.tidy, si_units=False, ) if options.latitude and options.longitude: if options.rank: stations = stations.filter_by_rank( latitude=float(options.latitude), longitude=float(options.longitude), rank=int(options.rank), ) elif options.distance: stations = stations.filter_by_distance( latitude=float(options.latitude), longitude=float(options.longitude), distance=int(options.distance), ) else: raise DocoptExit( "Geospatial queries need either --rank or --distance") results = stations elif options.station: results = stations.filter_by_station_id(read_list(options.station)) else: results = stations.all() df = pd.DataFrame() if options.stations: pass elif options["values"]: try: # TODO: Add stream-based processing here. results = results.values.all() except ValueError as ex: log.exception(ex) sys.exit(1) df = results.df if df.empty: log.error("No data available for given constraints") sys.exit(1) # Filter readings by datetime expression. if options["values"] and options.date: results.filter_by_date(options.date) # Apply filtering by SQL. if options.sql: if options.tidy: log.error("Combining SQL filtering with tidy format not possible") sys.exit(1) log.info(f"Filtering with SQL: {options.sql}") results.filter_by_sql(options.sql) # Emit to data sink, e.g. write to database. if options.target: results.to_target(options.target) return # Render to output format. try: if options.format == "json": output = results.to_json() elif options.format == "csv": output = results.to_csv() elif options.format == "geojson": output = results.to_geojson() else: raise KeyError("Unknown output format") except KeyError as ex: log.error( f'{ex}. Output format must be one of "json", "geojson", "csv".') sys.exit(1) print(output)
def run(): """ Usage: wetterdienst dwd observations stations --parameter=<parameter> --resolution=<resolution> --period=<period> [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--number=<number>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] wetterdienst dwd observations readings --parameter=<parameter> --resolution=<resolution> --station=<station> [--period=<period>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd observations readings --parameter=<parameter> --resolution=<resolution> --latitude=<latitude> --longitude=<longitude> [--period=<period>] [--number=<number>] [--distance=<distance>] [--tidy] [--date=<date>] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd forecasts stations [--date=<date>] [--station=<station>] [--latitude=<latitude>] [--longitude=<longitude>] [--number=<number>] [--distance=<distance>] [--sql=<sql>] [--format=<format>] wetterdienst dwd forecasts readings --mosmix-type=<mosmix-type> --station=<station> [--parameter=<parameter>] [--date=<date>] [--tidy] [--sql=<sql>] [--format=<format>] [--target=<target>] wetterdienst dwd about [parameters] [resolutions] [periods] wetterdienst dwd about coverage [--parameter=<parameter>] [--resolution=<resolution>] [--period=<period>] wetterdienst dwd about fields --parameter=<parameter> --resolution=<resolution> --period=<period> [--language=<language>] wetterdienst service [--listen=<listen>] wetterdienst --version wetterdienst (-h | --help) Options: --parameter=<parameter> Parameter Set/Parameter, e.g. "kl" or "precipitation_height", etc. --resolution=<resolution> Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1" --period=<period> Dataset period: "historical", "recent", "now" --station=<station> Comma-separated list of station identifiers --latitude=<latitude> Latitude for filtering by geoposition. --longitude=<longitude> Longitude for filtering by geoposition. --number=<number> Number of nearby stations when filtering by geoposition. --distance=<distance> Maximum distance in km when filtering by geoposition. --date=<date> Date for filtering data. Can be either a single date(time) or an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. --mosmix-type=<mosmix-type> type of mosmix, either 'small' or 'large' --sql=<sql> SQL query to apply to DataFrame. --format=<format> Output format. [Default: json] --target=<target> Output target for storing data into different data sinks. --language=<language> Output language. [Default: en] --version Show version information --debug Enable debug messages --listen=<listen> HTTP server listen address. [Default: localhost:7890] -h --help Show this screen Examples requesting stations: # Get list of all stations for daily climate summary data in JSON format wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent # Get list of all stations in CSV format wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --format=csv # Get list of specific stations wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 # Get list of specific stations in GeoJSON format wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson Examples requesting readings: # Get daily climate summary data for specific stations wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent # Optionally save/restore to/from disk in order to avoid asking upstream servers each time wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent # Limit output to specific date wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01 # Limit output to specified date range in ISO-8601 time interval format wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05 # The real power horse: Acquire data across historical+recent data sets wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=historical,recent --date=1969-01-01/2020-06-11 # Acquire monthly data for 2020-05 wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=monthly --period=recent,historical --date=2020-05 # Acquire monthly data from 2017-01 to 2019-12 wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=monthly --period=recent,historical --date=2017-01/2019-12 # Acquire annual data for 2019 wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=annual --period=recent,historical --date=2019 # Acquire annual data from 2010 to 2020 wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=annual --period=recent,historical --date=2010/2020 # Acquire hourly data wetterdienst dwd readings --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12 Examples using geospatial features: # Acquire stations and readings by geoposition, request specific number of nearby stations. wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 wetterdienst dwd readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 --date=2020-06-30 # Acquire stations and readings by geoposition, request stations within specific radius. wetterdienst dwd stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 wetterdienst dwd readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30 Examples using SQL filtering: # Find stations by state. wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'" # Find stations by name (LIKE query). wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')" # Find stations by name (regexp query). wetterdienst dwd stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))" # Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees. wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE element='temperature_air_max_200' AND value < 2.0;" Examples for inquiring metadata: # Display list of available parameters (air_temperature, precipitation, pressure, ...) wetterdienst dwd about parameters # Display list of available resolutions (10_minutes, hourly, daily, ...) wetterdienst dwd about resolutions # Display list of available periods (historical, recent, now) wetterdienst dwd about periods # Display coverage/correlation between parameters, resolutions and periods. # This can answer questions like ... wetterdienst dwd about coverage # Tell me all periods and resolutions available for 'air_temperature'. wetterdienst dwd about coverage --parameter=air_temperature # Tell me all parameters available for 'daily' resolution. wetterdienst dwd about coverage --resolution=daily Examples for exporting data to databases: # Shortcut command for fetching readings from DWD alias fetch="wetterdienst dwd readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent" # Store readings to DuckDB fetch --target="duckdb://database=dwd.duckdb&table=weather" # Store readings to InfluxDB fetch --target="influxdb://localhost/?database=dwd&table=weather" # Store readings to CrateDB fetch --target="crate://localhost/?database=dwd&table=weather" Run as HTTP service: wetterdienst dwd service wetterdienst dwd service --listen=0.0.0.0:9999 """ appname = f"{__appname__} {__version__}" # Read command line options. options = normalize_options(docopt(run.__doc__, version=appname)) # Setup logging. debug = options.get("debug") log_level = logging.INFO if debug: # pragma: no cover log_level = logging.DEBUG setup_logging(log_level) # Run service. if options.service: # pragma: no cover listen_address = options.listen log.info(f"Starting {appname}") log.info(f"Starting web service on {listen_address}") from wetterdienst.service import start_service start_service(listen_address) return # Output domain information. if options.about: about(options) return # Sanity checks. if (options.readings or options.forecasts) and options.format == "geojson": raise KeyError("GeoJSON format only available for stations output") # Acquire station list, also used for readings if required. # Filtering applied for distance (a.k.a. nearby) and pre-selected stations df = get_stations(options) if options.stations and df.empty: log.error("No data available for given constraints") sys.exit(1) # Acquire observations. if options.readings: # Use list of station identifiers. if options.station: station_ids = read_list(options.station) elif options.latitude and options.longitude: try: station_ids = df.STATION_ID.unique() except AttributeError: station_ids = df.WMO_ID.unique() else: raise KeyError( "Either --station or --latitude, --longitude required") # Funnel all parameters to the workhorse. if options.observations: readings = DWDObservationData( station_ids=station_ids, parameters=read_list(options.parameter), resolution=options.resolution, periods=read_list(options.period), humanize_parameters=True, tidy_data=options.tidy, ) elif options.forecasts: readings = DWDMosmixData( station_ids=station_ids, parameters=read_list(options.parameter), mosmix_type=options.mosmix_type, humanize_parameters=True, tidy_data=options.tidy, ) # Collect data and merge together. try: df = readings.all() except ValueError as ex: log.exception(ex) sys.exit(1) # Sanity checks. if df.empty: log.error("No data available") sys.exit(1) # Filter readings by datetime expression. if options.readings and options.date: resolution = None if options.observations: resolution = readings.resolution df = df.dwd.filter_by_date(options.date, resolution) # Make column names lowercase. df = df.dwd.lower() # Apply filtering by SQL. if options.sql: log.info(f"Filtering with SQL: {options.sql}") df = df.io.sql(options.sql) # Emit to data sink, e.g. write to database. if options.target: log.info(f"Writing data to target {options.target}") df.io.export(options.target) return # Render to output format. try: output = df.dwd.format(options.format) except KeyError as ex: log.error( f'{ex}. Output format must be one of "json", "geojson", "csv", "excel".' ) sys.exit(1) print(output)