"""
from meteostat import Normals
from jasper import Jasper
from jasper.helpers import get_stations
# General configuration
Normals.max_age = 0
STATIONS_PER_CYCLE = 10
# Create Jasper instance
jsp = Jasper("task.inventory.normals")
stations = get_stations(
jsp,
"SELECT `id` FROM `stations`",
STATIONS_PER_CYCLE,
)
if len(stations) > 0:
for station in stations:
try:
# Get daily data from Meteostat
data = Normals(station[0], "all").fetch()
# Get start & end dates of time series
start = data.index.get_level_values("start").min()
end = data.index.get_level_values("end").max()
start = f"{start}-01-01"
end = f"{end}-12-31"
# Return DataFrame
return df
except BaseException:
return pd.DataFrame()
# Get some stations
stations = get_stations(
jsp,
"""
SELECT
`id`,
`national_id`,
`altitude`
FROM
`stations`
WHERE
`country` = 'CA' AND
`national_id` IS NOT NULL
""",
STATIONS_PER_CYCLE,
)
# List of datasets
datasets = []
# Go through all stations
if len(stations) > 0:
for station in stations:
# Go through all years
import pandas as pd
from jasper import Jasper
from jasper.actions import persist
from jasper.helpers import get_stations, read_file
from jasper.schema import hourly_national
# General configuration
STATIONS_PER_CYCLE = 36
# Create Jasper instance
jsp = Jasper("import.noaa.hourly.national_metar")
# Get weather stations
stations = get_stations(
jsp, read_file("national_metar_stations.sql"), STATIONS_PER_CYCLE
)
# DataFrame which holds all data
df_full = None
# Import data for each weather station
if len(stations) > 0:
for station in stations:
try:
# Create request for JSON file
url = f"https://api.weather.gov/stations/{station['icao']}/observations/latest"
req = request.Request(
url, headers={"User-Agent": "meteostat.net [email protected]"}
)
# General configuration
STATIONS_PER_CYCLE = 10
# Create Jasper instance
jsp = Jasper("export.bulk.daily.legacy")
# Get weather station(s)
stations = get_stations(
jsp,
"""
SELECT
`stations`.`id` AS `id`,
`stations`.`tz` AS `tz`
FROM `stations`
WHERE
`stations`.`id` IN (
SELECT DISTINCT `station`
FROM `inventory`
WHERE
`mode` IN ('D', 'H')
)
""",
STATIONS_PER_CYCLE,
)
# Export data for each weather station
for station in stations:
result = jsp.query(
"""
SET STATEMENT
max_statement_time=60
from jasper import Jasper
from jasper.helpers import read_file, get_stations
from jasper.actions import export_csv
# General configuration
STATIONS_PER_CYCLE = 11
# Create Jasper instance
jsp = Jasper("export.bulk.monthly")
# Get weather station(s)
stations = get_stations(
jsp,
read_file("monthly_stations.sql"),
STATIONS_PER_CYCLE,
)
# Export data for each weather station
for station in stations:
result = jsp.query(
read_file("monthly.sql"), {"station": station[0], "timezone": station[1]}
)
if result.rowcount > 0:
# Fetch data
data = result.fetchall()
# Export data dump
export_csv(
The code is licensed under the MIT license.
"""
from jasper import Jasper
from jasper.helpers import read_file, get_stations
from jasper.actions import export_csv
# General configuration
STATIONS_PER_CYCLE = 10
# Create Jasper instance
jsp = Jasper("export.bulk.daily")
# Get weather station(s)
stations = get_stations(jsp, read_file("daily_stations.sql"),
STATIONS_PER_CYCLE)
# Export data for each weather station
for station in stations:
result = jsp.query(read_file("daily.sql"), {
"station": station[0],
"timezone": station[1]
})
if result.rowcount > 0:
# Fetch data
data = result.fetchall()
# Export data dump
export_csv(jsp, list(map(lambda d: d[:11], data)),
f"/daily/{station[0]}.csv.gz")
[
"".join(sorted(list(set(flag)))) if flag is not None else None
for flag in d[13:]
]
),
data,
)
),
f"{path}/{station}.map.csv.gz",
)
# Get weather stations
stations = get_stations(
jsp,
read_file(f"hourly_stations_{MODE}.sql"),
STATIONS_PER_CYCLE,
)
# Start & end year
now = datetime.now()
start_year = now.year - 1 if MODE in ("recent", "live") else 1890
end_year = now.year + 1
# Export data for each weather station
for station in stations:
result = jsp.query(
read_file("hourly.sql"),
{
"station": station[0],
"start_datetime": f"{start_year}-01-01 00:00:00",