Ejemplo n.º 1
0
"""

from meteostat import Normals
from jasper import Jasper
from jasper.helpers import get_stations

# General configuration
Normals.max_age = 0
STATIONS_PER_CYCLE = 10

# Create Jasper instance
jsp = Jasper("task.inventory.normals")

stations = get_stations(
    jsp,
    "SELECT `id` FROM `stations`",
    STATIONS_PER_CYCLE,
)

if len(stations) > 0:
    for station in stations:
        try:
            # Get daily data from Meteostat
            data = Normals(station[0], "all").fetch()

            # Get start & end dates of time series
            start = data.index.get_level_values("start").min()
            end = data.index.get_level_values("end").max()
            start = f"{start}-01-01"
            end = f"{end}-12-31"
Ejemplo n.º 2
0
        # Return DataFrame
        return df

    except BaseException:
        return pd.DataFrame()


# Get some stations
stations = get_stations(
    jsp,
    """
    SELECT
        `id`,
        `national_id`,
        `altitude`
    FROM
        `stations`
    WHERE
        `country` = 'CA' AND
        `national_id` IS NOT NULL
    """,
    STATIONS_PER_CYCLE,
)

# List of datasets
datasets = []

# Go through all stations
if len(stations) > 0:
    for station in stations:
        # Go through all years
Ejemplo n.º 3
0
import pandas as pd
from jasper import Jasper
from jasper.actions import persist
from jasper.helpers import get_stations, read_file
from jasper.schema import hourly_national


# General configuration
STATIONS_PER_CYCLE = 36

# Create Jasper instance
jsp = Jasper("import.noaa.hourly.national_metar")

# Get weather stations
stations = get_stations(
    jsp, read_file("national_metar_stations.sql"), STATIONS_PER_CYCLE
)

# DataFrame which holds all data
df_full = None

# Import data for each weather station
if len(stations) > 0:
    for station in stations:
        try:
            # Create request for JSON file
            url = f"https://api.weather.gov/stations/{station['icao']}/observations/latest"
            req = request.Request(
                url, headers={"User-Agent": "meteostat.net [email protected]"}
            )
Ejemplo n.º 4
0
# General configuration
STATIONS_PER_CYCLE = 10

# Create Jasper instance
jsp = Jasper("export.bulk.daily.legacy")

# Get weather station(s)
stations = get_stations(
    jsp,
    """
		SELECT
				`stations`.`id` AS `id`,
				`stations`.`tz` AS `tz`
		FROM `stations`
		WHERE
				`stations`.`id` IN (
						SELECT DISTINCT `station`
						FROM `inventory`
						WHERE
								`mode` IN ('D', 'H')
				)
""",
    STATIONS_PER_CYCLE,
)

# Export data for each weather station
for station in stations:
    result = jsp.query(
        """
			SET STATEMENT
				max_statement_time=60
Ejemplo n.º 5
0
from jasper import Jasper
from jasper.helpers import read_file, get_stations
from jasper.actions import export_csv


# General configuration
STATIONS_PER_CYCLE = 11

# Create Jasper instance
jsp = Jasper("export.bulk.monthly")

# Get weather station(s)
stations = get_stations(
    jsp,
    read_file("monthly_stations.sql"),
    STATIONS_PER_CYCLE,
)

# Export data for each weather station
for station in stations:
    result = jsp.query(
        read_file("monthly.sql"), {"station": station[0], "timezone": station[1]}
    )

    if result.rowcount > 0:
        # Fetch data
        data = result.fetchall()

        # Export data dump
        export_csv(
Ejemplo n.º 6
0
The code is licensed under the MIT license.
"""

from jasper import Jasper
from jasper.helpers import read_file, get_stations
from jasper.actions import export_csv

# General configuration
STATIONS_PER_CYCLE = 10

# Create Jasper instance
jsp = Jasper("export.bulk.daily")

# Get weather station(s)
stations = get_stations(jsp, read_file("daily_stations.sql"),
                        STATIONS_PER_CYCLE)

# Export data for each weather station
for station in stations:
    result = jsp.query(read_file("daily.sql"), {
        "station": station[0],
        "timezone": station[1]
    })

    if result.rowcount > 0:
        # Fetch data
        data = result.fetchall()

        # Export data dump
        export_csv(jsp, list(map(lambda d: d[:11], data)),
                   f"/daily/{station[0]}.csv.gz")
Ejemplo n.º 7
0
                    [
                        "".join(sorted(list(set(flag)))) if flag is not None else None
                        for flag in d[13:]
                    ]
                ),
                data,
            )
        ),
        f"{path}/{station}.map.csv.gz",
    )


# Get weather stations
stations = get_stations(
    jsp,
    read_file(f"hourly_stations_{MODE}.sql"),
    STATIONS_PER_CYCLE,
)

# Start & end year
now = datetime.now()
start_year = now.year - 1 if MODE in ("recent", "live") else 1890
end_year = now.year + 1

# Export data for each weather station
for station in stations:
    result = jsp.query(
        read_file("hourly.sql"),
        {
            "station": station[0],
            "start_datetime": f"{start_year}-01-01 00:00:00",