def write_dump(data: list, station: str, year: int = None) -> None:
"""
Convert DataFrame to CSV and export to bulk server
"""
# The file path
path = "/hourly"
# Filter rows by year if set
if year is not None:
path += f"/{year}"
data = list(filter(lambda row: int(row[0].year) == year, data))
# Export data dump
export_csv(jsp, list(map(lambda d: d[:13], data)), f"{path}/{station}.csv.gz")
# Export source map
# pylint: disable=consider-using-generator
export_csv(
jsp,
list(
map(
lambda d: d[:2]
+ tuple(
[
"".join(sorted(list(set(flag)))) if flag is not None else None
for flag in d[13:]
]
),
data,
)
),
f"{path}/{station}.map.csv.gz",
)
def write_dump(data: list, station: str, year: int = None) -> None:
"""
Convert DataFrame to CSV and export to bulk server
"""
# The file path
path = "/hourly/obs"
# Filter rows by year if set
if year is not None:
path += f"/{year}"
data = list(filter(lambda row: int(row[0].year) == year, data))
# Export data dump
export_csv(jsp, list(map(lambda d: d[:13], data)),
f"{path}/{station}.csv.gz")
) AS `daily_derived`
WHERE
(
`tavg` IS NOT NULL OR
`tmin` IS NOT NULL OR
`tmax` IS NOT NULL OR
`prcp` IS NOT NULL
) AND
`date` <= DATE_ADD(CURRENT_DATE(), INTERVAL 10 DAY)
GROUP BY
`date`
ORDER BY
`date`
""",
{
"station": station[0],
"timezone": station[1]
},
)
if result.rowcount > 0:
# Fetch data
data = result.fetchall()
# Export data dump
export_csv(jsp, list(map(lambda d: d[:11], data)),
f"/daily/obs/{station[0]}.csv.gz")
# Close Jasper instance
jsp.close()
STATIONS_PER_CYCLE,
)
# Export data for each weather station
for station in stations:
result = jsp.query(
read_file("monthly.sql"), {"station": station[0], "timezone": station[1]}
)
if result.rowcount > 0:
# Fetch data
data = result.fetchall()
# Export data dump
export_csv(
jsp, list(map(lambda d: d[:9], data)), f"/monthly/{station[0]}.csv.gz"
)
# Export source map
# pylint: disable=consider-using-generator
export_csv(
jsp,
list(
map(
lambda d: d[:2]
+ tuple(
[
"".join(sorted(list(set(flag))))
if flag is not None
else None
for flag in d[9:]
}
# Add to full dump
full.append(station_object)
# Check if any data is available
if (
record[14] is not None
or record[16] is not None
or record[18] is not None
or record[20] is not None
or record[22] is not None
):
lite.append(station_object)
# Convert to list
record = record.values()
# Add slim rows
slim_cols = [0, 1, 3, 4, 6, 7, 8, 9, 10, 11, 15, 16, 17, 18, 19, 20]
slim.append([record[i] for i in slim_cols])
# Write JSON dumps
export_json(jsp, full, "/stations/full.json.gz")
export_json(jsp, lite, "/stations/lite.json.gz")
# Write CSV dumps
export_csv(jsp, slim, "/stations/slim.csv.gz")
export_csv(jsp, [row[0:13] for row in slim], "/stations/lib.csv.gz")
# Close Jasper instance
jsp.close()
GROUP BY
EXTRACT(YEAR_MONTH FROM `date`)
)
) AS `monthly_derived`
WHERE
`tavg` IS NOT NULL
OR `tmin` IS NOT NULL
OR `tmax` IS NOT NULL
OR `prcp` IS NOT NULL
GROUP BY
`year`, `month`
ORDER BY
`year`, `month`
""",
{
"station": station[0],
"timezone": station[1]
},
)
if result.rowcount > 0:
# Fetch data
data = result.fetchall()
# Export data dump
export_csv(jsp, list(map(lambda d: d[:11], data)),
f"/monthly/full/{station[0]}.csv.gz")
# Close Jasper instance
jsp.close()