# Multi-thread processing
if len(datasets) > 1:
# Create process pool
with ThreadPool(THREADS) as pool:
# Process datasets in pool
output = pool.starmap(load, datasets)
# Wait for Pool to finish
pool.close()
pool.join()
# DataFrame structure
base = pd.DataFrame(columns=list(PARAMETERS.values()))
# Full DataFrame
full = pd.concat([base, *output])
# Set index
full.set_index(["station", "time"], inplace=True)
# Drop NaN-only rows
full = full.dropna(how="all")
if full.index.size > 0:
# Write into database
persist(jsp, full, hourly_national)
# Close Jasper instance
jsp.close()
"rhum": data["relativeHumidity"]["value"],
"wdir": data["windDirection"]["value"],
"wspd": data["windSpeed"]["value"],
"pres": data["seaLevelPressure"]["value"] / 100
if data["seaLevelPressure"]["value"]
else None,
}
# Create DataFrame
df = pd.DataFrame([record])
# Set index
df["station"] = station[0]
df["time"] = pd.to_datetime(df["time"])
df = df.set_index(["station", "time"])
# Append data to full DataFrame
if df_full is None:
df_full = df
else:
df_full = df_full.append(df)
except error.HTTPError:
pass
# Write DataFrame into Meteostat database
persist(jsp, df_full.round(1), hourly_national)
# Close Jasper instance
jsp.close()
# Calculate humidity data
# pylint: disable=unnecessary-lambda
df["rhum"] = df.apply(lambda row: temp_dwpt_to_rhum(row),
axis=1)
# Drop dew point column
# pylint: disable=no-member
df = df.drop("dwpt", axis=1)
# Add station column
df["station"] = station["id"]
# Set index
df = df.set_index(["station", "time"])
# Round decimals
df = df.round(1)
# Write data into Meteostat database
persist(jsp, df, hourly_global)
except BaseException:
pass
# Quit FTP connection
ftp.quit()
# Close Jasper instance
jsp.close()
obs.press.value("HPA") if obs.press is not None else None,
"coco":
get_condicode(obs.weather)
if obs.weather is not None else None,
})
except BaseException:
pass
# List -> DataFrame
df = pd.DataFrame.from_records(data)
# Calculate humidity data
# pylint: disable=unnecessary-lambda
df["rhum"] = df.apply(lambda row: temp_dwpt_to_rhum(row), axis=1)
# Drop dew point column
df = df.drop("dwpt", axis=1)
# Set index
df = df.set_index(["station", "time"])
# Round decimals
df = df.round(1)
# Write DataFrame into Meteostat database
persist(jsp, df, hourly_metar)
# Close Jasper instance
jsp.close()
# Rename columns
df = df.rename(columns=NAMES)
# Snow cm -> mm
df["snow"] = df["snow"].multiply(10)
# Change coco
df["coco"] = df["coco"].apply(get_condicode)
# Add station column
df["station"] = station["id"]
# Set index
df = df.set_index(["station", "time"])
# Append data to full DataFrame
if df_full is None:
df_full = df
else:
df_full = df_full.append(df)
except BaseException:
pass
# Write DataFrame into Meteostat database
persist(jsp, df_full, hourly_synop)
# Close Jasper instance
jsp.close()
)
# Rename columns
df = df.rename(columns=parameter["stubnames"])
# Translate from wide to long
df = pd.wide_to_long(df,
stubnames=parameter["name"],
i="station",
j="month")
# Sunshine hours to minutes
if parameter["name"] == "tsun":
df["tsun"] = df["tsun"] * 60
# Append data to full DataFrame
if df_full is None:
df_full = df
else:
df_full = df_full.join(df)
# Add start & end year
df_full["start"] = START
df_full["end"] = END
# Write DataFrame into Meteostat database
persist(jsp, df_full, normals_global)
# Close Jasper instance
jsp.close()
df["tavg"] = df["tavg"].div(10)
df["tmin"] = df["tmin"].div(10)
df["tmax"] = df["tmax"].div(10)
df["prcp"] = df["prcp"].div(10)
df["wspd"] = df["wspd"].div(10).apply(ms_to_kmh)
df["wpgt"] = df["wpgt"].div(10).apply(ms_to_kmh)
# Add station column
df["station"] = station["id"]
# Set index
df = df.set_index(["station", "time"])
# Append data to full DataFrame
if df_full is None:
df_full = df
else:
df_full = df_full.append(df)
except BaseException:
pass
# Write DataFrame into Meteostat database
persist(jsp, df_full, daily_global)
# Quit FTP connection
ftp.quit()
# Close Jasper instance
jsp.close()
# Convert PRES to MSL
df["pres"] = df.apply(lambda row, alt=altitude: pres_to_msl(row, alt), axis=1)
df["wpgt"] = df["wpgt"].apply(ms_to_kmh)
df["wspd"] = df["wspd"].apply(ms_to_kmh)
df["tsun"] = df["tsun"] * 60
df["snow"] = df["snow"] * 10
# Add weather station ID
df["station"] = station
# Set index
df = df.set_index(["station", "time"])
# Round decimals
df = df.round(1)
# Append data to full DataFrame
if df_full is None:
df_full = df
else:
df_full = df_full.append(df)
except BaseException:
pass
# Write DataFrame into Meteostat database
persist(jsp, df_full, daily_national)
# Close Jasper instance
jsp.close()
# Create DataFrame
df = pd.DataFrame(map(map_data, data["properties"]["timeseries"]))
# Set index
df["station"] = station[0]
df = df.set_index(["station", "time"])
# Shift prcp and coco columns by 1 (as they refer to the next hour)
df["prcp"] = df["prcp"].shift(1)
if not station[4]:
df["coco"] = df["coco"].shift(1)
# Append data to full DataFrame
if df_full is None:
df_full = df
else:
df_full = df_full.append(df)
except error.HTTPError:
pass
# Sleep
sleep(SLEEP_TIME)
# Write DataFrame into Meteostat database
persist(jsp, df_full, hourly_model)
# Close Jasper instance
jsp.close()
j="month",
)
# Sunshine hours to minutes
if parameter["name"] == "tsun":
df["tsun"] = df["tsun"] * 60
# Append data to full DataFrame
if df_station is None:
df_station = df
else:
df_station = df_station.join(df)
except BaseException:
pass
# Append data to full DataFrame
if df_full is None:
df_full = df_station
else:
df_full = df_full.append(df_station)
except BaseException:
pass
# Write DataFrame into Meteostat database
persist(jsp, df_full, monthly_global)
# Close Jasper instance
jsp.close()