# Multi-thread processing if len(datasets) > 1: # Create process pool with ThreadPool(THREADS) as pool: # Process datasets in pool output = pool.starmap(load, datasets) # Wait for Pool to finish pool.close() pool.join() # DataFrame structure base = pd.DataFrame(columns=list(PARAMETERS.values())) # Full DataFrame full = pd.concat([base, *output]) # Set index full.set_index(["station", "time"], inplace=True) # Drop NaN-only rows full = full.dropna(how="all") if full.index.size > 0: # Write into database persist(jsp, full, hourly_national) # Close Jasper instance jsp.close()
"rhum": data["relativeHumidity"]["value"], "wdir": data["windDirection"]["value"], "wspd": data["windSpeed"]["value"], "pres": data["seaLevelPressure"]["value"] / 100 if data["seaLevelPressure"]["value"] else None, } # Create DataFrame df = pd.DataFrame([record]) # Set index df["station"] = station[0] df["time"] = pd.to_datetime(df["time"]) df = df.set_index(["station", "time"]) # Append data to full DataFrame if df_full is None: df_full = df else: df_full = df_full.append(df) except error.HTTPError: pass # Write DataFrame into Meteostat database persist(jsp, df_full.round(1), hourly_national) # Close Jasper instance jsp.close()
# Calculate humidity data # pylint: disable=unnecessary-lambda df["rhum"] = df.apply(lambda row: temp_dwpt_to_rhum(row), axis=1) # Drop dew point column # pylint: disable=no-member df = df.drop("dwpt", axis=1) # Add station column df["station"] = station["id"] # Set index df = df.set_index(["station", "time"]) # Round decimals df = df.round(1) # Write data into Meteostat database persist(jsp, df, hourly_global) except BaseException: pass # Quit FTP connection ftp.quit() # Close Jasper instance jsp.close()
obs.press.value("HPA") if obs.press is not None else None, "coco": get_condicode(obs.weather) if obs.weather is not None else None, }) except BaseException: pass # List -> DataFrame df = pd.DataFrame.from_records(data) # Calculate humidity data # pylint: disable=unnecessary-lambda df["rhum"] = df.apply(lambda row: temp_dwpt_to_rhum(row), axis=1) # Drop dew point column df = df.drop("dwpt", axis=1) # Set index df = df.set_index(["station", "time"]) # Round decimals df = df.round(1) # Write DataFrame into Meteostat database persist(jsp, df, hourly_metar) # Close Jasper instance jsp.close()
# Rename columns df = df.rename(columns=NAMES) # Snow cm -> mm df["snow"] = df["snow"].multiply(10) # Change coco df["coco"] = df["coco"].apply(get_condicode) # Add station column df["station"] = station["id"] # Set index df = df.set_index(["station", "time"]) # Append data to full DataFrame if df_full is None: df_full = df else: df_full = df_full.append(df) except BaseException: pass # Write DataFrame into Meteostat database persist(jsp, df_full, hourly_synop) # Close Jasper instance jsp.close()
) # Rename columns df = df.rename(columns=parameter["stubnames"]) # Translate from wide to long df = pd.wide_to_long(df, stubnames=parameter["name"], i="station", j="month") # Sunshine hours to minutes if parameter["name"] == "tsun": df["tsun"] = df["tsun"] * 60 # Append data to full DataFrame if df_full is None: df_full = df else: df_full = df_full.join(df) # Add start & end year df_full["start"] = START df_full["end"] = END # Write DataFrame into Meteostat database persist(jsp, df_full, normals_global) # Close Jasper instance jsp.close()
df["tavg"] = df["tavg"].div(10) df["tmin"] = df["tmin"].div(10) df["tmax"] = df["tmax"].div(10) df["prcp"] = df["prcp"].div(10) df["wspd"] = df["wspd"].div(10).apply(ms_to_kmh) df["wpgt"] = df["wpgt"].div(10).apply(ms_to_kmh) # Add station column df["station"] = station["id"] # Set index df = df.set_index(["station", "time"]) # Append data to full DataFrame if df_full is None: df_full = df else: df_full = df_full.append(df) except BaseException: pass # Write DataFrame into Meteostat database persist(jsp, df_full, daily_global) # Quit FTP connection ftp.quit() # Close Jasper instance jsp.close()
# Convert PRES to MSL df["pres"] = df.apply(lambda row, alt=altitude: pres_to_msl(row, alt), axis=1) df["wpgt"] = df["wpgt"].apply(ms_to_kmh) df["wspd"] = df["wspd"].apply(ms_to_kmh) df["tsun"] = df["tsun"] * 60 df["snow"] = df["snow"] * 10 # Add weather station ID df["station"] = station # Set index df = df.set_index(["station", "time"]) # Round decimals df = df.round(1) # Append data to full DataFrame if df_full is None: df_full = df else: df_full = df_full.append(df) except BaseException: pass # Write DataFrame into Meteostat database persist(jsp, df_full, daily_national) # Close Jasper instance jsp.close()
# Create DataFrame df = pd.DataFrame(map(map_data, data["properties"]["timeseries"])) # Set index df["station"] = station[0] df = df.set_index(["station", "time"]) # Shift prcp and coco columns by 1 (as they refer to the next hour) df["prcp"] = df["prcp"].shift(1) if not station[4]: df["coco"] = df["coco"].shift(1) # Append data to full DataFrame if df_full is None: df_full = df else: df_full = df_full.append(df) except error.HTTPError: pass # Sleep sleep(SLEEP_TIME) # Write DataFrame into Meteostat database persist(jsp, df_full, hourly_model) # Close Jasper instance jsp.close()
j="month", ) # Sunshine hours to minutes if parameter["name"] == "tsun": df["tsun"] = df["tsun"] * 60 # Append data to full DataFrame if df_station is None: df_station = df else: df_station = df_station.join(df) except BaseException: pass # Append data to full DataFrame if df_full is None: df_full = df_station else: df_full = df_full.append(df_station) except BaseException: pass # Write DataFrame into Meteostat database persist(jsp, df_full, monthly_global) # Close Jasper instance jsp.close()