def get_meteoswiss(location="schwarzsee19"):
SITE, FOLDER = config(location)
if location == "schwarzsee19":
location = "plaffeien19"
location = location[:-2]
df = pd.read_csv(
os.path.join(FOLDER["raw"], location + "_meteoswiss.txt"),
# sep="\s+",
sep=";",
skiprows=2,
)
for col in df.columns:
if meteoswiss_parameter(col):
df[col] = pd.to_numeric(df[col], errors="coerce")
df = df.rename(columns={col: meteoswiss_parameter(col)["name"]})
# logger.info("%s from meteoswiss" % meteoswiss_parameter(col)["name"])
else:
df = df.drop(columns=col)
df["When"] = pd.to_datetime(df["When"], format="%Y%m%d%H%M")
# df["Prec"] = df["Prec"] / (10 * 60) # ppt rate mm/s
df_out = (df.set_index("When").resample("H").mean().reset_index())
df_out["Prec"] = (
df.set_index("When").resample("H").sum().reset_index()["Prec"])
# mask = (df["When"] >= SITE["start_date"]) & (df["When"] <= SITE["end_date"])
# df = df.loc[mask]
return df_out
def field(location="schwarzsee19"):
SITE, FOLDER = config(location)
if location == "gangles21":
df_in = pd.read_csv(FOLDER["input"] + SITE["name"] +
"_input_model.csv",
header=0,
parse_dates=["When"])
df = df_in.round(3)
# CSV output
logger.info(df_in.head())
logger.info(df_in.tail())
# df.to_csv(input_folder + SITE["name"] + "_input_field.csv")
mask = (df["When"] >= SITE["start_date"]) & (df["When"] <=
SITE["end_date"])
df = df.loc[mask]
df = df.reset_index()
return df
def __init__(self, location, ignore=False):
super(UQ_Icestupa, self).__init__(
# labels=["Time (days)", "Ice Volume ($m^3$)"],
interpolate=False,
suppress_graphics=False,
logger_level="debug",
ignore=ignore)
SITE, FOLDER = config(location)
initial_data = [SITE, FOLDER]
diff = SITE["end_date"] - SITE["start_date"]
days, seconds = diff.days, diff.seconds
self.total_hours = days * 24 + seconds // 3600
# Initialise all variables of dictionary
for dictionary in initial_data:
for key in dictionary:
setattr(self, key, dictionary[key])
# logger.info(f"%s -> %s" % (key, str(dictionary[key])))
self.read_input()
self.self_attributes()
self.df_c = pd.read_hdf(FOLDER["input"] + "model_input.h5", "df_c")
self.df_c = self.df_c.iloc[1:]
self.y_true = self.df_c.DroneV.values
if location == 'gangles21':
self.z_true = [0]
else:
self.df_cam = pd.read_hdf(FOLDER["input"] + "model_input.h5",
"df_cam")
self.df_cam = self.df_cam.reset_index()
# self.df_cam = self.df_cam.iloc[1180:]
self.z_true = self.df_cam.cam_temp.values
print("Ice volume measurements for %s are %s\n" %
(self.name, len(self.y_true)))
print("Surface temp measurements for %s are %s\n" %
(self.name, len(self.z_true)))
def __init__(self, location, ignore=False):
super(UQ_Icestupa,
self).__init__(labels=["Time (days)", "Ice Volume ($m^3$)"],
interpolate=False,
suppress_graphics=False,
logger_level="debug",
ignore=ignore)
SITE, FOLDER = config(location)
initial_data = [SITE, FOLDER]
diff = SITE["end_date"] - SITE["start_date"]
days, seconds = diff.days, diff.seconds
self.total_hours = days * 24 + seconds // 3600
# Initialise all variables of dictionary
for dictionary in initial_data:
for key in dictionary:
setattr(self, key, dictionary[key])
# logger.info(f"%s -> %s" % (key, str(dictionary[key])))
self.read_input()
self.self_attributes()
def __init__(self, location="Guttannen 2021", params="default"):
SITE, FOLDER = config(location)
diff = SITE["end_date"] - SITE["start_date"]
days, seconds = diff.days, diff.seconds
self.total_hours = days * 24 + seconds // 3600
if params == "best":
with open(FOLDER["sim"] + "best_params.pkl", "rb") as f:
best_params = pickle.load(f)
initial_data = [SITE, FOLDER, best_params]
else:
initial_data = [SITE, FOLDER]
# Initialise all variables of dictionary
for dictionary in initial_data:
for key in dictionary:
setattr(self, key, dictionary[key])
logger.info(f"%s -> %s" % (key, str(dictionary[key])))
# Initialize input dataset
input_file = self.input + self.name + "_input_model.csv"
self.df = pd.read_csv(input_file,
sep=",",
header=0,
parse_dates=["When"])
# Drops garbage columns
self.df = self.df[self.df.columns.drop(
list(self.df.filter(regex="Unnamed")))]
# Reset date range
self.df = self.df.set_index("When")
self.df = self.df[SITE["start_date"]:SITE["end_date"]]
self.df = self.df.reset_index()
logger.debug(self.df.head())
logger.debug(self.df.tail())
def __init__(self,
name="guttannen21",
DX=0.020,
DT=60 * 60,
A_I=0.25,
A_S=0.85,
IE=0.97,
T_PPT=1,
T_F=1.5,
A_DECAY=17.5,
Z=0.0025,
SA_corr=1.5):
super(Icestupa, self).__init__()
print("Initializing classifier:\n")
args, _, _, values = inspect.getargvalues(inspect.currentframe())
values.pop("self")
for arg, val in values.items():
setattr(self, arg, val)
# print("{} = {}".format(arg,val))
SITE, FOLDER = config(location=self.name)
initial_data = [SITE, FOLDER]
diff = SITE["end_date"] - SITE["start_date"]
days, seconds = diff.days, diff.seconds
self.total_hours = days * 24 + seconds // 3600
# Initialise all variables of dictionary
for dictionary in initial_data:
for key in dictionary:
setattr(self, key, dictionary[key])
self.read_input()
self.self_attributes()
self.diff = self.total_hours
grey = '#ced4da'
CB91_Blue = "#2CBDFE"
CB91_Green = "#47DBCD"
CB91_Pink = "#F3A0F2"
CB91_Purple = "#9D2EC5"
CB91_Violet = "#661D98"
CB91_Amber = "#F5B14C"
# index = pd.date_range(start ='1-1-2022',
# end ='1-1-2024', freq ='D', name= "When")
# df_out = pd.DataFrame(columns=locations,index=index)
fig, ax = plt.subplots(len(locations), 1, sharex='col')
for i, location in enumerate(locations):
SITE, FOLDER = config(location)
icestupa = Icestupa(location)
icestupa.read_output()
icestupa.self_attributes()
variance = []
mean = []
evaluations = []
data = un.Data()
# filename1 = FOLDER['sim']+ "SE_full.h5"
filename1 = FOLDER['sim'] + "full.h5"
filename2 = FOLDER['sim'] + "fountain.h5"
# filename1 = FOLDER['sim']+ "efficiency.h5"
# print(data)
'IE': np.arange(0.9, 0.99, 0.01).tolist(),
'A_I': np.arange(0.3, 0.4, 0.01).tolist(),
'A_S': np.arange(0.8, 0.9, 0.01).tolist(),
'T_RAIN': np.arange(0, 2, 1).tolist(),
'T_DECAY': np.arange(1, 22, 1).tolist(),
# 'v_a_limit': np.arange(4, 10, 1).tolist(),
# 'Z_I': np.arange(0.0010, 0.0020, 0.0001).tolist(),
}
experiments = []
for params in ParameterGrid(param_grid):
experiments.append(params)
for location in locations:
SITE, FOLDER, df_h = config(location)
model = Tune_Icestupa(location)
results = []
with Pool(multiprocessing.cpu_count()) as executor:
for result in executor.map(model.run, experiments):
results.append(result)
results = pd.DataFrame(results)
results.round(3).to_csv(FOLDER["sim"] + "/Full_tune_sim.csv", sep=",")
print(results)
def era5(location="schwarzsee19"):
if location in ["gangles21"]:
df_in3 = pd.read_csv(
"/home/suryab/work/ERA5/outputs/leh_2021.csv",
sep=",",
header=0,
parse_dates=["When"],
)
df_in3 = df_in3.set_index("When")
df_in3 = df_in3.reset_index()
SITE, FOLDER = config(location)
mask = (df_in3["When"] >= SITE["start_date"]) & (df_in3["When"] <=
SITE["end_date"])
df_in3 = df_in3.loc[mask]
df_in3 = df_in3.reset_index(drop="True")
time_steps = 60 * 60
df_in3["ssrd"] /= time_steps
df_in3["strd"] /= time_steps
df_in3["fdir"] /= time_steps
df_in3["v_a"] = np.sqrt(df_in3["u10"]**2 + df_in3["v10"]**2)
# Derive RH
df_in3["t2m"] -= 273.15
df_in3["d2m"] -= 273.15
df_in3["t2m_RH"] = df_in3["t2m"]
df_in3["d2m_RH"] = df_in3["d2m"]
df_in3 = df_in3.apply(lambda x: e_sat(x) if x.name == "t2m_RH" else x)
df_in3 = df_in3.apply(lambda x: e_sat(x) if x.name == "d2m_RH" else x)
df_in3["RH"] = 100 * df_in3["d2m_RH"] / df_in3["t2m_RH"]
df_in3["sp"] = df_in3["sp"] / 100
df_in3["tp"] = df_in3["tp"] * 1000 / 3600 # mm/s
df_in3["SW_diffuse"] = df_in3["ssrd"] - df_in3["fdir"]
df_in3 = df_in3.set_index("When")
# CSV output
df_in3.rename(
columns={
"t2m": "T_a",
"sp": "p_a",
"tp": "Prec",
"fdir": "SW_direct",
"strd": "LW_in",
},
inplace=True,
)
df_in3 = df_in3[[
"T_a",
"RH",
"Prec",
"v_a",
"SW_direct",
"SW_diffuse",
"LW_in",
"p_a",
]]
df_in3 = df_in3.round(3)
upsampled = df_in3.resample("15T")
interpolated = upsampled.interpolate(method="linear")
interpolated = interpolated.reset_index()
df_in3 = interpolated[[
"When",
"T_a",
"RH",
"v_a",
"SW_direct",
"SW_diffuse",
"LW_in",
"p_a",
"Prec",
]]
df_in3 = df_in3.reset_index()
mask = (df_in3["When"] >= SITE["start_date"]) & (df_in3["When"] <=
SITE["end_date"])
df_in3 = df_in3.loc[mask]
df_in3 = df_in3.reset_index()
df_in3.to_csv(FOLDER["input"] + SITE["name"] + "_input_ERA5.csv")
df_ERA5 = interpolated[[
"When",
"T_a",
"v_a",
"RH",
"SW_direct",
"SW_diffuse",
"LW_in",
"p_a",
"Prec",
]]
# logger.info(df_ERA5.head())
# logger.info(df_ERA5.tail())
return df_ERA5, df_in3
# logger.info(df_ERA5.head())
# logger.info(df_ERA5.tail())
return df_ERA5, df_in3
if __name__ == "__main__":
logger = logging.getLogger(__name__)
coloredlogs.install(
fmt="%(name)s %(levelname)s %(message)s",
level=logging.INFO,
logger=logger,
)
location = "gangles21"
SITE, FOLDER = config("gangles21")
df = field(location)
df_ERA5, df_in3 = era5(SITE["name"])
# print(df_ERA5.SW_direct.describe())
# print(df.SW_direct.describe())
pp = PdfPages(FOLDER["input"] + "compare.pdf")
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.scatter(df.SW_direct, df_ERA5.SW_direct, s=2)
ax1.set_xlabel("Field SW")
ax1.set_ylabel("ERA5 SW")
ax1.grid()
lims = [
def aws(location="diavolezza21"):
SITE, FOLDER, df_h = config(location="Diavolezza 2021")
# Get header colun list
df = pd.read_csv(
FOLDER["raw"] + location + "_aws.csv",
# sep=";",
sep="\t",
header=0,
# parse_dates=['date_time']
)
print(df.columns)
logger.info(df['date_time'].head())
logger.info(df['date_time'].tail())
df["When"] = pd.to_datetime(df["date_time"], format="%d/%m/%y %H:%M")
df.rename(
columns={
# "date_time": "When",
" WndScMae": "v_a",
" Tair_act_AVG": "T_a",
" Rhum_act_AVG": "RH",
"SWdown_cor": "SW_out",
# " SWdown_AVG": "SW_out",
"SWup_cor": "SW_in",
# " SWup_AVG": "SW_in",
# " LWdown_AVG": "LW_out",
"LWdown_cor": "LW_out",
# " LWup_AVG": "LW_in",
"LWup_cor": "LW_in",
" HS_act": "HS",
" Baro_act_AVG": "p_a",
},
inplace=True,
)
df = df[[
"When", "v_a", "T_a", "RH", "SW_out", "SW_in", "LW_out", "LW_in", "HS",
"p_a"
]]
# df = df[["When", "v_a", "T_a", "RH", "HS", "p_a"]]
df["a"] = df["SW_out"] / df["SW_in"]
df.loc[df.a > 1, "a"] = np.NaN
df.loc[:, "a"] = df["a"].interpolate()
df = df.set_index("When").sort_index()
df = df.resample(pd.offsets.Minute(n=15)).mean().reset_index()
# df["Prec"]= df.HS.diff()
# df["Prec"] = df["Prec"] *10 / (15 * 60) # ppt rate mm/s
# df.loc[df.Prec < 0, "Prec"] = np.NaN
# df.loc[df.Prec *15*60 > 500, "Prec"] = np.NaN
# df.loc[df.Prec *15*60 < 4, "Prec"] = 0
# logger.error(df.loc[df.Prec.isna(), "HS"].head())
# df.loc[:, "Prec"] = df["Prec"].interpolate()
# diffuse_fraction=0.25
# df["SW_direct"] = (1-diffuse_fraction) * (df["SW_in"])
# df["SW_diffuse"] = diffuse_fraction * (df["SW_in"])
logger.warning(df.head())
logger.warning(df.tail())
fig, ax1 = plt.subplots()
skyblue = "#9bc4f0"
blue = "#0a4a97"
x = df.When
# y = df.Prec *15*60
y = df.a
ax1.plot(
x,
y,
linestyle="-",
color=blue,
)
# ax1.set_ylabel("Temperature[$l\\, min^{-1}$]")
ax1.set_ylabel("AWS ppt[$mm$]")
ax1.xaxis.set_major_locator(mdates.WeekdayLocator())
ax1.xaxis.set_major_formatter(mdates.DateFormatter("%b %d"))
ax1.xaxis.set_minor_locator(mdates.DayLocator())
fig.autofmt_xdate()
plt.savefig(FOLDER["input"] + SITE["name"] + "test1.png")
mask = (df["When"] >= SITE["start_date"]) & (df["When"] <=
SITE["end_date"])
df = df.loc[mask]
df = df.reset_index(drop=True)
df = df.set_index("When")
logger.info(
pd.date_range(start=df.index[0], end=df.index[-1],
freq='15T').difference(df.index))
df = df.reset_index()
return df
def field(location="schwarzsee19"):
if location == "diavolezza21":
path = "/home/suryab/ownCloud/Sites/Diavolezza/diavolezza_sdcard/"
all_files = glob.glob(path + "*.txt")
SITE, FOLDER, df_h = config(location="Diavolezza 2021")
li = []
ctr = 0
# Get header colun list
df = pd.read_csv(
all_files[10],
sep=";",
skiprows=3,
header=0,
encoding="latin-1",
)
names = df.columns
names = names[:-3]
for file in all_files:
try:
df = pd.read_csv(
file,
sep=";",
skiprows=3,
header=0,
encoding="latin-1",
usecols=names,
)
df = df[1:].reset_index(drop=True)
df = df[["Q_Wasser ", "T_Luft", "r_Luft", "w_Luft"]]
for col in df.columns:
df[col] = df[col].astype(float)
df = df.round(2)
df.rename(
columns={
"w_Luft": "v_a",
"T_Luft": "T_a",
"r_Luft": "RH",
"Q_Wasser ": "Discharge",
},
inplace=True,
)
a_file = open(file, encoding='latin-1')
line = a_file.readline()
line = line.split(";")[-1]
date = line.split("+")[0]
date = date[3:]
date = datetime.strptime(date, ' %d %b %Y %I:%M:%S %p ')
df["When"] = pd.to_datetime([
date + timedelta(seconds=10 * h)
for h in range(0, df.shape[0])
])
li.append(df)
except:
ctr += 1
a_file = open(file, encoding='latin-1')
line = a_file.readline()
line = line.split(";")[-1]
date = line.split("+")[0]
date = date[3:]
date = datetime.strptime(date, ' %d %b %Y %I:%M:%S %p ')
print(date)
the_type, the_value, the_traceback = sys.exc_info()
print(the_type)
pass
df = pd.concat(li, axis=0, ignore_index=True)
df = df.set_index("When").sort_index().reset_index()
df = df.set_index("When").resample(
pd.offsets.Minute(n=15)).mean().reset_index()
mask = (df["When"] >= SITE["start_date"]) & (df["When"] <=
SITE["end_date"])
df = df.loc[mask]
df = df.reset_index(drop=True)
print("Number of hours missing : %s" % ctr)
# CSV output
df.to_csv(FOLDER["raw"] + SITE["name"] + "_field.csv")
return df
print("Number of hours missing : %s" % ctr)
# CSV output
df.to_csv(FOLDER["raw"] + SITE["name"] + "_field.csv")
return df
if __name__ == "__main__":
logger = logging.getLogger(__name__)
coloredlogs.install(
fmt="%(name)s %(levelname)s %(message)s",
level=logging.INFO,
logger=logger,
)
SITE, FOLDER, *args = config("Diavolezza 2021")
df = aws()
# sdcard = True
sdcard = False
if sdcard:
df_field = field("diavolezza21")
else:
df_field = pd.read_csv(
FOLDER["raw"] + SITE["name"] + "_field.csv",
sep=",",
header=0,
parse_dates=["When"],
)
def get_field(location="schwarzsee19"):
SITE, FOLDER= config(location)
if location == "gangles21":
col_list = [
"TIMESTAMP",
"AirTC_Avg",
"RH",
"WS",
]
cols = ["T_a", "RH", "v_a"]
df_in = pd.read_csv(
FOLDER["raw"] + "/Gangles_Table15Min.dat",
sep=",",
skiprows=[0, 2, 3, 4],
parse_dates=["TIMESTAMP"],
)
df_in = df_in[col_list]
df_in.rename(
columns={
"TIMESTAMP": "When",
"AirTC_Avg": "T_a",
"RH_probe_Avg": "RH",
"WS": "v_a",
},
inplace=True,
)
df_in1 = pd.read_csv(
FOLDER["raw"] + "/Gangles_Table60Min.dat",
sep=",",
skiprows=[0, 2, 3],
parse_dates=["TIMESTAMP"],
)
df_in1.rename(
columns={
"TIMESTAMP": "When",
"BP_mbar": "p_a", # mbar same as hPa
},
inplace=True,
)
for col in df_in1:
if col != "When":
df_in1[col] = pd.to_numeric(df_in1[col], errors="coerce")
df_in = df_in.set_index("When")
df_in1 = df_in1.set_index("When")
df_in1 = df_in1.reindex(
pd.date_range(df_in1.index[0], df_in1.index[-1], freq="15Min"),
fill_value=np.NaN,
)
df_in = df_in.replace("NAN", np.NaN)
df_in1 = df_in1.replace("NAN", np.NaN)
df_in1 = df_in1.resample("15Min").interpolate("linear")
df_in.loc[:, "p_a"] = df_in1["p_a"]
df_in = df_in.replace("NAN", np.NaN)
if df_in.isnull().values.any():
print("Warning: Null values present")
print(df_in[cols].isnull().sum())
df_in = df_in.round(3)
df_in = df_in.reset_index()
df_in.rename(
columns={
"index": "When",
},
inplace=True,
)
start_date = datetime(2020, 12, 14)
df_in = df_in.set_index("When")
df_in = df_in[start_date:]
df1 = pd.read_csv(
FOLDER["raw"] + "/HIAL_input_field.csv",
sep=",",
parse_dates=["When"],
)
df = df_in
df1 = df1.set_index("When")
cols = ["SW_global"]
for col in cols:
df.loc[:, col] = df1[col]
df = df.reset_index()
df = df[df.columns.drop(list(df.filter(regex="Unnamed")))]
df = df.dropna()
# df.to_csv("outputs/" + location + "_input_field.csv")
mask = df["SW_global"] < 0
mask_index = df[mask].index
df.loc[mask_index, 'SW_global'] = 0
# diffuse_fraction = 0
# df["SW_diffuse"] = diffuse_fraction * df.SW_global
# df["SW_direct"] = (1-diffuse_fraction)* df.SW_global
df = (
df.set_index("When")
.resample("H")
.mean()
.reset_index()
)
df["Prec"] = 0
df["missing_type"] ='-'
df["cld"] = 0
df.to_csv(
# "/home/suryab/work/air_model/data/"
# + location
# + "/interim/"
FOLDER["input"]
+ location
+ "_input_model.csv"
)
return df
if location == "guttannen20":
df_in = pd.read_csv(
FOLDER["raw"] + SITE["name"] + "_field.txt",
header=None,
encoding="latin-1",
skiprows=7,
sep="\\s+",
index_col=False,
names=[
"Date",
"Time",
"Discharge",
"Wind Direction",
"Wind Speed",
"Maximum Wind Speed",
"Temperature",
"Humidity",
"Pressure",
"Pluviometer",
],
)
types_dict = {
"Date": str,
"Time": str,
"Discharge": float,
"Wind Direction": float,
"Wind Speed": float,
"Temperature": float,
"Humidity": float,
"Pressure": float,
"Pluviometer": float,
}
for col, col_type in types_dict.items():
df_in[col] = df_in[col].astype(col_type)
df_in["When"] = pd.to_datetime(df_in["Date"] + " " + df_in["Time"])
df_in["When"] = pd.to_datetime(df_in["When"], format="%Y.%m.%d %H:%M:%S")
df_in = df_in.drop(["Pluviometer", "Date", "Time"], axis=1)
df_in = df_in.set_index("When").resample("H").mean().reset_index()
mask = (df_in["When"] >= SITE["start_date"]) & (
df_in["When"] <= SITE["end_date"]
)
df_in = df_in.loc[mask]
df_in = df_in.reset_index()
days = pd.date_range(start=SITE["start_date"], end=SITE["end_date"], freq="H")
days = pd.DataFrame({"When": days})
df = pd.merge(
df_in[
[
"When",
"Discharge",
"Wind Speed",
"Temperature",
"Humidity",
"Pressure",
]
],
days,
on="When",
)
df = df.round(3)
# CSV output
df.rename(
columns={
"Wind Speed": "v_a",
"Temperature": "T_a",
"Humidity": "RH",
"Pressure": "p_a",
},
inplace=True,
)
logger.info(df_in.head())
logger.info(df_in.tail())
df.to_csv(FOLDER["input"] + SITE["name"] + "_input_field.csv")
if location == "guttannen21":
df_in = pd.read_csv(
FOLDER["raw"] + SITE["name"] + "_field.txt",
header=None,
encoding="latin-1",
skiprows=7,
sep="\\s+",
names=[
"Date",
"Time",
"Wind Direction",
"Wind Speed",
"Maximum Wind Speed",
"Temperature",
"Humidity",
"Pressure",
"Pluviometer",
],
)
types_dict = {
"Date": str,
"Time": str,
"Wind Direction": float,
"Wind Speed": float,
"Temperature": float,
"Humidity": float,
"Pressure": float,
"Pluviometer": float,
}
for col, col_type in types_dict.items():
df_in[col] = df_in[col].astype(col_type)
df_in["When"] = pd.to_datetime(df_in["Date"] + " " + df_in["Time"])
df_in["When"] = pd.to_datetime(df_in["When"], format="%Y.%m.%d %H:%M:%S")
df_in = df_in.drop(["Pluviometer", "Date", "Time"], axis=1)
logger.debug(df_in.head())
logger.debug(df_in.tail())
df_in = df_in.set_index("When").resample("H").mean().reset_index()
mask = (df_in["When"] >= SITE["start_date"]) & (
df_in["When"] <= SITE["end_date"]
)
df_in = df_in.loc[mask]
df_in = df_in.reset_index()
days = pd.date_range(start=SITE["start_date"], end=SITE["end_date"], freq="H")
days = pd.DataFrame({"When": days})
df = pd.merge(
df_in[
[
"When",
"Wind Speed",
"Temperature",
"Humidity",
"Pressure",
]
],
days,
on="When",
)
df = df.round(3)
# CSV output
df.rename(
columns={
"Wind Speed": "v_a",
"Temperature": "T_a",
"Humidity": "RH",
"Pressure": "p_a",
},
inplace=True,
)
df.to_csv(FOLDER["input"] + SITE["name"] + "_input_field.csv")
if location == "schwarzsee19":
df_in = pd.read_csv(
FOLDER["raw"] + SITE["name"][:-2] + "_aws.txt",
header=None,
encoding="latin-1",
skiprows=7,
sep="\\s+",
names=[
"Date",
"Time",
"Discharge",
"Wind Direction",
"Wind Speed",
"Maximum Wind Speed",
"Temperature",
"Humidity",
"Pressure",
"Pluviometer",
],
)
df_in = df_in.drop(["Pluviometer"], axis=1)
df_in["When"] = pd.to_datetime(df_in["Date"] + " " + df_in["Time"])
df_in["When"] = pd.to_datetime(df_in["When"], format="%Y.%m.%d %H:%M:%S")
# Correct datetime errors
for i in tqdm(range(1, df_in.shape[0])):
if str(df_in.loc[i, "When"].year) != "2019":
df_in.loc[i, "When"] = df_in.loc[i - 1, "When"] + pd.Timedelta(
minutes=5
)
df_in = df_in.set_index("When").resample("H").last().reset_index()
mask = (df_in["When"] >= SITE["start_date"]) & (
df_in["When"] <= SITE["end_date"]
)
df_in = df_in.loc[mask]
df_in = df_in.reset_index()
days = pd.date_range(start=SITE["start_date"], end=SITE["end_date"], freq="H")
days = pd.DataFrame({"When": days})
df = pd.merge(
days,
df_in[
[
"When",
"Discharge",
"Wind Speed",
"Maximum Wind Speed",
"Wind Direction",
"Temperature",
"Humidity",
"Pressure",
]
],
on="When",
)
# Include Spray time
df_nights = pd.read_csv(
FOLDER["raw"] + "schwarzsee_fountain_time.txt",
sep="\\s+",
)
df_nights["Start"] = pd.to_datetime(
df_nights["Date"] + " " + df_nights["start"]
)
df_nights["End"] = pd.to_datetime(df_nights["Date"] + " " + df_nights["end"])
df_nights["Start"] = pd.to_datetime(
df_nights["Start"], format="%Y-%m-%d %H:%M:%S"
)
df_nights["End"] = pd.to_datetime(df_nights["End"], format="%Y-%m-%d %H:%M:%S")
df_nights["Date"] = pd.to_datetime(df_nights["Date"], format="%Y-%m-%d")
df["Fountain"] = 0
for i in range(0, df_nights.shape[0]):
df_nights.loc[i, "Start"] = df_nights.loc[i, "Start"] - pd.Timedelta(days=1)
df.loc[
(df["When"] >= df_nights.loc[i, "Start"])
& (df["When"] <= df_nights.loc[i, "End"]),
"Fountain",
] = 1
# CSV output
df.rename(
columns={
"Wind Speed": "v_a",
"Temperature": "T_a",
"Humidity": "RH",
"Pressure": "p_a",
},
inplace=True,
)
df.Discharge = df.Fountain * df.Discharge
df.to_csv(FOLDER["input"] + SITE["name"] + "_input_field.csv")
df = df.set_index("When").resample("H").mean().reset_index()
return df
import glob
dirname = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
sys.path.append(dirname)
from src.utils.settings import config
from src.data.make_dataset import era5, linreg, meteoswiss, meteoswiss_parameter
if __name__ == "__main__":
logger = logging.getLogger(__name__)
coloredlogs.install(
fmt="%(name)s %(levelname)s %(message)s",
level=logging.INFO,
logger=logger,
)
SITE, FOLDER, *args = config("ravat20")
df_ERA5, df_in3 = era5(SITE["name"])
print(df_ERA5.describe())
df_ERA5 = df_ERA5.set_index("When")
df = df_ERA5
df['missing_type'] = 'NA'
# Fill from ERA5
logger.warning("Temperature NaN percent: %0.2f" %
(df["T_a"].isna().sum() / df.shape[0] * 100))
logger.warning("wind NaN percent: %0.2f" %
(df["v_a"].isna().sum() / df.shape[0] * 100))
df = df.reset_index()
# Main logger
logger = logging.getLogger(__name__)
coloredlogs.install(
fmt="%(funcName)s %(levelname)s %(message)s",
level=logging.ERROR,
logger=logger,
)
answers = dict(
# location="Schwarzsee 2019",
location="Guttannen 2021",
# location="Gangles 2021",
trigger="Manual",
)
# Get settings for given location and trigger
SITE, FOUNTAIN, FOLDER, *args = config(answers["location"])
# Initialise icestupa object
icestupa = Icestupa(answers["location"])
cmap = plt.cm.rainbow # define the colormap
norm = mpl.colors.Normalize(vmin=-100, vmax=0)
sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)
sm.set_array([])
filename = FOLDER["sim"] + "/DX_sim.csv"
filename2 = FOLDER["sim"] + "/DX_sim.h5"
figures = FOLDER["sim"] + "/DX_sim.pdf"
filename = FOLDER["sim"] + "/Tune_sim.csv"
figures = FOLDER["sim"] + "/Tune_sim.pdf"
