def main():
start_year = 1979
end_year = 1988
field_names = ["TT", "PR", "AH", "AV", "STFL", "STFA", "TRAF", "TDRA"]
file_name_prefixes = ["dm", "pm", "pm", "pm", "pm", "pm", "pm", "pm"]
sim_name = "crcm5-hcd-rl-intfl"
rpn_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup".format(
sim_name)
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
nc_sim_folder = os.path.join(nc_db_folder, sim_name)
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True)
pmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="pm",
all_files_in_samples_folder=True)
for field_name, fname_prefix in zip(field_names, file_name_prefixes):
if fname_prefix == "pm":
manager = pmManager
elif fname_prefix == "dm":
manager = dmManager
else:
raise Exception("Unknown file type...")
do_export(start_year,
end_year,
var_name=field_name,
dataManager=manager,
nc_sim_folder=nc_sim_folder)
pass
def plot_mean_2d_fields():
names = [
"STFL",
]
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_260x260_wo_lakes_and_with_lakeroff",
]
run_id_list = [
"w/o lakes, with lake roff., high res",
]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
stfl_bounds = [
0, 1, 10, 100, 250, 500, 850, 900, 950, 1000, 1200, 1500, 1800, 2000,
3000, 4000, 10000, 20000
]
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1990, 12, 31),
out_img="lake_roff_and_lake_rout_effect_stfl_1.png",
var_name="STFL",
level=-1,
bounds=stfl_bounds)
pass
def main():
import matplotlib.pyplot as plt
lons, lats, descr = get_lat_lons()
#needed for the model domain
manager = Crcm5ModelDataManager(
samples_folder_path=
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
b = manager.get_rotpole_basemap()
x, y = b(lons, lats)
start_year = 1979
end_year = 1988
data = calculate_seasonal_mean_field(months=[6, 7, 8],
start_year=start_year,
end_year=end_year)
levels = np.arange(0, 7, 0.25)
data = np.ma.masked_where(data == descr["nodata_value"], data)
b.contourf(x, y, data, levels=levels)
b.colorbar()
b.drawcoastlines()
plt.show()
pass
def compare_sim_with_obs():
start_year = 1990
end_year = 1986
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_lowres_003",
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_86x86_0.5deg_wo_lakes_and_wo_lakeroff"
]
run_id_list = [
"003, 10 soil layers [{0}-{1}]".format(start_year, end_year),
"wo lakes and wo lakeroff"
]
colors = ["m", "k"] #"b" is reserved for observations
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
compare_hydrographs_at_stations(
data_managers,
start_date=datetime(start_year, 1, 1),
end_date=datetime(end_year, 12, 31),
img_path="hydrographs_003_and_wo_lakes_and_wo_lakeroff.png",
colors=colors)
def main():
from_guillimin_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/"
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
#interflow experiment (crcm5-hcd-rl-intfl)
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup3/all_files_in_one_folder"
#(crcm5-hcd-rl) lakes and rivers interacting no interflow
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup"
#lakes are full of land (crcm5-r)
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
#lakes and rivers not interacting
data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-r_spinup2/all_files"
hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf"
#Lake residence time decreased 5 times
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl-kd5_spinup/all_files_in_one_folder"
#latest interflow
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small/all_files_in_one_dir"
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf"
#interflow and ecoclimap
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap/all_files_in_one_dir"
#ecoclimap and era075
#data_folder = "quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075/all_files_in_one_dir"
#data_folder = os.path.join(from_guillimin_folder, data_folder)
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf"
#soil anisotropy 10000
#data_folder = os.path.join(from_guillimin_folder, "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf"
#soil anisotropy 10000 and do not care about bulk field capacity
#data_folder = os.path.join(from_guillimin_folder,
# "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf"
data_folder = os.path.join(from_guillimin_folder,
"quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS")
hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5"
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
#var_names = ["STFL", "PR", "TT", "AV", "AH", "TRAF", "TDRA", "I5", "I0", "I1", "I2", "IMAV", "AS", "INTF"]
#var_names = [ "I0", "I1", "I2", "IMAV"]
#var_names = ["AS", ]
#var_names = ["QQ", ]
#var_names = ["INTF", ]
var_names = []
dm.export_to_hdf(var_list=var_names, file_path=hdf_file_path, mode="a")
#export_static_fields_to_hdf(
# hdf_file= hdf_file_path, data_folder=data_folder
#)
pass
def main():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3_old_snc",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_diff_lk_types_crcm_lk_fractions",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3_sturm_snc",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_test_lake_level_260x260_1",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_test_lake_level_260x260_1_lakes_off",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_lakes_bowling_function",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_river_ice"
]
run_id_list = [
"w/o lakes", "with lakes", "w/o lakes sn.cond by Sturm",
"high res. with lakes", "high res. w/o lakes", "Bowling lake func.",
"river ice"
]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
compare_hydrographs_at_stations(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1988, 12, 31),
img_path="hydrographs_all_runs.png")
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1988, 12, 31),
impose_min=0,
out_img="lake_roff_and_lake_rout_effect_ice.png")
pass
def export_static_fields_to_hdf(
hdf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf",
data_folder="/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-r_spinup2/all_files",
overwrite=False):
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
dm.export_static_fields_to_hdf(file_path=hdf_file, overwrite=overwrite)
pass
def skynet3_test():
#To test dictionary to recarray to pytables implementation
data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl-kd5_spinup/all_files_in_one_folder"
hdf_file_path = "/skynet3_rech1/huziy/hdf_store/test.hdf"
var_names = [
"INTF",
]
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
dm.export_to_hdf(var_list=var_names, file_path=hdf_file_path, mode="w")
def main():
rpn_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
start_year = 1979
end_year = 1988
sim_name = "crcm5-r"
#dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True)
pmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="pm",
all_files_in_samples_folder=True)
#plot results
assert isinstance(pmManager, Crcm5ModelDataManager)
lons, lats = pmManager.lons2D, pmManager.lats2D
stfl_mask = pmManager.cbf < 0
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons, lats2d=lats)
x, y = basemap(lons, lats)
print(x.shape)
month_dates = [
datetime(year, month, 15) for year in range(start_year, end_year + 1)
for month in range(1, 13)
]
print(len(month_dates), " month of animation ")
nc_data_folder = os.path.join(nc_db_folder, sim_name)
dsDict = {}
var_names = ["STFL", "PR", "TT"]
for i, vName in enumerate(var_names):
path = os.path.join(nc_data_folder, "{0}.nc4".format(vName))
dsDict[vName] = Dataset(path)
aniObj = Animator(var_names, dsDict, basemap, x, y, stfl_mask=stfl_mask)
#fa = animation.FuncAnimation(fig, aniObj.animate, month_dates, interval=50)
temp_folder = "for_anim_{0}".format(sim_name)
if os.path.isdir(temp_folder):
shutil.rmtree(temp_folder)
os.mkdir(temp_folder)
for d in month_dates:
aniObj.animate(d)
aniObj.saveFrame(tmp_folder=temp_folder)
def show_lake_and_lakeroff_effect():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_wo_lakes_and_wo_lakeroff",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_diff_lk_types_crcm_lk_fractions",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_river_ice",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_quebec_river_ice_gwrestime0",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_lake_level_260x260_1_lakes_off",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_260x260_wo_lakes_and_with_lakeroff",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_river_ice_1yrspnp_const_manning",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_260x260_wo_lakes_and_with_lakeroff_nogw"
]
run_id_list = [
"w/o lakes, w/o lake roff.",
# "with lakes, with lake roff.",
# "with ice",
# "with ice, no ground water",
# "high res. lakes off",
# "high res wo lakes with lake rof",
# "low res, const manning, riv ice",
"high res., nogw, wo lakes, with lakerof"
]
cmap = cm.get_cmap("jet", len(run_id_list))
# colors = [cmap(i / float(len(run_id_list))) for i in range(len(run_id_list)) ]
colors = ["r", "k", "g", "y", "m"]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
# compare_hydrographs_at_stations(data_managers, start_date=datetime(1986,1,1), end_date=datetime(1990,12, 31),
# img_path="hydrograph_lake_and_lakeroff_effect_test_df.png", colors = colors
# )
stfl_bounds = [
0, 1, 10, 100, 250, 500, 850, 900, 950, 1000, 1200, 1500, 1800, 2000,
3000, 4000, 10000, 20000
]
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1990, 12, 31),
out_img="lake_roff_and_lake_rout_effect_stfl.png",
var_name="STFL",
level=-1,
bounds=stfl_bounds)
def main():
manager = Crcm5ModelDataManager(
samples_folder_path="/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
hdf_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
mean_field = manager.hdf_get_climatology_for_season(months=[6, 7, 8],
hdf_db_path=hdf_path,
var_name="TRAF", level=5,
start_year=1979,
end_year=1988)
plt.contourf(mean_field)
plt.show()
pass
def main():
var_name = "STFL"
data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3_old_snc"
data_manager = Crcm5ModelDataManager(samples_folder_path=data_path,
var_name=var_name,
all_files_in_samples_folder=True,
file_name_prefix="pm")
lons2D, lats2D = data_manager.lons2D, data_manager.lats2D
basemap = Basemap(projection="omerc",
no_rot=True,
lon_1=-68,
lat_1=52,
lon_2=16.65,
lat_2=0,
llcrnrlon=lons2D[0, 0],
llcrnrlat=lats2D[0, 0],
urcrnrlon=lons2D[-1, -1],
urcrnrlat=lats2D[-1, -1],
resolution="l")
fig = plt.figure()
data = data_manager.get_date_to_field_dict()
print(list(data.keys()))
print(len(list(data.keys())))
times = list(
sorted([
datetime.strptime(s, "%Y-%m-%d %H:%M") for s in list(data.keys())
]))
times = [t.strftime("%Y-%m-%d %H:%M") for t in times]
ims = []
x, y = basemap(lons2D, lats2D)
aniObj = Animator(data, basemap, x, y, times[0])
fa = animation.FuncAnimation(fig, aniObj.animate, times, interval=50)
plt.show()
#fa.save("animateflow.mpg")
data.close()
pass
def main():
stations = cehq_station.load_from_hydat_db(natural=True, province="QC")
dm = Crcm5ModelDataManager(
samples_folder_path=
"/skynet3_rech1/huziy/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
basemap = dm.get_rotpole_basemap()
lons = [s.longitude for s in stations]
lats = [s.latitude for s in stations]
n_cont_years = [len(s.get_list_of_complete_years()) for s in stations]
x, y = basemap(lons, lats)
basemap.scatter(x, y, c=n_cont_years)
basemap.drawcoastlines()
basemap.colorbar()
plt.show()
def show_lake_effect():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v4_old_snc",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_diff_lk_types_crcm_lk_fractions"
]
run_id_list = ["w/o lakes, with lake roff.", "with lakes, with lake roff."]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
# compare_hydrographs_at_stations(data_managers, start_date=datetime(1986,1,1), end_date=datetime(1988,12, 31))
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1988, 12, 31),
out_img="lake_effect_2d.png",
impose_min=0)
def compare_level_num_influence():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_lowres_001",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_lowres_002",
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_lowres_004"
]
run_id_list = [
"001, 10 soil layers", "002, 3 soil layers", "004, interflow"
]
colors = ["r", "k", "m"] #"b" is reserved for observations
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
compare_hydrographs_at_stations(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1990, 12, 31),
img_path="cmp_001_002_004.png",
colors=colors)
def validate_daily_climatology():
"""
"""
#years are inclusive
start_year = 1979
end_year =1988
sim_name_list = ["crcm5-r", "crcm5-hcd-r", "crcm5-hcd-rl"]
rpn_folder_path_form = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup"
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
#select stations
selected_ids = None
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
print("start reading cehq obs data")
# stations = cehq_station.read_station_data(selected_ids = selected_ids,
# start_date=start_date, end_date=end_date
# )
stations = []
print("start reading hydat obs data")
stations.extend(cehq_station.read_hydat_station_data(folder_path="/home/huziy/skynet3_rech1/HYDAT",
start_date = start_date, end_date = end_date))
print("finished reading station data")
varname = "STFL"
sim_name_to_manager = {}
sim_name_to_station_to_model_point = {}
dmManager = None
for sim_name in sim_name_list:
print(sim_name)
rpn_folder = rpn_folder_path_form.format(sim_name)
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm",
all_files_in_samples_folder=True, need_cell_manager=True)
sim_name_to_manager[sim_name] = dmManager
nc_sim_folder = os.path.join(nc_db_folder, sim_name)
nc_path = os.path.join(nc_sim_folder, "{0}_all.nc".format(varname))
print("get model points for the stations")
st_to_mp = dmManager.get_model_points_for_stations(stations, nc_path=nc_path, npoints=1,
nc_sim_folder=nc_sim_folder
)
sim_name_to_station_to_model_point[sim_name] = st_to_mp
common_lake_fractions = dmManager.lake_fraction
#for tests
#test(sim_name_to_station_to_model_point)
print("finished reading data in memory")
from matplotlib.backends.backend_pdf import PdfPages
pp = PdfPages("comp_with_obs_daily_clim.pdf")
stations_to_plot = [] #only stations that are compared with model are needed on a map
day_stamps = Station.get_stamp_days(2001)
for s in stations:
plt.figure()
assert isinstance(s, Station)
years = s.get_list_of_complete_years()
print(s)
if len(years) < 6: continue
dates, obs_data = s.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=day_stamps, years=years)
obs_ann_mean = np.mean(obs_data)
plt.plot( dates, obs_data, label = "Obs: ann.mean = {0:.1f}".format( obs_ann_mean ) )
mp = None
for sim_name in sim_name_list:
manager = sim_name_to_manager[sim_name]
if s not in sim_name_to_station_to_model_point[sim_name]:
continue
mp = sim_name_to_station_to_model_point[sim_name][s]
for mp in sim_name_to_station_to_model_point[sim_name][s]:
assert isinstance(mp, ModelPoint)
dates, values = mp.get_daily_climatology_for_complete_years(stamp_dates=day_stamps, varname = "STFL")
plt.plot(dates, values , label = "{0}: {1:.2f} \n ann.mean = {2:.1f}, dist = {3:.1f} km".format( sim_name,
manager.lake_fraction[mp.flow_in_mask == 1].mean(), np.mean(values), mp.distance_to_station / 1000.0) )
ax = plt.gca()
assert isinstance(ax, Axes)
ax.xaxis.set_major_formatter(DateFormatter("%d/%b"))
ax.xaxis.set_major_locator(MonthLocator(bymonth=list(range(1,13,3)), bymonthday=15 ))
plt.legend(prop = FontProperties(size=8))
if mp is None: continue
stations_to_plot.append(s)
plt.title("{0}: point lake fraction={1:.4f}".format(s.id, common_lake_fractions[mp.ix, mp.jy] ) )
pp.savefig()
#plot station positions
plt.figure()
bm = dmManager.get_rotpole_basemap_using_lons_lats(lons2d=dmManager.lons2D, lats2d=dmManager.lats2D, resolution="i")
bm.drawcoastlines(linewidth=0.1)
bm.drawrivers(linewidth=0.1)
lons_list = [s.longitude for s in stations_to_plot]
lats_list = [s.latitude for s in stations_to_plot]
x_list, y_list = bm(lons_list, lats_list)
bm.scatter(x_list, y_list, linewidths=0, s=0.5, zorder = 3)
ax = plt.gca()
for s, the_x, the_y in zip(stations, x_list, y_list):
ax.annotate(s.id, xy=(the_x, the_y),xytext=(3, 3), textcoords='offset points',
font_properties = FontProperties(size = 4), bbox = dict(facecolor = 'w', alpha = 1),
ha = "left", va = "bottom", zorder = 2)
pp.savefig()
pp.close()
def main():
#TODO: implement
var_name_to_ij = {
"TT": (0, 0),
"PR": (0, 1),
"AU": (1, 0),
"AV": (1, 1),
"STFA": (0, 2)
}
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True)
basemap = dmManager.get_omerc_basemap()
lons, lats = dmManager.lons2D, dmManager.lats2D
x, y = basemap(lons, lats)
lkfr = dmManager.lake_fraction
acc_area = dmManager.accumulation_area_km2
i_arr, j_arr = np.where((lkfr > 0.4) & (lkfr < 0.6) & (acc_area > 10000))
name_to_data = {}
for sim in sim_names:
name_to_data[sim] = _get_values(sim, "STFL").mean(axis=0)
fig = plt.figure()
k = 0
nrows = 3
ncols = 3
gs = gridspec.GridSpec(nrows, ncols)
name_to_handle = {}
for row in range(nrows):
for col in range(ncols):
i, j = i_arr[k], j_arr[k]
ax = fig.add_subplot(gs[row, col])
for sim in sim_names:
xnew = np.linspace(1, 13, 100)
power_smooth = spline(np.arange(1, 13),
name_to_data[sim][:, i, j],
xnew,
order=3)
h = ax.plot(np.arange(1, 13),
name_to_data[sim][:, i, j],
label=sim,
lw=2)
ax.xaxis.set_ticks(np.arange(1, 13))
ax.set_title("{0}, {1}".format(lons[i, j], lats[i, j]))
if not k:
name_to_handle[sim] = h[0]
if row == 1 and col == 0:
ax.set_ylabel("Streamflow ($m^3/s$)")
if row == 2 and col == 1:
ax.set_xlabel("Month")
k += 4
names = sorted(list(name_to_handle.keys()), key=lambda x: len(x))
fig.legend((name_to_handle[name] for name in names), names)
plt.show()
pass
def main():
from_guillimin_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/"
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
# interflow experiment (crcm5-hcd-rl-intfl)
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup3/all_files_in_one_folder"
# (crcm5-hcd-rl) lakes and rivers interacting no interflow
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup"
# lakes are full of land (crcm5-r)
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
# hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
# lakes and rivers not interacting
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-r_spinup2/all_files"
# hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf"
# Lake residence time decreased 5 times
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl-kd5_spinup/all_files_in_one_folder"
# latest interflow
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small/all_files_in_one_dir"
# hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf"
# interflow and ecoclimap
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap/all_files_in_one_dir"
# ecoclimap and era075
#data_folder = "quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075/all_files_in_one_dir"
#data_folder = os.path.join(from_guillimin_folder, data_folder)
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf"
#soil anisotropy 10000
#data_folder = os.path.join(from_guillimin_folder, "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf"
#soil anisotropy 10000 and do not care about bulk field capacity
#data_folder = os.path.join(from_guillimin_folder,
# "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf"
# On guillimin: crcm5-r
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-r/all_files_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-r.hdf5"
# On guillimin: crcm5-hcd-r
# data_folder = "/sb/project/xgk-345-ab/huziy/Output/quebec_0.1_crcm5-hcd-r/all_files_in_one_dir"
# hdf_file_path = "/sb/project/xgk-345-ab/huziy/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
# On guillimin: crcm5-hcd-rl
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-hcd-rl/all_files_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
# Skynet2: crcm5-hcd-rl at 0.4 deg
data_folder = "/b1_fs1/huziy/quebec-analysis-driven-crcm5-sims/quebec_0.4deg_crcm5-hcd-rl/all_files_in_one_dir"
hdf_file_path = "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.4_crcm5-hcd-rl.hdf5"
# Interflow spinup
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS/all_files_in_one_dir"
# hdf_file_path = \
# "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5"
# Interflow
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/all_files_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
# Interflow avoiding truncation
# data_folder = "/gs/project/ugh-612-aa/huziy/Output/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation/all_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5"
# For debugging on skynet
# data_folder = "/RESCUE/skynet3_rech1/huziy/test_export_to_hdf/test"
# hdf_file_path = "/RESCUE/skynet3_rech1/huziy/test_export_to_hdf/test.hdf5"
start_year = end_year = None
if len(sys.argv) >= 3:
data_folder = sys.argv[1]
hdf_file_path = sys.argv[2]
if len(sys.argv) >= 5:
start_year = int(sys.argv[3])
end_year = int(sys.argv[4])
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
var_names = [
"STFA", "PR", "TT", "AV", "AH", "TRAF", "TDRA", "I5", "I0", "I1", "I2",
"IMAV", "AS", "QQ", "UU", "VV", "WW", "GZ", "HR", "HU", "CLDP", "LC",
"LD", "AL", "L1"
]
# var_names = [ "I0", "I1", "I2", "IMAV"]
# var_names = ["AS", ]
# var_names = ["QQ", ]
# var_names = ["INTF", ]
# var_names = ["AL",]
# var_names = ["L1", "IMAV", "I5", "I1", "I2"]
# var_names = ["TRAF", ]
# var_names = ["STFA", ]
dm.export_to_hdf(var_list=var_names,
file_path=hdf_file_path,
mode="w",
start_year=start_year,
end_year=end_year)
export_static_fields_to_hdf(hdf_file=hdf_file_path,
data_folder=data_folder)
pass
def main():
path1 = "/home/huziy/skynet3_exec1/from_guillimin/new_outputs/quebec_86x86_0.5deg_wo_lakes_and_wo_lakeroff"
base_data_manager1 = Crcm5ModelDataManager(
samples_folder_path=path1, all_files_in_samples_folder=True)
basemap = base_data_manager1.get_omerc_basemap()
x, y = basemap(base_data_manager1.lons2D, base_data_manager1.lats2D)
start_date = datetime(1986, 1, 1)
end_date = datetime(1990, 12, 31)
months = list(range(4, 6))
traf1 = base_data_manager1.get_monthly_climatology(start_date=start_date,
end_date=end_date,
months=months,
varname="TRAF",
level=1)
traf1 = np.mean(traf1, axis=0)
tdra1 = base_data_manager1.get_monthly_climatology(start_date=start_date,
end_date=end_date,
months=months,
varname="TDRA",
level=1)
tdra1 = np.mean(tdra1, axis=0)
ratio1 = tdra1.copy()
denom_pos = (traf1 + tdra1) > 0
ratio1[denom_pos] /= (traf1 + tdra1)[denom_pos]
ratio1 = np.ma.masked_where(~denom_pos, ratio1)
print(np.min(ratio1), np.max(ratio1))
print(np.min(tdra1), np.max(tdra1))
print(np.min(traf1), np.max(traf1))
path2 = "/home/huziy/skynet3_exec1/from_guillimin/new_outputs/quebec_lowres_005"
base_data_manager2 = Crcm5ModelDataManager(
samples_folder_path=path2, all_files_in_samples_folder=True)
traf2 = base_data_manager2.get_monthly_climatology(start_date=start_date,
end_date=end_date,
months=months,
varname="TRAF",
level=1)
traf2 = np.mean(traf2, axis=0)
tdra2 = base_data_manager2.get_monthly_climatology(start_date=start_date,
end_date=end_date,
months=months,
varname="TDRA",
level=1)
tdra2 = np.mean(tdra2, axis=0)
ratio2 = tdra2.copy()
denom_pos = (traf2 + tdra2) > 0
ratio2[denom_pos] /= (traf2 + tdra2)[denom_pos]
ratio2 = np.ma.masked_where(~denom_pos, ratio2)
levels = [
0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.1, 0.15, 0.18, 0.20, 0.4, 0.6,
0.8, 1.0
]
cMap = get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cMap.N)
plt.figure()
min_change = 0.01
change = (ratio2 - ratio1) / ratio1
change = np.ma.masked_where(change <= min_change, change)
basemap.pcolormesh(x,
y,
change,
vmin=levels[0],
vmax=levels[-1],
norm=bn,
cmap=cMap)
plt.title("(intfl - no_intfl)/no_intfl")
plt.colorbar()
basemap.drawcoastlines()
plt.figure()
basemap.pcolormesh(x, y, ratio2)
plt.title("intfl")
plt.colorbar()
basemap.drawcoastlines()
plt.figure()
basemap.pcolormesh(x, y, ratio1)
plt.title("no_intfl")
plt.colorbar()
basemap.drawcoastlines()
plt.show()
#TODO: implement
pass
def main():
var_name_liquid = "I1"
var_name_solid = "I2"
#peirod of interest
start_year = 1979
end_year = 1988
#spatial averaging will be done over upstream points to the stations
selected_ids = [
"092715", "080101", "074903", "050304", "080104", "081007", "061905",
"041903", "040830", "093806", "090613", "081002", "093801", "080718"
]
selected_ids = ["074903"]
#simulation names corresponding to the paths
sim_names = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"]
sim_labels = [x.upper() for x in sim_names]
colors = ["blue", "violet"]
layer_widths = [
0.1, 0.2, 0.3, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5,
1.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0
]
layer_depths = np.cumsum(layer_widths)
paths = [
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup",
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples_all_in_one"
]
seasons = [[12, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
season_names = ["DJF", "MAM", "JJA", "SON"]
managers = [
Crcm5ModelDataManager(samples_folder_path=path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=(i == 0))
for i, path in enumerate(paths)
]
#share the cell manager
a_data_manager = managers[0]
assert isinstance(a_data_manager, Crcm5ModelDataManager)
cell_manager = a_data_manager.cell_manager
assert isinstance(cell_manager, CellManager)
for m in managers[1:]:
assert isinstance(m, Crcm5ModelDataManager)
m.cell_manager = cell_manager
#share the lake fraction field
lake_fraction = a_data_manager.lake_fraction
#selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905",
# "041903", "040830", "093806", "090613", "081002", "093801", "080718"]
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
stations = cehq_station.read_station_data(selected_ids=selected_ids,
start_date=start_date,
end_date=end_date)
#stations with corresponding model points
station_to_mp = a_data_manager.get_dataless_model_points_for_stations(
stations)
#figure out levels in soil
sim_label_to_profiles = {}
for s, mp in station_to_mp.items():
assert isinstance(mp, ModelPoint)
mask = (mp.flow_in_mask == 1) & (lake_fraction < 0.6)
fig = plt.figure()
fmt = ScalarFormatter(useMathText=True)
fmt.set_powerlimits([-2, 2])
print(mp.ix, mp.jy, s.id)
for m, label, color in zip(managers, sim_labels, colors):
assert isinstance(m, Crcm5ModelDataManager)
monthly_means_liquid = _get_cached_monthly_mean_fields(
label, start_year, end_year, var_name_liquid)
if monthly_means_liquid is None:
monthly_means_liquid = m.get_monthly_climatology_of_3d_field(
var_name=var_name_liquid,
start_year=start_year,
end_year=end_year)
_cache_monthly_mean_fields(monthly_means_liquid, label,
start_year, end_year,
var_name_liquid)
monthly_means_solid = _get_cached_monthly_mean_fields(
label, start_year, end_year, var_name_solid)
if monthly_means_solid is None:
monthly_means_solid = m.get_monthly_climatology_of_3d_field(
var_name=var_name_solid,
start_year=start_year,
end_year=end_year)
_cache_monthly_mean_fields(monthly_means_solid, label,
start_year, end_year,
var_name_solid)
profiles = [
monthly_means_liquid[i][mask, :].mean(axis=0) +
monthly_means_solid[i][mask, :].mean(axis=0) for i in range(12)
]
sim_label_to_profiles[label] = np.array(profiles)
x = [date2num(datetime(2001, month, 1)) for month in range(1, 13)]
y = layer_depths
y2d, x2d = np.meshgrid(y, x)
delta = (sim_label_to_profiles[sim_labels[1]] -
sim_label_to_profiles[sim_labels[0]]
) / sim_label_to_profiles[sim_labels[0]] * 100
#delta = np.ma.masked_where(delta < 0.1, delta)
cmap = my_colormaps.get_cmap_from_ncl_spec_file(
path="colormap_files/BlueRed.rgb", ncolors=10)
the_min = -6.0
the_max = 6.0
step = (the_max - the_min) / float(cmap.N)
plt.pcolormesh(x2d[:, :8],
y2d[:, :8],
delta[:, :8],
cmap=cmap,
vmin=the_min,
vmax=the_max) #, levels = np.arange(-6,7,1))
plt.gca().invert_yaxis()
plt.colorbar(ticks=np.arange(the_min, the_max + step, step))
plt.gca().set_ylabel("Depth (m)")
plt.gca().xaxis.set_major_formatter(DateFormatter("%b"))
#fig.tight_layout()
fig.savefig("soil_profile_upstream_of_{0}.pdf".format(s.id))
pass
def validate_as_is():
#years are inclusive
start_year = 1979
end_year = 1988
sim_name_list = ["crcm5-r", "crcm5-hcd-r", "crcm5-hcd-rl"]
rpn_folder_path_form = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup"
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
#select stations
selected_ids = None
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
stations = cehq_station.read_station_data(selected_ids=selected_ids,
start_date=start_date,
end_date=end_date)
varname = "STFL"
sim_name_to_manager = {}
sim_name_to_station_to_model_point = {}
dmManager = None
for sim_name in sim_name_list:
rpn_folder = rpn_folder_path_form.format(sim_name)
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True,
need_cell_manager=True)
sim_name_to_manager[sim_name] = dmManager
nc_path = os.path.join(nc_db_folder, sim_name)
nc_path = os.path.join(nc_path, "{0}_all.nc".format(varname))
st_to_mp = dmManager.get_model_points_for_stations(stations,
nc_path=nc_path,
varname=varname)
sim_name_to_station_to_model_point[sim_name] = st_to_mp
common_lake_fractions = dmManager.lake_fraction
from matplotlib.backends.backend_pdf import PdfPages
pp = PdfPages('comp_with_obs_as_is.pdf')
for s in stations:
#check the availability of the data
assert isinstance(s, Station)
#s.get_longest_continuous_series()
plt.figure()
obs_data = [s.date_to_value[d] for d in s.dates]
obs_ann_mean = np.mean(obs_data)
plt.plot(s.dates, [s.date_to_value[d] for d in s.dates],
label="Obs \n ann.mean = {0:.1f}".format(obs_ann_mean))
mp = None
for sim_name in sim_name_list:
manager = sim_name_to_manager[sim_name]
if s not in sim_name_to_station_to_model_point[sim_name]:
continue
mp = sim_name_to_station_to_model_point[sim_name][s]
plt.plot(mp.time,
mp.data[:, 0],
label="{0}: {1:.2f} \n ann.mean = {2:.1f}".format(
sim_name,
manager.lake_fraction[mp.flow_in_mask == 1].mean(),
mp.data[:, 0].mean()))
plt.legend()
if mp is None: continue
plt.title("{0}: point lake fraction={1:.4f}".format(
s.id, common_lake_fractions[mp.ix, mp.jy]))
pp.savefig()
pp.close()
def main(months=None):
start_year = 1979
end_year = 1988
import matplotlib.pyplot as plt
fig = plt.figure()
var_name_to_ij = {
"TT": (1, 0),
"PR": (2, 0),
"AH": (1, 1),
"AV": (2, 1),
"STFL": (0, 1),
"TRAF": (1, 2),
"TDRA": (2, 2)
}
fmt = ScalarFormatter(useMathText=True)
fmt.set_powerlimits((-2, 3))
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True)
basemap = dmManager.get_rotpole_basemap()
lons, lats = dmManager.lons2D, dmManager.lats2D
x, y = basemap(lons, lats)
lkfr = dmManager.lake_fraction
gs = GridSpec(3, 3, width_ratios=[1, 1, 1], height_ratios=[1, 1, 1])
#fig.suptitle("{0} minus {1}".format(sim_name2, sim_name1))
month_dates = [datetime(2001, m, 1) for m in months]
ax = None
for var_name in field_names:
coef = 1
if var_name == "PR":
coef = 24 * 60 * 60 * 1000
elif var_name in ["TDRA", "TRAF"]:
coef = 24 * 60 * 60
elif var_name == "AV":
coef = 3 * 60 * 60
#cmap.set_bad("0.6")
print(var_name)
v1 = _get_values(sim_name1,
var_name,
start_year,
end_year,
months=months)
v2 = _get_values(sim_name2,
var_name,
start_year,
end_year,
months=months)
#calculate annual means, for each year
v1 = v1.mean(axis=1)
v1m = v1.mean(axis=0)
v2 = v2.mean(axis=1)
v2m = v2.mean(axis=0)
i, j = var_name_to_ij[var_name]
ax = fig.add_subplot(gs[i, j])
dv = (v2m - v1m) * coef
t, p = stats.ttest_ind(v1, v2)
if var_name in ["STFL"]:
dv = np.ma.masked_where((lkfr >= 0.6), dv)
print(months)
i_interest, j_interest = np.where(dv > 53)
print(i_interest, j_interest)
dv = maskoceans(lons, lats, dv)
print(10 * "*")
elif var_name in ["TDRA", "TRAF"]:
dv = maskoceans(lons, lats, dv)
else:
pass
dv = np.ma.masked_where(
p > 1, dv) #mask changes not significant to the 10 % level
if not np.all(dv.mask):
print("{0}: min = {1}; max = {2}".format(var_name, dv.min(),
dv.max()))
print("")
max_abs = np.ma.max(np.abs(dv))
delta = max_abs
the_power = np.log10(delta)
the_power = np.ceil(the_power) if the_power >= 0 else np.floor(
the_power)
delta = np.ceil(delta / 10.0**the_power) * 10.0**the_power
while delta > 2 * max_abs:
delta /= 2.0
while 0.8 * delta > max_abs:
delta *= 0.8
step = delta / 5.0 #10 ** (the_power - 1) * 2 if the_power >= 0 else 10 ** the_power * 2
if var_name == "TT":
step = 0.06
delta = 0.66
levels = np.arange(-delta, delta + step, step)
cmap = my_colormaps.get_cmap_from_ncl_spec_file(
path="colormap_files/BlueRed.rgb", ncolors=len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N) if len(levels) > 0 else None
print("delta={0}; step={1}; the_power={2}".format(
delta, step, the_power))
the_img = basemap.pcolormesh(x,
y,
dv,
cmap=cmap,
vmin=-delta,
vmax=delta,
norm=bn)
#add colorbar
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", "5%", pad="3%")
cb = plt.colorbar(the_img, cax=cax, format=fmt)
#coast lines
basemap.drawcoastlines(ax=ax, linewidth=0.5)
assert isinstance(ax, Axes)
ax.set_title("$\Delta$ " + field_name_to_long_name[var_name] + "\n" +
"({0})".format(field_name_to_units[var_name]))
bbox_props = dict(boxstyle="rarrow,pad=0.3", fc="wheat", ec="b", lw=2)
label = "-".join([d.strftime("%b") for d in month_dates
]) + "\n({0}-{1})".format(start_year, end_year)
ax.annotate(label,
xy=(0.1, 0.85),
xycoords="figure fraction",
bbox=bbox_props,
font_properties=FontProperties(size=15, weight="bold"))
#fig.tight_layout()
if months is None:
fig.savefig("annual_mean_diffs_{0}_minus_{1}.jpeg".format(
sim_name2, sim_name1))
else:
print(month_dates, months)
fig.savefig("seasonal_mean_{0}_diffs_{1}_minus_{2}.png".format(
"_".join(map(str, months)), sim_name2, sim_name1))
pass
def main():
data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_198501_198612_0.1deg"
coord_file = os.path.join(data_path, "pm1985010100_00000000p")
manager = Crcm5ModelDataManager(samples_folder_path=data_path,
file_name_prefix="pm",
all_files_in_samples_folder=True)
selected_ids = [
"104001", "103715", "093806", "093801", "092715", "081006", "061502",
"040830", "080718"
]
start_date = datetime(1986, 1, 1)
end_date = datetime(1986, 12, 31)
stations = cehq_station.read_station_data(selected_ids=selected_ids,
start_date=start_date,
end_date=end_date)
plot_utils.apply_plot_params(width_pt=None,
height_cm=30.0,
width_cm=16,
font_size=10)
fig = plt.figure()
#two columns
gs = GridSpec(len(stations) // 2 + len(stations) % 2,
2,
hspace=0.4,
wspace=0.4)
line_model, line_obs = None, None
stations.sort(key=lambda x: x.latitude, reverse=True)
for i, s in enumerate(stations):
model_ts = manager.get_streamflow_timeseries_for_station(
s, start_date=start_date, end_date=end_date)
ax = fig.add_subplot(gs[i // 2, i % 2])
assert isinstance(model_ts, TimeSeries)
#[t, m_data] = model_ts.get_daily_normals()
#[t, s_data] = s.get_daily_normals()
assert isinstance(s, Station)
#climatologies
#line_model = ax.plot(t, m_data, label = "Model (CRCM5)", lw = 3, color = "b")
#line_obs = ax.plot(t, s_data, label = "Observation", lw = 3, color = "r")
model_ts = model_ts.get_ts_of_daily_means()
print(model_ts.time[0], model_ts.time[-1])
print(model_ts.data[0:10])
mod_vals = model_ts.get_data_for_dates(s.dates)
print(mod_vals[:20])
print("+" * 20)
assert len(mod_vals) == len(s.dates)
#model_acorr = [1] + [ np.corrcoef([mod_vals[i:], mod_vals[:-i]])[0,1] for i in range(1,len(mod_vals)) ]
#obs_acorr = [1] + [ np.corrcoef([s.values[i:], s.values[:-i]])[0,1] for i in range(1,len(mod_vals)) ]
npoints = np.array(list(range(len(mod_vals), 0, -1)))
model_acorr = np.correlate(mod_vals, mod_vals, mode="full")
model_acorr = model_acorr[len(model_acorr) / 2:] / max(model_acorr)
model_acorr /= npoints
obs_acorr = np.correlate(s.values, s.values, mode="full")
obs_acorr = obs_acorr[len(obs_acorr) / 2:] / max(obs_acorr)
obs_acorr /= npoints
print(len(model_acorr), len(s.dates))
line_model = ax.plot(s.dates,
model_acorr,
label="Model (CRCM5)",
lw=1,
color="b")
line_obs = ax.plot(s.dates,
obs_acorr,
label="Observation",
lw=3,
color="r",
alpha=0.5)
#ax.annotate( "r = {0:.2f}".format( float( np.corrcoef([mod_vals, s.values])[0,1] )), xy = (0.7,0.8), xycoords= "axes fraction")
ax.set_title(
"%s: da_diff=%.2f %%, dist = %.1f" %
(s.id, (-s.drainage_km2 + model_ts.metadata["acc_area_km2"]) /
s.drainage_km2 * 100.0, model_ts.metadata["distance_to_obs"]))
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.xaxis.set_major_locator(MonthLocator(bymonth=list(range(1, 13, 2))))
lines = (line_model, line_obs)
labels = ("Model (CRCM5)", "Observation")
fig.legend(lines, labels)
fig.savefig("acorr_without_lakes_0.1deg_1year.png")
pass
def validate_yearmax_ice_cover_from_hostetler_with_glerl(
path="/home/huziy/skynet3_rech1/CRCM_GL_simulation/all_files",
start_year=2003,
end_year=2009):
model_manager = Crcm5ModelDataManager(samples_folder_path=path,
all_files_in_samples_folder=True)
varname = "LC"
hl_icecov_yearly_max = model_manager.get_yearmax_fields(
start_year=start_year, end_year=end_year, var_name=varname)
hl_icecov_yearly_max_clim = np.mean(list(hl_icecov_yearly_max.values()),
axis=0)
# Get Nemo manager here only for coordinates and mask
nemo_manager = NemoYearlyFilesManager(
folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012",
suffix="icemod.nc")
lon2d, lat2d, bmp = nemo_manager.get_coords_and_basemap()
# Interpolate hostetler's lake fraction to the model's grid
hl_icecov_yearly_max_clim = model_manager.interpolate_data_to(
hl_icecov_yearly_max_clim, lon2d, lat2d)
hl_icecov_yearly_max_clim = np.ma.masked_where(~nemo_manager.lake_mask,
hl_icecov_yearly_max_clim)
model_lake_avg_ts = []
for the_year in range(start_year, end_year + 1):
model_lake_avg_ts.append(
hl_icecov_yearly_max[the_year][nemo_manager.lake_mask].mean())
# plt.figure()
xx, yy = bmp(lon2d.copy(), lat2d.copy())
# im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc)
# bmp.colorbar(im)
# Read and interpolate obs
path_to_obs = "/RESCUE/skynet3_rech1/huziy/nemo_obs_for_validation/glerl_icecov.nc"
obs_varname = "ice_cover"
obs_lake_avg_ts = []
with Dataset(path_to_obs) as ds:
time_var = ds.variables["time"]
lons_obs = ds.variables["lon"][:]
lats_obs = ds.variables["lat"][:]
dates = num2date(time_var[:], time_var.units)
nx, ny = lons_obs.shape
data = ds.variables[obs_varname][:]
data = np.ma.masked_where((data > 100) | (data < 0), data)
print(data.min(), data.max())
panel = pd.Panel(data=data,
items=dates,
major_axis=range(nx),
minor_axis=range(ny))
panel = panel.select(lambda d: start_year <= d.year <= end_year)
the_max_list = []
for key, g in panel.groupby(lambda d: d.year, axis="items"):
the_max_field = np.ma.max(np.ma.masked_where(
(g.values > 100) | (g.values < 0), g.values),
axis=0)
obs_lake_avg_ts.append(the_max_field.mean())
the_max_list.append(the_max_field)
obs_yearmax_ice_conc = np.ma.mean(the_max_list, axis=0) / 100.0
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_obs.flatten(),
lats_obs.flatten())
ktree = cKDTree(list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lon2d.flatten(),
lat2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)))
obs_yearmax_ice_conc_interp = obs_yearmax_ice_conc.flatten(
)[inds].reshape(lon2d.shape)
obs_yearmax_ice_conc_interp = np.ma.masked_where(
~nemo_manager.lake_mask, obs_yearmax_ice_conc_interp)
# plt.figure()
# b = Basemap()
# xx, yy = b(lons_obs, lats_obs)
# im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc)
# b.colorbar(im)
# b.drawcoastlines()
# Plot as usual: model, obs, model - obs
img_folder = Path("nemo/hostetler")
if not img_folder.is_dir():
img_folder.mkdir()
img_file = img_folder.joinpath(
"validate_yearmax_icecov_hl_vs_glerl_{}-{}.pdf".format(
start_year, end_year))
fig = plt.figure()
gs = GridSpec(2, 4, width_ratios=[1, 1, 1, 0.05])
all_axes = []
cmap = cm.get_cmap("jet", 10)
diff_cmap = cm.get_cmap("RdBu_r", 10)
# Model
ax = fig.add_subplot(gs[0, 0])
ax.set_title("Hostetler")
bmp.pcolormesh(xx,
yy,
hl_icecov_yearly_max_clim,
cmap=cmap,
vmin=0,
vmax=1)
all_axes.append(ax)
# Obs
ax = fig.add_subplot(gs[0, 2])
ax.set_title("GLERL")
im = bmp.pcolormesh(xx,
yy,
obs_yearmax_ice_conc_interp,
cmap=cmap,
vmin=0,
vmax=1)
all_axes.append(ax)
plt.colorbar(im, cax=fig.add_subplot(gs[0, -1]))
# Biases
ax = fig.add_subplot(gs[1, :])
ax.set_title("Hostetler - GLERL")
im = bmp.pcolormesh(xx,
yy,
hl_icecov_yearly_max_clim -
obs_yearmax_ice_conc_interp,
cmap=diff_cmap,
vmin=-1,
vmax=1)
bmp.colorbar(im, ax=ax)
all_axes.append(ax)
for the_ax in all_axes:
bmp.drawcoastlines(ax=the_ax)
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
# Plot lake aversged ice concentrations
fig = plt.figure()
plt.gca().get_xaxis().get_major_formatter().set_useOffset(False)
plt.plot(range(start_year, end_year + 1),
model_lake_avg_ts,
"b",
lw=2,
label="Hostetler")
plt.plot(range(start_year, end_year + 1),
np.asarray(obs_lake_avg_ts) / 100.0,
"r",
lw=2,
label="GLERL")
plt.grid()
plt.legend(loc=3)
fig.savefig(str(
img_folder.joinpath(
"lake_avg_iceconc_hostetler_offline_vs_GLERL.pdf")),
bbox_inches="tight")
def main():
swe_obs_manager = SweDataManager(var_name="SWE")
data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3"
coord_file = os.path.join(data_path, "pm1985050100_00000000p")
managerLowRes = Crcm5ModelDataManager(samples_folder_path=data_path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=True)
data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_highres_spinup_12_month_without_lakes_v3"
coord_file = os.path.join(data_path, "pm1985050100_00000000p")
managerHighRes = Crcm5ModelDataManager(samples_folder_path=data_path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=True)
start_year = 1987
end_year = 1987
months = [1, 2, 12]
rot_lat_lon = RotatedLatLon(lon1=-68, lat1=52, lon2=16.65, lat2=0.0)
basemap = Basemap(projection="omerc",
llcrnrlon=managerHighRes.lons2D[0, 0],
llcrnrlat=managerHighRes.lats2D[0, 0],
urcrnrlon=managerHighRes.lons2D[-1, -1],
urcrnrlat=managerHighRes.lats2D[-1, -1],
lat_1=rot_lat_lon.lat1,
lat_2=rot_lat_lon.lat2,
lon_1=rot_lat_lon.lon1,
lon_2=rot_lat_lon.lon2,
no_rot=True)
swe_obs = swe_obs_manager.get_mean(start_year, end_year, months=months)
obs_ihr = swe_obs_manager.interpolate_data_to(swe_obs,
managerHighRes.lons2D,
managerHighRes.lats2D,
nneighbours=1)
obs_ilr = swe_obs_manager.interpolate_data_to(swe_obs,
managerLowRes.lons2D,
managerLowRes.lats2D,
nneighbours=1)
lowResSwe = managerLowRes.get_mean_field(start_year,
end_year,
months=months,
var_name="I5")
lowResErr = (lowResSwe - obs_ilr)
lowResErr[obs_ilr > 0] /= obs_ilr[obs_ilr > 0]
lowResErr = np.ma.masked_where(obs_ilr <= 0, lowResErr)
highResSwe = managerHighRes.get_mean_field(start_year,
end_year,
months=months,
var_name="I5")
highResErr = (highResSwe - obs_ihr)
highResErr[obs_ihr > 0] /= obs_ihr[obs_ihr > 0]
highResErr = np.ma.masked_where(obs_ihr <= 0, highResErr)
upscaledHighResSwe = upscale(managerHighRes, managerLowRes, highResSwe)
upscaledHighResErr = upscaledHighResSwe - obs_ilr
good_points = obs_ilr > 0
upscaledHighResErr[good_points] /= obs_ilr[good_points]
upscaledHighResErr = np.ma.masked_where(~good_points, upscaledHighResErr)
plot_and_compare_2fields(lowResSwe,
"low res",
upscaledHighResSwe,
"high res (upscaled)",
basemap=basemap,
manager1=managerLowRes,
manager2=managerLowRes)
plot_and_compare_2fields(lowResErr,
"low res err",
upscaledHighResErr,
"high res (upscaled) err",
basemap=basemap,
manager1=managerLowRes,
manager2=managerLowRes,
clevs=np.arange(-1, 1.2, 0.2))
plot_and_compare_2fields(lowResSwe,
"low res",
highResSwe,
"high res",
basemap=basemap,
manager1=managerLowRes,
manager2=managerHighRes)
plot_and_compare_2fields(lowResErr,
"low res err",
highResErr,
"high res err",
basemap=basemap,
manager1=managerLowRes,
manager2=managerHighRes,
clevs=np.arange(-1, 1.2, 0.2))
plt.show()
def validate_seas_mean_lswt_from_hostetler_and_nemo_with_homa(
hl_data_path="/home/huziy/skynet3_rech1/CRCM_GL_simulation/all_files",
start_year=2003, end_year=2009):
crcm5_model_manager = Crcm5ModelDataManager(samples_folder_path=hl_data_path, all_files_in_samples_folder=True)
varname = "sohefldo"
season = "Summer"
season_to_months = {season: range(6, 9)}
season_months = list(season_to_months[season])
# Get Nemo manager here only for coordinates and mask
nemo_offline_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012",
suffix="grid_T.nc")
nemo_crcm5_manager = NemoYearlyFilesManager(
folder="/home/huziy/skynet3_rech1/one_way_coupled_nemo_outputs_1979_1985",
suffix="grid_T.nc")
lon2d, lat2d, bmp = nemo_offline_manager.get_coords_and_basemap()
# Interpolate
# get nemo sst
nemo_offline_sst = nemo_offline_manager.get_seasonal_clim_field(start_year=start_year,
end_year=end_year,
season_to_months=season_to_months,
varname=varname)
nemo_crcm5_sst = nemo_crcm5_manager.get_seasonal_clim_field(start_year=start_year,
end_year=end_year,
season_to_months=season_to_months,
varname=varname)
nemo_offline_sst = np.ma.masked_where(~nemo_offline_manager.lake_mask, nemo_offline_sst[season])
nemo_crcm5_sst = np.ma.masked_where(~nemo_offline_manager.lake_mask, nemo_crcm5_sst[season])
# plt.figure()
xx, yy = bmp(lon2d.copy(), lat2d.copy())
# im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc)
# bmp.colorbar(im)
vmin = min(nemo_offline_sst.min(), nemo_crcm5_sst.min())
vmax = max(nemo_offline_sst.max(), nemo_crcm5_sst.max())
# plt.figure()
# b = Basemap()
# xx, yy = b(lons_obs, lats_obs)
# im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc)
# b.colorbar(im)
# b.drawcoastlines()
# Plot as usual: model, obs, model - obs
img_folder = Path("nemo")
if not img_folder.is_dir():
img_folder.mkdir()
img_file = img_folder.joinpath("{}_{}_nemo-offline_vs_nemo-crcm5_{}-{}.png".format(season.lower(),
varname, start_year, end_year))
fig = plt.figure()
gs = GridSpec(1, 3, width_ratios=[1, 1, 0.05])
all_axes = []
cmap = cm.get_cmap("jet", 10)
locator = MaxNLocator(nbins=10)
ticks = locator.tick_values(vmin=vmin, vmax=vmax)
norm = BoundaryNorm(boundaries=ticks, ncolors=len(ticks) - 1)
# Model, Hostetler
ax = fig.add_subplot(gs[0, 0])
ax.set_title("NEMO+CRCM5")
bmp.pcolormesh(xx, yy, nemo_crcm5_sst, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
#
ax = fig.add_subplot(gs[0, 1])
ax.set_title("NEMO-offline")
im = bmp.pcolormesh(xx, yy, nemo_offline_sst, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
# Obs: Homa
# ax = fig.add_subplot(gs[0, 2])
# ax.set_title("MODIS")
# im = bmp.pcolormesh(xx, yy, obs_sst_clim, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
# all_axes.append(ax)
plt.colorbar(im, cax=fig.add_subplot(gs[0, -1]), ticks=ticks)
for the_ax in all_axes:
bmp.drawcoastlines(ax=the_ax)
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
def diagnose(station_ids=None, model_data_path=None):
manager = Crcm5ModelDataManager(samples_folder_path=model_data_path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=True)
nx, ny = manager.lons2D.shape
rot_lat_lon = RotatedLatLon(lon1=-68, lat1=52, lon2=16.65, lat2=0.0)
x00, y00 = rot_lat_lon.toProjectionXY(manager.lons2D[0, 0],
manager.lats2D[0, 0])
x10, y10 = rot_lat_lon.toProjectionXY(manager.lons2D[1, 0],
manager.lats2D[1, 0])
x01, y01 = rot_lat_lon.toProjectionXY(manager.lons2D[0, 1],
manager.lats2D[0, 1])
dx = x10 - x00
dy = y01 - y00
print("dx, dy = {0}, {1}".format(dx, dy))
areas = rot_lat_lon.get_areas_of_gridcells(
dx, dy, nx, ny, y00, 1) #1 -since the index is starting from 1
print(areas[0, 0])
start_date = datetime(1986, 1, 1)
end_date = datetime(1986, 12, 31)
stations = cehq_station.read_station_data(selected_ids=station_ids,
start_date=start_date,
end_date=end_date)
stations.sort(key=lambda x: x.latitude, reverse=True)
for i, s in enumerate(stations):
fig = plt.figure()
#3 columns
gs = GridSpec(5,
3,
hspace=0.2,
wspace=0.2,
right=0.98,
left=0.1,
top=0.98)
model_ts = manager.get_streamflow_timeseries_for_station(
s, start_date=start_date, end_date=end_date, nneighbours=9)
print(model_ts.time[0], model_ts.time[-1])
i_model0, j_model0 = model_ts.metadata["ix"], model_ts.metadata["jy"]
mask = manager.get_mask_for_cells_upstream(i_model0, j_model0)
#hydrographs
ax = fig.add_subplot(gs[0, 0])
plot_streamflows(ax, s, model_ts)
#relative error
ax = fig.add_subplot(gs[1, 0])
plot_streamflow_re(ax, s, model_ts)
#directions
plot_directions_and_positions(fig.add_subplot(gs[:2, 1]),
s,
model_ts,
manager,
rot_lat_lon,
mask=mask)
#runoff
ax = fig.add_subplot(gs[2, 0])
plot_runoff(ax, manager, areas, model_ts, mask=mask)
#runoff from gldas
ax = fig.add_subplot(gs[2, 1])
#plot_gldas_runoff(ax, manager, areas, model_ts, mask = mask)
#temperature
ax_temp = fig.add_subplot(gs[3, 0])
ax_prec = fig.add_subplot(gs[4, 0])
plot_total_precip_and_temp_re_1d(ax_prec,
ax_temp,
manager,
rot_lat_lon,
areas,
model_ts,
mask=mask)
#swe timeseries
ax = fig.add_subplot(gs[3, 1])
plot_swe_timeseries(ax, manager, areas, model_ts, mask=mask)
#print np.where(mask == 1)
print("(i, j) = ({0}, {1})".format(model_ts.metadata["ix"],
model_ts.metadata["jy"]))
fig.savefig("diagnose_{0}_{1:.2f}deg.pdf".format(s.id, dx))
def validate_daily_climatology():
"""
"""
#years are inclusive
start_year = 1979
end_year = 1988
#sim_name_list = ["crcm5-r", "crcm5-hcd-r", "crcm5-hcd-rl"]
sim_name_list = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"]
rpn_folder_paths = [
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup".format(sim_name_list[0]),
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup2/Samples_all_in_one_folder".format(
sim_name_list[1])
]
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
#select stations
selected_ids = None
selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905",
"041903", "040830", "093806", "090613", "081002", "093801", "080718"]
selected_ids = ["074903", ]
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
selected_ids = None
stations = cehq_station.read_station_data(selected_ids=selected_ids,
start_date=start_date, end_date=end_date
)
stations_hydat = cehq_station.read_hydat_station_data(folder_path="/home/huziy/skynet3_rech1/HYDAT",
start_date=start_date, end_date=end_date)
stations.extend(stations_hydat)
varname = "STFL"
sim_name_to_manager = {}
sim_name_to_station_to_model_point = {}
day_stamps = Station.get_stamp_days(2001)
sweManager = SweDataManager(var_name="SWE")
cruTempManager = CRUDataManager(lazy=True)
cruPreManager = CRUDataManager(var_name="pre", lazy=True,
path="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.pre.dat.nc")
#common lake fractions when comparing simulations on the same grid
all_model_points = []
cell_manager = None
for sim_name, rpn_folder in zip(sim_name_list, rpn_folder_paths):
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm",
all_files_in_samples_folder=True, need_cell_manager=cell_manager is None)
#here using the fact that all the simulations are on the same grid
if cell_manager is None:
cell_manager = dmManager.cell_manager
else:
dmManager.cell_manager = cell_manager
#determine comon lake fractions, so it is not taken from the trivial case lf = 0, but note
#this has only sense when all the simulations were performed on the same grid
sim_name_to_manager[sim_name] = dmManager
nc_sim_folder = os.path.join(nc_db_folder, sim_name)
nc_path = os.path.join(nc_sim_folder, "{0}_all.nc4".format(varname))
#In general there are several model points corresponding to a given station
st_to_mp = dmManager.get_model_points_for_stations(stations, sim_name=sim_name,
nc_path=nc_path,
nc_sim_folder=nc_sim_folder,
set_data_to_model_points=True)
print("got model points for stations")
sim_name_to_station_to_model_point[sim_name] = st_to_mp
#save model points to a list of all points
for s, mps in st_to_mp.items():
assert isinstance(s, Station)
for mp in mps:
assert isinstance(mp, ModelPoint)
#calculate upstream swe if needed
if s.mean_swe_upstream_daily_clim is None:
s.mean_swe_upstream_daily_clim = sweManager.get_mean_upstream_timeseries_daily(mp, dmManager,
stamp_dates=day_stamps)
#These are taken from CRU dataset, only monthly data are available
s.mean_temp_upstream_monthly_clim = cruTempManager.get_mean_upstream_timeseries_monthly(mp,
dmManager)
s.mean_prec_upstream_monthly_clim = cruPreManager.get_mean_upstream_timeseries_monthly(mp,
dmManager)
print("Calculated observed upstream mean values...")
all_model_points.extend(mps)
print("imported input data successfully, plotting ...")
#for tests
#test(sim_name_to_station_to_model_point)
#select only stations which have corresponding model points
stations = list(sim_name_to_station_to_model_point[sim_name_list[0]].keys())
from matplotlib.backends.backend_pdf import PdfPages
for s in stations:
years = s.get_list_of_complete_years()
if len(years) < 6: continue #skip stations with less than 6 continuous years of data
pp = PdfPages("nc_diagnose_{0}.pdf".format(s.id))
#plot hydrographs
fig = plt.figure()
gs = gridspec.GridSpec(3, 3, left=0.05, hspace=0.3, wspace=0.2)
ax_stfl = fig.add_subplot(gs[0, 0])
labels, handles = plot_hydrographs(ax_stfl, s, sim_name_to_station_to_model_point,
day_stamps=day_stamps, sim_names=sim_name_list
)
plt.setp(ax_stfl.get_xticklabels(), visible=False) #do not show ticklabels for upper rows
fig.legend(handles, labels, "lower right")
#plot swe 1d compare with obs
ax_swe = fig.add_subplot(gs[1, 0], sharex=ax_stfl)
plot_swe_1d_compare_with_obs(ax_swe, s, sim_name_to_station_to_model_point,
day_stamps=day_stamps, sim_names=sim_name_list)
#plot mean temp 1d compare with obs -- here plot biases directly...??
ax_temp = fig.add_subplot(gs[0, 2])
plot_temp_1d_compare_with_obs(ax_temp, s, sim_name_to_station_to_model_point, sim_names=sim_name_list)
plt.setp(ax_temp.get_xticklabels(), visible=False) #do not show ticklabels for upper rows
#plot mean precip 1d compare with obs -- here plot biases directly...??
ax = fig.add_subplot(gs[1, 2], sharex=ax_temp)
plot_precip_1d_compare_with_obs(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list)
#plot mean Surface and subsurface runoff
ax = fig.add_subplot(gs[0, 1], sharex=ax_stfl)
plot_surf_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list)
plt.setp(ax.get_xticklabels(), visible=False) #do not show ticklabels for upper rows
ax = fig.add_subplot(gs[1, 1], sharex=ax_stfl)
plot_subsurf_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list)
plt.setp(ax.get_xticklabels(), visible=False) #do not show ticklabels for upper rows
ax = fig.add_subplot(gs[2, 1], sharex=ax_stfl)
plot_total_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list)
pp.savefig()
#plot flow direction and basin boundaries
fig = plt.figure()
gs = gridspec.GridSpec(1, 2, right=0.99, bottom=0.001)
ax = fig.add_subplot(gs[0, 1])
plot_flow_directions_and_basin_boundaries(ax, s, sim_name_to_station_to_model_point,
sim_name_to_manager=sim_name_to_manager)
pp.savefig()
#plot 2d correlation between wind speed and measured streamflow at the station
pp.close()
def main():
import matplotlib.pyplot as plt
#path = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/test_with_lakeroff"
#path = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup"
path = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r/all_in_one_folder"
manager = Crcm5ModelDataManager(samples_folder_path=path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=True)
allData = {}
field2d, allStfl = manager.get_mean_2d_field_and_all_data(var_name="STFL")
allData["FACC"] = manager.accumulation_area_km2
traf2d, allTraf = manager.get_mean_2d_field_and_all_data(
var_name="TRAF", level=5, level_kind=level_kinds.ARBITRARY)
tdra2d, allTdra = manager.get_mean_2d_field_and_all_data(
var_name="TDRA", level=5, level_kind=level_kinds.ARBITRARY)
swe2d, allSwe = manager.get_mean_2d_field_and_all_data(
var_name="I5", level=-1, level_kind=level_kinds.ARBITRARY)
pre2d, allPr = manager.get_mean_2d_field_and_all_data(var_name="PR")
swsr2d, allSwsr = manager.get_mean_2d_field_and_all_data(var_name="SWSR")
swsl2d, allSwsl = manager.get_mean_2d_field_and_all_data(var_name="SWSL")
upin2d, allUpin = manager.get_mean_2d_field_and_all_data(var_name="UPIN")
#gwdi2d,allGwdi = manager.get_mean_2d_field_and_all_data(var_name="GWDI")
swst2d, allSwst = manager.get_mean_2d_field_and_all_data(var_name="SWST")
for k, v in allTraf.items():
allTraf[k] = v * manager.cell_area * 1.0e-3
for k, v in allTdra.items():
allTdra[k] = v * manager.cell_area * 1.0e-3
for k, v in allPr.items():
allPr[k] = v * manager.cell_area
#for k,v in allImav.iteritems():
# allImav[k] = v * manager.cell_area * 1.0e-3
for k, v in allSwe.items():
allSwe[k] *= v * manager.cell_area * 1.0e-3 * 1e-6
allData["TRAF"] = allTraf
allData["TDRA"] = allTdra
allData["PR"] = allPr
allData["I5"] = allSwe
allData["STFL"] = allStfl
allData["SWSR"] = allSwsr
allData["SWSL"] = allSwsl
allData["UPIN"] = allUpin
#allData["GWDI"] = allGwdi
allData["SWST"] = allSwst
#read in slope and channel length from the geophysics file
r = RPN(
"/home/huziy/skynet3_rech1/geof_lake_infl_exp/geophys_Quebec_0.1deg_260x260_with_dd_v6"
)
slp = r.get_first_record_for_name("SLOP")[20:-20, 20:-20]
slp[slp < 1e-4] = 1e-4
allData["SLOP"] = slp
allData["LENG"] = r.get_first_record_for_name("LENG")[20:-20, 20:-20]
allData["LKOU"] = r.get_first_record_for_name("LKOU")[20:-20, 20:-20]
r.close()
basemap = manager.get_omerc_basemap(resolution="c")
lon, lat = manager.lons2D, manager.lats2D
fig = plt.figure()
x, y = basemap(lon, lat)
field2d = np.ma.masked_where(field2d <= 0.1, field2d)
#field2d = np.ma.masked_where((field2d < 10000) | (upin2d > 100000), field2d)
basemap.pcolormesh(x, y, field2d)
plt.colorbar()
basemap.drawcoastlines()
ax = plt.gca()
TimeseriesPlotter(name_to_date_to_field=allData,
basemap=basemap,
lons2d=lon,
lats2d=lat,
ax=ax,
cell_manager=manager.cell_manager,
data_manager=manager)
fig = plt.figure()
x, y = basemap(lon, lat)
fig = plt.figure()
x, y = basemap(lon, lat)
basemap.pcolormesh(x, y, traf2d)
plt.title("Total traf")
plt.colorbar()
basemap.drawcoastlines()
fig = plt.figure()
x, y = basemap(lon, lat)
basemap.pcolormesh(x, y, swst2d)
plt.title("SWST")
plt.colorbar()
basemap.drawcoastlines()
traf2dsoil, dummy = manager.get_mean_2d_field_and_all_data(
var_name="TRAF", level=1, level_kind=level_kinds.ARBITRARY)
traf2dlake, dummy = manager.get_mean_2d_field_and_all_data(
var_name="TRAF", level=6, level_kind=level_kinds.ARBITRARY)
fig = plt.figure()
x, y = basemap(lon, lat)
basemap.pcolormesh(x, y, upin2d)
plt.title("UPIN")
plt.colorbar()
basemap.drawcoastlines()
plt.show()
pass
def plot_at_indices(ix, jy):
var_name_liquid = "I1"
var_name_solid = "I2"
#peirod of interest
start_year = 1979
end_year = 1988
#simulation names corresponding to the paths
sim_names = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"]
sim_labels = [x.upper() for x in sim_names]
layer_widths = [
0.1, 0.2, 0.3, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5,
1.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0
]
layer_depths = np.cumsum(layer_widths)
paths = [
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup",
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples_all_in_one"
]
managers = [
Crcm5ModelDataManager(samples_folder_path=path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=(i == 0))
for i, path in enumerate(paths)
]
#share the cell manager
a_data_manager = managers[0]
assert isinstance(a_data_manager, Crcm5ModelDataManager)
cell_manager = a_data_manager.cell_manager
assert isinstance(cell_manager, CellManager)
for m in managers[1:]:
assert isinstance(m, Crcm5ModelDataManager)
m.cell_manager = cell_manager
#share the lake fraction field
lake_fraction = a_data_manager.lake_fraction
selected_ids = [
"092715", "080101", "074903", "050304", "080104", "081007", "061905",
"041903", "040830", "093806", "090613", "081002", "093801", "080718"
]
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
stations = cehq_station.read_station_data(selected_ids=selected_ids,
start_date=start_date,
end_date=end_date)
#stations with corresponding model points
station_to_mp = a_data_manager.get_dataless_model_points_for_stations(
stations)
#figure out levels in soil
sim_label_to_profiles = {}
fig = plt.figure()
fmt = ScalarFormatter(useMathText=True)
fmt.set_powerlimits([-2, 2])
for m, label in zip(managers, sim_labels):
assert isinstance(m, Crcm5ModelDataManager)
monthly_means_liquid = _get_cached_monthly_mean_fields(
label, start_year, end_year, var_name_liquid)
if monthly_means_liquid is None:
monthly_means_liquid = m.get_monthly_climatology_of_3d_field(
var_name=var_name_liquid,
start_year=start_year,
end_year=end_year)
_cache_monthly_mean_fields(monthly_means_liquid, label, start_year,
end_year, var_name_liquid)
monthly_means_solid = _get_cached_monthly_mean_fields(
label, start_year, end_year, var_name_solid)
if monthly_means_solid is None:
monthly_means_solid = m.get_monthly_climatology_of_3d_field(
var_name=var_name_solid,
start_year=start_year,
end_year=end_year)
_cache_monthly_mean_fields(monthly_means_solid, label, start_year,
end_year, var_name_solid)
profiles = [
monthly_means_liquid[i][ix, jy, :] +
monthly_means_solid[i][ix, jy, :] for i in range(12)
]
sim_label_to_profiles[label] = np.array(profiles)
x = list(range(12))
y = layer_depths
y2d, x2d = np.meshgrid(y, x)
plt.contourf(
x2d, y2d, sim_label_to_profiles[sim_labels[1]] -
sim_label_to_profiles[sim_labels[0]])
plt.gca().invert_yaxis()
plt.colorbar()
#fig.tight_layout()
fig.savefig("soil_profile_at_ix={0};jy={1}.pdf".format(ix, jy))
