def main():
start_year = 1979
end_year = 1988
field_names = ["TT", "PR", "AH", "AV", "STFL", "STFA", "TRAF", "TDRA"]
file_name_prefixes = ["dm", "pm", "pm", "pm", "pm", "pm", "pm", "pm"]
sim_name = "crcm5-hcd-rl-intfl"
rpn_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup".format(
sim_name)
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
nc_sim_folder = os.path.join(nc_db_folder, sim_name)
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True)
pmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="pm",
all_files_in_samples_folder=True)
for field_name, fname_prefix in zip(field_names, file_name_prefixes):
if fname_prefix == "pm":
manager = pmManager
elif fname_prefix == "dm":
manager = dmManager
else:
raise Exception("Unknown file type...")
do_export(start_year,
end_year,
var_name=field_name,
dataManager=manager,
nc_sim_folder=nc_sim_folder)
pass
def calculate_daily_mean_fields():
dates, clim_fields_hcd_rl = Crcm5ModelDataManager.hdf_get_daily_climatological_fields(
hdf_db_path="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf",
var_name="STFL", level_index=None, use_grouping=True, start_year=1979, end_year=1988)
dates, clim_fields_hcd_rl_intfl = Crcm5ModelDataManager.hdf_get_daily_climatological_fields(
hdf_db_path="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup.hdf",
var_name="STFL", level_index=None, use_grouping=True, start_year=1979, end_year=1988)
# Calculate mean timeseries and take a difference
ts_hcd_rl = []
for field in clim_fields_hcd_rl:
field = np.asarray(field)
ts_hcd_rl.append(field[field >= 0].mean())
ts_hcd_rl = np.asarray(ts_hcd_rl)
ts_hcd_rl_intfl = []
for field in clim_fields_hcd_rl_intfl:
field = np.asarray(field)
ts_hcd_rl_intfl.append(field[field >= 0].mean())
ts_hcd_rl_intfl = np.asarray(ts_hcd_rl_intfl)
daily_diff_data = (ts_hcd_rl_intfl - ts_hcd_rl) / ts_hcd_rl * 100
daily_diff_ts = pd.TimeSeries(data=daily_diff_data, index=dates)
monthly_diff_ts = daily_diff_ts.resample("M", how="mean")
month_vals = np.asarray([d.month for d in dates])
month_mean_for_day = np.zeros(len(month_vals))
fig = plt.figure(figsize=(20, 6))
ax = plt.gca()
assert isinstance(ax, Axes)
ax.set_ylabel("$\left(Q_{\\rm hcd-rl-intfl} - Q_{\\rm hcd-rl}\\right)/Q_{\\rm hcd-rl} \\times 100\%$")
ax.plot(dates, daily_diff_data)
ax.plot(ax.get_xlim(), [0, 0], "k-")
# plot a mean for each month
for the_month in range(1, 13):
month_mean_for_day[month_vals == the_month] = monthly_diff_ts[the_month - 1]
month_dates = list(filter(lambda d: d.month == the_month, dates))
month_vals = np.ones((len(month_dates),)) * monthly_diff_ts[the_month - 1]
ax.plot(month_dates, month_vals, "r", lw=1.5)
ax.grid("on")
ax.xaxis.set_major_formatter(DateFormatter("%b/%d"))
ax.xaxis.set_major_locator(DayLocator(bymonthday=1))
plt.tight_layout()
plt.savefig("intfl_diff.png")
def main():
rpn_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
start_year = 1979
end_year = 1988
sim_name = "crcm5-r"
#dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True)
pmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="pm",
all_files_in_samples_folder=True)
#plot results
assert isinstance(pmManager, Crcm5ModelDataManager)
lons, lats = pmManager.lons2D, pmManager.lats2D
stfl_mask = pmManager.cbf < 0
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons, lats2d=lats)
x, y = basemap(lons, lats)
print(x.shape)
month_dates = [
datetime(year, month, 15) for year in range(start_year, end_year + 1)
for month in range(1, 13)
]
print(len(month_dates), " month of animation ")
nc_data_folder = os.path.join(nc_db_folder, sim_name)
dsDict = {}
var_names = ["STFL", "PR", "TT"]
for i, vName in enumerate(var_names):
path = os.path.join(nc_data_folder, "{0}.nc4".format(vName))
dsDict[vName] = Dataset(path)
aniObj = Animator(var_names, dsDict, basemap, x, y, stfl_mask=stfl_mask)
#fa = animation.FuncAnimation(fig, aniObj.animate, month_dates, interval=50)
temp_folder = "for_anim_{0}".format(sim_name)
if os.path.isdir(temp_folder):
shutil.rmtree(temp_folder)
os.mkdir(temp_folder)
for d in month_dates:
aniObj.animate(d)
aniObj.saveFrame(tmp_folder=temp_folder)
def plot_mean_2d_fields():
names = [
"STFL",
]
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_260x260_wo_lakes_and_with_lakeroff",
]
run_id_list = [
"w/o lakes, with lake roff., high res",
]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
stfl_bounds = [
0, 1, 10, 100, 250, 500, 850, 900, 950, 1000, 1200, 1500, 1800, 2000,
3000, 4000, 10000, 20000
]
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1990, 12, 31),
out_img="lake_roff_and_lake_rout_effect_stfl_1.png",
var_name="STFL",
level=-1,
bounds=stfl_bounds)
pass
def get_daily_climatology(path_to_hdf_file="", var_name="STFL", level=None, start_year=None, end_year=None):
# if the difference of 2 variables is requested
opsign = "-" if "-" in var_name else "+" if "+" in var_name else None
if "-" in var_name or "+" in var_name:
v1name, v2name = var_name.replace(" ", "").split(opsign)
dates, v1data = get_daily_climatology(path_to_hdf_file=path_to_hdf_file, level=level,
start_year=start_year, end_year=end_year, var_name=v1name)
_, v2data = get_daily_climatology(path_to_hdf_file=path_to_hdf_file, level=level,
start_year=start_year, end_year=end_year, var_name=v2name)
return dates, v1data - v2data if opsign == "-" else v1data + v2data if opsign == "+" else None
if var_name.endswith("_min"):
return get_daily_min_climatology(path_to_hdf_file=path_to_hdf_file, var_name=var_name, level=level,
start_year=start_year, end_year=end_year)
elif var_name.endswith("_max"):
return get_daily_max_climatology(path_to_hdf_file=path_to_hdf_file, var_name=var_name, level=level,
start_year=start_year, end_year=end_year)
else:
return Crcm5ModelDataManager.hdf_get_daily_climatological_fields(hdf_db_path=path_to_hdf_file,
start_year=start_year,
end_year=end_year,
var_name=var_name,
level_index=level,
use_grouping=True)
def main():
import matplotlib.pyplot as plt
lons, lats, descr = get_lat_lons()
#needed for the model domain
manager = Crcm5ModelDataManager(
samples_folder_path=
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
b = manager.get_rotpole_basemap()
x, y = b(lons, lats)
start_year = 1979
end_year = 1988
data = calculate_seasonal_mean_field(months=[6, 7, 8],
start_year=start_year,
end_year=end_year)
levels = np.arange(0, 7, 0.25)
data = np.ma.masked_where(data == descr["nodata_value"], data)
b.contourf(x, y, data, levels=levels)
b.colorbar()
b.drawcoastlines()
plt.show()
pass
def main():
path = "/skynet3_rech1/huziy/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples/quebec_crcm5-hcd-rl-intfl_197901/pm1979010100_00000000p"
rObj = RPN(path)
sani = rObj.get_first_record_for_name("SANI")
lons, lats = rObj.get_longitudes_and_latitudes_for_the_last_read_rec()
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons, lats2d=lats)
x, y = basemap(lons, lats)
sani = np.ma.masked_where(sani < 2, sani)
levels = [2, 3, 4, 5, 6, 7, 8, 10, 15, 20, 25, 30, 40]
cmap = cm.get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
fig = plt.figure()
basemap.contourf(x, y, sani, levels=levels, cmap=cmap, norm=bn)
basemap.drawcoastlines()
basemap.colorbar(ticks=levels)
fig.tight_layout()
fig.savefig("soil_anis.jpeg")
pass
def main():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3_old_snc",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_diff_lk_types_crcm_lk_fractions",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3_sturm_snc",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_test_lake_level_260x260_1",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_test_lake_level_260x260_1_lakes_off",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_lakes_bowling_function",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_river_ice"
]
run_id_list = [
"w/o lakes", "with lakes", "w/o lakes sn.cond by Sturm",
"high res. with lakes", "high res. w/o lakes", "Bowling lake func.",
"river ice"
]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
compare_hydrographs_at_stations(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1988, 12, 31),
img_path="hydrographs_all_runs.png")
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1988, 12, 31),
impose_min=0,
out_img="lake_roff_and_lake_rout_effect_ice.png")
pass
def main():
from_guillimin_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/"
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
#interflow experiment (crcm5-hcd-rl-intfl)
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup3/all_files_in_one_folder"
#(crcm5-hcd-rl) lakes and rivers interacting no interflow
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup"
#lakes are full of land (crcm5-r)
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
#lakes and rivers not interacting
data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-r_spinup2/all_files"
hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf"
#Lake residence time decreased 5 times
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl-kd5_spinup/all_files_in_one_folder"
#latest interflow
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small/all_files_in_one_dir"
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf"
#interflow and ecoclimap
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap/all_files_in_one_dir"
#ecoclimap and era075
#data_folder = "quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075/all_files_in_one_dir"
#data_folder = os.path.join(from_guillimin_folder, data_folder)
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf"
#soil anisotropy 10000
#data_folder = os.path.join(from_guillimin_folder, "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf"
#soil anisotropy 10000 and do not care about bulk field capacity
#data_folder = os.path.join(from_guillimin_folder,
# "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf"
data_folder = os.path.join(from_guillimin_folder,
"quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS")
hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5"
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
#var_names = ["STFL", "PR", "TT", "AV", "AH", "TRAF", "TDRA", "I5", "I0", "I1", "I2", "IMAV", "AS", "INTF"]
#var_names = [ "I0", "I1", "I2", "IMAV"]
#var_names = ["AS", ]
#var_names = ["QQ", ]
#var_names = ["INTF", ]
var_names = []
dm.export_to_hdf(var_list=var_names, file_path=hdf_file_path, mode="a")
#export_static_fields_to_hdf(
# hdf_file= hdf_file_path, data_folder=data_folder
#)
pass
def compare_sim_with_obs():
start_year = 1990
end_year = 1986
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_lowres_003",
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_86x86_0.5deg_wo_lakes_and_wo_lakeroff"
]
run_id_list = [
"003, 10 soil layers [{0}-{1}]".format(start_year, end_year),
"wo lakes and wo lakeroff"
]
colors = ["m", "k"] #"b" is reserved for observations
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
compare_hydrographs_at_stations(
data_managers,
start_date=datetime(start_year, 1, 1),
end_date=datetime(end_year, 12, 31),
img_path="hydrographs_003_and_wo_lakes_and_wo_lakeroff.png",
colors=colors)
def get_basemap_from_hdf(file_path="", resolution="l"):
"""
:param file_path:
:return: lons(2d), lats(2), basemap - corresponding to the data in the file
"""
h = tb.open_file(file_path)
# Extract 2d longitudes and latitudes
lons = h.getNode("/", "longitude")[:]
lats = h.getNode("/", "latitude")[:]
rotpoletable = h.getNode("/", "rotpole")
assert isinstance(rotpoletable, tb.Table)
params = {}
for row in rotpoletable:
print(row["name"], row["value"])
params[row["name"].decode()] = row["value"].decode() if isinstance(row["value"], bytes) else row["value"]
rotpoletable.close()
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons, lats2d=lats,
lon_1=params["lon1"], lon_2=params["lon2"],
lat_1=params["lat1"], lat_2=params["lat2"], resolution=resolution
)
h.close()
return lons, lats, basemap
def main():
path = "/skynet3_rech1/huziy/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples/quebec_crcm5-hcd-rl-intfl_197901/pm1979010100_00000000p"
rObj = RPN(path)
sani = rObj.get_first_record_for_name("SANI")
lons, lats = rObj.get_longitudes_and_latitudes_for_the_last_read_rec()
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(lons2d=lons, lats2d=lats)
x, y = basemap(lons, lats)
sani = np.ma.masked_where(sani < 2, sani)
levels = [2, 3, 4, 5, 6, 7, 8, 10, 15, 20, 25, 30, 40]
cmap = cm.get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
fig = plt.figure()
basemap.contourf(x, y, sani, levels=levels, cmap=cmap, norm=bn)
basemap.drawcoastlines()
basemap.colorbar(ticks=levels)
fig.tight_layout()
fig.savefig("soil_anis.jpeg")
pass
def get_daily_min_climatology(path_to_hdf_file="", var_name="STFL", level=None,
start_year=None, end_year=None):
var_name_min = "{0}_min".format(var_name) if not var_name.endswith("_min") else var_name
return Crcm5ModelDataManager.hdf_get_daily_extreme_climatological_fields(
hdf_db_path=path_to_hdf_file,
start_year=start_year, end_year=end_year, var_name=var_name_min,
level=level, maximum=False)
def export_static_fields_to_hdf(
hdf_file="/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf",
data_folder="/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-r_spinup2/all_files",
overwrite=False):
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
dm.export_static_fields_to_hdf(file_path=hdf_file, overwrite=overwrite)
pass
def correct_proj_table():
data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup3"
from rpn.rpn import RPN
hdf_file = "/home/huziy/skynet3_rech1/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup3.hdf"
from . import hdf_table_schemes
for fName in os.listdir(data_folder):
if not fName.startswith("dm"):
continue
fpath = os.path.join(data_folder, fName)
r = RPN(fpath)
tt = r.get_first_record_for_name("TT")
projData = r.get_proj_parameters_for_the_last_read_rec()
Crcm5ModelDataManager.export_grid_properties_to_hdf(file_path=hdf_file, grid_params=projData,
table_scheme=hdf_table_schemes.projection_table_scheme)
r.close()
return
def skynet3_test():
#To test dictionary to recarray to pytables implementation
data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl-kd5_spinup/all_files_in_one_folder"
hdf_file_path = "/skynet3_rech1/huziy/hdf_store/test.hdf"
var_names = [
"INTF",
]
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
dm.export_to_hdf(var_list=var_names, file_path=hdf_file_path, mode="w")
def correct_proj_table():
data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup3"
from rpn.rpn import RPN
hdf_file = "/home/huziy/skynet3_rech1/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup3.hdf"
from . import hdf_table_schemes
for fName in os.listdir(data_folder):
if not fName.startswith("dm"):
continue
fPath = os.path.join(data_folder, fName)
r = RPN(fPath)
tt = r.get_first_record_for_name("TT")
projData = r.get_proj_parameters_for_the_last_read_rec()
Crcm5ModelDataManager.export_grid_properties_to_hdf(
file_path=hdf_file,
grid_params=projData,
table_scheme=hdf_table_schemes.projection_table_scheme)
r.close()
return
def get_mean_2d_fields_for_months(path="", var_name="", level=None, months=None,
start_year=None,
end_year=None):
"""
Return means over the specified months for each year
:param path: path to the hdf file with data
:param var_name:
:param level:
"""
params = dict(path_to_hdf=path, months=months, level=level,
start_year=start_year, end_year=end_year)
if ("-" in var_name) or ("+" in var_name):
var_name1, var_name2 = re.split(r"[+\-]", var_name)
v1 = Crcm5ModelDataManager.hdf_get_seasonal_means(var_name=var_name1, **params)
v2 = Crcm5ModelDataManager.hdf_get_seasonal_means(var_name=var_name2, **params)
return v1 - v2 if "-" in var_name else v1 + v2
else:
return Crcm5ModelDataManager.hdf_get_seasonal_means(var_name=var_name, **params)
def main():
rpn_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
start_year = 1979
end_year = 1988
sim_name = "crcm5-r"
#dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True)
pmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="pm", all_files_in_samples_folder=True)
#plot results
assert isinstance(pmManager, Crcm5ModelDataManager)
lons, lats = pmManager.lons2D, pmManager.lats2D
stfl_mask = pmManager.cbf < 0
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons, lats2d = lats
)
x, y = basemap(lons, lats)
print(x.shape)
month_dates = [ datetime(year, month, 15) for year in range(start_year, end_year + 1) for month in range(1,13) ]
print(len(month_dates), " month of animation ")
nc_data_folder = os.path.join(nc_db_folder, sim_name)
dsDict = {}
var_names = [ "STFL", "PR", "TT"]
for i, vName in enumerate(var_names):
path = os.path.join(nc_data_folder, "{0}.nc4".format(vName))
dsDict[vName] = Dataset(path)
aniObj = Animator(var_names, dsDict, basemap, x, y, stfl_mask = stfl_mask)
#fa = animation.FuncAnimation(fig, aniObj.animate, month_dates, interval=50)
temp_folder = "for_anim_{0}".format(sim_name)
if os.path.isdir(temp_folder):
shutil.rmtree(temp_folder)
os.mkdir(temp_folder)
for d in month_dates:
aniObj.animate(d)
aniObj.saveFrame(tmp_folder=temp_folder)
def main():
AFRIC = 1
QUEBEC = 2
varname = "drainage_density"
region = QUEBEC
if region == QUEBEC:
data_path = "/home/huziy/skynet3_rech1/Netbeans Projects/Java/DDM/directions_with_drainage_density/directions_qc_dx0.1deg_5.nc"
out_path = "qc_{0}_0.1deg.pdf".format(varname)
elif region == AFRIC:
data_path = "/home/huziy/skynet3_rech1/Netbeans Projects/Java/DDM/directions_africa_dx0.44deg.v3.nc"
out_path = "af_{0}_0.44deg.pdf".format(varname)
else:
raise Exception("Unknown region...")
#
ds = Dataset(data_path)
data = ds.variables[varname][20:-20, 20:-20]
lons = ds.variables["lon"][20:-20, 20:-20]
lats = ds.variables["lat"][20:-20, 20:-20]
slope = ds.variables["slope"][20:-20, 20:-20]
fig = plt.figure()
print(data.min(), data.max())
ax = plt.gca()
data = np.ma.masked_where(slope < 0, data)
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(lons2d=lons, lats2d=lats)
lons[lons > 180] -= 360
x, y = basemap(lons, lats)
data = maskoceans(lons, lats, data, inlands=False)
img = basemap.contourf(x, y, data, cmap=cm.get_cmap("jet", 10))
ax.set_title("Drainage density")
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", "5%", pad="3%")
cb = fig.colorbar(img, cax=cax)
cax.set_title("(km**-1)")
basemap.drawcoastlines(ax=ax)
fig.tight_layout()
fig.savefig(out_path)
def show_lake_and_lakeroff_effect():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_wo_lakes_and_wo_lakeroff",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_diff_lk_types_crcm_lk_fractions",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_river_ice",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_quebec_river_ice_gwrestime0",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_lake_level_260x260_1_lakes_off",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_260x260_wo_lakes_and_with_lakeroff",
# "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_river_ice_1yrspnp_const_manning",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_260x260_wo_lakes_and_with_lakeroff_nogw"
]
run_id_list = [
"w/o lakes, w/o lake roff.",
# "with lakes, with lake roff.",
# "with ice",
# "with ice, no ground water",
# "high res. lakes off",
# "high res wo lakes with lake rof",
# "low res, const manning, riv ice",
"high res., nogw, wo lakes, with lakerof"
]
cmap = cm.get_cmap("jet", len(run_id_list))
# colors = [cmap(i / float(len(run_id_list))) for i in range(len(run_id_list)) ]
colors = ["r", "k", "g", "y", "m"]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
# compare_hydrographs_at_stations(data_managers, start_date=datetime(1986,1,1), end_date=datetime(1990,12, 31),
# img_path="hydrograph_lake_and_lakeroff_effect_test_df.png", colors = colors
# )
stfl_bounds = [
0, 1, 10, 100, 250, 500, 850, 900, 950, 1000, 1200, 1500, 1800, 2000,
3000, 4000, 10000, 20000
]
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1990, 12, 31),
out_img="lake_roff_and_lake_rout_effect_stfl.png",
var_name="STFL",
level=-1,
bounds=stfl_bounds)
def main():
manager = Crcm5ModelDataManager(
samples_folder_path="/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
hdf_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
mean_field = manager.hdf_get_climatology_for_season(months=[6, 7, 8],
hdf_db_path=hdf_path,
var_name="TRAF", level=5,
start_year=1979,
end_year=1988)
plt.contourf(mean_field)
plt.show()
pass
def main():
var_name = "STFL"
data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v3_old_snc"
data_manager = Crcm5ModelDataManager(samples_folder_path=data_path,
var_name=var_name,
all_files_in_samples_folder=True,
file_name_prefix="pm")
lons2D, lats2D = data_manager.lons2D, data_manager.lats2D
basemap = Basemap(projection="omerc",
no_rot=True,
lon_1=-68,
lat_1=52,
lon_2=16.65,
lat_2=0,
llcrnrlon=lons2D[0, 0],
llcrnrlat=lats2D[0, 0],
urcrnrlon=lons2D[-1, -1],
urcrnrlat=lats2D[-1, -1],
resolution="l")
fig = plt.figure()
data = data_manager.get_date_to_field_dict()
print(list(data.keys()))
print(len(list(data.keys())))
times = list(
sorted([
datetime.strptime(s, "%Y-%m-%d %H:%M") for s in list(data.keys())
]))
times = [t.strftime("%Y-%m-%d %H:%M") for t in times]
ims = []
x, y = basemap(lons2D, lats2D)
aniObj = Animator(data, basemap, x, y, times[0])
fa = animation.FuncAnimation(fig, aniObj.animate, times, interval=50)
plt.show()
#fa.save("animateflow.mpg")
data.close()
pass
def main():
stations = cehq_station.load_from_hydat_db(natural=True, province="QC")
dm = Crcm5ModelDataManager(
samples_folder_path=
"/skynet3_rech1/huziy/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
basemap = dm.get_rotpole_basemap()
lons = [s.longitude for s in stations]
lats = [s.latitude for s in stations]
n_cont_years = [len(s.get_list_of_complete_years()) for s in stations]
x, y = basemap(lons, lats)
basemap.scatter(x, y, c=n_cont_years)
basemap.drawcoastlines()
basemap.colorbar()
plt.show()
def show_lake_effect():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_without_lakes_v4_old_snc",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_86x86_0.5deg_with_diff_lk_types_crcm_lk_fractions"
]
run_id_list = ["w/o lakes, with lake roff.", "with lakes, with lake roff."]
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
# compare_hydrographs_at_stations(data_managers, start_date=datetime(1986,1,1), end_date=datetime(1988,12, 31))
compare_means_2d(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1988, 12, 31),
out_img="lake_effect_2d.png",
impose_min=0)
def compare_level_num_influence():
path_list = [
"/home/huziy/skynet3_exec1/from_guillimin/quebec_lowres_001",
"/home/huziy/skynet3_exec1/from_guillimin/quebec_lowres_002",
"/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_lowres_004"
]
run_id_list = [
"001, 10 soil layers", "002, 3 soil layers", "004, interflow"
]
colors = ["r", "k", "m"] #"b" is reserved for observations
data_managers = []
for the_path, the_id in zip(path_list, run_id_list):
theManager = Crcm5ModelDataManager(samples_folder_path=the_path,
all_files_in_samples_folder=True)
theManager.run_id = the_id
data_managers.append(theManager)
compare_hydrographs_at_stations(data_managers,
start_date=datetime(1986, 1, 1),
end_date=datetime(1990, 12, 31),
img_path="cmp_001_002_004.png",
colors=colors)
def main(months=None):
start_year = 1979
end_year = 1988
import matplotlib.pyplot as plt
fig = plt.figure()
var_name_to_ij = {
"TT": (1, 0),
"PR": (2, 0),
"AH": (1, 1),
"AV": (2, 1),
"STFL": (0, 1),
"TRAF": (1, 2),
"TDRA": (2, 2)
}
fmt = ScalarFormatter(useMathText=True)
fmt.set_powerlimits((-2, 3))
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True)
basemap = dmManager.get_rotpole_basemap()
lons, lats = dmManager.lons2D, dmManager.lats2D
x, y = basemap(lons, lats)
lkfr = dmManager.lake_fraction
gs = GridSpec(3, 3, width_ratios=[1, 1, 1], height_ratios=[1, 1, 1])
#fig.suptitle("{0} minus {1}".format(sim_name2, sim_name1))
month_dates = [datetime(2001, m, 1) for m in months]
ax = None
for var_name in field_names:
coef = 1
if var_name == "PR":
coef = 24 * 60 * 60 * 1000
elif var_name in ["TDRA", "TRAF"]:
coef = 24 * 60 * 60
elif var_name == "AV":
coef = 3 * 60 * 60
#cmap.set_bad("0.6")
print(var_name)
v1 = _get_values(sim_name1,
var_name,
start_year,
end_year,
months=months)
v2 = _get_values(sim_name2,
var_name,
start_year,
end_year,
months=months)
#calculate annual means, for each year
v1 = v1.mean(axis=1)
v1m = v1.mean(axis=0)
v2 = v2.mean(axis=1)
v2m = v2.mean(axis=0)
i, j = var_name_to_ij[var_name]
ax = fig.add_subplot(gs[i, j])
dv = (v2m - v1m) * coef
t, p = stats.ttest_ind(v1, v2)
if var_name in ["STFL"]:
dv = np.ma.masked_where((lkfr >= 0.6), dv)
print(months)
i_interest, j_interest = np.where(dv > 53)
print(i_interest, j_interest)
dv = maskoceans(lons, lats, dv)
print(10 * "*")
elif var_name in ["TDRA", "TRAF"]:
dv = maskoceans(lons, lats, dv)
else:
pass
dv = np.ma.masked_where(
p > 1, dv) #mask changes not significant to the 10 % level
if not np.all(dv.mask):
print("{0}: min = {1}; max = {2}".format(var_name, dv.min(),
dv.max()))
print("")
max_abs = np.ma.max(np.abs(dv))
delta = max_abs
the_power = np.log10(delta)
the_power = np.ceil(the_power) if the_power >= 0 else np.floor(
the_power)
delta = np.ceil(delta / 10.0**the_power) * 10.0**the_power
while delta > 2 * max_abs:
delta /= 2.0
while 0.8 * delta > max_abs:
delta *= 0.8
step = delta / 5.0 #10 ** (the_power - 1) * 2 if the_power >= 0 else 10 ** the_power * 2
if var_name == "TT":
step = 0.06
delta = 0.66
levels = np.arange(-delta, delta + step, step)
cmap = my_colormaps.get_cmap_from_ncl_spec_file(
path="colormap_files/BlueRed.rgb", ncolors=len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N) if len(levels) > 0 else None
print("delta={0}; step={1}; the_power={2}".format(
delta, step, the_power))
the_img = basemap.pcolormesh(x,
y,
dv,
cmap=cmap,
vmin=-delta,
vmax=delta,
norm=bn)
#add colorbar
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", "5%", pad="3%")
cb = plt.colorbar(the_img, cax=cax, format=fmt)
#coast lines
basemap.drawcoastlines(ax=ax, linewidth=0.5)
assert isinstance(ax, Axes)
ax.set_title("$\Delta$ " + field_name_to_long_name[var_name] + "\n" +
"({0})".format(field_name_to_units[var_name]))
bbox_props = dict(boxstyle="rarrow,pad=0.3", fc="wheat", ec="b", lw=2)
label = "-".join([d.strftime("%b") for d in month_dates
]) + "\n({0}-{1})".format(start_year, end_year)
ax.annotate(label,
xy=(0.1, 0.85),
xycoords="figure fraction",
bbox=bbox_props,
font_properties=FontProperties(size=15, weight="bold"))
#fig.tight_layout()
if months is None:
fig.savefig("annual_mean_diffs_{0}_minus_{1}.jpeg".format(
sim_name2, sim_name1))
else:
print(month_dates, months)
fig.savefig("seasonal_mean_{0}_diffs_{1}_minus_{2}.png".format(
"_".join(map(str, months)), sim_name2, sim_name1))
pass
def main():
#TODO: implement
var_name_to_ij = {
"TT": (0, 0),
"PR": (0, 1),
"AU": (1, 0),
"AV": (1, 1),
"STFA": (0, 2)
}
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder,
file_name_prefix="dm",
all_files_in_samples_folder=True)
basemap = dmManager.get_omerc_basemap()
lons, lats = dmManager.lons2D, dmManager.lats2D
x, y = basemap(lons, lats)
lkfr = dmManager.lake_fraction
acc_area = dmManager.accumulation_area_km2
i_arr, j_arr = np.where((lkfr > 0.4) & (lkfr < 0.6) & (acc_area > 10000))
name_to_data = {}
for sim in sim_names:
name_to_data[sim] = _get_values(sim, "STFL").mean(axis=0)
fig = plt.figure()
k = 0
nrows = 3
ncols = 3
gs = gridspec.GridSpec(nrows, ncols)
name_to_handle = {}
for row in range(nrows):
for col in range(ncols):
i, j = i_arr[k], j_arr[k]
ax = fig.add_subplot(gs[row, col])
for sim in sim_names:
xnew = np.linspace(1, 13, 100)
power_smooth = spline(np.arange(1, 13),
name_to_data[sim][:, i, j],
xnew,
order=3)
h = ax.plot(np.arange(1, 13),
name_to_data[sim][:, i, j],
label=sim,
lw=2)
ax.xaxis.set_ticks(np.arange(1, 13))
ax.set_title("{0}, {1}".format(lons[i, j], lats[i, j]))
if not k:
name_to_handle[sim] = h[0]
if row == 1 and col == 0:
ax.set_ylabel("Streamflow ($m^3/s$)")
if row == 2 and col == 1:
ax.set_xlabel("Month")
k += 4
names = sorted(list(name_to_handle.keys()), key=lambda x: len(x))
fig.legend((name_to_handle[name] for name in names), names)
plt.show()
pass
def main():
from_guillimin_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/"
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
# interflow experiment (crcm5-hcd-rl-intfl)
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup3/all_files_in_one_folder"
# (crcm5-hcd-rl) lakes and rivers interacting no interflow
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup"
# lakes are full of land (crcm5-r)
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup"
# hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r_spinup.hdf"
# lakes and rivers not interacting
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-r_spinup2/all_files"
# hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r_spinup2.hdf"
# Lake residence time decreased 5 times
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl-kd5_spinup/all_files_in_one_folder"
# latest interflow
# data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small/all_files_in_one_dir"
# hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf"
# interflow and ecoclimap
#data_folder = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap/all_files_in_one_dir"
# ecoclimap and era075
#data_folder = "quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075/all_files_in_one_dir"
#data_folder = os.path.join(from_guillimin_folder, data_folder)
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf"
#soil anisotropy 10000
#data_folder = os.path.join(from_guillimin_folder, "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf"
#soil anisotropy 10000 and do not care about bulk field capacity
#data_folder = os.path.join(from_guillimin_folder,
# "quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc/all_files_in_one_dir")
#hdf_file_path = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf"
# On guillimin: crcm5-r
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-r/all_files_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-r.hdf5"
# On guillimin: crcm5-hcd-r
# data_folder = "/sb/project/xgk-345-ab/huziy/Output/quebec_0.1_crcm5-hcd-r/all_files_in_one_dir"
# hdf_file_path = "/sb/project/xgk-345-ab/huziy/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
# On guillimin: crcm5-hcd-rl
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-hcd-rl/all_files_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
# Skynet2: crcm5-hcd-rl at 0.4 deg
data_folder = "/b1_fs1/huziy/quebec-analysis-driven-crcm5-sims/quebec_0.4deg_crcm5-hcd-rl/all_files_in_one_dir"
hdf_file_path = "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.4_crcm5-hcd-rl.hdf5"
# Interflow spinup
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS/all_files_in_one_dir"
# hdf_file_path = \
# "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5"
# Interflow
# data_folder = "/home/huziy/current_project/Output/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/all_files_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5"
# Interflow avoiding truncation
# data_folder = "/gs/project/ugh-612-aa/huziy/Output/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation/all_in_one_dir"
# hdf_file_path = "/home/huziy/current_project/PythonProjects/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5"
# For debugging on skynet
# data_folder = "/RESCUE/skynet3_rech1/huziy/test_export_to_hdf/test"
# hdf_file_path = "/RESCUE/skynet3_rech1/huziy/test_export_to_hdf/test.hdf5"
start_year = end_year = None
if len(sys.argv) >= 3:
data_folder = sys.argv[1]
hdf_file_path = sys.argv[2]
if len(sys.argv) >= 5:
start_year = int(sys.argv[3])
end_year = int(sys.argv[4])
dm = Crcm5ModelDataManager(samples_folder_path=data_folder,
all_files_in_samples_folder=True)
var_names = [
"STFA", "PR", "TT", "AV", "AH", "TRAF", "TDRA", "I5", "I0", "I1", "I2",
"IMAV", "AS", "QQ", "UU", "VV", "WW", "GZ", "HR", "HU", "CLDP", "LC",
"LD", "AL", "L1"
]
# var_names = [ "I0", "I1", "I2", "IMAV"]
# var_names = ["AS", ]
# var_names = ["QQ", ]
# var_names = ["INTF", ]
# var_names = ["AL",]
# var_names = ["L1", "IMAV", "I5", "I1", "I2"]
# var_names = ["TRAF", ]
# var_names = ["STFA", ]
dm.export_to_hdf(var_list=var_names,
file_path=hdf_file_path,
mode="w",
start_year=start_year,
end_year=end_year)
export_static_fields_to_hdf(hdf_file=hdf_file_path,
data_folder=data_folder)
pass
def diagnose(station_ids=None, model_data_path=None):
manager = Crcm5ModelDataManager(samples_folder_path=model_data_path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=True)
nx, ny = manager.lons2D.shape
rot_lat_lon = RotatedLatLon(lon1=-68, lat1=52, lon2=16.65, lat2=0.0)
x00, y00 = rot_lat_lon.toProjectionXY(manager.lons2D[0, 0],
manager.lats2D[0, 0])
x10, y10 = rot_lat_lon.toProjectionXY(manager.lons2D[1, 0],
manager.lats2D[1, 0])
x01, y01 = rot_lat_lon.toProjectionXY(manager.lons2D[0, 1],
manager.lats2D[0, 1])
dx = x10 - x00
dy = y01 - y00
print("dx, dy = {0}, {1}".format(dx, dy))
areas = rot_lat_lon.get_areas_of_gridcells(
dx, dy, nx, ny, y00, 1) #1 -since the index is starting from 1
print(areas[0, 0])
start_date = datetime(1986, 1, 1)
end_date = datetime(1986, 12, 31)
stations = cehq_station.read_station_data(selected_ids=station_ids,
start_date=start_date,
end_date=end_date)
stations.sort(key=lambda x: x.latitude, reverse=True)
for i, s in enumerate(stations):
fig = plt.figure()
#3 columns
gs = GridSpec(5,
3,
hspace=0.2,
wspace=0.2,
right=0.98,
left=0.1,
top=0.98)
model_ts = manager.get_streamflow_timeseries_for_station(
s, start_date=start_date, end_date=end_date, nneighbours=9)
print(model_ts.time[0], model_ts.time[-1])
i_model0, j_model0 = model_ts.metadata["ix"], model_ts.metadata["jy"]
mask = manager.get_mask_for_cells_upstream(i_model0, j_model0)
#hydrographs
ax = fig.add_subplot(gs[0, 0])
plot_streamflows(ax, s, model_ts)
#relative error
ax = fig.add_subplot(gs[1, 0])
plot_streamflow_re(ax, s, model_ts)
#directions
plot_directions_and_positions(fig.add_subplot(gs[:2, 1]),
s,
model_ts,
manager,
rot_lat_lon,
mask=mask)
#runoff
ax = fig.add_subplot(gs[2, 0])
plot_runoff(ax, manager, areas, model_ts, mask=mask)
#runoff from gldas
ax = fig.add_subplot(gs[2, 1])
#plot_gldas_runoff(ax, manager, areas, model_ts, mask = mask)
#temperature
ax_temp = fig.add_subplot(gs[3, 0])
ax_prec = fig.add_subplot(gs[4, 0])
plot_total_precip_and_temp_re_1d(ax_prec,
ax_temp,
manager,
rot_lat_lon,
areas,
model_ts,
mask=mask)
#swe timeseries
ax = fig.add_subplot(gs[3, 1])
plot_swe_timeseries(ax, manager, areas, model_ts, mask=mask)
#print np.where(mask == 1)
print("(i, j) = ({0}, {1})".format(model_ts.metadata["ix"],
model_ts.metadata["jy"]))
fig.savefig("diagnose_{0}_{1:.2f}deg.pdf".format(s.id, dx))
def main():
dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True)
pmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="pm", all_files_in_samples_folder=True)
#export monthly means to netcdf files if necessary
if export_to_nc:
for varname, prefix in zip( field_names, file_name_prefixes ):
manager = None
if prefix == "dm":
manager = dmManager
elif prefix == "pm":
manager = pmManager
level = -1
level_kind = -1
if varname == "TT":
level = 1
level_kind = level_kinds.HYBRID
if varname in ["TRAF", "TDRA"]:
level = 1
level_kind = level_kinds.ARBITRARY
if varname == "STFA": continue
export_monthly_means_to_ncdb(manager, varname, level= level,
level_kind= level_kind, rewrite = True)
#plot results
assert isinstance(pmManager, Crcm5ModelDataManager)
lons, lats = pmManager.lons2D, pmManager.lats2D
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons, lats2d = lats
)
x, y = basemap(lons, lats)
nc_data_folder = os.path.join(nc_db_folder, sim_name)
import matplotlib.pyplot as plt
all_axes = []
ax_to_levels = {}
imgs = []
gs = gridspec.GridSpec(2,3, height_ratios=[1,1], width_ratios=[1,1,1])
fig = plt.figure()
#fig.suptitle("({0} - {1})".format(start_year, end_year))
#plot Temp
varname = "TT"
levels = [-30, -25, -10, -5, -2, 0, 2, 5,10, 15, 20, 25]
cmap = cm.get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
ds = Dataset(os.path.join(nc_data_folder, "{0}.nc4".format(varname)))
years = ds.variables["year"][:]
sel = np.where((start_year <= years) & (years <= end_year))[0]
tt = ds.variables[varname][sel,:,:,:].mean(axis = 0).mean(axis = 0)
ds.close()
ax = fig.add_subplot(gs[0,0])
ax.set_title("Temperature (${\\rm ^\circ C}$)")
img = basemap.contourf(x, y, tt, levels = levels, cmap = cmap, norm = bn)
all_axes.append(ax)
imgs.append(img)
ax_to_levels[ax] = levels
#plot precip
varname = "PR"
levels = np.arange(0, 6.5, 0.5)
cmap = cm.get_cmap("jet_r", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
ds = Dataset(os.path.join(nc_data_folder, "{0}.nc4".format(varname)))
years = ds.variables["year"][:]
sel = np.where((start_year <= years) & (years <= end_year))[0]
pr = ds.variables[varname][sel,:,:,:].mean(axis = 0).mean(axis = 0)
convert_factor = 1000.0 * 24 * 60 * 60 #m/s to mm/day
pr *= convert_factor
ds.close()
ax = fig.add_subplot(gs[0,1])
ax.set_title("Precip (mm/day)")
img = basemap.contourf(x, y, pr, levels = levels, cmap = cmap, norm = bn)
all_axes.append(ax)
imgs.append(img)
ax_to_levels[ax] = levels
#plot AH
varname = "AH"
ds = Dataset(os.path.join(nc_data_folder, "{0}.nc4".format(varname)))
years = ds.variables["year"][:]
sel = np.where((start_year <= years) & (years <= end_year))[0]
ah = ds.variables[varname][sel,:,:,:].mean(axis = 0).mean(axis = 0)
ds.close()
levels = np.arange(-60, 160, 20)# np.linspace(au.min(), au.max(), 10)
cmap = cm.get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
ax = fig.add_subplot(gs[1,0])
ax.set_title("Sensible heat flux (${\\rm W/m^2}$)")
img = basemap.contourf(x, y, ah, cmap = cmap, norm = bn, levels = levels)
all_axes.append(ax)
imgs.append(img)
ax_to_levels[ax] = levels
#plot AV
varname = "AV"
ds = Dataset(os.path.join(nc_data_folder, "{0}.nc4".format(varname)))
years = ds.variables["year"][:]
sel = np.where((start_year <= years) & (years <= end_year))[0]
av = ds.variables[varname][sel,:,:,:].mean(axis = 0).mean(axis = 0)
ds.close()
coef = 3600* 3
levels = np.arange(-40, 220, 20)
cmap = cm.get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
ax = fig.add_subplot(gs[1,1])
ax.set_title("Latent heat flux (${\\rm W/m^2}$)")
img = basemap.contourf(x, y, av * coef, levels = levels, cmap = cmap, norm = bn)
all_axes.append(ax)
imgs.append(img)
ax_to_levels[ax] = levels
#plot stfl
varname = "STFL"
ds = Dataset(os.path.join(nc_data_folder, "{0}.nc4".format(varname)))
years = ds.variables["year"][:]
sel = np.where((start_year <= years) & (years <= end_year))[0]
stfl = ds.variables[varname][sel,:,:,:].mean(axis = 0).mean(axis = 0)
ds.close()
levels = [0,50,100,200,300,500,750,1000, 1500,2000,5000,10000,15000]
stfl = np.ma.masked_where(stfl < 0.01, stfl)
cmap = cm.get_cmap("jet", len(levels) - 1)
bn = BoundaryNorm(levels, cmap.N)
ax = fig.add_subplot(gs[0,2])
ax.set_title("Streamflow (${\\rm m^3/s}$)")
img = basemap.contourf(x, y, stfl, levels = levels, cmap = cmap, norm = bn)
all_axes.append(ax)
imgs.append(img)
ax_to_levels[ax] = levels
sf = ScalarFormatter(useMathText=True)
sf.set_powerlimits([-3,4])
#draw coast lines
for the_ax, the_img in zip(all_axes, imgs):
basemap.drawcoastlines(ax = the_ax)
divider = make_axes_locatable(the_ax)
cax = divider.append_axes("right", "5%", pad="3%")
cb = plt.colorbar(the_img, cax = cax, ticks = ax_to_levels[the_ax])
assert isinstance(cax, Axes)
title = cax.get_title()
fig.tight_layout()
fig.savefig("{0}-mean-annual-fields.pdf".format(sim_name))
def main():
data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_198501_198612_0.1deg"
coord_file = os.path.join(data_path, "pm1985010100_00000000p")
manager = Crcm5ModelDataManager(samples_folder_path=data_path,
file_name_prefix="pm",
all_files_in_samples_folder=True)
selected_ids = [
"104001", "103715", "093806", "093801", "092715", "081006", "061502",
"040830", "080718"
]
start_date = datetime(1986, 1, 1)
end_date = datetime(1986, 12, 31)
stations = cehq_station.read_station_data(selected_ids=selected_ids,
start_date=start_date,
end_date=end_date)
plot_utils.apply_plot_params(width_pt=None,
height_cm=30.0,
width_cm=16,
font_size=10)
fig = plt.figure()
#two columns
gs = GridSpec(len(stations) // 2 + len(stations) % 2,
2,
hspace=0.4,
wspace=0.4)
line_model, line_obs = None, None
stations.sort(key=lambda x: x.latitude, reverse=True)
for i, s in enumerate(stations):
model_ts = manager.get_streamflow_timeseries_for_station(
s, start_date=start_date, end_date=end_date)
ax = fig.add_subplot(gs[i // 2, i % 2])
assert isinstance(model_ts, TimeSeries)
#[t, m_data] = model_ts.get_daily_normals()
#[t, s_data] = s.get_daily_normals()
assert isinstance(s, Station)
#climatologies
#line_model = ax.plot(t, m_data, label = "Model (CRCM5)", lw = 3, color = "b")
#line_obs = ax.plot(t, s_data, label = "Observation", lw = 3, color = "r")
model_ts = model_ts.get_ts_of_daily_means()
print(model_ts.time[0], model_ts.time[-1])
print(model_ts.data[0:10])
mod_vals = model_ts.get_data_for_dates(s.dates)
print(mod_vals[:20])
print("+" * 20)
assert len(mod_vals) == len(s.dates)
#model_acorr = [1] + [ np.corrcoef([mod_vals[i:], mod_vals[:-i]])[0,1] for i in range(1,len(mod_vals)) ]
#obs_acorr = [1] + [ np.corrcoef([s.values[i:], s.values[:-i]])[0,1] for i in range(1,len(mod_vals)) ]
npoints = np.array(list(range(len(mod_vals), 0, -1)))
model_acorr = np.correlate(mod_vals, mod_vals, mode="full")
model_acorr = model_acorr[len(model_acorr) / 2:] / max(model_acorr)
model_acorr /= npoints
obs_acorr = np.correlate(s.values, s.values, mode="full")
obs_acorr = obs_acorr[len(obs_acorr) / 2:] / max(obs_acorr)
obs_acorr /= npoints
print(len(model_acorr), len(s.dates))
line_model = ax.plot(s.dates,
model_acorr,
label="Model (CRCM5)",
lw=1,
color="b")
line_obs = ax.plot(s.dates,
obs_acorr,
label="Observation",
lw=3,
color="r",
alpha=0.5)
#ax.annotate( "r = {0:.2f}".format( float( np.corrcoef([mod_vals, s.values])[0,1] )), xy = (0.7,0.8), xycoords= "axes fraction")
ax.set_title(
"%s: da_diff=%.2f %%, dist = %.1f" %
(s.id, (-s.drainage_km2 + model_ts.metadata["acc_area_km2"]) /
s.drainage_km2 * 100.0, model_ts.metadata["distance_to_obs"]))
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.xaxis.set_major_locator(MonthLocator(bymonth=list(range(1, 13, 2))))
lines = (line_model, line_obs)
labels = ("Model (CRCM5)", "Observation")
fig.legend(lines, labels)
fig.savefig("acorr_without_lakes_0.1deg_1year.png")
pass
def get_omerc_basemap_quebec(lons, lats, lon1=-68, lat1=52, lon2=16.65, lat2=0):
return Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(lons, lats,
lon_1=lon1, lat_1=lat1, lon_2=lon2, lat_2=lat2)
def main():
sim1 = "crcm5-r"
sim2 = "crcm5-hcd-r"
start_date = datetime(1985,1,1)
end_date = datetime(1985,12,31)
sims = [sim1, sim2]
season_months = [4,5,6]
ncdb_path = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
fname_pattern = "{0}_all.nc4"
varname = "TT"
figure = plt.figure()
file_paths = [ os.path.join(ncdb_path, the_sim, fname_pattern.format(varname)) for the_sim in sims ]
dsList = [ Dataset(fPath) for fPath in file_paths ]
lons2d, lats2d, time = dsList[0].variables["lon"][:], dsList[0].variables["lat"][:], dsList[0].variables["time"][:]
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(lons2d = lons2d, lats2d = lats2d)
varsDict = {}
time = num2date(time, dsList[0].variables["time"].units)
sel_times = np.where([t.month in season_months for t in time])[0]
sim_name_to_mean = {}
for sim, ds in zip( sims, dsList ):
varsDict["{0} ({1})".format(varname, sim)] = ds.variables[varname]
data = ds.variables[varname][sel_times,0,:,:].mean(axis = 0)
sim_name_to_mean[sim] = data
x, y = basemap(lons2d, lats2d)
#plot clickable field
ax = plt.gca()
basemap.contourf(x, y, sim_name_to_mean[sim2] - sim_name_to_mean[sim1])
basemap.colorbar()
basemap.drawcoastlines()
ax.set_title("({0})-({1})".format(sim2, sim1).upper())
TimeSeriesPlotter(ax, basemap, lons2d, lats2d, varsDict, time, start_date, end_date)
plt.show()
pass
def main():
import matplotlib.pyplot as plt
#path = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/test_with_lakeroff"
#path = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup"
path = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-r/all_in_one_folder"
manager = Crcm5ModelDataManager(samples_folder_path=path,
file_name_prefix="pm",
all_files_in_samples_folder=True,
need_cell_manager=True)
allData = {}
field2d, allStfl = manager.get_mean_2d_field_and_all_data(var_name="STFL")
allData["FACC"] = manager.accumulation_area_km2
traf2d, allTraf = manager.get_mean_2d_field_and_all_data(
var_name="TRAF", level=5, level_kind=level_kinds.ARBITRARY)
tdra2d, allTdra = manager.get_mean_2d_field_and_all_data(
var_name="TDRA", level=5, level_kind=level_kinds.ARBITRARY)
swe2d, allSwe = manager.get_mean_2d_field_and_all_data(
var_name="I5", level=-1, level_kind=level_kinds.ARBITRARY)
pre2d, allPr = manager.get_mean_2d_field_and_all_data(var_name="PR")
swsr2d, allSwsr = manager.get_mean_2d_field_and_all_data(var_name="SWSR")
swsl2d, allSwsl = manager.get_mean_2d_field_and_all_data(var_name="SWSL")
upin2d, allUpin = manager.get_mean_2d_field_and_all_data(var_name="UPIN")
#gwdi2d,allGwdi = manager.get_mean_2d_field_and_all_data(var_name="GWDI")
swst2d, allSwst = manager.get_mean_2d_field_and_all_data(var_name="SWST")
for k, v in allTraf.items():
allTraf[k] = v * manager.cell_area * 1.0e-3
for k, v in allTdra.items():
allTdra[k] = v * manager.cell_area * 1.0e-3
for k, v in allPr.items():
allPr[k] = v * manager.cell_area
#for k,v in allImav.iteritems():
# allImav[k] = v * manager.cell_area * 1.0e-3
for k, v in allSwe.items():
allSwe[k] *= v * manager.cell_area * 1.0e-3 * 1e-6
allData["TRAF"] = allTraf
allData["TDRA"] = allTdra
allData["PR"] = allPr
allData["I5"] = allSwe
allData["STFL"] = allStfl
allData["SWSR"] = allSwsr
allData["SWSL"] = allSwsl
allData["UPIN"] = allUpin
#allData["GWDI"] = allGwdi
allData["SWST"] = allSwst
#read in slope and channel length from the geophysics file
r = RPN(
"/home/huziy/skynet3_rech1/geof_lake_infl_exp/geophys_Quebec_0.1deg_260x260_with_dd_v6"
)
slp = r.get_first_record_for_name("SLOP")[20:-20, 20:-20]
slp[slp < 1e-4] = 1e-4
allData["SLOP"] = slp
allData["LENG"] = r.get_first_record_for_name("LENG")[20:-20, 20:-20]
allData["LKOU"] = r.get_first_record_for_name("LKOU")[20:-20, 20:-20]
r.close()
basemap = manager.get_omerc_basemap(resolution="c")
lon, lat = manager.lons2D, manager.lats2D
fig = plt.figure()
x, y = basemap(lon, lat)
field2d = np.ma.masked_where(field2d <= 0.1, field2d)
#field2d = np.ma.masked_where((field2d < 10000) | (upin2d > 100000), field2d)
basemap.pcolormesh(x, y, field2d)
plt.colorbar()
basemap.drawcoastlines()
ax = plt.gca()
TimeseriesPlotter(name_to_date_to_field=allData,
basemap=basemap,
lons2d=lon,
lats2d=lat,
ax=ax,
cell_manager=manager.cell_manager,
data_manager=manager)
fig = plt.figure()
x, y = basemap(lon, lat)
fig = plt.figure()
x, y = basemap(lon, lat)
basemap.pcolormesh(x, y, traf2d)
plt.title("Total traf")
plt.colorbar()
basemap.drawcoastlines()
fig = plt.figure()
x, y = basemap(lon, lat)
basemap.pcolormesh(x, y, swst2d)
plt.title("SWST")
plt.colorbar()
basemap.drawcoastlines()
traf2dsoil, dummy = manager.get_mean_2d_field_and_all_data(
var_name="TRAF", level=1, level_kind=level_kinds.ARBITRARY)
traf2dlake, dummy = manager.get_mean_2d_field_and_all_data(
var_name="TRAF", level=6, level_kind=level_kinds.ARBITRARY)
fig = plt.figure()
x, y = basemap(lon, lat)
basemap.pcolormesh(x, y, upin2d)
plt.title("UPIN")
plt.colorbar()
basemap.drawcoastlines()
plt.show()
pass
def plot_flow_directions_and_basin_boundaries(ax, s, sim_name_to_station_to_model_point,
sim_name_to_manager=None):
assert isinstance(ax, Axes)
assert isinstance(s, Station)
assert isinstance(sim_name_to_station_to_model_point, dict)
mp_list = list(sim_name_to_station_to_model_point.items())[0][1][s]
#selecting only one (the first model point)
mp = mp_list[0]
manager = list(sim_name_to_manager.items())[0][1]
assert isinstance(manager, Crcm5ModelDataManager)
flow_in_mask = manager.get_mask_for_cells_upstream(mp.ix, mp.jy)
lons2d, lats2d = manager.lons2D, manager.lats2D
i_upstream, j_upstream = np.where(flow_in_mask == 1)
nx_total, ny_total = lons2d.shape
imin, imax = np.min(i_upstream), np.max(i_upstream)
jmin, jmax = np.min(j_upstream), np.max(j_upstream)
margin = 8
imin = max(0, imin - margin)
jmin = max(0, jmin - margin)
imax = min(nx_total - 1, imax + margin)
jmax = min(ny_total - 1, jmax + margin)
sub_lons2d = lons2d[imin:imax + 1, jmin:jmax + 1]
sub_lats2d = lats2d[imin:imax + 1, jmin:jmax + 1]
sub_flow_directions = manager.flow_directions[imin:imax + 1, jmin:jmax + 1]
sub_flow_in_mask = flow_in_mask[imin:imax + 1, jmin:jmax + 1]
sub_i_upstream, sub_j_upstream = np.where(sub_flow_in_mask == 1)
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=sub_lons2d, lats2d=sub_lats2d, resolution="h"
)
#plot all stations
#stations = sim_name_to_station_to_model_point.items()[0][1].keys()
x_list = [the_station.longitude for the_station in (s,)]
y_list = [the_station.latitude for the_station in (s,)]
basemap_big = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons2d, lats2d=lats2d
)
x_list, y_list = basemap_big(x_list, y_list)
basemap_big.scatter(x_list, y_list, c="r", s=40, linewidths=0, ax=ax)
basemap_big.drawcoastlines(ax=ax)
basemap_big.drawrivers(ax=ax)
ax.annotate(s.id, xy=basemap(s.longitude, s.latitude), xytext=(3, 3), textcoords='offset points',
font_properties=FontProperties(weight="bold"), bbox=dict(facecolor='w', alpha=1),
ha="left", va="bottom", zorder=2)
x_big, y_big = basemap_big(manager.lons2D, manager.lats2D)
####zoom to the area of interest
#axins = zoomed_inset_axes(ax, 3, loc=2)
displayCoords = ax.transData.transform((x_big[imin, jmin], y_big[imin, jmin]))
x_shift_fig, y_shift_fig = ax.figure.transFigure.inverted().transform(displayCoords)
print("After transData", displayCoords)
print("xshift and yshift", x_shift_fig, y_shift_fig)
#ax.annotate("yuyu", xy = ( 0.733264985153, 0.477182994408), xycoords = "figure fraction" )
rect = [0.1, y_shift_fig + 0.1, 0.4, 0.4]
axins = ax.figure.add_axes(rect)
#assert isinstance(axins, Axes)
x, y = basemap(sub_lons2d, sub_lats2d)
x1d_start = x[sub_flow_in_mask == 1]
y1d_start = y[sub_flow_in_mask == 1]
fld1d = sub_flow_directions[sub_flow_in_mask == 1]
from util import direction_and_value
ishift, jshift = direction_and_value.flowdir_values_to_shift(fld1d)
sub_i_upstream_next = sub_i_upstream + ishift
sub_j_upstream_next = sub_j_upstream + jshift
u = x[sub_i_upstream_next, sub_j_upstream_next] - x1d_start
v = y[sub_i_upstream_next, sub_j_upstream_next] - y1d_start
u2d = np.ma.masked_all_like(x)
v2d = np.ma.masked_all_like(y)
u2d[sub_i_upstream, sub_j_upstream] = u
v2d[sub_i_upstream, sub_j_upstream] = v
basemap.quiver(x, y, u2d, v2d, angles="xy", scale_units="xy", scale=1, ax=axins)
x_list = [the_station.longitude for the_station in (s,)]
y_list = [the_station.latitude for the_station in (s,)]
x_list, y_list = basemap(x_list, y_list)
basemap.scatter(x_list, y_list, c="r", s=40, linewidths=0)
basemap.drawcoastlines(ax=axins)
basemap.drawrivers(ax=axins)
#read waterbase file, and plot only the polygons which contain at least one upstream point
shp_path = "/skynet3_exec1/huziy/Netbeans Projects/Java/DDM/data/shape/waterbase/na_bas_ll_r500m/na_bas_ll_r500m.shp"
c = fiona.open(shp_path)
hits = c.filter(bbox=(sub_lons2d[0, 0], sub_lats2d[0, 0], sub_lons2d[-1, -1], sub_lats2d[-1, -1]))
points = [Point(the_x, the_y) for the_x, the_y in zip(x1d_start, y1d_start)]
selected_polygons = []
for feature in hits:
new_coords = []
old_coords = feature["geometry"]["coordinates"]
#transform to the map coordinates
for ring in old_coords:
x1 = [tup[0] for tup in ring]
y1 = [tup[1] for tup in ring]
x2, y2 = basemap(x1, y1)
new_coords.append(list(zip(x2, y2)))
feature["geometry"]["coordinates"] = new_coords
poly = shape(feature["geometry"])
#print poly, type(poly)
#print feature.keys()
#print feature["properties"]
prep_poly = prep.prep(poly)
hits = list(filter(prep_poly.contains, points))
if len(hits) > 2:
selected_polygons.append(feature["geometry"])
for p in selected_polygons:
axins.add_patch(PolygonPatch(p, fc="none", ec="b", lw=1.5))
zoom_lines_color = "#6600FF"
#set color of the frame
for child in axins.get_children():
if isinstance(child, Spine):
child.set_color(zoom_lines_color)
child.set_linewidth(3)
# draw a bbox of the region of the inset axes in the parent axes and
# connecting lines between the bbox and the inset axes area
mark_inset(ax, axins, loc1=1, loc2=3, fc="none", ec=zoom_lines_color, lw=3)
#basemap.readshapefile(, "basin")
pass
