def plot_comparisons_of_seasonal_sst_with_homa_obs(self, start_year=None, end_year=None, season_to_months=None,
exp_label=""):
model_data = self.get_seasonal_mean_lst(season_to_months=season_to_months,
start_year=start_year, end_year=end_year)
obs_sst_path = os.path.expanduser("~/skynet3_rech1/nemo_obs_for_validation/GreatLakes_2003_5km-2/sst-glk.nc")
obs_data = self.read_and_interpolate_homa_data(path=obs_sst_path, start_year=start_year, end_year=end_year,
season_to_months=season_to_months)
plot_utils.apply_plot_params(font_size=10, width_pt=None, width_cm=20, height_cm=10)
# calculate climatologic differences
diff = {}
for season in list(season_to_months.keys()):
diff[season] = np.mean(
[model_data[y][season] - obs_data[y][season] for y in range(start_year, end_year + 1)], axis=0)
diff[season] = np.ma.masked_where(~self.lake_mask, diff[season])
the_field = diff[season]
print("diff stats({}): min={}; max={}; avg={}".format(
season, the_field.min(), the_field.max(), the_field.mean()))
# plot seasonal biases
xx, yy = self.basemap(self.lons.copy(), self.lats.copy())
# calculate difference ranges
diff_max = 0
for season, the_diff in diff.items():
diff_max = max(np.percentile(np.abs(the_diff[~the_diff.mask]), 90), diff_max)
diff_max = 5
locator = MaxNLocator(nbins=12, symmetric=True)
bounds = locator.tick_values(-diff_max, diff_max)
bn = BoundaryNorm(bounds, len(bounds) - 1)
cmap = cm.get_cmap("RdBu_r", len(bounds) - 1)
im = None
fig = plt.figure()
ncols = 2
# fig.suptitle(r"LST $\left({\rm ^\circ C}\right)$", font_properties=FontProperties(weight="bold"))
gs = GridSpec(len(season_to_months) // ncols, ncols + 1, width_ratios=[1.0, ] * ncols + [0.05, ])
for i, season in enumerate(season_to_months.keys()):
ax = fig.add_subplot(gs[i // ncols, i % ncols])
im = self.basemap.pcolormesh(xx, yy, diff[season][:], ax=ax, cmap=cmap, norm=bn)
ax.set_title(season)
self.basemap.drawcoastlines(ax=ax, linewidth=0.5)
if not i:
ax.set_ylabel("NEMO - Obs")
cb = plt.colorbar(im, ticks=locator, cax=fig.add_subplot(gs[:, -1]), extend="both")
nemo_img_dir = "nemo"
if not os.path.isdir(nemo_img_dir):
os.mkdir(nemo_img_dir)
plt.tight_layout()
fig.savefig(os.path.join(nemo_img_dir, "sst_homa_validation_{}.pdf".format(exp_label)))
plt.show()
def get_boundary_norm_using_all_vals(to_plot, ncolors):
vmin = np.percentile(to_plot[~to_plot.mask], 5)
vmax = np.percentile(to_plot[~to_plot.mask], 95)
med = np.median(to_plot[~to_plot.mask])
locator = MaxNLocator(ncolors)
bounds = locator.tick_values(vmin, vmax)
return BoundaryNorm(bounds, ncolors=ncolors), bounds, bounds[0], bounds[-1]
def update_ticks(axes, coord, components, is_log):
"""
Changes the axes to have the proper tick formatting based on the type of
component.
Returns `None` or the number of categories if components is Categorical.
Parameters
----------
axes : `~matplotlib.axes.Axes`
A matplotlib axis object to alter
coord : { 'x' | 'y' }
The coordinate axis on which to update the ticks
components : iterable
A list of components that are plotted along this axis
if_log : boolean
Whether the axis has a log-scale
"""
if coord == 'x':
axis = axes.xaxis
elif coord == 'y':
axis = axes.yaxis
else:
raise TypeError("coord must be one of x,y")
is_cat = any(comp.categorical for comp in components)
is_date = any(comp.datetime for comp in components)
if is_date:
loc = AutoDateLocator()
fmt = AutoDateFormatter(loc)
axis.set_major_locator(loc)
axis.set_major_formatter(fmt)
elif is_log:
axis.set_major_locator(LogLocator())
axis.set_major_formatter(LogFormatterMathtext())
elif is_cat:
all_categories = np.empty((0,), dtype=np.object)
for comp in components:
all_categories = np.union1d(comp.categories, all_categories)
locator = MaxNLocator(10, integer=True)
locator.view_limits(0, all_categories.shape[0])
format_func = partial(tick_linker, all_categories)
formatter = FuncFormatter(format_func)
axis.set_major_locator(locator)
axis.set_major_formatter(formatter)
return all_categories.shape[0]
else:
axis.set_major_locator(AutoLocator())
axis.set_major_formatter(ScalarFormatter())
def get_boundary_norm(vmin, vmax, ncolors, exclude_zero=False, varname = None, difference = False):
bounds = None
# custom variable norms
if difference:
pass
else:
if varname == "TRAF":
bounds = [0, 0.1, 0.5] + list(range(1, ncolors - 3)) + [vmax, ]
# temperature
# Do not do anything if bounds were already calculated
if bounds is None:
if vmin * vmax >= 0:
locator = MaxNLocator(ncolors)
bounds = np.asarray(locator.tick_values(vmin, vmax))
elif exclude_zero:
# implies that this is the case for difference
delta = max(abs(vmax), abs(vmin))
assert ncolors % 2 == 1
d = 2.0 * delta / float(ncolors)
print(d, np.log10(d))
ndec = -int(np.floor(np.log10(d)))
print(ndec)
d = np.round(d, decimals=ndec)
assert d > 0
print("ncolors = {0}".format(ncolors))
negats = [-d / 2.0 - d * i for i in range((ncolors - 1) / 2)]
bounds = negats[::-1] + [-the_bound for the_bound in negats]
assert 0 not in bounds
assert bounds[0] == -bounds[-1]
else:
locator = MaxNLocator(nbins=ncolors, symmetric=True)
bounds = np.asarray(locator.tick_values(vmin, vmax))
return BoundaryNorm(bounds, ncolors=ncolors), bounds, bounds[0], bounds[-1]
def get_best_map_ticks(ax, transform=None):
"""
Use the matplotlib.ticker class to automatically set nice values for the major and minor ticks.
Log axes generally come out nicely spaced without needing manual intervention. For particularly narrow latitude
vs longitude plots the ticks can come out overlapped, so an exception is included to deal with this.
"""
from matplotlib.ticker import MaxNLocator
max_x_bins = 9
max_y_bins = 7 # as plots are wider rather than taller
lon_steps = [1, 3, 6, 9, 10]
lat_steps = [1, 3, 6, 9, 10]
variable_step = [1, 2, 4, 5, 10]
xmin, xmax, ymin, ymax = ax.get_extent(crs=transform)
# ymin, ymax = ax.get_ylim()
if (xmax - xmin) < 5:
lon_steps = variable_step
if (ymax - ymin) < 5:
lat_steps = variable_step
# We need to make a special exception for particularly narrow and wide plots, which will be lat vs lon
# preserving the aspect ratio. This gives more options for the spacing to try and find something that can use
# the maximum number of bins.
if (ymax - ymin) > 2.2 * (xmax - xmin):
max_x_bins = 4
max_y_bins = 11
elif (xmax - xmin) > 2.2 * (ymax - ymin):
max_x_bins = 14
max_y_bins = 4
lon_locator = MaxNLocator(nbins=max_x_bins, steps=lon_steps)
lat_locator = MaxNLocator(nbins=max_y_bins, steps=lat_steps)
lon_ticks = lon_locator.tick_values(xmin, xmax)
# Prune any large longitude ticks
lon_ticks = [t for t in lon_ticks if abs(t) <= 360.0]
lat_ticks = lat_locator.tick_values(ymin, ymax)
# Prune any large latitude ticks
lat_ticks = [t for t in lat_ticks if abs(t) <= 90.0]
return lon_ticks, lat_ticks
def update_ticks(axes, coord, components, is_log):
"""
Changes the axes to have the proper tick formatting based on the type of
component.
:param axes: A matplotlib axis object to alter
:param coord: 'x' or 'y'
:param components: A list() of components that are plotted along this axis
:param is_log: Boolean for log-scale.
:kwarg max_categories: The maximum number of categories to display.
:return: None or #categories if components is Categorical
"""
if coord == 'x':
axis = axes.xaxis
elif coord == 'y':
axis = axes.yaxis
else:
raise TypeError("coord must be one of x,y")
is_cat = all(comp.categorical for comp in components)
if is_log:
axis.set_major_locator(LogLocator())
axis.set_major_formatter(LogFormatterMathtext())
elif is_cat:
all_categories = np.empty((0,), dtype=np.object)
for comp in components:
all_categories = np.union1d(comp.categories, all_categories)
locator = MaxNLocator(10, integer=True)
locator.view_limits(0, all_categories.shape[0])
format_func = partial(tick_linker, all_categories)
formatter = FuncFormatter(format_func)
axis.set_major_locator(locator)
axis.set_major_formatter(formatter)
return all_categories.shape[0]
else:
axis.set_major_locator(AutoLocator())
axis.set_major_formatter(ScalarFormatter())
def get_diff_levels(key_to_data, ncolors=20, varname=""):
"""
get nice levels for the contourf plot
:param field:
:type key_to_data: dict
"""
locator = MaxNLocator(nbins=ncolors, symmetric=True)
if varname in ["TT", ]:
return locator.tick_values(-7, 7)
elif varname in ["PR", ]:
return locator.tick_values(-3, 3)
delta = -1
for k, field in key_to_data.items():
if hasattr(field, "mask"):
good_data = field[~field.mask]
else:
good_data = field.flatten()
delta = max(np.round(np.percentile(np.abs(good_data), 95)), delta)
return locator.tick_values(-delta, delta)
def get_colormap_and_norm_for(var_name, to_plot=None, ncolors=10, vmin=None, vmax=None):
"""
If vmin or vmax is None then to_plot parameter is required
:param var_name:
:param ncolors: Number of discrete colors in the colorbar, try to take a good number like 10, 5, ...
:return:
Note: when `var_name` is STFL, the parameter ncolors is ignored
"""
if None in [vmin, vmax]:
vmin, vmax = to_plot.min(), to_plot.max()
locator = MaxNLocator(ncolors)
clevs = locator.tick_values(vmin, vmax)
if var_name in ["STFL", "STFA"]:
upper = 1000
bounds = [0, 100, 200, 500, 1000]
while upper <= vmax:
upper += 1000
bounds.append(upper)
ncolors = len(bounds) - 1
cmap = cm.get_cmap("Blues", ncolors)
norm = BoundaryNorm(bounds, ncolors=ncolors) # LogNorm(vmin=10 ** (pmax - ncolors), vmax=10 ** pmax)
else:
reverse = True
# if var_name in ["PR"]:
# reverse = False
cmap = cm.get_cmap("Spectral_r" if reverse else "Spectral", len(clevs) - 1)
# norm, bounds, vmin_nice, vmax_nice = get_boundary_norm_using_all_vals(to_plot, ncolors)
norm = BoundaryNorm(clevs, len(clevs) - 1)
return cmap, norm
def bin_boundaries(self, vmin, vmax):
"""Return bin boundaries (a.k.a. tick locations)."""
# The calculation of the label width should actually depend on the
# tick Formatter instance.
label_width = max(len('%g' % v) for v in (vmin, vmax))
label_px_width = 1.2 * TextPath((0, 0), '0' * (label_width + 2)).get_extents().width
w, _ = self.axis.axes.figure.canvas.get_width_height()
if not isinstance(self.axis, maxis.XAxis):
raise NotImplementedError("Currently only XAxis is supported.")
# Use figure width to calculate the number of bins---only valid for x-axis.
# The following nbins calculation is approximate since it should vary
# with subplot parameters---i.e. padding on the left and right of
# subplots and multiple columns in a subplot grid.
self._nbins = min(self._nbins_cached, int(w / label_px_width))
return MaxNLocator.bin_boundaries(self, vmin, vmax)
class mpl_breaks:
"""
Compute breaks using MPL's default locator
See :class:`~matplotlib.ticker.MaxNLocator` for the
parameter descriptions
Examples
--------
>>> x = range(10)
>>> limits = (0, 9)
>>> mpl_breaks()(limits)
array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])
>>> mpl_breaks(nbins=2)(limits)
array([ 0., 5., 10.])
"""
def __init__(self, *args, **kwargs):
self.locator = MaxNLocator(*args, **kwargs)
def __call__(self, limits):
"""
Compute breaks
Parameters
----------
limits : tuple
Minimum and maximum values
Returns
-------
out : array_like
Sequence of breaks points
"""
if any(np.isinf(limits)):
return []
if limits[0] == limits[1]:
return np.array([limits[0]])
return self.locator.tick_values(limits[0], limits[1])
def _plot_row(vname="", level=0, config_dict=None, plot_cc_only_for=None, mark_significance=True):
"""
if plot_cc_only_for is not None, should be equal to the label of the simulation to be plotted
"""
lons, lats = config_dict.lons, config_dict.lats
bmp = config_dict.basemap
"""
:type bmp: mpl_toolkits.basemap.Basemap
"""
xx, yy = bmp(lons, lats)
lons[lons > 180] -= 360
fig = config_dict.fig
gs = config_dict.gs
""":type : matplotlib.gridspec.GridSpec """
nrows_subplots, ncols_subplots = gs.get_geometry()
label_base = config_dict.label_base
label_modif = config_dict.label_modif
the_row = config_dict.the_row
season_to_months = config_dict.season_to_months
if "+" in vname or "-" in vname:
op = "+" if "+" in vname else "-"
vname1, vname2 = vname.split(op)
vname1 = vname1.strip()
vname2 = vname2.strip()
current_base = {}
future_base = {}
current_modif = {}
future_modif = {}
# vname1
current_base1 = compute_seasonal_means_for_each_year(config_dict["Current"][label_base], var_name=vname1,
level=level,
season_to_months=season_to_months)
future_base1 = compute_seasonal_means_for_each_year(config_dict["Future"][label_base], var_name=vname1,
level=level,
season_to_months=season_to_months)
current_modif1 = compute_seasonal_means_for_each_year(config_dict["Current"][label_modif], var_name=vname1,
level=level,
season_to_months=season_to_months)
future_modif1 = compute_seasonal_means_for_each_year(config_dict["Future"][label_modif], var_name=vname1,
level=level,
season_to_months=season_to_months)
# vname2
current_base2 = compute_seasonal_means_for_each_year(config_dict["Current"][label_base], var_name=vname2,
level=level,
season_to_months=season_to_months)
future_base2 = compute_seasonal_means_for_each_year(config_dict["Future"][label_base], var_name=vname2,
level=level,
season_to_months=season_to_months)
current_modif2 = compute_seasonal_means_for_each_year(config_dict["Current"][label_modif], var_name=vname2,
level=level,
season_to_months=season_to_months)
future_modif2 = compute_seasonal_means_for_each_year(config_dict["Future"][label_modif], var_name=vname2,
level=level,
season_to_months=season_to_months)
for season in current_base1:
current_base[season] = eval("current_base2[season]{}current_base1[season]".format(op))
future_base[season] = eval("future_base2[season]{}future_base1[season]".format(op))
current_modif[season] = eval("current_modif2[season]{}current_modif1[season]".format(op))
future_modif[season] = eval("future_modif2[season]{}future_modif1[season]".format(op))
else:
current_base = compute_seasonal_means_for_each_year(config_dict["Current"][label_base], var_name=vname,
level=level,
season_to_months=season_to_months)
future_base = compute_seasonal_means_for_each_year(config_dict["Future"][label_base], var_name=vname,
level=level,
season_to_months=season_to_months)
current_modif = compute_seasonal_means_for_each_year(config_dict["Current"][label_modif], var_name=vname,
level=level,
season_to_months=season_to_months)
future_modif = compute_seasonal_means_for_each_year(config_dict["Future"][label_modif], var_name=vname,
level=level,
season_to_months=season_to_months)
# Calculate the differences in cc signal
season_to_diff = OrderedDict()
season_to_plot_diff = OrderedDict()
diff_max = 0
print(list(current_base.keys()))
# Get the ranges for colorbar and calculate p-values
print("------------------ impacts on projected changes to {} -----------------------".format(vname))
season_to_pvalue = OrderedDict()
for season in list(current_base.keys()):
_, pvalue_current = ttest_ind(current_modif[season], current_base[season], axis=0, equal_var=False)
_, pvalue_future = ttest_ind(future_modif[season], future_base[season], axis=0, equal_var=False)
if plot_cc_only_for is None:
season_to_pvalue[season] = np.minimum(pvalue_current, pvalue_future)
season_to_diff[season] = (future_modif[season] - current_modif[season]) - \
(future_base[season] - current_base[season])
else:
if plot_cc_only_for == label_base:
_, season_to_pvalue[season] = ttest_ind(future_base[season], current_base[season], axis=0, equal_var=False)
c_data = current_base[season]
f_data = future_base[season]
else:
_, season_to_pvalue[season] = ttest_ind(future_modif[season], current_modif[season], axis=0, equal_var=False)
c_data = current_modif[season]
f_data = future_modif[season]
season_to_diff[season] = f_data - c_data
# Convert units if required
if vname in config_dict.multipliers:
season_to_diff[season] *= config_dict.multipliers[vname]
field_to_plot = infovar.get_to_plot(vname, season_to_diff[season].mean(axis=0), lons=lons, lats=lats)
season_to_plot_diff[season] = field_to_plot
print("{}: {}".format(season, season_to_plot_diff[season].mean()))
if hasattr(field_to_plot, "mask"):
diff_max = max(np.percentile(np.abs(field_to_plot[~field_to_plot.mask]), 95), diff_max)
else:
diff_max = max(np.percentile(np.abs(field_to_plot), 95), diff_max)
print("--------------------------------------------------------")
img = None
locator = MaxNLocator(nbins=10, symmetric=True)
clevels = locator.tick_values(-diff_max, diff_max)
bn = BoundaryNorm(clevels, len(clevels) - 1)
cmap = cm.get_cmap("RdBu_r", len(clevels) - 1)
for col, season in enumerate(current_base.keys()):
ax = fig.add_subplot(gs[the_row, col])
if not col:
ax.set_ylabel(infovar.get_long_display_label_for_var(vname))
if not the_row:
ax.set_title(season)
img = bmp.pcolormesh(xx, yy, season_to_plot_diff[season].copy(),
vmin=-diff_max, vmax=diff_max,
cmap=cmap, norm=bn, ax=ax)
# logging
good_vals = season_to_plot_diff[season]
good_vals = good_vals[~good_vals.mask]
print("------" * 10)
print("{}: min={}; max={}; area-avg={};".format(season, good_vals.min(), good_vals.max(), good_vals.mean()))
bmp.readshapefile(quebec_info.BASIN_BOUNDARIES_DERIVED_10km[:-4], "basin_edge", ax=ax)
p = season_to_pvalue[season]
if hasattr(season_to_plot_diff[season], "mask"):
p = np.ma.masked_where(season_to_plot_diff[season].mask, p)
if plot_cc_only_for is not None and mark_significance:
cs = bmp.contourf(xx, yy, p, hatches=["..."], levels=[0.05, 1], colors='none')
if (col == ncols_subplots - 2) and (the_row == nrows_subplots - 1):
# create a legend for the contour set
artists, labels = cs.legend_elements()
labels = ["not significant"]
ax.legend(artists, labels, handleheight=1, loc="upper right",
bbox_to_anchor=(1.0, -0.05), borderaxespad=0., frameon=False)
bmp.drawcoastlines(ax=ax, linewidth=0.4)
if vname in ["I5"] and season.lower() in ["summer"]:
ax.set_visible(False)
cb = plt.colorbar(img, cax=fig.add_subplot(gs[the_row, len(current_base)]), extend="both")
if hasattr(config_dict, "name_to_units") and vname in config_dict.name_to_units:
cb.ax.set_title(config_dict.name_to_units[vname])
else:
cb.ax.set_title(infovar.get_units(vname))
def bin_boundaries(self, vmin, vmax):
self._guess_steps(vmin, vmax)
return MaxNLocator.bin_boundaries(self, vmin, vmax)
def _raw_ticks(self, vmin, vmax):
self._guess_steps(vmin, vmax)
return MaxNLocator._raw_ticks(self, vmin, vmax)
def set_params(self, **kwargs):
"""Set parameters within this locator."""
MaxNLocator.set_params(self, **kwargs)
if 'dms' in kwargs:
self._dms = kwargs['dms']
def plot(self,
property: typing.Union[str, typing.List[str]] = 'energies',
key: str = None,
filename=None,
baselines: typing.Dict[str, float] = None,
*args, **kwargs):
"""
Convenience function to plot the progress of the optimizer
:param filename: if given plot to file, otherwise plot to terminal
:param property: the property to plot, given as string
:param key: for properties like angles and gradients you can specifiy which one you want to plot
if set to none all keys are plotted. You can pass down single keys or lists of keys. DO NOT use tuples of keys or any other hashable list types
give key as list if you want to plot multiple properties with different keys
"""
from matplotlib import pyplot as plt
from matplotlib.ticker import MaxNLocator
fig = plt.figure()
fig.gca().xaxis.set_major_locator(MaxNLocator(integer=True))
import pickle
if baselines is not None:
for k, v in baselines.items():
plt.axhline(y=v, label=k)
if hasattr(property, "lower"):
properties = [property.lower()]
else:
properties = property
labels = None
if 'labels' in kwargs:
labels = kwargs['labels']
elif 'label' in kwargs:
labels = kwargs['label']
if hasattr(labels, "lower"):
labels = [labels] * len(properties)
for k, v in kwargs.items():
if hasattr(plt, k):
f = getattr(plt, k)
if callable(f):
f(v)
else:
f = v
if key is None:
keys = [[k for k in self.angles[-1].keys()]] * len(properties)
elif isinstance(key, typing.Hashable):
keys = [[assign_variable(key)]] * len(properties)
else:
key = [assign_variable(k) for k in key]
keys = [key] * len(properties)
for i, p in enumerate(properties):
try:
label = labels[i]
except:
label = p
if p == "energies":
data = getattr(self, "extract_" + p)()
plt.plot(list(data.keys()), list(data.values()), label=str(label), marker='o', linestyle='--')
else:
for k in keys[i]:
data = getattr(self, "extract_" + p)(key=k)
plt.plot(list(data.keys()), list(data.values()), label=str(label) + " " + str(k), marker='o',
linestyle='--')
loc = 'best'
if 'loc' in kwargs:
loc = kwargs['loc']
plt.legend(loc=loc)
if filename is None:
plt.show()
else:
pickle.dump(fig, open(filename + ".pickle", "wb"))
plt.savefig(fname=filename + ".pdf", **kwargs)
def fit_Spline(mainDic,x,y,yerr,infilename,outfilename,biasDic,outliersline,outliersdist,observedIntraInRangeSum, possibleIntraInRangeCount, possibleInterAllCount, observedIntraAllSum, observedInterAllSum, resolution, passNo):
'''
infilename = name of interactionCounts file
'''
with open(logfile, 'a') as log:
log.write("\nFitting a univariate spline to the probability means\n"),
log.write("------------------------------------------------------------------------------------\n"),
splineX = None
newSplineY = None
residual = None
FDRx = None
FDRy = None
if not interOnly:
if outliersdist != None:
y = [f for _, f in sorted(zip(x,y), key=lambda pair: pair[0])]
x.sort()
for i in range(1,len(x)):
if x[i]<=x[i-1]:
print("ERROR in spline fitting. Distances do not decrease across bins. Ensure interaction file is correct.")
print("Avg. distance of bin(i-1)... %s" % x[i-1])
print("Avg. distance of bin(i)... %s" % x[i])
sys.exit(2)
# maximum residual allowed for spline is set to min(y)^2
splineError=min(y)*min(y)
# use fitpack2 method -fit on the real x and y from equal occupancy binning
ius = UnivariateSpline(x, y, s=splineError)
tempMaxX=max(x)
tempMinX=min(x)
tempList=sorted([dis for dis in mainDic])
splineX=[]
### The below for loop will make sure nothing is out of range of [min(x) max(x)]
### Therefore everything will be within the range where the spline is defined
for i in tempList:
if tempMinX<=i<=tempMaxX:
splineX.append(i)
splineY=ius(splineX)
#print(splineY)
#print(yerr)
ir = IsotonicRegression(increasing=False)
newSplineY = ir.fit_transform(splineX,splineY)
#print(newSplineY)
residual =sum([i*i for i in (y - ius(x))])
if visual==True:
xi = np.linspace(min(x),max(x),5*len(x))
yi = ius(xi)
print("Plotting %s" % (outfilename + ".png"))
plt.clf()
fig = plt.figure()
ax = fig.add_subplot(2,1,1)
plt.plot(myUtils.scale_a_list(splineX,toKb), myUtils.scale_a_list(newSplineY,toProb),'g-',label="spline-"+str(passNo),linewidth=2)
plt.errorbar(myUtils.scale_a_list(x,toKb),myUtils.scale_a_list(y,toProb),myUtils.scale_a_list(yerr,toProb),fmt='r.',label="Mean with std. error",linewidth=2)
#plt.ylabel('Contact probability (x10$^{-5}$)',fontsize='large')
#plt.xlabel('Genomic distance (kb)',fontsize='large')
plt.ylabel('Contact probability (x10$^{-5}$)')
plt.xlabel('Genomic distance (kb)')
if distLowThres>0 and distUpThres<float("inf"):
plt.xlim(myUtils.scale_a_list([distLowThres, distUpThres],toKb))
plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 3, prune=None))
ax.legend(loc="upper right")
ax = fig.add_subplot(2,1,2)
plt.loglog(splineX,newSplineY,'g-')
plt.errorbar(x, y, yerr=yerr, fmt='r.') # Data
if distLowThres>0 and distUpThres<float("inf"):
plt.xlim([distLowThres, distUpThres])
plt.ylabel('Contact probability (log-scale)')
plt.xlabel('Genomic distance (log-scale)')
plt.savefig(outfilename+'.png')
# NOW write the calculated pvalues and corrected pvalues in a file
infile = gzip.open(infilename, 'rt')
intraInRangeCount=0
intraOutOfRangeCount=0
intraVeryProximalCount=0
interCount=0
discardCount=0
p_vals=[]
q_vals=[]
biasl=[]
biasr=[]
for line in infile:
ch1,mid1,ch2,mid2,contactCount=line.rstrip().split()
contactCount = float(contactCount)
interxn=myUtils.Interaction([ch1, int(mid1), ch2, int(mid2)])
interxn.setCount(contactCount)
mid1 = int(mid1); mid2 = int(mid2)
interactionType = interxn.getType(distLowThres,distUpThres)
bias1=1.0; bias2=1.0; # assumes there is no bias to begin with
# if the biasDic is not null sets the real bias values
if biasDic:
if ch1 in biasDic and mid1 in biasDic[ch1]:
bias1=biasDic[ch1][mid1]
if ch2 in biasDic and mid2 in biasDic[ch2]:
bias2=biasDic[ch2][mid2]
biasl.append(bias1)
biasr.append(bias2)
if (bias1<0 or bias2<0) and interactionType !='inter':
prior_p=1.0
p_val=1.0
discardCount+=1
elif interactionType=='intraInRange' and not interOnly:
distToLookUp=max(interxn.getDistance(),min(x))
distToLookUp=min(distToLookUp,max(x))
i=min(bisect.bisect_left(splineX, distToLookUp),len(splineX)-1)
prior_p=newSplineY[i]*(bias1*bias2)
# prior_p=newSplineY[i]
p_val=scsp.bdtrc(interxn.getCount()-1,observedIntraInRangeSum,prior_p)
intraInRangeCount +=1
# print(interxn.getCount() - 1, observedIntraInRangeSum, prior_p)
elif interactionType =='intraShort' and not interOnly:
prior_p=1.0
p_val=1.0
intraVeryProximalCount += 1
elif interactionType =='intraLong' and not interOnly:
prior_p=1.0
#p_val=scsp.bdtrc(interxn.getCount()-1, observedIntraAllSum,prior_p) ##RUNBY
p_val=1.0
intraOutOfRangeCount += 1
else:
if allReg or interOnly:
prior_p=interChrProb*(bias1*bias2)
p_val=scsp.bdtrc(interxn.getCount()-1,observedInterAllSum,prior_p)
interCount += 1
else:
p_val=1.0
#p_vals.append(p_val)
p_vals.append(p_val)
infile.close()
outlierThres = 0
# Do the BH FDR correction
if allReg:
outlierThres=1.0/(possibleIntraInRangeCount+possibleInterAllCount)
q_vals=myStats.benjamini_hochberg_correction(p_vals, possibleInterAllCount+possibleIntraInRangeCount)
elif interOnly and not allReg:
outlierThres = 1.0/possibleInterAllCount
q_vals=myStats.benjamini_hochberg_correction(p_vals, possibleInterAllCount)
else:
outlierThres = 1.0/possibleIntraInRangeCount
q_vals=myStats.benjamini_hochberg_correction(p_vals, possibleIntraInRangeCount)
print("Outlier threshold is... %s" % (outlierThres))
#now we write the values back to the file
infile =gzip.open(infilename, 'rt')
if resolution:
outfile =gzip.open(outfilename+'.res'+str(resolution)+'.significances.txt.gz', 'wt')
else:
outfile =gzip.open(outfilename+'.significances.txt.gz', 'wt')
print("Writing p-values and q-values to file %s" % (outfilename + ".significances.txt"))
outfile.write("chr1\tfragmentMid1\tchr2\tfragmentMid2\tcontactCount\tp-value\tq-value\tbias1\tbias2\n")
count=0
for line in infile:
words=line.rstrip().split()
chr1=words[0]
midPoint1=int(words[1])
chr2=words[2]
midPoint2=int(words[3])
interactionCount=float(words[4])
p_val=p_vals[count]
q_val=q_vals[count]
bias1=biasl[count]
bias2=biasr[count]
if (allReg or interOnly) and chr1!=chr2:
outfile.write("%s\t%d\t%s\t%d\t%d\t%e\t%e\t%e\t%e\n" % (str(chr1), midPoint1, str(chr2), midPoint2, interactionCount, p_val, q_val, bias1, bias2))
if (allReg or not interOnly) and chr1==chr2:
interactionDistance = abs(midPoint1-midPoint2)
if myUtils.in_range_check(interactionDistance,distLowThres, distUpThres):
outfile.write("%s\t%d\t%s\t%d\t%d\t%e\t%e\t%e\t%e\n" % (str(chr1), midPoint1, str(chr2), midPoint2, interactionCount, p_val, q_val, bias1, bias2))
if p_val<outlierThres:
outliersline.add(count)
outliersdist.add(abs(midPoint1-midPoint2))
count+=1
outfile.close()
infile.close()
if visual == True:
print("Plotting q-values to file %s" % outfilename + ".qplot.png")
minFDR=0.0
maxFDR=0.05
increment=0.001
FDRx,FDRy=plot_qvalues(q_vals,minFDR,maxFDR,increment,outfilename+".qplot")
with open(logfile, 'a') as log:
log.write("Spline successfully fit\n"),
log.write("\n"),
log.write("\n"),
return [splineX, newSplineY, residual, outliersline, outliersdist, FDRx, FDRy] # from fit_Spline
def predict_and_plot_sstaging(data_path, data_file):
print("I am here", os.getcwd())
model = keras.models.load_model('flaskapp/model')
data_path = Path(data_path)
record = data_file
annotation_dict = defaultdict(lambda: 5, {
'1': 0,
'2': 1,
'3': 2,
'4': 2,
'R': 3
})
classes = defaultdict(lambda: '6', {
'1000': '1',
'0100': '2',
'0010': '3',
'0001': '4',
'0000': '5'
})
class_count = defaultdict(lambda: 0, {
'1000': 0,
'0100': 0,
'0010': 0,
'0001': 0,
'0000': 0
})
annsamp = wfdb.rdann(str(data_path / record),
extension='st', summarize_labels=True)
signal = wfdb.rdrecord(str(data_path / record), channels=[2])
physical_signal = signal.p_signal
physical_signal = preprocessing.scale(physical_signal)
number_annotations = len(annsamp.aux_note)
starting_index = int((len(physical_signal) / 7500) -
number_annotations)*7500
physical_signal = physical_signal[starting_index:]
inputs = np.split(physical_signal, number_annotations)
annots = list()
target = [[0]*4 for _ in range(number_annotations)]
for annotation_index in range(number_annotations):
labels = annsamp.aux_note[annotation_index].split(' ')
labels = [_l.strip('\x00') if type(_l) == str else str(_l)
for _l in labels]
for label in labels:
if annotation_dict[label] != 5:
target[annotation_index][annotation_dict[label]] = 1
event_class = ''.join([str(v) for v in target[annotation_index]])
event_class = classes[event_class]
annots.append(event_class)
annots_c = to_categorical(annots)
inputs = np.array(inputs)
y_pred = model.predict(inputs)
y_pred_c = y_pred.argmax(axis=1)
y_pred_c = y_pred_c.tolist()
S1 = 0
S2 = 0
S3 = 0
R = 0
W = 0
for i in y_pred_c:
if i == 1:
S1 += 1
elif i == 2:
S2 += 1
elif i == 3:
S3 += 1
elif i == 4:
R += 1
else:
W += 1
annots = np.array(annots).astype(np.int)
annotation_dict1 = defaultdict(
lambda: 5, {
'S1': 1,
'S2': 2,
'S3': 3,
'R': 4,
'W': 5
}
)
matplotlib.rcParams['agg.path.chunksize'] = 10000
fig, ax = plt.subplots(figsize=(50, 20))
plt.rcParams["axes.linewidth"] = 3
plt.xticks(fontsize=50)
plt.yticks(fontsize=50)
plt.title('Hypno Çizimi', fontsize=50)
plt.tick_params(axis='both', pad=30)
plt.yticks(range(1, len(annotation_dict1.keys())+1),
annotation_dict1.keys())
plt.plot(y_pred_c, color='blue', linewidth=5)
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0)
print(figfile)
global figure
figure = base64.b64encode(figfile.getvalue()).decode('ascii')
plt.savefig("./flaskapp/static/images"+"/" +
record+"_hypno.png", format='png')
plt.clf()
plt.cla()
gc.collect()
fs = 250
times = create_times(len(physical_signal)/fs, fs)
plt.rcParams['axes.linewidth'] = 5
plt.rcParams["figure.figsize"] = (40, 50)
plt.title('Epoch Çizimi', fontsize=50)
plt.tick_params(axis='both', pad=45)
plt.xticks(fontsize=50)
plt.yticks(fontsize=50)
# plt.xlabel('time (s)', fontsize=100, labelpad=50)
# plt.ylabel('Voltage (mV)', fontsize=100, labelpad=50)
plt.gca().yaxis.set_major_locator(MaxNLocator(prune='lower'))
plt.plot(times, physical_signal, linewidth=6)
figfile1 = BytesIO()
plt.savefig(figfile1, format='png')
figfile1.seek(0)
global figure2
figure2 = base64.b64encode(figfile1.getvalue()).decode('ascii')
plt.savefig("./flaskapp/static/images"+"/" +
record + "_epochs.png", format='png')
plt.clf()
plt.cla()
gc.collect()
total = len(annots)
eq = 0
for i in range(len(annots)):
if annots[i] == y_pred_c[i]:
eq += 1
f = open("./flaskapp/"+data_file + ".txt", "w")
f.write("Toplam uyku süresi:" + str(total/2) + " dakika\n")
f.write("S1:" + str(S1) + " dakika\n")
f.write("S2:" + str(S2) + " dakika\n")
f.write("S3:" + str(S3) + " dakika\n")
f.write("R:" + str(R) + " dakika\n")
f.write("W:" + str(W) + " dakika\n")
f.close()
global read_file
read = open("./flaskapp/"+data_file + ".txt")
global info
info = read.read()
read_file = read.readlines()
ss_labels = ["S1", "S2", "S3", "R", "W"]
ss_values = [S1, S2, S3, R, W]
ss_explode = (0, 0, 0.1, 0, 0)
fig1, ax1 = plt.subplots(figsize=(40, 30))
ax1.pie(ss_values, explode=ss_explode, labels=ss_labels, autopct='%1.1f%%',
shadow=False, startangle=90, textprops={'fontsize': 60})
ax1.axis('equal')
figfile2 = BytesIO()
plt.savefig(figfile2, format='png')
figfile2.seek(0)
global figure3
figure3 = base64.b64encode(figfile2.getvalue()).decode('ascii')
plt.savefig("./flaskapp/static/images"+"/" +
record + "_pie.png", format='png')
plt.clf()
plt.cla()
gc.collect()
def plot_split_value_histogram(
booster: Union[Booster, LGBMModel],
feature: Union[int, str],
bins: Union[int, str, None] = None,
ax=None,
width_coef: float = 0.8,
xlim: Optional[Tuple[float, float]] = None,
ylim: Optional[Tuple[float, float]] = None,
title: Optional[str] = 'Split value histogram for feature with @index/name@ @feature@',
xlabel: Optional[str] = 'Feature split value',
ylabel: Optional[str] = 'Count',
figsize: Optional[Tuple[float, float]] = None,
dpi: Optional[int] = None,
grid: bool = True,
**kwargs: Any
) -> Any:
"""Plot split value histogram for the specified feature of the model.
Parameters
----------
booster : Booster or LGBMModel
Booster or LGBMModel instance of which feature split value histogram should be plotted.
feature : int or str
The feature name or index the histogram is plotted for.
If int, interpreted as index.
If str, interpreted as name.
bins : int, str or None, optional (default=None)
The maximum number of bins.
If None, the number of bins equals number of unique split values.
If str, it should be one from the list of the supported values by ``numpy.histogram()`` function.
ax : matplotlib.axes.Axes or None, optional (default=None)
Target axes instance.
If None, new figure and axes will be created.
width_coef : float, optional (default=0.8)
Coefficient for histogram bar width.
xlim : tuple of 2 elements or None, optional (default=None)
Tuple passed to ``ax.xlim()``.
ylim : tuple of 2 elements or None, optional (default=None)
Tuple passed to ``ax.ylim()``.
title : str or None, optional (default="Split value histogram for feature with @index/name@ @feature@")
Axes title.
If None, title is disabled.
@feature@ placeholder can be used, and it will be replaced with the value of ``feature`` parameter.
@index/name@ placeholder can be used,
and it will be replaced with ``index`` word in case of ``int`` type ``feature`` parameter
or ``name`` word in case of ``str`` type ``feature`` parameter.
xlabel : str or None, optional (default="Feature split value")
X-axis title label.
If None, title is disabled.
ylabel : str or None, optional (default="Count")
Y-axis title label.
If None, title is disabled.
figsize : tuple of 2 elements or None, optional (default=None)
Figure size.
dpi : int or None, optional (default=None)
Resolution of the figure.
grid : bool, optional (default=True)
Whether to add a grid for axes.
**kwargs
Other parameters passed to ``ax.bar()``.
Returns
-------
ax : matplotlib.axes.Axes
The plot with specified model's feature split value histogram.
"""
if MATPLOTLIB_INSTALLED:
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
else:
raise ImportError('You must install matplotlib and restart your session to plot split value histogram.')
if isinstance(booster, LGBMModel):
booster = booster.booster_
elif not isinstance(booster, Booster):
raise TypeError('booster must be Booster or LGBMModel.')
hist, split_bins = booster.get_split_value_histogram(feature=feature, bins=bins, xgboost_style=False)
if np.count_nonzero(hist) == 0:
raise ValueError('Cannot plot split value histogram, '
f'because feature {feature} was not used in splitting')
width = width_coef * (split_bins[1] - split_bins[0])
centred = (split_bins[:-1] + split_bins[1:]) / 2
if ax is None:
if figsize is not None:
_check_not_tuple_of_2_elements(figsize, 'figsize')
_, ax = plt.subplots(1, 1, figsize=figsize, dpi=dpi)
ax.bar(centred, hist, align='center', width=width, **kwargs)
if xlim is not None:
_check_not_tuple_of_2_elements(xlim, 'xlim')
else:
range_result = split_bins[-1] - split_bins[0]
xlim = (split_bins[0] - range_result * 0.2, split_bins[-1] + range_result * 0.2)
ax.set_xlim(xlim)
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
if ylim is not None:
_check_not_tuple_of_2_elements(ylim, 'ylim')
else:
ylim = (0, max(hist) * 1.1)
ax.set_ylim(ylim)
if title is not None:
title = title.replace('@feature@', str(feature))
title = title.replace('@index/name@', ('name' if isinstance(feature, str) else 'index'))
ax.set_title(title)
if xlabel is not None:
ax.set_xlabel(xlabel)
if ylabel is not None:
ax.set_ylabel(ylabel)
ax.grid(grid)
return ax
def ps(dosave=True, fname='figures/domains.png', lont=None, latt=None, ht=None, dd=None):
'''
Plot Bathymetry of Puget Sound, Admiralty Inlet, and Admiralty Head
Inputs:
dosave Save figure
fname File name for figure
lont, latt transect points to plot if they are input
ht Depth along transect, if input
dd Distance in meters along transect
'''
# download bathymetry, which can be found at: http://figshare.com/preview/_preview/1165560 (27.3MB)
# Read in bathymetry
mat = scipy.io.loadmat('cascadia_gridded.mat')
# x and y limits for these plots
lonlimsPS = np.array([-124., -122.15]) #-123.21, -122.15])
latlimsPS = np.array([47.02, 48.82])
lonlimsAI = np.array([-122.85, -122.535])
latlimsAI = np.array([47.9665, 48.228])
lonlimsAH = np.array([-122.72, -122.64])
latlimsAH = np.array([48.12, 48.18])
# Functionality copied from https://github.com/clawpack/geoclaw/blob/master/src/python/geoclaw/topotools.py#L873
land_cmap = plt.get_cmap('Greens_r')
sea_cmap = plt.get_cmap('Blues_r')
cmapPS = colormaps.add_colormaps((land_cmap, sea_cmap), data_limits=[-375,2500], data_break=0.0)
cmapAI = 'Blues_r'
cmapAH = 'Blues_r'
# levels to plot
levsPS = np.concatenate((np.arange(-375, 0, 25), np.arange(0,3000,500)))
levsAI = np.arange(-200, 20, 20)
levsAH = np.arange(-120, 15, 15)
# use basemap
basemapPS = Basemap(llcrnrlon=lonlimsPS[0], llcrnrlat=latlimsPS[0],
urcrnrlon=lonlimsPS[1], urcrnrlat=latlimsPS[1],
lat_0=latlimsPS.mean(), lon_0=lonlimsPS.mean(),
projection='lcc', resolution='f',
area_thresh=0.)
xPS, yPS = basemapPS(mat['lon_topo'], mat['lat_topo'])
xlimsAI, ylimsAI = basemapPS(lonlimsAI, latlimsAI)
xlimsAH, ylimsAH = basemapPS(lonlimsAH, latlimsAH)
# Make Puget Sound plot
fig = plt.figure(figsize=(16,16))
axPS = fig.add_subplot(111)
basemapPS.drawcoastlines(ax=axPS)
mappablePS = axPS.contourf(xPS, yPS, mat['z_topo'], cmap=cmapPS, levels=levsPS, zorder=2)
locator = MaxNLocator(6) # if you want no more than 10 contours
locator.create_dummy_axis()
locator.set_bounds(lonlimsPS[0], lonlimsPS[1])
pars = locator()
locator = MaxNLocator(6) # if you want no more than 10 contours
locator.create_dummy_axis()
locator.set_bounds(latlimsPS[0], latlimsPS[1])
mers = locator()
basemapPS.drawparallels(mers, dashes=(1, 1), linewidth=0.15, labels=[1,0,0,0], ax=axPS)#, zorder=3)
basemapPS.drawmeridians(pars, dashes=(1, 1), linewidth=0.15, labels=[0,0,0,1], ax=axPS)#, zorder=3)
cbPS = fig.colorbar(mappablePS, pad=0.015, aspect=35)
cbPS.set_label('Height/depth [m]')
# Label
axPS.text(0.8, 0.025, 'Puget Sound', transform=axPS.transAxes, color='0.15')
# Inset magnified plot of Admiralty Inlet
axAI = zoomed_inset_axes(axPS, 2, loc=1)
basemapPS.drawcoastlines(ax=axAI)
basemapPS.fillcontinents('darkgreen', ax=axAI)
mappableAI = axAI.contourf(xPS, yPS, mat['z_topo'], cmap=cmapAI, levels=levsAI)
axAI.set_xlim(xlimsAI)
axAI.set_ylim(ylimsAI)
# Inlaid colorbar
caxAI = fig.add_axes([0.581, 0.665, 0.011, 0.1])
cbAI = plt.colorbar(mappableAI, cax=caxAI, orientation='vertical')
cbAI.ax.tick_params(labelsize=12)
# draw a bbox of the region of the inset axes in the parent axes and
# connecting lines between the bbox and the inset axes area
mark_inset(axPS, axAI, loc1=2, loc2=4, fc="none", ec="0.3", lw=1.5, zorder=5)
# Label
axAI.text(0.41, 0.83, 'Admiralty\n Inlet', transform=axAI.transAxes, color='0.15', fontsize=16)
# Inset magnified plot of Admiralty Head
axAH = zoomed_inset_axes(axPS, 9, loc=3)
basemapPS.drawcoastlines(ax=axAH)
basemapPS.fillcontinents('darkgreen', ax=axAH)
mappableAH = axAH.contourf(xPS, yPS, mat['z_topo'], cmap=cmapAH, levels=levsAH)
axAH.set_xlim(xlimsAH)
axAH.set_ylim(ylimsAH)
if plt.is_numlike(lont):
# add points if you have some
xt, yt = basemapPS(lont, latt)
axAH.plot(xt, yt, 'k', lw=3)
# Inlaid colorbar
caxAH = fig.add_axes([0.398, 0.116, 0.012, 0.15])
cbAH = plt.colorbar(mappableAH, cax=caxAH, orientation='vertical')
cbAH.ax.tick_params(labelsize=12)
# draw a bbox of the region of the inset axes in the parent axes and
# connecting lines between the bbox and the inset axes area
mark_inset(axPS, axAH, loc1=2, loc2=4, fc="none", ec="0.3", lw=1.5, zorder=5)
# Label
axAH.text(0.47, 0.92, 'Admiralty Head', transform=axAH.transAxes, color='0.15', fontsize=16)
# pdb.set_trace()
if plt.is_numlike(lont):
# Add axes to plot transect depths
axdepths = fig.add_axes([0.28, 0.39, 0.14, 0.075], zorder=11)
axdepths.plot((np.arange(lont.size)*dd)/1000., -ht, '0.2', lw=2, zorder=12)
axdepths.tick_params(axis='both', colors='0.1', top='off', right='off', width=2, length=4, labelsize=12, labelcolor='0.1')
axdepths.spines['bottom'].set_color('none')
axdepths.spines['top'].set_color('none')
axdepths.spines['left'].set_color('none')
axdepths.spines['right'].set_color('none')
axdepths.set_xlabel('Distance along transect [km]', fontsize=14, color='0.1')
axdepths.set_ylabel('Transect depth [m]', fontsize=14, color='0.1')
axdepths.patch.set_alpha(0.0) # make bg transparent
fig.show()
# Save figure
if dosave:
fig.savefig(fname, bbox_inches='tight')
fig.show()
def _plot_residuals_to_ax(data_all,
model_ML,
ax,
e_unit=u.eV,
sed=True,
errorbar_opts={}):
"""Function to compute and plot residuals in units of the uncertainty"""
if "group" not in data_all.keys():
data_all["group"] = np.zeros(len(data_all))
groups = np.unique(data_all["group"])
MLf_unit, MLsedf = sed_conversion(model_ML[0], model_ML[1].unit, sed)
MLene = model_ML[0].to(e_unit)
MLflux = (model_ML[1] * MLsedf).to(MLf_unit)
ax.axhline(0, color="k", lw=1, ls="--")
interp = False
if data_all["energy"].size != MLene.size or not np.allclose(
data_all["energy"].value, MLene.value):
interp = True
from scipy.interpolate import interp1d
modelfunc = interp1d(MLene.value, MLflux.value, bounds_error=False)
for g in groups:
groupidx = np.where(data_all["group"] == g)
data = data_all[groupidx]
notul = ~data["ul"]
df_unit, dsedf = sed_conversion(data["energy"], data["flux"].unit, sed)
ene = data["energy"].to(e_unit)
xerr = u.Quantity((data["energy_error_lo"], data["energy_error_hi"]))
flux = (data["flux"] * dsedf).to(df_unit)
dflux = (data["flux_error_lo"] + data["flux_error_hi"]) / 2.0
dflux = (dflux * dsedf).to(df_unit)[notul]
if interp:
difference = flux[notul] - modelfunc(ene[notul]) * flux.unit
else:
difference = flux[notul] - MLflux[groupidx][notul]
# wrap around color and marker cycles
color = color_cycle[int(g) % len(color_cycle)]
marker = marker_cycle[int(g) % len(marker_cycle)]
opts = dict(
zorder=100,
marker=marker,
ls="",
elinewidth=2,
capsize=0,
mec=color,
mew=0.1,
ms=6,
color=color,
)
opts.update(errorbar_opts)
ax.errorbar(ene[notul].value, (difference / dflux).decompose().value,
yerr=(dflux / dflux).decompose().value,
xerr=xerr[:, notul].to(e_unit).value,
**opts)
from matplotlib.ticker import MaxNLocator
ax.yaxis.set_major_locator(
MaxNLocator(5, integer="True", prune="upper", symmetric=True))
ax.set_ylabel(r"$\Delta\sigma$")
ax.set_xscale("log")
def __init__(self, *args, **kwargs):
self.locator = MaxNLocator(*args, **kwargs)
def _plot_var(vname="", level=0, config_dict=None, data_dict=None):
modif_label = config_dict.label_modif
base_label = config_dict.label_base
base_config_c, modif_config_c = [config_dict["Current"][the_label] for the_label in [base_label, modif_label]]
base_config_f, modif_config_f = [config_dict["Future"][the_label] for the_label in [base_label, modif_label]]
bmp = config_dict.basemap
lons = config_dict.lons
lons[lons > 180] -= 360
lats = config_dict.lats
xx, yy = bmp(lons, lats)
i_to_label = {
0: base_label,
1: modif_label,
2: "{}\n--\n{}".format(modif_label, base_label)
}
j_to_title = {
0: "Current ({}-{})".format(base_config_c.start_year, base_config_c.end_year),
1: "Future ({}-{})".format(base_config_f.start_year, base_config_f.end_year),
2: "Future - Current"
}
# Create a folder for images
img_folder = os.path.join("cc_paper", "{}_vs_{}".format(modif_label, base_label))
if not os.path.isdir(img_folder):
os.makedirs(img_folder)
nrows = ncols = 3
plot_utils.apply_plot_params(font_size=10, width_pt=None, width_cm=27, height_cm=20)
field_cmap = cm.get_cmap("jet", 20)
diff_cmap = cm.get_cmap("RdBu_r", 20)
for season in list(data_dict[base_config_c].keys()):
fig = plt.figure()
fig.suptitle(
"{} ({})".format(infovar.get_long_display_label_for_var(vname), season),
font_properties=FontProperties(weight="bold"))
gs = GridSpec(nrows=nrows, ncols=ncols)
mean_c_base = data_dict[base_config_c][season].mean(axis=0)
mean_f_base = data_dict[base_config_f][season].mean(axis=0)
mean_c_modif = data_dict[modif_config_c][season].mean(axis=0)
mean_f_modif = data_dict[modif_config_f][season].mean(axis=0)
print("------------ {}, {} --------------".format(vname, season))
print("data_dict[base_config_c][season].shape = {}".format(data_dict[base_config_c][season].shape))
_, p_signif_base_cc = ttest_ind(data_dict[base_config_c][season], data_dict[base_config_f][season], axis=0, equal_var=False)
_, p_signif_current = ttest_ind(data_dict[base_config_c][season], data_dict[modif_config_c][season], axis=0, equal_var=False)
print("Base p-value ranges: {} to {}".format(p_signif_base_cc.min(), p_signif_base_cc.max()))
_, p_signif_modif_cc = ttest_ind(data_dict[modif_config_c][season], data_dict[modif_config_f][season], axis=0, equal_var=False)
_, p_signif_future = ttest_ind(data_dict[base_config_f][season], data_dict[modif_config_f][season], axis=0, equal_var=False)
ij_to_pvalue = {
(0, 2): p_signif_base_cc,
(1, 2): p_signif_modif_cc,
(2, 0): p_signif_current,
(2, 1): p_signif_future,
# (2, 2): np.minimum(p_signif_current, p_signif_future)
}
ij_to_data = {
(0, 0): mean_c_base, (1, 0): mean_c_modif,
(0, 1): mean_f_base, (1, 1): mean_f_modif
}
# Apply multipliers for unit conversion
for k in list(ij_to_data.keys()):
ij_to_data[k] *= multiplier_dict.get(vname, 1)
# mask oceans
mask_inland = vname in ["STFL", "STFA"] or vname.startswith("I")
ij_to_data[k] = maskoceans(lonsin=lons, latsin=lats, datain=ij_to_data[k], inlands=mask_inland)
# get all means to calculate the ranges of mean fields
all_means = []
for vals in ij_to_data.values():
all_means.extend(vals[~vals.mask])
if vname == "PR":
# mm/day
minval = 0
maxval = 5
extend_val = "max"
else:
minval = np.percentile(all_means, 1)
maxval = np.percentile(all_means, 95)
extend_val = "neither"
d1 = ij_to_data[0, 1] - ij_to_data[0, 0]
d2 = ij_to_data[1, 1] - ij_to_data[1, 0]
all_cc_diffs = np.ma.asarray([d1, d2])
mindiff_cc = np.percentile(all_cc_diffs[~all_cc_diffs.mask], 1)
maxdiff_cc = np.percentile(all_cc_diffs[~all_cc_diffs.mask], 99)
maxdiff_cc = max(np.ma.abs(maxdiff_cc), np.ma.abs(mindiff_cc))
# Limit the temperature change scale
if vname == "TT":
maxdiff_cc = min(maxdiff_cc, 10)
mindiff_cc = -maxdiff_cc
# Add all differences due to changed processes in a list
all_proc_diffs = []
d1 = ij_to_data[1, 0] - ij_to_data[0, 0]
d2 = ij_to_data[1, 1] - ij_to_data[0, 1]
all_proc_diffs.extend([d1, d2])
all_proc_diffs.append(d2 - d1)
all_proc_diffs = np.ma.asarray(all_proc_diffs)
mindiff_proc = np.percentile(all_proc_diffs[~all_proc_diffs.mask], 1)
maxdiff_proc = np.percentile(all_proc_diffs[~all_proc_diffs.mask], 95)
maxdiff_proc = max(np.abs(maxdiff_proc), np.abs(mindiff_proc))
mindiff_proc = -maxdiff_proc
plot_data = []
for row in range(nrows - 1):
row_data = []
for col in range(ncols - 1):
row_data.append(ij_to_data[row, col])
plot_data.append(row_data + [0, ])
# calculate differences between different model configurations
row = 2
plot_data.append([0, 0, 0])
for col in range(2):
plot_data[row][col] = plot_data[1][col] - plot_data[0][col]
# calculate CC
col = 2
for row in range(3):
plot_data[row][col] = plot_data[row][1] - plot_data[row][0]
# Do the plotting
for i in range(nrows):
for j in range(ncols):
ax = fig.add_subplot(gs[i, j])
plotting_values = (i < 2) and (j < 2)
if plotting_values:
the_min, the_max = minval, maxval
elif i == nrows - 1:
the_min, the_max = mindiff_proc, maxdiff_proc
else:
the_min, the_max = mindiff_cc, maxdiff_cc
the_cmap = field_cmap if plotting_values else diff_cmap
locator = MaxNLocator(nbins=the_cmap.N // 2, symmetric=not plotting_values)
bounds = locator.tick_values(the_min, the_max)
if vname in ["STFA", "STFL"] and plotting_values:
the_min = 1.0e-5
the_max = 1.0e4
locator = LogLocator(numticks=11)
bounds = locator.tick_values(the_min, the_max)
print(bounds)
norm = LogNorm(vmin=bounds[0], vmax=bounds[-1])
plot_data[i][j] = np.ma.masked_where(plot_data[i][j] <= the_min, plot_data[i][j])
print("the_max = {}".format(the_max))
the_cmap = cm.get_cmap("jet", len(bounds) - 1)
else:
norm = BoundaryNorm(bounds, the_cmap.N)
extend = "both" if not plotting_values else extend_val
if plotting_values:
print(i, j, the_min, the_max)
im = bmp.pcolormesh(xx, yy, plot_data[i][j][:, :], cmap=the_cmap, norm=norm)
if (i, j) in ij_to_pvalue:
p = ij_to_pvalue[i, j]
# not_signif = (p > 0.2)
# not_signif = np.ma.masked_where(~not_signif, not_signif) * 0.75
# not_signif = np.ma.masked_where(plot_data[i][j].mask, not_signif)
# bmp.pcolormesh(xx, yy, not_signif, vmin=0, vmax=1, cmap=cm.get_cmap("gray", 20))
p = np.ma.masked_where(plot_data[i][j].mask, p)
cs = bmp.contourf(xx, yy, p, hatches=["..."], levels=[0.1, 1], colors='none')
# create a legend for the contour set
# artists, labels = cs.legend_elements()
# ax.legend(artists, labels, handleheight=2)
bmp.colorbar(im, ticks=bounds, extend=extend)
bmp.drawcoastlines(ax=ax, linewidth=0.5)
ax.set_title(j_to_title.get(j, "") if i == 0 else "")
ax.set_ylabel(i_to_label.get(i, "") if j == 0 else "")
# Save the image to the file
img_name = "{}_{}_{}-{}_{}-{}.png".format(
vname + str(level), season,
base_config_f.start_year, base_config_f.end_year,
base_config_c.start_year, base_config_c.end_year)
img_path = os.path.join(img_folder, img_name)
fig.savefig(img_path, bbox_inches="tight")
plt.close(fig)
def _before_plot(self, region):
super(ScalarDataPlot, self)._before_plot(region)
self.ax.set_yscale(self.yscale)
self.ax.yaxis.set_major_locator(
MaxNLocator(nbins=self.n_yticks - 1, min_n_ticks=self.n_yticks))
def __init__(self, nbins=9, steps=[1, 2, 5, 10], **kwargs):
MaxNLocator.__init__(self, nbins=nbins, steps=steps, **kwargs )
n = args.chromn
if n <= 0:
fig = plotG_all(r, clens, drawraw=args.drawraw,rawcolor=rawclr,
smoothcolor=smoothclr, smoothwidth=smoothwidth,
maxgap=args.maxgap)
else:
fig = plotG_one(n-1, r, clens, drawraw=args.drawraw,rawcolor=rawclr,
smoothcolor=smoothclr, smoothwidth=smoothwidth,
maxgap=args.maxgap)
ax = fig.gca()
if 'coords' in vars(args):
ax.set_xlim(*args.coords)
if n > 0:
if 'nticks' in vars(args):
m = MaxNLocator(*args.nticks)
m.view_limits(*ax.get_xlim())
ax.xaxis.set_major_locator(m)
else:
ax.xaxis.set_major_locator(MaxNLocator(4))
if 'ylim' in vars(args):
ax.set_ylim(*args.ylim)
if 'figsize' in vars(args):
fig.set_size_inches(*args.figsize)
if args.threshold > 0:
ax.hlines(args.threshold, 0, tlen, color=args.thresholdclr, linestyle='dashed')
name,ext = os.path.splitext(args.outfile.name)
# check extension of outfile
def auto_set_ticks(self):
"""
Use the matplotlib.ticker class to automatically set nice values for the major and minor ticks.
Log axes generally come out nicely spaced without needing manual intervention. For particularly narrow latitude
vs longitude plots the ticks can come out overlapped, so an exception is included to deal with this.
"""
from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter
y_variable = self.plot_args['y_variable'].lower()
x_variable = self.plot_args['x_variable'].lower()
ymin, ymax = self.matplotlib.ylim()
# Matplotlib xlim doesn't work with cartopy plots
xmin, xmax = self.xmin, self.xmax
# xmin, xmax = self.matplotlib.xlim()
max_x_bins = 9
max_y_bins = 9
xsteps = self.plot_args['xrange'].get('xstep', None)
ysteps = self.plot_args['yrange'].get('ystep', None)
lon_steps = [1, 3, 6, 9, 10]
lat_steps = [1, 3, 6, 9, 10]
variable_step = [1, 2, 4, 5, 10]
if (xmax - xmin) < 5:
lon_steps = variable_step
if (ymax - ymin) < 5:
lat_steps = variable_step
# We need to make a special exception for particularly narrow and wide plots, which will be lat vs lon
# preserving the aspect ratio. This gives more options for the spacing to try and find something that can use
# the maximum number of bins.
if x_variable.startswith('lon') and y_variable.startswith('lat'):
max_y_bins = 7 # as plots are wider rather than taller
if (ymax - ymin) > 2.2 * (xmax - xmin):
max_x_bins = 4
max_y_bins = 11
elif (xmax - xmin) > 2.2 * (ymax - ymin):
max_x_bins = 14
max_y_bins = 4
lat_or_lon = 'lat', 'lon'
if xsteps is None and not self.plot_args['logx']:
if self.plot_args['x_variable'].lower().startswith(lat_or_lon):
lon_locator = MaxNLocator(nbins=max_x_bins, steps=lon_steps)
if self.is_map():
self.cartopy_axis.set_xticks(lon_locator.tick_values(xmin, xmax), crs=self.transform)
self.cartopy_axis.xaxis.set_major_formatter(LongitudeFormatter())
else:
self.matplotlib.axes().xaxis.set_major_locator(lon_locator)
else:
self.matplotlib.axes().xaxis.set_major_locator(MaxNLocator(nbins=max_x_bins, steps=variable_step))
if not self.is_map():
self.matplotlib.axes().xaxis.set_minor_locator(AutoMinorLocator())
self.matplotlib.axes().xaxis.grid(False, which='minor')
if ysteps is None and not self.plot_args['logy']:
if y_variable.startswith(lat_or_lon):
lat_locator = MaxNLocator(nbins=max_y_bins, steps=lat_steps)
if self.is_map():
self.cartopy_axis.set_yticks(lat_locator.tick_values(ymin, ymax), crs=self.transform)
self.cartopy_axis.yaxis.set_major_formatter(LatitudeFormatter())
else:
self.matplotlib.axes().yaxis.set_major_locator(lat_locator)
else:
self.matplotlib.axes().yaxis.set_major_locator(MaxNLocator(nbins=max_y_bins, steps=variable_step))
if not self.is_map():
self.matplotlib.axes().yaxis.set_minor_locator(AutoMinorLocator())
self.matplotlib.axes().yaxis.grid(False, which='minor')
prev_overlap = overlap[-1]
if np.linalg.norm(st_prev[0] - s_t) == 0:
overlap.append(prev_overlap + 1)
else:
overlap.append(prev_overlap)
st_prev[0] = s_t
else:
overlap.append(0)
st_prev.append(s_t)
cp.minimize_frank_wolfe(
f.f_grad,
x0,
l1_ball.lmo,
callback=trace,
max_iter=int(1e4),
step=step,
verbose=True,
lipschitz=f.lipschitz,
)
ax.plot(overlap, label=label, marker=marker, markevery=7 + i)
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
ax.legend()
ax.set_xlabel("number of iterations")
ax.set_ylabel("LMO overlap")
ax.set_title(dataset_title)
fig.tight_layout() # otherwise the right y-label is slightly clipped
ax.grid()
# plt.legend()
plt.show()
def validate_seas_mean_lswt_from_hostetler_and_nemo_with_homa(
hl_data_path="/home/huziy/skynet3_rech1/CRCM_GL_simulation/all_files",
start_year=2003, end_year=2006):
crcm5_model_manager = Crcm5ModelDataManager(samples_folder_path=hl_data_path, all_files_in_samples_folder=True)
varname = "L1"
season = "Fall"
season_to_months = {season: range(9, 11)}
season_months = list(season_to_months[season])
print(season_months, season_to_months)
hl_lake_temp_clim = crcm5_model_manager.get_mean_field(start_year=start_year, end_year=end_year, var_name=varname,
level=1.1, months=season_months)
hl_lake_temp_clim = hl_lake_temp_clim[13:-13, 13:-13]
print("hl_lake_temp_clim.shape = ", hl_lake_temp_clim.shape)
lon_hl, lat_hl = crcm5_model_manager.lons2D, crcm5_model_manager.lats2D
print("lon_hl.shape = ", lon_hl.shape)
print(lon_hl.min(), lon_hl.max())
print(lat_hl.min(), lat_hl.max())
# Get Nemo manager here only for coordinates and mask
nemo_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012",
suffix="icemod.nc")
lon2d, lat2d, bmp = nemo_manager.get_coords_and_basemap()
# Interpolate hostetler's lake fraction to the model's grid
hl_lake_temp_clim -= 273.15
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lon_hl.flatten(), lat_hl.flatten())
print(xs.shape)
ktree = cKDTree(data=list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lon2d.flatten(), lat2d.flatten())
dists, inds = ktree.query(np.asarray(list(zip(xt, yt, zt))))
print(inds[:20])
print(type(inds))
print(inds.min(), inds.max())
hl_lake_temp_clim = hl_lake_temp_clim.flatten()[inds].reshape(lon2d.shape)
# get nemo and observed sst
obs_sst, nemo_sst, _, _, _ = nemo_manager.get_nemo_and_homa_seasonal_mean_sst(start_year=start_year,
end_year=end_year,
season_to_months=season_to_months)
obs_sst_clim = np.ma.mean([obs_sst[y][season] for y in range(start_year, end_year + 1)], axis=0)
nemo_sst_clim = np.ma.mean([nemo_sst[y][season] for y in range(start_year, end_year + 1)], axis=0)
obs_sst_clim = np.ma.masked_where(~nemo_manager.lake_mask, obs_sst_clim)
nemo_sst_clim = np.ma.masked_where(~nemo_manager.lake_mask, nemo_sst_clim)
hl_lake_temp_clim = np.ma.masked_where(~nemo_manager.lake_mask, hl_lake_temp_clim)
# plt.figure()
xx, yy = bmp(lon2d.copy(), lat2d.copy())
# im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc)
# bmp.colorbar(im)
vmin = min(obs_sst_clim.min(), nemo_sst_clim.min(), hl_lake_temp_clim.min())
vmax = max(obs_sst_clim.max(), nemo_sst_clim.max(), hl_lake_temp_clim.max())
print("vmin={}; vmax={}".format(vmin, vmax))
# plt.figure()
# b = Basemap()
# xx, yy = b(lons_obs, lats_obs)
# im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc)
# b.colorbar(im)
# b.drawcoastlines()
# Plot as usual: model, obs, model - obs
img_folder = Path("nemo/hostetler")
if not img_folder.is_dir():
img_folder.mkdir()
img_file = img_folder.joinpath("validate_{}_lswt_hostetler_nemo_vs_homa_{}-{}.png".format(
season, start_year, end_year))
fig = plt.figure()
gs = GridSpec(1, 4, width_ratios=[1, 1, 1, 0.05])
all_axes = []
cmap = cm.get_cmap("jet", 10)
locator = MaxNLocator(nbins=10)
ticks = locator.tick_values(vmin=vmin, vmax=vmax)
norm = BoundaryNorm(boundaries=ticks, ncolors=len(ticks) - 1)
# Model, Hostetler
ax = fig.add_subplot(gs[0, 0])
ax.set_title("Hostetler+CRCM5")
bmp.pcolormesh(xx, yy, hl_lake_temp_clim, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
#
ax = fig.add_subplot(gs[0, 1])
ax.set_title("NEMO-offline")
im = bmp.pcolormesh(xx, yy, nemo_sst_clim, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
# Obs: Homa
ax = fig.add_subplot(gs[0, 2])
ax.set_title("MODIS")
im = bmp.pcolormesh(xx, yy, obs_sst_clim, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
plt.colorbar(im, cax=fig.add_subplot(gs[0, -1]), ticks=ticks)
for the_ax in all_axes:
bmp.drawcoastlines(ax=the_ax)
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
def validate_seas_mean_lswt_from_hostetler_and_nemo_with_homa(
hl_data_path="/home/huziy/skynet3_rech1/CRCM_GL_simulation/all_files",
start_year=2003, end_year=2009):
crcm5_model_manager = Crcm5ModelDataManager(samples_folder_path=hl_data_path, all_files_in_samples_folder=True)
varname = "sohefldo"
season = "Summer"
season_to_months = {season: range(6, 9)}
season_months = list(season_to_months[season])
# Get Nemo manager here only for coordinates and mask
nemo_offline_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012",
suffix="grid_T.nc")
nemo_crcm5_manager = NemoYearlyFilesManager(
folder="/home/huziy/skynet3_rech1/one_way_coupled_nemo_outputs_1979_1985",
suffix="grid_T.nc")
lon2d, lat2d, bmp = nemo_offline_manager.get_coords_and_basemap()
# Interpolate
# get nemo sst
nemo_offline_sst = nemo_offline_manager.get_seasonal_clim_field(start_year=start_year,
end_year=end_year,
season_to_months=season_to_months,
varname=varname)
nemo_crcm5_sst = nemo_crcm5_manager.get_seasonal_clim_field(start_year=start_year,
end_year=end_year,
season_to_months=season_to_months,
varname=varname)
nemo_offline_sst = np.ma.masked_where(~nemo_offline_manager.lake_mask, nemo_offline_sst[season])
nemo_crcm5_sst = np.ma.masked_where(~nemo_offline_manager.lake_mask, nemo_crcm5_sst[season])
# plt.figure()
xx, yy = bmp(lon2d.copy(), lat2d.copy())
# im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc)
# bmp.colorbar(im)
vmin = min(nemo_offline_sst.min(), nemo_crcm5_sst.min())
vmax = max(nemo_offline_sst.max(), nemo_crcm5_sst.max())
# plt.figure()
# b = Basemap()
# xx, yy = b(lons_obs, lats_obs)
# im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc)
# b.colorbar(im)
# b.drawcoastlines()
# Plot as usual: model, obs, model - obs
img_folder = Path("nemo")
if not img_folder.is_dir():
img_folder.mkdir()
img_file = img_folder.joinpath("{}_{}_nemo-offline_vs_nemo-crcm5_{}-{}.png".format(season.lower(),
varname, start_year, end_year))
fig = plt.figure()
gs = GridSpec(1, 3, width_ratios=[1, 1, 0.05])
all_axes = []
cmap = cm.get_cmap("jet", 10)
locator = MaxNLocator(nbins=10)
ticks = locator.tick_values(vmin=vmin, vmax=vmax)
norm = BoundaryNorm(boundaries=ticks, ncolors=len(ticks) - 1)
# Model, Hostetler
ax = fig.add_subplot(gs[0, 0])
ax.set_title("NEMO+CRCM5")
bmp.pcolormesh(xx, yy, nemo_crcm5_sst, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
#
ax = fig.add_subplot(gs[0, 1])
ax.set_title("NEMO-offline")
im = bmp.pcolormesh(xx, yy, nemo_offline_sst, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
all_axes.append(ax)
# Obs: Homa
# ax = fig.add_subplot(gs[0, 2])
# ax.set_title("MODIS")
# im = bmp.pcolormesh(xx, yy, obs_sst_clim, cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)
# all_axes.append(ax)
plt.colorbar(im, cax=fig.add_subplot(gs[0, -1]), ticks=ticks)
for the_ax in all_axes:
bmp.drawcoastlines(ax=the_ax)
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
def __init__(self, nbins=9, steps=[1, 2, 5, 10]):
MaxNLocator.__init__(self, nbins=nbins, steps=steps)
self._nbins_cached = nbins
def plot_profiles():
obs_base_dir = Path("/home/huziy/skynet3_rech1/nemo_obs_for_validation/data_from_Ram_Yerubandi/ADCP-profiles")
obs_dir_list = [
str(obs_base_dir.joinpath("105.317")),
str(obs_base_dir.joinpath("155.289"))
]
obs_var_col = AdcpProfileObs.vmag_col
model_var_name = FLOW_SPEED
model_folder = "/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012"
manager_nemo_u = NemoYearlyFilesManager(folder=model_folder, suffix="_U.nc")
manager_nemo_v = NemoYearlyFilesManager(folder=model_folder, suffix="_V.nc")
manager_nemo_w = NemoYearlyFilesManager(folder=model_folder, suffix="_W.nc")
fig = plt.figure()
gs = GridSpec(len(obs_dir_list), 5, width_ratios=[1, 1, 0.05, 1, 0.05])
cmap = cm.get_cmap("jet", 10)
diff_cmap = cm.get_cmap("RdBu_r", 10)
for i, obs_dir in enumerate(obs_dir_list):
adcp = AdcpProfileObs()
dates, levels, obs_data = adcp.get_acdp_profiles(folder=obs_dir, data_column=obs_var_col)
dates_m, levs_m, u_cs = manager_nemo_u.get_tz_crosssection_for_the_point(
lon=adcp.longitude, lat=adcp.latitude,
start_date=dates[0], end_date=dates[-1],
var_name="vozocrtx", zlist=levels
)
dates_m, levs_m, v_cs = manager_nemo_v.get_tz_crosssection_for_the_point(
lon=adcp.longitude, lat=adcp.latitude,
start_date=dates[0], end_date=dates[-1],
var_name="vomecrty", zlist=levels
)
dates_m, levs_m, w_cs = manager_nemo_w.get_tz_crosssection_for_the_point(
lon=adcp.longitude, lat=adcp.latitude,
start_date=dates[0], end_date=dates[-1],
var_name="vovecrtz", zlist=levels
)
numdates = date2num(dates.tolist())
print("Obs dates are: {} ... {}".format(dates[0], dates[-1]))
print([num2date([n for n in numdates if n not in dates_m])])
zz, tt = np.meshgrid(levels, numdates)
umag_mod_cs = (u_cs ** 2 + v_cs ** 2 + w_cs ** 2) ** 0.5 * 100.0
all_axes = []
# Obs
ax = fig.add_subplot(gs[i, 0])
ax.set_title("Obs")
cs = ax.contourf(tt, zz, obs_data, cmap=cmap)
ax.set_ylabel("({:.1f}, {:.1f})".format(adcp.longitude, adcp.latitude))
all_axes.append(ax)
# Model
ax = fig.add_subplot(gs[i, 1])
ax.set_title("NEMO-offline")
cs = ax.contourf(tt, zz, umag_mod_cs, levels=cs.levels, cmap=cmap)
all_axes.append(ax)
plt.colorbar(cs, cax=fig.add_subplot(gs[i, 2]))
# Bias
ax = fig.add_subplot(gs[i, 3])
ax.set_title("Model - Obs.")
delta = umag_mod_cs - obs_data
vmax = np.abs(delta).max()
vmin = -vmax
locator = MaxNLocator(nbins=diff_cmap.N, symmetric=True)
cs = ax.contourf(tt, zz, delta, levels=locator.tick_values(vmin, vmax), cmap=diff_cmap)
plt.colorbar(cs, cax=fig.add_subplot(gs[i, 4]))
all_axes.append(ax)
for the_ax in all_axes:
the_ax.xaxis.set_major_formatter(DateFormatter("%b"))
the_ax.xaxis.set_major_locator(MonthLocator())
the_ax.invert_yaxis()
img_folder = Path("nemo/adcp")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_file = img_folder.joinpath("adcp_profiles.pdf")
fig.tight_layout()
fig.savefig(str(img_file), bbox_inches="tight")
def __init__(self):
"""
Create a new locator instance.
"""
# Select divisions that are round minutes
MaxNLocator.__init__(self, nbins=8, steps=[1, 5/3., 10/3., 5, 10])
def main():
start_year_c = 1980
end_year_c = 2010
img_folder = "cc_paper"
current_path = "/RESCUE/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5"
base_label = "CRCM5-LI"
# Need to read land fraction
geo_file_path = "/RESCUE/skynet3_rech1/huziy/hdf_store/pm1979010100_00000000p"
r_obj = RPN(geo_file_path)
mg_field = r_obj.get_first_record_for_name("MG")
lons, lats = r_obj.get_longitudes_and_latitudes_for_the_last_read_rec()
r_obj.close()
future_shift_years = 90
params = dict(
data_path=current_path,
start_year=start_year_c, end_year=end_year_c,
label=base_label
)
base_config_c = RunConfig(**params)
base_config_f = base_config_c.get_shifted_config(future_shift_years)
varname = "INTF"
level = 0
daily_dates, intf_c = analysis.get_daily_climatology(path_to_hdf_file=base_config_c.data_path,
var_name=varname, level=level,
start_year=base_config_c.start_year,
end_year=base_config_c.end_year)
_, intf_f = analysis.get_daily_climatology(path_to_hdf_file=base_config_f.data_path,
var_name=varname, level=level,
start_year=base_config_f.start_year,
end_year=base_config_f.end_year)
mg_fields = np.asarray([mg_field for d in daily_dates])
mg_crit = 0.0001
the_mask = mg_fields <= mg_crit
# Convert to mm/day as well
intf_c = _avg_along_lon(intf_c, the_mask) * 24 * 3600
intf_f = _avg_along_lon(intf_f, the_mask) * 24 * 3600
lats_agg = lats.mean(axis=0)
num_dates = date2num(daily_dates)
lats_agg_2d, num_dates_2d = np.meshgrid(lats_agg, num_dates)
# Do the plotting
fig = plt.figure()
gs = GridSpec(2, 3, width_ratios=[1, 1, 0.05])
norm = SymLogNorm(5e-5)
all_axes = []
# Current
ax = fig.add_subplot(gs[0, 0])
cs = ax.contourf(num_dates_2d, lats_agg_2d, intf_c[:], 60, norm=norm)
ax.set_title("Current ({}-{})".format(
base_config_c.start_year, base_config_c.end_year))
all_axes.append(ax)
# Future
ax = fig.add_subplot(gs[0, 1])
ax.set_title("Future ({}-{})".format(
base_config_f.start_year, base_config_f.end_year))
cs = ax.contourf(num_dates_2d, lats_agg_2d, intf_f[:], levels=cs.levels, norm=norm)
all_axes.append(ax)
# Colorbar for value plots
cax = fig.add_subplot(gs[0, 2])
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-1, 2))
plt.colorbar(cs, cax=cax, format=sfmt)
cax.set_xlabel("mm/day")
cax.yaxis.get_offset_text().set_position((-2, 10))
# CC
diff_cmap = cm.get_cmap("RdBu_r", 20)
diff = (intf_f - intf_c) / (0.5 * (intf_c + intf_f)) * 100
diff[(intf_f == 0) & (intf_c == 0)] = 0
print(np.min(diff), np.max(diff))
print(np.any(diff.mask))
print(np.any(intf_c.mask))
print(np.any(intf_f.mask))
delta = 200
vmin = -delta
vmax = delta
locator = MaxNLocator(nbins=20, symmetric=True)
clevs = locator.tick_values(vmin=vmin, vmax=vmax)
ax = fig.add_subplot(gs[1, 1])
cs = ax.contourf(num_dates_2d, lats_agg_2d, diff, cmap=diff_cmap,
levels=clevs, extend="both")
ax.set_title("Future - Current")
# ax.set_aspect("auto")
all_axes.append(ax)
cb = plt.colorbar(cs, cax=fig.add_subplot(gs[1, -1]))
cb.ax.set_xlabel(r"%")
for i, the_ax in enumerate(all_axes):
the_ax.xaxis.set_minor_formatter(FuncFormatter(lambda d, pos: num2date(d).strftime("%b")[0]))
the_ax.xaxis.set_major_formatter(FuncFormatter(lambda d, pos: ""))
the_ax.xaxis.set_minor_locator(MonthLocator(bymonthday=15))
the_ax.xaxis.set_major_locator(MonthLocator())
the_ax.grid()
if i != 1:
the_ax.set_ylabel(r"Latitude ${\rm \left(^\circ N \right)}$")
# identify approximately the melting period and lower latitudes
march1 = date2num(datetime(daily_dates[0].year, 3, 1))
june1 = date2num(datetime(daily_dates[0].year, 6, 1))
sel_mask = (num_dates_2d >= march1) & (num_dates_2d < june1) & (lats_agg_2d <= 50)
print("Mean interflow decrease in the southern regions: {}%".format(diff[sel_mask].mean()))
# identify the regions of max interflow rates in current and future climates
lat_min = 55
lat_max = 57.5
may1 = date2num(datetime(daily_dates[0].year, 5, 1))
july1 = date2num(datetime(daily_dates[0].year, 7, 1))
mean_max_current = intf_c[(lats_agg_2d >= lat_min) & (lats_agg_2d <= lat_max) & (num_dates_2d <= july1) & (num_dates_2d >= june1)].mean()
mean_max_future = intf_f[(lats_agg_2d >= lat_min) & (lats_agg_2d <= lat_max) & (num_dates_2d <= june1) & (num_dates_2d >= may1)].mean()
print("Mean change in the maximum interflow rate: {} %".format((mean_max_future - mean_max_current) * 100 / mean_max_current))
img_file = Path(img_folder).joinpath("INTF_rate_longit_avg.png")
fig.tight_layout()
from crcm5.analyse_hdf import common_plot_params
fig.savefig(str(img_file), bbox_inches="tight", transparent=True, dpi=common_plot_params.FIG_SAVE_DPI)