def _remove_ensemble_mean_trend(da, dim="init_date", ensemble_dim="ensemble"):
"""Remove ensemble mean trend along given dimension.
Parameters
----------
da : xarray DataArray
Input data array
dim : str, default init_date
Dimension over which to calculate and remove trend
init_dim: str, default 'init_date'
Name of the initial date dimension to create in the output
ensemble_dim : str, default 'ensemble'
Name of the ensemble member dimension in da
Returns
-------
da_detrended : xarray DataArray
Detrended data array
"""
ensmean_trend = da.mean(ensemble_dim).polyfit(dim=dim, deg=1)
ensmean_trend_line = xr.polyval(da[dim], ensmean_trend["polyfit_coefficients"])
ensmean_trend_line_anomaly = ensmean_trend_line - ensmean_trend_line.isel({dim: 0})
da_detrended = da - ensmean_trend_line_anomaly
return da_detrended
def test_polyval(use_dask, use_datetime):
if use_dask and not has_dask:
pytest.skip("requires dask")
if use_datetime:
xcoord = xr.DataArray(pd.date_range("2000-01-01", freq="D",
periods=10),
dims=("x", ),
name="x")
x = xr.core.missing.get_clean_interp_index(xcoord, "x")
else:
xcoord = x = np.arange(10)
da = xr.DataArray(
np.stack((1.0 + x + 2.0 * x**2, 1.0 + 2.0 * x + 3.0 * x**2)),
dims=("d", "x"),
coords={
"x": xcoord,
"d": [0, 1]
},
)
coeffs = xr.DataArray(
[[2, 1, 1], [3, 2, 1]],
dims=("d", "degree"),
coords={
"d": [0, 1],
"degree": [2, 1, 0]
},
)
if use_dask:
coeffs = coeffs.chunk({"d": 2})
da_pv = xr.polyval(da.x, coeffs)
xr.testing.assert_allclose(da, da_pv.T)
def odyn_loc(SCE, MOD, DIR_O, DIR_OG, LOC, \
ref_steric, ye, N, ys, Gam, NormD, LowPass):
'''Compute the ocean dynamics and thermal expansion contribution to local sea
level using KNMI14 files.'''
nb_MOD = len(MOD)
nb_y2 = ye - ys + 1
lat_N, lat_S, lon_W, lon_E = LOC
#For KNMI files the SSH has a different name for each scenario
SSH_VAR_dic = {'rcp45': 'ZOSH45', 'rcp85': 'ZOS85'}
SSH_VAR = SSH_VAR_dic[SCE]
# Read sterodynamics and global steric contributions
for m in range(nb_MOD):
fi = xr.open_dataset(f'{DIR_O}{MOD[m]}_{SCE}.nc', use_cftime=True)
fig = xr.open_dataset(f'{DIR_OG}{MOD[m]}_{SCE}.nc', use_cftime=True)
fi = fi.assign_coords(TIME=fi['TIME.year'])
fig = fig.assign_coords(time=fig['time.year']).squeeze()
sd_da = fi[SSH_VAR].assign_coords(model=MOD[m])
st_da = fig[SSH_VAR].assign_coords(model=MOD[m])
if m == 0:
full_sd_da = sd_da
full_st_da = st_da
else:
full_sd_da = xr.concat([full_sd_da, sd_da], dim='model')
full_st_da = xr.concat([full_st_da, st_da], dim='model')
full_sd_da = full_sd_da.rename({'TIME': 'time'})
full_sd_da = full_sd_da.sel(time=slice(ref_steric[0], ye),
latitude=slice(lat_S, lat_N),
longitude=slice(lon_W, lon_E))
MAT = full_sd_da.mean(dim=['latitude', 'longitude'])
MAT_G = full_st_da.sel(time=slice(ref_steric[0], ye))
MAT = MAT - MAT.sel(time=slice(ref_steric[0], ref_steric[1])).mean(
dim='time')
if LowPass:
fit_coeff = MAT.polyfit('time', 3)
MAT = xr.polyval(coord=MAT.time, coeffs=fit_coeff.polyfit_coefficients)
MAT_G = MAT_G - MAT_G.sel(time=slice(ref_steric[0], ref_steric[1])).mean(
dim='time')
MAT_A = MAT - MAT_G
list_da = [MAT, MAT_G, MAT_A]
X_O_out = np.zeros([3, N, nb_y2])
for idx, da in enumerate(list_da):
# Select years after the reference period and convert from m to cm
da = da.sel(time=slice(ys, None)) * 100
X_O_out[idx, :, :] = misc.normal_distrib(da, Gam, NormD)
return X_O_out
def _get_trend(self, grpd, dim="time"):
# Estimate trend over da
trend = xr.polyval(coord=grpd[dim], coeffs=grpd.polyfit_coefficients)
if self.preserve_mean:
trend = apply_correction(trend,
invert(trend.mean(dim=dim), self.kind),
self.kind)
return trend
def _get_trend(self, da):
# Estimate trend over da
trend = xr.polyval(coord=da[self._fitted_dim],
coeffs=self._fitds.polyfit_coefficients)
if self.preserve_mean:
trend = apply_correction(
trend, invert(trend.mean(dim=self._fitted_dim), self.kind),
self.kind)
return trend
def rm_poly(ds: Union[xr.Dataset, xr.DataArray],
dim: str = "time",
deg: int = 2,
**kwargs: Any) -> Union[xr.Dataset, xr.DataArray]:
"""Remove degree polynomial of degree ``deg`` along dimension ``dim``."""
coefficients = ds.polyfit(dim, deg=deg, **kwargs)
coord = ds[dim]
fits = []
if isinstance(ds, xr.Dataset):
for v in coefficients:
name = v.replace("_polyfit_coefficients", "")
fit = xr.polyval(coord, coefficients[v]).rename(name)
fits.append(fit)
fits = xr.merge(fits)
elif isinstance(ds, xr.DataArray):
name = ds.name
v = list(coefficients.data_vars)[0]
fits = xr.polyval(coord, coefficients[v]).rename(name)
with xr.set_options(keep_attrs=True):
ds_rm_poly = ds - fits
return ds_rm_poly
def _polydetrend_get_trend(da, *, dim, degree, preserve_mean, kind):
"""Polydetrend, atomic func on 1 group."""
if len(dim) > 1:
da = da.mean(dim[1:])
dim = dim[0]
pfc = da.polyfit(dim=dim, deg=degree)
trend = xr.polyval(coord=da[dim], coeffs=pfc.polyfit_coefficients)
if preserve_mean:
trend = apply_correction(trend, invert(trend.mean(dim=dim), kind),
kind)
out = trend.rename("trend").to_dataset()
return out
def detrend(self, dim=['time'], degree=1):
"""Remove a polynomial trend from each dimension, one at a time.
"""
if type(dim) is not type([]): dim = [dim]
assert set(dim).issubset(set(self.ds[self.varname].dims))
for d in dim:
p = self.ds[self.varname].polyfit(dim=d, deg=degree)
fit = xr.polyval(self.ds[self.varname][d], p.polyfit_coefficients)
self.ds[self.varname] -= fit
if 'ilamb' not in self.ds[self.varname].attrs:
self.ds[self.varname].attrs['ilamb'] = ''
dim = ["'%s'" % d for d in dim]
self.ds[
self.varname].attrs['ilamb'] += "detrend(dim=[%s],degree=%d); " % (
",".join(dim), degree)
return self
def detrend_array(arr, dim="time", deg=1):
"""Detrend an Xarray DataArray
Parameters
----------
arr : xarray.DataArray
Input data array
dim : str, optional
Name of time dimension, by default "time"
deg : int, optional
Number of polynomial coefficients for fit, by default 1 (linear)
Returns
-------
xarray.DataArray
Detrended data array
"""
coeffs = arr.polyfit(dim=dim, deg=deg)
fitted = xr.polyval(arr[dim], coeffs.polyfit_coefficients)
return arr - fitted
# Read ocean dynamics and extrapolate if necessary
DIR_CDE = '/Users/dewilebars/Projects/Project_ProbSLR/CMIP_SeaLevel/outputs/'
fcde = misc.read_zos_ds(data_dir, SCE)
if cde_opt == 'mean':
CDE = fcde.CorrectedReggrided_zos.mean(dim='model')
else:
CDE = fcde.CorrectedReggrided_zos.sel(model=cde_opt)
if YEARS[-1] > 2100:
new_time = xr.DataArray(np.array(YEARS + 0.5),
dims='time',
coords=[np.array(YEARS + 0.5)],
name='time')
fit_coeff = CDE.polyfit('time', 2)
CDE = xr.polyval(coord=new_time, coeffs=fit_coeff.polyfit_coefficients)
# Read global sea level projection
#fcomp = xr.open_dataset(f'../outputs/ref_proj/SeaLevelPerc_{namelist_name}_{SCE}.nc')
fcomp = xr.open_dataset(f'../outputs/SeaLevelPerc_{namelist_name}_{SCE}.nc')
# Read fingerprints
f_gic = xr.open_dataset(f'{DIR_F}Relative_GLACIERS.nc', decode_times=False)
f_gic = f_gic.assign_coords({'time': np.arange(1986, 2101)})
f_gic = f_gic.rename({'longitude': 'lon', 'latitude': 'lat'})
f_gic = misc.rotate_longitude(f_gic, 'lon')
f_ic = xr.open_dataset(
f'{DIR_F}Relative_icesheets.nc') / 100 # Convert from % to fraction
f_ic = f_ic.rename({'longitude': 'lon', 'latitude': 'lat'})
f_ic = misc.rotate_longitude(f_ic, 'lon')
def peakmem_polyval(self, nx, ndeg):
x = self.xs[nx]
c = self.coeffs[ndeg]
xr.polyval(x, c).compute()
def tglob_cmip(data_dir, temp_opt, sce, start_date, ye, LowPass=False):
'''Read the text files of monthly temperature for each CMIP5 model and store
yearly averged values in an xarray DataArray.
Output data is in degree Kelvin'''
nb_y = ye - start_date + 1
if temp_opt in ['CMIP5', 'AR5']:
temp_data_dir = f'{data_dir}Data_AR5/Tglobal/'
if temp_opt == 'CMIP5':
path = f'{temp_data_dir}global_tas_Amon_*_{sce}_r1i1p1.dat'
files = glob.glob(path)
elif temp_opt == 'AR5':
files = temp_path_AR5(temp_data_dir, sce)
model_names = [f[86:-17] for f in files]
df = pd.DataFrame({
'clean_model_names': model_names,
'file_names': files
})
elif temp_opt == 'CMIP6':
temp_data_dir = f'{data_dir}Data_cmip6/tas_global_averaged_climate_explorer/'
df = select_tglob_cmip6_files(temp_data_dir, sce)
else:
print(f'ERROR: Value of TEMP: {mip} not recognized')
col_names = [
'Year', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',
'Oct', 'Nov', 'Dec'
]
for i in range(len(df)):
TEMP = pd.read_csv(df.file_names.iloc[i],
comment='#',
delim_whitespace=True,
names=col_names)
TEMP = TEMP.set_index('Year')
TGLOBi = xr.DataArray(TEMP.mean(axis=1))
mod = df.clean_model_names.iloc[i]
TGLOBi = TGLOBi.expand_dims({'model': [mod]})
if i == 0:
TGLOB = TGLOBi
else:
TGLOB = xr.concat([TGLOB, TGLOBi], dim='model')
TGLOB = TGLOB.rename({'Year': 'time'})
TGLOB = TGLOB.sel(time=slice(start_date, ye))
if LowPass:
new_time = xr.DataArray(np.arange(start_date, ye + 1),
dims='time',
coords=[np.arange(start_date, ye + 1)],
name='time')
fit_coeff = TGLOB.polyfit('time', 2)
TGLOB = xr.polyval(coord=new_time,
coeffs=fit_coeff.polyfit_coefficients)
TGLOB.name = 'GSAT'
return TGLOB
def snowfall_approximation(
pr: xr.DataArray,
tas: xr.DataArray,
thresh: str = "0 degC",
method: str = "binary",
) -> xr.DataArray:
"""Snowfall approximation from total precipitation and temperature.
Solid precipitation estimated from precipitation and temperature according to a given method.
Parameters
----------
pr : xarray.DataArray
Mean daily precipitation flux.
tas : xarray.DataArray, optional
Mean, maximum, or minimum daily temperature.
thresh : str,
Threshold temperature, used by method "binary".
method : {"binary", "brown", "auer"}
Which method to use when approximating snowfall from total precipitation. See notes.
Returns
-------
xarray.DataArray, [same units as pr]
Solid precipitation flux.
Notes
-----
The following methods are available to approximate snowfall and are drawn from the
Canadian Land Surface Scheme (CLASS, [Verseghy09]_).
- ``'binary'`` : When the temperature is under the freezing threshold, precipitation
is assumed to be solid. The method is agnostic to the type of temperature used
(mean, maximum or minimum).
- ``'brown'`` : The phase between the freezing threshold goes from solid to liquid linearly
over a range of 2°C over the freezing point.
- ``'auer'`` : The phase between the freezing threshold goes from solid to liquid as a degree six
polynomial over a range of 6°C over the freezing point.
References
----------
.. [Verseghy09] Diana Verseghy (2009), CLASS – The Canadian Land Surface Scheme (Version 3.4), Technical
Documentation (Version 1.1), Environment Canada, Climate Research Division, Science and Technology Branch.
https://gitlab.com/cccma/classic/-/blob/master/src/atmosphericVarsCalc.f90
"""
if method == "binary":
thresh = convert_units_to(thresh, tas)
prsn = pr.where(tas <= thresh, 0)
elif method == "brown":
# Freezing point + 2C in the native units
upper = convert_units_to(convert_units_to(thresh, "degC") + 2, tas)
thresh = convert_units_to(thresh, tas)
# Interpolate fraction over temperature (in units of tas)
t = xr.DataArray([-np.inf, thresh, upper, np.inf],
dims=("tas", ),
attrs={"units": "degC"})
fraction = xr.DataArray([1.0, 1.0, 0.0, 0.0],
dims=("tas", ),
coords={"tas": t})
# Multiply precip by snowfall fraction
prsn = pr * fraction.interp(tas=tas, method="linear")
elif method == "auer":
dtas = convert_units_to(tas, "degK") - convert_units_to(thresh, "degK")
# Create nodes for the snowfall fraction: -inf, thresh, ..., thresh+6, inf [degC]
t = np.concatenate([[-273.15],
np.linspace(0, 6, 100, endpoint=False), [6, 1e10]])
t = xr.DataArray(t, dims="tas", name="tas", coords={"tas": t})
# The polynomial coefficients, valid between thresh and thresh + 6 (defined in CLASS)
coeffs = xr.DataArray(
[100, 4.6664, -15.038, -1.5089, 2.0399, -0.366, 0.0202],
dims=("degree", ),
coords={"degree": range(7)},
)
fraction = xr.polyval(t.tas, coeffs).clip(0, 100) / 100
fraction[0] = 1
fraction[-2:] = 0
# Convert snowfall fraction coordinates to native tas units
prsn = pr * fraction.interp(tas=dtas, method="linear")
else:
raise ValueError(
f"Method {method} not one of 'binary', 'brown' or 'auer'.")
prsn.attrs["units"] = pr.attrs["units"]
return prsn
order=trend_order,
year_min=1850,
year_max=2100,
conv_pic_hist=conv_pic_hist,
gap=None,
rmv_disc=False,
verbose=verbose)
try:
# This breaks when Trend_pic_coeff does not contain values.
# It hapens for BCC-CSM2-MR for which polyfit does not return
# coefficients but does not crash
test = Trend_pic_coeff.values
# Build polynomial from coefficients
Trend_pic = xr.polyval(coord=y_ds.time, coeffs=Trend_pic_coeff)
# Remove the average over the reference period
Trend_pic = Trend_pic - Trend_pic.sel(
time=slice(ref_p_min, ref_p_max)).mean(dim='time')
except:
print(
'!!! WARNING: Detrending from piControl for this model does not' +
' work !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
branching_method = 'no_detrending'
Trend_pic = xr.DataArray(np.zeros(len(years_s)),
coords=[years_s],
dims=["time"])
MAT_CorrectedZOS_reg = np.zeros(
[len(years_s), len(mask_ds.lat),
def odyn_cmip(SCE, data_dir, LOC, ref_steric, ye, N, ys, Gam, NormD, LowPass,
BiasCorr):
'''Read the CMIP5 and CMIP6 global steric and ocean dynamics contribution
and compute a probability distribution.'''
nb_y2 = ye - ys + 1
# Read global steric
full_st_da = misc.read_zostoga_ds(data_dir, SCE)
# Convert from m to cm
MAT_G = full_st_da['zostoga_corrected'].sel(
time=slice(ref_steric[0], ye)) * 100
if LOC:
lat_N, lat_S, lon_W, lon_E = LOC
zos_ds = misc.read_zos_ds(data_dir, SCE)
full_sd_da = zos_ds['CorrectedReggrided_zos'].sel(
time=slice(ref_steric[0], ye),
lat=slice(lat_S, lat_N),
lon=slice(lon_W, lon_E))
# There are more models available for zos than for zostoga
# Here we select the intersection
model_list = list(
set(full_sd_da.model.values) & set(MAT_G.model.values))
MAT_A = full_sd_da.sel(model=model_list).mean(dim=['lat', 'lon'])
if BiasCorr:
# Use a bias correction based on the comparison of model and
# observations between 1979 and 2018
MAT_A = MAT_A * BiasCorr
MAT = MAT_A + MAT_G
else:
MAT = MAT_G
MAT_A = xr.zeros_like(MAT_G)
if LowPass:
new_time = xr.DataArray(np.arange(ys, ye + 1) + 0.5,
dims='time',
coords=[np.arange(ys, ye + 1) + 0.5],
name='time')
fit_coeff = MAT.polyfit('time', 2)
MAT = xr.polyval(coord=new_time, coeffs=fit_coeff.polyfit_coefficients)
fit_coeff = MAT_G.polyfit('time', 2)
MAT_G = xr.polyval(coord=new_time,
coeffs=fit_coeff.polyfit_coefficients)
MAT_A = MAT - MAT_G
list_da = [MAT, MAT_G, MAT_A]
X_O_out = np.zeros([3, N, nb_y2])
for idx, da in enumerate(list_da):
# Select years after the reference period
da = da.sel(time=slice(ys, None))
if not (LowPass):
# Add a year at the end because data is available until 2099 while code
# goes up to 2100
dal = da.isel(time=-1).assign_coords(
time=da.time[-1] + 1) # Last year with new coords
da = xr.concat([da, dal], dim='time')
X_O_out[idx, :, :] = misc.normal_distrib(da, Gam, NormD)
return X_O_out
def detrend_dim(da, dim, deg=1):
# detrend along a single dimension
p = da.polyfit(dim=dim, deg=deg)
fit = xr.polyval(da[dim], p.polyfit_coefficients)
return da - fit
attrs = {'units': hist_ds.attrs['parent_time_units']}
time_flt = [float(hist_ds.attrs['branch_time_in_parent'])]
time_ds = xr.Dataset({'time': ('time', time_flt, attrs)})
time_ds = xr.decode_cf(time_ds, use_cftime=True)
conv_pic_hist = float(VAR1a.time[0]) - time_ds.time.dt.year.values[0]
except:
# Pick a random large value that makes sure branching is not used in
# trend_zos_pic_cmip5
conv_pic_hist = -9999
Trend_pic_coeff, branching_method = pic.trend_pic(
MIP, VAR, ModelList.iloc[i], order=1, year_min=1850, year_max=2100,
conv_pic_hist=conv_pic_hist, gap=gap, rmv_disc=True, verbose=verbose)
# Build polynomial from coefficients
Trend_pic = xr.polyval(coord=VAR1a.time, coeffs=Trend_pic_coeff)
trend_da[i,:] = Trend_pic.sel(time=slice(year_min_min,year_max))
da[i,:] = VAR1a.sel(time=slice(year_min_min,year_max))
ds[VAR+'_corrected'] = da
ds['trend_picontrol'] = trend_da
### Remove missing models
ds = ds.where(ds.zostoga_corrected!=0)
ds = ds.dropna('model',how='all')
if year_max == 2300: #TODO Old code bellow
print('### List of models that run to 2300')
for i in range(0,dimMod-1):
tot_mis = np.sum(np.isnan(AVAR1[0,i,:]))
