def models_for_experiments(dic,
variable,
table,
experiments,
excluded_models=[],
included_models=None):
"""
Provided with DIC a dictionnary of data versions, and a VARIABLE name, returns
a list of (model,variant) pairs that have data for VARIABLE and for all EXPERIMENTS
(according to DIC), and the same variant index for all EXPERIMENTS for one given model
(excluding for piControl re. variant)
There is only one variant returned per model, as choosen by function `choose_variant()`
The structure of the dictionnary should be as follow
dic[experiment][variable][table][model][realization] = (grid,anything,anything)
"""
if included_models == "None":
included_models = None
# Store available realizations per model and exp
variants = dict()
for exp in experiments:
for model in dic[exp][variable][table]:
if included_models is None or model in included_models:
for real in dic[exp][variable][table][model].keys():
#feed_dic(variants,dic[exp][variable][table][model][real],model.encode('ascii'),exp,real)
feed_dic(variants, dic[exp][variable][table][model][real],
model, exp, real)
return choose_variant(variants, experiments, excluded_models,
included_models)
def read_aggregates(ref_period, proj_period, tag, cache_dir):
"""
This is an ancillary function for `change_figure_with_caching`. See its doc
"""
d = dict()
import glob
files = glob.glob(cache_dir + "/*=*=*=*=*=*=*=%s.nc" % tag)
for f in files:
bn = f.split("/")[-1]
bn = bn.split(".")[0]
bn = bn.split("=")
season, variable, exp, field, derive, rperiod, pperiod, _ = bn
if pperiod == proj_period and rperiod == ref_period:
dataset = fds(f, period="1850-2100")
# Need to erase cached value
cdrop(dataset)
feed_dic(d, dataset, variable, exp, season, field, derive)
return d
def models_for_experiments_multi_var(dic,
variable_table_pairs,
experiments,
excluded_models=[],
included_models=None):
"""
Same as function `models_for_experiments()` but for a list of (variable,table) pairs
"""
if included_models == "None":
included_models = None
# Store available realizations per model and exp which have data for all variables
# We use first variable,table pair as pivot
variable, table = variable_table_pairs[0]
variants = dict()
#
for exp in experiments:
try:
a = dic[exp][variable][table]
except:
print(exp, variable, table)
raise ValueError("")
for model in dic[exp][variable][table]:
for real in dic[exp][variable][table][model].keys():
ok = True
for var, tab in variable_table_pairs[1:]:
try:
a = dic[exp][var][tab][model][real]
except:
ok = False
if ok:
feed_dic(variants, dic[exp][variable][table][model][real],
model, exp, real)
return choose_variant(variants, experiments, excluded_models,
included_models)
def stats_of_basins_changes(model_changes,
ref_experiment,
scenario,
ref_period,
variable,
table,
time_stat,
data_versions,
slices,
stats_list,
basins_data,
fprint=True,
excluded_models=[],
included_models=None,
must_have_vars=[],
relative=True,
compute=False,
house_keeping=False):
"""
Feed dic MODEL_CHANGES with change values for TIME_STAT of VARIABLE spatially averaged
over basins listed in BASINS_DATA["basins"] (which can include "globe" and "land"); this with
respect to a REF_PERIOD and for a series of time SLICES in a given SCENARIO; and for the list of
models which, according to dict DATA_VERSIONS, provide VARIABLE for both experiments,
List INCLUDED_MODELS, if not None, limits models used
List EXCLUDED_MODELS allows to discard models (even those included in INCLUDED_MODELS
TIME_STAT can be "mean" or "std" and is computed by function `cancillary.mean_or_std`
Structure of MODEL_CHANGES (which forgets about TABLE) is :
model_changes[scenario][variable][time_stat][basin][period][model]=change_value
Also returns a dict of ensemble stats of changes across the ensemble of models, for STATS_LIST :
>>> ens_changes[basin][ensemble_stat][period] = stat_value
For syntax and semantics of STATS_LIST, see function `cancillary.ensemble_stat()`
BASINS_DATA is a dict with keys :
- basins : list of basin names for those basins to process
- basins_file : filename for the file coding basins
- basins_key : a dict of correspondance between basin names and basin numbers in basins_file
Change values are relative changes, except if RELATIVE=False
"""
lbasins = basins_data["basins"]
# Compute variables values on ref_period and slices,
values = dict()
if fprint:
print("%6s %s %s %s" % (scenario, variable, table, time_stat), end='')
pairs = must_have_vars + [(variable, table)]
models = models_for_experiments_multi_var(data_versions, pairs,
[scenario, ref_experiment],
excluded_models, included_models)
for model, variant in models:
if fprint: print(" %s" % (model), end='')
for period in [ref_period] + slices:
#if fprint : print "%s"%period[0:3],
for basin in lbasins:
if basin == "globe":
operator = ccdo_fast
args = {"operator": "fldmean"}
else:
operator = basin_average
args = {
"model": model,
"basin": basin,
"basins": basins_data,
"compute": True
}
value = cvalue(
mean_or_std(scenario,
ref_experiment,
model,
variant,
"anm",
variable,
time_stat,
table,
period,
data_versions,
operator,
args,
compute=compute,
house_keeping=house_keeping))
feed_dic(values, value, model, basin, period)
if fprint: print()
# Compute relative or absolute changes, and store it with model as last dict key
model_changes = dict()
for period in slices:
for model in values:
for basin in values[model]:
pro = values[model][basin][period]
ref = values[model][basin][ref_period]
if relative:
change = 100 * (pro - ref) / ref
else:
change = pro - ref
feed_dic(model_changes, change, basin, period, model)
# Compute stats on models ensemble (inside a scenario)
ens_changes = dict()
for period in slices:
# Next on other variables, per basin
for basin in model_changes:
# Access relevant models ensemble
ens = model_changes[basin][period]
for ens_stat in stats_list:
stat_value = ensemble_stat(ens, ens_stat)
feed_dic(ens_changes, stat_value, basin, ens_stat, period)
#
return ens_changes, models
def feed(name, field):
feed_dic(dic, field, experiment, season, name, derivation_label, model)
def feed(value, key):
feed_dic(aggregates, value, variable, experiment, season, key,
derivation_label)
def change_fields(
project,
variable,
experiments,
seasons,
ref_period,
projection_period,
models_to_plot,
data_versions,
derivation_label=None,
relative=False,
standardized=False,
print_statistics=False,
ref_experiment="historical",
variab_sampling_args={
"house_keeping": True,
"compute": True,
"detrend": True,
"shift": 100,
"nyears": 20,
"number": 20
},
common_grid=None,
table=None,
deep=None,
threshold=None,
low_change_agree_threshold=1.645,
sign_agree_threshold=0.9):
"""Computes a series of change fields for one
VARIABLE and a list of EXPERIMENTS, for period PROJECTION_PERIOD
with respect to REF_EXPERIMENT for REF_PERIOD. Computed fields
are e.g. mean change and median relative change. See 'aggregates' below for
the list of fields.
The computation is performed for all seasons listed in arg SEASONS (e.g. ['DJF', 'ANN'])
The variable may undergo some transformation rather than plain averaging over
the period(s); such transforations are found in dict `changes.derivations` and must be
designated by a label (DERIVATION_LABEL)
See examples in the scripts of the various figures of AR6/WGI chapter 8
Also computes variability fields and other related fields (see below) using piControl
and some settings for sampling it. See explanations with function `variability.variability_AR5` and
`variability.stippling_hatching_masks_AR5`
MODELS_TO_PLOT is a dictionnary which keys are experiments and values are lists
of pairs(model,variant) to include for that experiment.
Toggles RELATIVE activate the computation of relative change (in addition to plain
change). Key suffix (in keys list below) is then : `_rchange`.
Toggles STANDARDIZED activate the computation of change standardized by its
interannual variability. Key suffix (in keys list below) is `_schange`.
The (multi-decadal) variability is then also standardized the same way.
Arg THRESHOLD (if not None) is used when relative is True, as a floor value of
the reference field below which the per-model relative changes are set to missing
Arg LOW_CHANGE_AGREE_THRESHOLD provides the threshold for
computing the field of fractional agreement on a low change; its
default value is the one set for AR6 hatching sheme; it is
compared, separately for each model, to the ratio of absolute value
of change by internal multi-decadal variability, for deciding for
a low change for that model. The fraction of agreement across
models is returned under key `agree_low` (see below), and mask
fields defined by AR6 'comprehensive' hatching scheme are returned under key
`lowchange` (for zones without an agreed significant change) and "conflicts"
(for zones with conflicting significant signals)
The fraction of sign agreement across models is returned under key
`agreement_fraction_on_sign` (see below)
The variable is sought in CMIP6 table Amon or Lmon (depending on the
variable), except specified otherwise using arg TABLE (e.g. value 'day' is
for processing variable 'pr', provided the derivation make sense for daily data)
Other details of the variable are defined using dict DATA_VERSIONS, which must be a
dictionnary with this structure (like provided by the notebook in sibling directory
data_versions) :
>>> data_versions[experiment][variable][table][model][variant]=(grid,version,data_period)
e.g.
>>> data_versions["ssp245"]["pr"]["Amon"]["MPI-ESM1-2-HR"]["r1i1p1f1"]=("gn","v20190710","2015-2100")
(data-period is used only for piControl)
**Returns** all results through a pair of dicts of dicts : aggregates and dic
Dict 'aggregates' provides ensemble statistics (e.g. mean or median among
models), while dict 'dic' returns model specific fields
Use it as :
>>> field=aggregates[variable][experiment][season][key][derivation_label]
where key has values :
* mean_change : ensemble mean value of the changes
* mean_rchange : ensemble mean value of the relative changes
* mean_schange : ensemble mean value of the interannual variability standardized changes
* median_change, median_rchange, median_schange : same as three keys above, but for ensemble median
* means_rchange : relative change of the ensemble means
* median_variability : ensemble median of the multi-decadal variability
* (as defined by AR5) : stippling, hatching : 0/1 masks for the AR5 scheme stippling and hatching
* (as defined by AR6 simple case) : agreement_fraction_on_sign : fraction of models which agree on sign (highest fraction among the two signs)
* (as defined by AR6 comprehensive case) : agree_low, conflict, lowchange : see explanation above with argument LOW_CHANGE_AGREE_THRESHOLD
>>> field=dic[experiment][season][key][derivation_label][model]
where key has values : reference, projection, change, rchange, schange, nchange, variability
* variability means : multi-decadal variability
* nchange means : change standardized by multi-decadal variability
"""
aggregates = dict()
dic = dict()
if standardized and (variab_sampling_args == {}
or variab_sampling_args is None):
raise Exception("Must provide variability sampling arguments when requesting "+\
"a variable standardized by inter_annual variability")
#
if print_statistics:
print(
"Values below are field means, except for last two columns. Last third is median and %. Last three columns are %"
)
print(
"exp. seas model variablity reference projection change|rel/std :mdn max p90"
)
print(120 * "-")
# a commodity function for storing resuts in dic aggregates
def feed(value, key):
feed_dic(aggregates, value, variable, experiment, season, key,
derivation_label)
for experiment in experiments:
#
if table is None:
table = table_for_var_and_experiment(variable, experiment)
#project=project_for_experiment(experiment)
control_models, models = models_to_plot[experiment]
do_variability= (variab_sampling_args != {} and variab_sampling_args is not None and \
len(control_models) > 0 )
#
for season in seasons:
if not print_statistics: print(experiment, season, end='')
#
#
for model, realization in models:
change_fields_internal(dic, project, model, realization,
variable, ref_period, projection_period,
ref_experiment, experiment, season,
derivation_label, relative,
standardized, print_statistics,
common_grid, table, data_versions, deep,
variab_sampling_args, threshold)
#
if do_variability:
if print_statistics: print("Variabilities :", end='')
for model, realization in control_models:
variability = variability_field(
project, model, realization, variable, season,
derivation_label, standardized, variab_sampling_args,
common_grid, table, data_versions, deep)
feed_dic(dic, variability, experiment, season,
"variability", derivation_label, model)
if print_statistics:
print(
"%s % 7.2g / " %
(model,
cvalue(ccdo_fast(variability,
operator="fldmean"))),
end='')
#
# Compute ratio of change to internal variability for models which allow for
# (done on common grid)
if model in [m for m, r in models]:
change = dic[experiment][season]["change"][
derivation_label][model]
nchange = ccdo2(change, variability, operator="div")
feed_dic(dic, nchange, experiment, season, "nchange",
derivation_label, model)
if print_statistics: print
# Compute ensemble statistics
if print_statistics:
print("%s medians :" % experiment, end='')
ensavg = ccdo_ens(cens(
dic[experiment][season]["change"][derivation_label]),
operator='ensmean')
feed(ensavg, "mean_change")
ensmdn = ccdo_ens(cens(
dic[experiment][season]["change"][derivation_label]),
operator='enspctl,50')
feed(ensmdn, "median_change")
if print_statistics:
print("change %7.2g" %
cvalue(ccdo_fast(ensmdn, operator="fldpctl,50")),
end='')
if relative:
rmean = ccdo_ens(cens(
dic[experiment][season]["rchange"][derivation_label]),
operator='ensmean')
feed(rmean, "mean_rchange")
rmedian = ccdo_ens(cens(
dic[experiment][season]["rchange"][derivation_label]),
operator='enspctl,50')
feed(rmedian, "median_rchange")
# Also compute the relative change of ensemble means
ref_ensavg = ccdo_ens(
cens(dic[experiment][season]["reference_remapped"]
[derivation_label]),
operator='ensmean')
proj_ensavg = ccdo_ens(
cens(dic[experiment][season]["projection_remapped"]
[derivation_label]),
operator='ensmean')
rmeans = ccdo2(ccdo2(proj_ensavg, ref_ensavg, operator="sub"),
ref_ensavg,
operator="mulc,100 -div")
feed(rmeans, "means_rchange")
if print_statistics:
print("relative %7.2g %%" %
cvalue(ccdo_fast(rmedian, operator="fldpctl,50")),
end='')
if standardized:
smean = ccdo_ens(cens(
dic[experiment][season]["schange"][derivation_label]),
operator='ensmean')
feed(smean, "mean_schange")
smedian = ccdo_ens(cens(
dic[experiment][season]["schange"][derivation_label]),
operator='enspctl,50')
feed(smedian, "median_schange")
if print_statistics:
csync()
print()
#
if standardized:
agree_field = "schange"
avg = smean
else:
agree_field = "change"
avg = ensavg
agreef = agreement_fraction_on_sign(
cens(dic[experiment][season][agree_field][derivation_label]))
feed(agreef, "agreement_fraction_on_sign")
#
# Compute stippling and hatching fields (a la AR5) for the figure
#
if do_variability:
medvar = ccdo_ens(cens(
dic[experiment][season]["variability"][derivation_label]),
operator='enspctl,50')
feed(medvar, "median_variability")
#
s, h = stippling_hatching_masks_AR5(avg, medvar, agreef)
feed(s, "stippling")
feed(h, "hatching")
#
# Agreement on low change wrt OWN internal variability
if "nchange" in dic[experiment][season]:
agree_low = agreement_fraction_on_lower(
cens(dic[experiment][season]["nchange"]
[derivation_label]), low_change_agree_threshold)
feed(agree_low, "agree_low")
(l , c) = lowchange_conflict_masks_AR6(\
agreef,agree_low,
fraction_on_sign=sign_agree_thresold)
feed(l, "lowchange")
feed(c, "conflict")
else:
print("No common model for ", variable, seasons,
experiments)
print("models: ", models)
print("control_models:", control_models)
#
print()
return aggregates, dic