def control_detrend(model,
variable,
begin,
length,
shift,
variant,
version,
compute=True,
house_keeping=True,
detrend=True):
# Detrend the requested variable for the piControl run
# If compute is True, actually evaluates the result and get rid
# of original data partial copy
# If house_keeping is True, don't compute and remove existing file from cache
from climaf.operators import ctrend, csubtrend
project = project_for_model(model)
institute = institute_for_model(model)
table = table_for_var_and_experiment(variable, "piControl")
control = dict(variable=variable,
institute=institute,
model=model,
project=project,
table=table,
experiment="piControl",
version=version,
realization=variant)
if project == "CMIP6": control.update(mip="CMIP")
d = ds(period="%d-%d" % (begin + shift, begin + shift + length - 1),
**control)
if detrend:
a = ctrend(d)
ap = ccdo_fast(a, operator="mulc,0"
) # Do not want to have a zero-mean detrended serie
detrended = csubtrend(d, ap, a.b)
if compute:
cfile(detrended)
if house_keeping:
cdrop(d) # Discard cache copy of original data
cdrop(a)
cdrop(a.b)
cdrop(ap)
return detrended
else:
return d
def land_average(data, model, compute=False, house_keeping=False, test=False):
"""
"""
# Try to access land fraction field sftlf
base_dict = dict(project="CMIP6",
experiment="piControl",
mip="CMIP",
model=model,
institute=institute_for_model(model),
version="*",
realization="*",
table="fx",
period="fx",
variable="sftlf")
#
if "GFDL" in model or "INM" in model:
base_dict.update(grid="gr1")
#
sftlf = ds(**base_dict)
try:
cfile(sftlf)
except:
base_dict["experiment"] = "piControl"
try:
cfile(sftlf)
except:
raise ValueError("No sftlf field for " + model)
# Use CDO for masking on land, and computing mean
mask = ccdo_fast(sftlf, operator="divc,100. -setvrange,100,100")
masked = ccdo2(data, mask, operator="mul")
average = ccdo_fast(masked, operator="fldmean")
if test:
return masked
if compute:
ceval(average)
if house_keeping:
cdrop(masked)
return average
def change_fields_internal(dic, project, model, realization, variable,
ref_period, projection_period, ref_experiment,
experiment, season, derivation_label, relative,
standardized, print_statistics, common_grid, table,
data_versions, deep, variab_sampling_args,
threshold):
"""
See function change_fields for documentation
"""
def feed(name, field):
feed_dic(dic, field, experiment, season, name, derivation_label, model)
if print_statistics:
print("%s %s %-20s" % (experiment, season, model), end='')
else:
print(model, end='')
#
#project=project_for_experiment(experiment)
grid, version, _ = data_versions[ref_experiment][variable][table][model][
realization]
roption = choose_regrid_option(variable, table, model, grid)
derivation = derivations[derivation_label]
#
base_dict = dict(project=project,
experiment=ref_experiment,
model=model,
institute=institute_for_model(model),
period=ref_period,
variable=variable,
table=table,
version=version,
grid=grid,
realization=realization)
#
# Compute reference time mean over requested season
reference_dict = base_dict.copy()
reference_ds = ds(**reference_dict)
reference = process_dataset(reference_ds, season, **derivation)
feed("reference", reference)
reference_remapped = regridn(reference, cdogrid=common_grid, **roption)
feed("reference_remapped", reference_remapped)
if relative:
if threshold is not None:
thresholded_reference = ccdo_fast(
reference, operator="setrtomiss,-1.e+10,%f" % threshold)
else:
thresholded_reference = reference
#
# Compute projection time mean over requested season
projection_dict = reference_dict.copy()
_, version, _ = data_versions[experiment][variable][table][model][
realization]
projection_dict.update(experiment=experiment,
period=projection_period,
realization=realization,
version=version)
projection_ds = ds(**projection_dict)
projection = process_dataset(projection_ds, season, **derivation)
feed("projection", projection)
feed("projection_remapped",
regridn(projection, cdogrid=common_grid, **roption))
#
# Compute absolute and relative changes, and regrid to common grid
change = ccdo2(projection, reference, operator="sub")
feed("change_orig", change)
change_remapped = regridn(change, cdogrid=common_grid, **roption)
feed("change", change_remapped)
if relative:
rchange = ccdo2(change,
thresholded_reference,
operator="mulc,100 -div")
feed("rchange_orig", rchange)
rchange_remapped = regridn(rchange, cdogrid=common_grid, **roption)
feed("rchange", rchange_remapped)
if standardized:
inter_ann_var = variability_field(None,
model,
realization,
variable,
season,
derivation_label,
standardized,
variab_sampling_args,
None,
table,
data_versions,
deep=None,
only_inter_annual=True)
schange = ccdo2(change, inter_ann_var, operator="div")
feed("schange_orig", schange)
schange_remapped = regridn(schange, cdogrid=common_grid, **roption)
feed("schange", schange_remapped)
#
if print_statistics:
variab_value = 0.
print(" %7.2g %7.2g %7.2g %7.2g"%(
variab_value,\
cvalue(ccdo_fast(reference,operator="fldmean")),\
cvalue(ccdo_fast(projection,operator="fldmean")),\
cvalue(ccdo_fast(change,operator="fldmean"))),end='')
if relative:
print(" %7.2g %7.2g %7.2g" %
(cvalue(ccdo_fast(rchange, operator="fldpctl,50")),
cvalue(ccdo_fast(rchange, operator="fldmax")),
cvalue(ccdo_fast(rchange, operator="fldpctl,90"))),
end='')
if standardized:
print(" %7.2g %7.2g %7.2g" %
(cvalue(ccdo_fast(schange, operator="fldmean")),
cvalue(ccdo_fast(schange, operator="fldmax")),
cvalue(ccdo_fast(schange, operator="fldpctl,90"))),
end='')
print()
return
def global_change(variable,
table,
experiment,
period,
ref_experiment,
ref_period,
models,
data_versions,
filter_length=21):
"""
Returns a CliMAF ensemble of 1D-vectors representing the global change for a
VARIABLE along a PERIOD of EXPERIMENT vs. a REF_PERIOD in another REF_EXPERIMENT,
for a list of pairs (model,realization) (arg MODELS). The period can begin during
REF_EXPERIMENT (but REF_EXPERIMENT must end at EXPERIMENTS's begin)
The values are first yearly averaged and filtered on FILTER_LENGTH years
DATA_VERSIONS, must be a dictionnary with this structure (like provided by the notebook
in sibling directory data_versions)
>>> data_versions[experiment][variable][table][model][variant]=(grid,version,data_period)
e.g.
>>> data_versions["ssp245"]["pr"]["Amon"]["MPI-ESM1-2-HR"]["r1i1p1f1"]=("gn","v20190710","2015-2100")
(data-period is used only for piControl)
Tuned for CMIP6 only (yet).
Assumes that, for each model in the provided list, realization
indices and grids are consistent across experiments
"""
#
GSAT = cens()
#
for model, realization in models:
grid, ref_version, _ = data_versions[ref_experiment][variable][table][
model][realization]
grid, version, _ = data_versions[experiment][variable][table][model][
realization]
print("Global_change - processing %20s %s %s" %
(model, realization, version),
end='')
dic = dict(project="CMIP6_extent",
experiment=ref_experiment,
extent_experiment=experiment,
model=model,
institute=institute_for_model(model),
period=period,
variable=variable,
table=table_for_var_and_experiment(variable, experiment),
mip="*",
realization=realization,
version=ref_version,
extent_version=version,
grid=grid)
filtered_tasmean = ccdo(ds(**dic),
operator="runmean,%d -yearmean -fldmean" %
filter_length)
#
_, version, _ = data_versions[ref_experiment][variable][table][model][
realization]
ref = dic.copy()
ref.update(experiment=ref_experiment,
mip=mip_for_experiment(ref_experiment),
period=ref_period,
version=version)
ref_tasmean = ccdo(ds(**ref), operator="timmean -fldmean")
cfile(ref_tasmean)
ref = cvalue(ref_tasmean)
#print("GTAS = %g"%ref)
#
GSAT[model] = ccdo_fast(filtered_tasmean, operator="subc,%g" % ref)
#
return GSAT
def variability_AR5(model,
realization,
variable,
table,
data_versions,
season="ANN",
project="CMIP6",
operator=None,
operator_args={},
post_operator=None,
post_operator_args={},
shift=100,
nyears=20,
number=20,
variability=True,
compute=True,
house_keeping=False,
detrend=True,
deep=None):
"""
Compute the variability according to AR5 Box 2.1 :
- select data time series in piControl for the whole of the samples (from
its begin+SHIFT, duration consistent with NUMBER samples of size NYEARS);
the data variant and version, and the begin date, are selected according to
dictionnary DATA_VERSIONS,
- transform this data using OPERATOR (and its OPERATOR_ARGS) that should produce
one value per year (default being to compute annual or seasonal means)
- detrend that data, if required (this is done by default)
- build an ensemble representing the samples (NUMBER * NYEARS)
- transform each member's result using POST_OPERATOR and POST_OPERATOR_ARGS (default
is to compute a time average)
- if arg VARIABILITY is False , returns that result (i.e. by defaut the time mean),
- otherwise computes and returns the variability as the ensemble standard deviation
multiplied by square root of 2
Arg MODELS_WITH_ENOUGH_SPINUP is the list of those models for which the required
SHIFT may be relaxed, because they are supposed to be already in a balanced state
from the start of published piControl data
The returned value is a CliMAF object (either a field or an ensemble, depending on VARIABILITY)
Arg COMPUTE, if set to True, drives an immediate lauch of the computation, of CliMAF object,
and then, if arg DEEP is True, re-compute all results from scratch, without using CliMAF cached
values for intermediate results.
Arg HOUSE_KEEPING, if set to True, allows to release CliMAF cache intermediate results, to keep
cache use as low as possible
Used e.g for variability of :
- plain variables
- walsh seasonnality index
- number of dry days per year
- year mean daily precipitation for non-dry days
- inter_annual variability for any variable, using :
* post_operator=inter_annual_variability
* post_operator_args={"factor" : 1.414}
This version yet tested only on CMIP6 models
"""
init_trend()
from climaf.operators import ctrend, csubtrend
if realization not in data_versions["piControl"][variable][table][model]:
realization = data_versions["piControl"][variable][table][model].keys(
)[0]
grid, version, data_period = data_versions["piControl"][variable][table][
model][realization]
duration = nyears * number
true_begin = int(data_period.split('-')[0][0:4])
end = int(data_period.split('-')[1][0:4])
begin = true_begin + shift
if begin + duration - 1 > end:
# In CMIP6, some models have enough spinup before piControl start,
# but a too short piControl length
# We assume that this has been dealt with at the stage of data selection, and allow
# to release the constraint on shift at the beginning of the data period
alt_begin = end - duration + 1
if alt_begin >= true_begin:
begin = alt_begin
else:
message="Duration for %s %s %s %s %s %s is too short : [%d - %d] even with no shift %d is shorter than %d years "%\
(model,variable,table,realization,version,grid,true_begin,end,shift,duration)
raise ValueError(message)
#
period = "%g-%g" % (begin, begin + duration - 1)
base_dict = dict(project=project,
experiment="piControl",
model=model,
institute=institute_for_model(model),
period=period,
variable=variable,
table=table,
version=version,
grid=grid,
realization=realization)
if project == "CMIP6":
base_dict.update(mip="CMIP")
# Basic dataset (e.g. precip)
basic = ds(**base_dict)
dat = basic
# Implement the operation if required, otherwise seasonal or yearly average
if operator is None:
if season in ["ann", "ANN", "anm"]:
dat_op = ccdo(dat, operator="yearmean")
else:
dat_op = ccdo_fast(dat, operator="selseason,%s -seasmean" % season)
else:
if season in ["ann", "ANN", "anm"]:
dat_op = operator(dat, **operator_args)
else:
dat_season = ccdo_fast(dat, operator="selseason,%s" % season)
dat_op = operator(dat_season, **operator_args)
dat = dat_op
# Detrend the data if required
if detrend:
a = ctrend(dat)
ap = ccdo_fast(a, operator="mulc,0"
) # Do not want to have a zero-mean detrended serie
detrended = csubtrend(dat, ap, a.b)
dat = detrended
# Build an ensemble which members are the slices
econtrol = cens()
slices = [
"%d-%d" % (begin + n * nyears, begin + (n + 1) * nyears - 1)
for n in range(0, number)
]
for period in slices:
econtrol[period] = ccdo_fast(dat,
operator="seldate," +
init_period(period).iso())
# On each slice, implement the required post operation, otherwise compute a plain average
if post_operator is not None:
cmeans = cens()
for member in econtrol:
cmeans[member] = post_operator(econtrol[member],
**post_operator_args)
else:
cmeans = ccdo_fast(econtrol, operator="timmean")
if variability is True:
# Compute variability over the slices ensemble
variab1 = ccdo_ens(cmeans, operator='ensstd1')
variab = ccdo_fast(variab1, operator="mulc,1.414") # cf. AR5 Box 2.1
#
if compute:
if variability: cfile(variab, deep=deep)
else: cfile(cmeans, deep=deep)
if house_keeping: # Discard intermediate data
cdrop(basic)
cdrop(dat_op)
if operator is not None and season not in ["ann", "ANN", "anm"]:
cdrop(dat_season)
if detrend:
cdrop(a)
cdrop(a.b)
cdrop(ap)
cdrop(detrended)
cdrop(dat)
for period in slices:
cdrop(econtrol[period])
#
if variability:
if house_keeping:
cdrop(cmeans)
cdrop(variab1)
return variab
else:
return cmeans
def mean_or_std(scenario,
ref_experiment,
model,
realization,
season,
variable,
stat,
table,
period,
data_versions,
operator=None,
operator_args={},
compute=False,
detrend=True,
house_keeping=False):
"""
Compute a STAT ("mean" or "std") of annual or seasonal means
of a VARIABLE over a PERIOD for a given MODEL and a virtual experiment
which merges REF_EXPERIMENT and SCENARIO (but REF_EXPERIMENT can be None)
Detrending is performed before computing standard deviation, except if requested
otherwise using arg 'detrend'
Additionnaly, a CliMAF OPERATOR can be applied after time averaging (and before
standard deviation computation if applicable).
Dict DATA_VERSIONS provide details (grid, version, realization) for the data to use
PERIOD can extend from some date in REF_EXPERIMENT to some other date of SCENARIO,
but then, the end of REF_EXPERIMENT must match the beginning of SCENARIO
Works for CMIP6 only, yet
"""
#
# Define relevant dataset
if ref_experiment is not None:
grid, version, _ = data_versions[ref_experiment][variable][table][
model][realization]
_, scenario_version, _ = data_versions[scenario][variable][table][
model][realization]
dic = dict(project="CMIP6_extent",
experiment=ref_experiment,
extent_experiment=scenario,
variable=variable,
period=period,
mip="*",
model=model,
institute=institute_for_model(model),
table=table_for_var_and_experiment(variable, scenario),
version=version,
extent_version=scenario_version,
grid=grid,
realization=realization)
else:
grid, version, _ = data_versions[scenario][variable][table][model][
realization]
dic = dict(project="CMIP6",
experiment=scenario,
variable=variable,
period=period,
mip="*",
model=model,
institute=institute_for_model(model),
table=table_for_var_and_experiment(variable, scenario),
version=version,
grid=grid,
realization=realization)
#
# Compute relevant stat
if stat == "mean":
# Average over years, possibly after selecting season
time_operation = "timmean"
if season != "anm": time_operation += "-selseason,%s" % season
rep = ccdo(ds(**dic), operator=time_operation)
if operator is not None:
rep = operator(rep, **operator_args)
#
elif stat == "std":
# First compute annual mean or seasonal mean for each year
time_operation = "yearmean"
if season != "anm": time_operation = "selseason,%s -seasmean" % season
cached = ccdo(ds(**dic), operator=time_operation)
if operator is not None:
cached = operator(cached, **operator_args)
if detrend:
# Must detrend
init_trend()
from climaf.operators import ctrend, csubtrend
a = ctrend(cached)
detrended = csubtrend(cached, a, a.b)
else:
detrended = cached
# Compute standard deviation of seasonal values series
rep = ccdo_fast(detrended, operator="timstd1")
else:
raise ValueError("stat %s cannot yet be processed by mean_or_std for season %s"+\
"and variable %s"%(stat,season,variable))
if compute: ceval(rep)
#
if house_keeping and stat == 'std':
cdrop(cached)
cdrop(detrended)
return rep
if house_keeping:
cdrop(regridded)
if test:
return masked
else:
return average
test_basin_average = False
#
if test_basin_average:
model = "CNRM-CM6-1"
dic = dict(project="CMIP6_extent",
experiment="ssp585",
model=model,
institute=institute_for_model(model),
period="2015-2020",
variable="mrro",
table="Lmon",
version="latest",
mip="ScenarioMIP",
realization=default_variant(model, "ssp585", "mrro"))
basin_mean = basin_average(ds(**dic), "", "Amazon", True, False, True)
ncview(basin_mean)
def land_average(data, model, compute=False, house_keeping=False, test=False):
"""
"""
# Try to access land fraction field sftlf