def control_detrend(model, variable, begin, length, shift, variant, version, compute=True, house_keeping=True, detrend=True): # Detrend the requested variable for the piControl run # If compute is True, actually evaluates the result and get rid # of original data partial copy # If house_keeping is True, don't compute and remove existing file from cache from climaf.operators import ctrend, csubtrend project = project_for_model(model) institute = institute_for_model(model) table = table_for_var_and_experiment(variable, "piControl") control = dict(variable=variable, institute=institute, model=model, project=project, table=table, experiment="piControl", version=version, realization=variant) if project == "CMIP6": control.update(mip="CMIP") d = ds(period="%d-%d" % (begin + shift, begin + shift + length - 1), **control) if detrend: a = ctrend(d) ap = ccdo_fast(a, operator="mulc,0" ) # Do not want to have a zero-mean detrended serie detrended = csubtrend(d, ap, a.b) if compute: cfile(detrended) if house_keeping: cdrop(d) # Discard cache copy of original data cdrop(a) cdrop(a.b) cdrop(ap) return detrended else: return d
def land_average(data, model, compute=False, house_keeping=False, test=False): """ """ # Try to access land fraction field sftlf base_dict = dict(project="CMIP6", experiment="piControl", mip="CMIP", model=model, institute=institute_for_model(model), version="*", realization="*", table="fx", period="fx", variable="sftlf") # if "GFDL" in model or "INM" in model: base_dict.update(grid="gr1") # sftlf = ds(**base_dict) try: cfile(sftlf) except: base_dict["experiment"] = "piControl" try: cfile(sftlf) except: raise ValueError("No sftlf field for " + model) # Use CDO for masking on land, and computing mean mask = ccdo_fast(sftlf, operator="divc,100. -setvrange,100,100") masked = ccdo2(data, mask, operator="mul") average = ccdo_fast(masked, operator="fldmean") if test: return masked if compute: ceval(average) if house_keeping: cdrop(masked) return average
def change_fields_internal(dic, project, model, realization, variable, ref_period, projection_period, ref_experiment, experiment, season, derivation_label, relative, standardized, print_statistics, common_grid, table, data_versions, deep, variab_sampling_args, threshold): """ See function change_fields for documentation """ def feed(name, field): feed_dic(dic, field, experiment, season, name, derivation_label, model) if print_statistics: print("%s %s %-20s" % (experiment, season, model), end='') else: print(model, end='') # #project=project_for_experiment(experiment) grid, version, _ = data_versions[ref_experiment][variable][table][model][ realization] roption = choose_regrid_option(variable, table, model, grid) derivation = derivations[derivation_label] # base_dict = dict(project=project, experiment=ref_experiment, model=model, institute=institute_for_model(model), period=ref_period, variable=variable, table=table, version=version, grid=grid, realization=realization) # # Compute reference time mean over requested season reference_dict = base_dict.copy() reference_ds = ds(**reference_dict) reference = process_dataset(reference_ds, season, **derivation) feed("reference", reference) reference_remapped = regridn(reference, cdogrid=common_grid, **roption) feed("reference_remapped", reference_remapped) if relative: if threshold is not None: thresholded_reference = ccdo_fast( reference, operator="setrtomiss,-1.e+10,%f" % threshold) else: thresholded_reference = reference # # Compute projection time mean over requested season projection_dict = reference_dict.copy() _, version, _ = data_versions[experiment][variable][table][model][ realization] projection_dict.update(experiment=experiment, period=projection_period, realization=realization, version=version) projection_ds = ds(**projection_dict) projection = process_dataset(projection_ds, season, **derivation) feed("projection", projection) feed("projection_remapped", regridn(projection, cdogrid=common_grid, **roption)) # # Compute absolute and relative changes, and regrid to common grid change = ccdo2(projection, reference, operator="sub") feed("change_orig", change) change_remapped = regridn(change, cdogrid=common_grid, **roption) feed("change", change_remapped) if relative: rchange = ccdo2(change, thresholded_reference, operator="mulc,100 -div") feed("rchange_orig", rchange) rchange_remapped = regridn(rchange, cdogrid=common_grid, **roption) feed("rchange", rchange_remapped) if standardized: inter_ann_var = variability_field(None, model, realization, variable, season, derivation_label, standardized, variab_sampling_args, None, table, data_versions, deep=None, only_inter_annual=True) schange = ccdo2(change, inter_ann_var, operator="div") feed("schange_orig", schange) schange_remapped = regridn(schange, cdogrid=common_grid, **roption) feed("schange", schange_remapped) # if print_statistics: variab_value = 0. print(" %7.2g %7.2g %7.2g %7.2g"%( variab_value,\ cvalue(ccdo_fast(reference,operator="fldmean")),\ cvalue(ccdo_fast(projection,operator="fldmean")),\ cvalue(ccdo_fast(change,operator="fldmean"))),end='') if relative: print(" %7.2g %7.2g %7.2g" % (cvalue(ccdo_fast(rchange, operator="fldpctl,50")), cvalue(ccdo_fast(rchange, operator="fldmax")), cvalue(ccdo_fast(rchange, operator="fldpctl,90"))), end='') if standardized: print(" %7.2g %7.2g %7.2g" % (cvalue(ccdo_fast(schange, operator="fldmean")), cvalue(ccdo_fast(schange, operator="fldmax")), cvalue(ccdo_fast(schange, operator="fldpctl,90"))), end='') print() return
def global_change(variable, table, experiment, period, ref_experiment, ref_period, models, data_versions, filter_length=21): """ Returns a CliMAF ensemble of 1D-vectors representing the global change for a VARIABLE along a PERIOD of EXPERIMENT vs. a REF_PERIOD in another REF_EXPERIMENT, for a list of pairs (model,realization) (arg MODELS). The period can begin during REF_EXPERIMENT (but REF_EXPERIMENT must end at EXPERIMENTS's begin) The values are first yearly averaged and filtered on FILTER_LENGTH years DATA_VERSIONS, must be a dictionnary with this structure (like provided by the notebook in sibling directory data_versions) >>> data_versions[experiment][variable][table][model][variant]=(grid,version,data_period) e.g. >>> data_versions["ssp245"]["pr"]["Amon"]["MPI-ESM1-2-HR"]["r1i1p1f1"]=("gn","v20190710","2015-2100") (data-period is used only for piControl) Tuned for CMIP6 only (yet). Assumes that, for each model in the provided list, realization indices and grids are consistent across experiments """ # GSAT = cens() # for model, realization in models: grid, ref_version, _ = data_versions[ref_experiment][variable][table][ model][realization] grid, version, _ = data_versions[experiment][variable][table][model][ realization] print("Global_change - processing %20s %s %s" % (model, realization, version), end='') dic = dict(project="CMIP6_extent", experiment=ref_experiment, extent_experiment=experiment, model=model, institute=institute_for_model(model), period=period, variable=variable, table=table_for_var_and_experiment(variable, experiment), mip="*", realization=realization, version=ref_version, extent_version=version, grid=grid) filtered_tasmean = ccdo(ds(**dic), operator="runmean,%d -yearmean -fldmean" % filter_length) # _, version, _ = data_versions[ref_experiment][variable][table][model][ realization] ref = dic.copy() ref.update(experiment=ref_experiment, mip=mip_for_experiment(ref_experiment), period=ref_period, version=version) ref_tasmean = ccdo(ds(**ref), operator="timmean -fldmean") cfile(ref_tasmean) ref = cvalue(ref_tasmean) #print("GTAS = %g"%ref) # GSAT[model] = ccdo_fast(filtered_tasmean, operator="subc,%g" % ref) # return GSAT
def variability_AR5(model, realization, variable, table, data_versions, season="ANN", project="CMIP6", operator=None, operator_args={}, post_operator=None, post_operator_args={}, shift=100, nyears=20, number=20, variability=True, compute=True, house_keeping=False, detrend=True, deep=None): """ Compute the variability according to AR5 Box 2.1 : - select data time series in piControl for the whole of the samples (from its begin+SHIFT, duration consistent with NUMBER samples of size NYEARS); the data variant and version, and the begin date, are selected according to dictionnary DATA_VERSIONS, - transform this data using OPERATOR (and its OPERATOR_ARGS) that should produce one value per year (default being to compute annual or seasonal means) - detrend that data, if required (this is done by default) - build an ensemble representing the samples (NUMBER * NYEARS) - transform each member's result using POST_OPERATOR and POST_OPERATOR_ARGS (default is to compute a time average) - if arg VARIABILITY is False , returns that result (i.e. by defaut the time mean), - otherwise computes and returns the variability as the ensemble standard deviation multiplied by square root of 2 Arg MODELS_WITH_ENOUGH_SPINUP is the list of those models for which the required SHIFT may be relaxed, because they are supposed to be already in a balanced state from the start of published piControl data The returned value is a CliMAF object (either a field or an ensemble, depending on VARIABILITY) Arg COMPUTE, if set to True, drives an immediate lauch of the computation, of CliMAF object, and then, if arg DEEP is True, re-compute all results from scratch, without using CliMAF cached values for intermediate results. Arg HOUSE_KEEPING, if set to True, allows to release CliMAF cache intermediate results, to keep cache use as low as possible Used e.g for variability of : - plain variables - walsh seasonnality index - number of dry days per year - year mean daily precipitation for non-dry days - inter_annual variability for any variable, using : * post_operator=inter_annual_variability * post_operator_args={"factor" : 1.414} This version yet tested only on CMIP6 models """ init_trend() from climaf.operators import ctrend, csubtrend if realization not in data_versions["piControl"][variable][table][model]: realization = data_versions["piControl"][variable][table][model].keys( )[0] grid, version, data_period = data_versions["piControl"][variable][table][ model][realization] duration = nyears * number true_begin = int(data_period.split('-')[0][0:4]) end = int(data_period.split('-')[1][0:4]) begin = true_begin + shift if begin + duration - 1 > end: # In CMIP6, some models have enough spinup before piControl start, # but a too short piControl length # We assume that this has been dealt with at the stage of data selection, and allow # to release the constraint on shift at the beginning of the data period alt_begin = end - duration + 1 if alt_begin >= true_begin: begin = alt_begin else: message="Duration for %s %s %s %s %s %s is too short : [%d - %d] even with no shift %d is shorter than %d years "%\ (model,variable,table,realization,version,grid,true_begin,end,shift,duration) raise ValueError(message) # period = "%g-%g" % (begin, begin + duration - 1) base_dict = dict(project=project, experiment="piControl", model=model, institute=institute_for_model(model), period=period, variable=variable, table=table, version=version, grid=grid, realization=realization) if project == "CMIP6": base_dict.update(mip="CMIP") # Basic dataset (e.g. precip) basic = ds(**base_dict) dat = basic # Implement the operation if required, otherwise seasonal or yearly average if operator is None: if season in ["ann", "ANN", "anm"]: dat_op = ccdo(dat, operator="yearmean") else: dat_op = ccdo_fast(dat, operator="selseason,%s -seasmean" % season) else: if season in ["ann", "ANN", "anm"]: dat_op = operator(dat, **operator_args) else: dat_season = ccdo_fast(dat, operator="selseason,%s" % season) dat_op = operator(dat_season, **operator_args) dat = dat_op # Detrend the data if required if detrend: a = ctrend(dat) ap = ccdo_fast(a, operator="mulc,0" ) # Do not want to have a zero-mean detrended serie detrended = csubtrend(dat, ap, a.b) dat = detrended # Build an ensemble which members are the slices econtrol = cens() slices = [ "%d-%d" % (begin + n * nyears, begin + (n + 1) * nyears - 1) for n in range(0, number) ] for period in slices: econtrol[period] = ccdo_fast(dat, operator="seldate," + init_period(period).iso()) # On each slice, implement the required post operation, otherwise compute a plain average if post_operator is not None: cmeans = cens() for member in econtrol: cmeans[member] = post_operator(econtrol[member], **post_operator_args) else: cmeans = ccdo_fast(econtrol, operator="timmean") if variability is True: # Compute variability over the slices ensemble variab1 = ccdo_ens(cmeans, operator='ensstd1') variab = ccdo_fast(variab1, operator="mulc,1.414") # cf. AR5 Box 2.1 # if compute: if variability: cfile(variab, deep=deep) else: cfile(cmeans, deep=deep) if house_keeping: # Discard intermediate data cdrop(basic) cdrop(dat_op) if operator is not None and season not in ["ann", "ANN", "anm"]: cdrop(dat_season) if detrend: cdrop(a) cdrop(a.b) cdrop(ap) cdrop(detrended) cdrop(dat) for period in slices: cdrop(econtrol[period]) # if variability: if house_keeping: cdrop(cmeans) cdrop(variab1) return variab else: return cmeans
def mean_or_std(scenario, ref_experiment, model, realization, season, variable, stat, table, period, data_versions, operator=None, operator_args={}, compute=False, detrend=True, house_keeping=False): """ Compute a STAT ("mean" or "std") of annual or seasonal means of a VARIABLE over a PERIOD for a given MODEL and a virtual experiment which merges REF_EXPERIMENT and SCENARIO (but REF_EXPERIMENT can be None) Detrending is performed before computing standard deviation, except if requested otherwise using arg 'detrend' Additionnaly, a CliMAF OPERATOR can be applied after time averaging (and before standard deviation computation if applicable). Dict DATA_VERSIONS provide details (grid, version, realization) for the data to use PERIOD can extend from some date in REF_EXPERIMENT to some other date of SCENARIO, but then, the end of REF_EXPERIMENT must match the beginning of SCENARIO Works for CMIP6 only, yet """ # # Define relevant dataset if ref_experiment is not None: grid, version, _ = data_versions[ref_experiment][variable][table][ model][realization] _, scenario_version, _ = data_versions[scenario][variable][table][ model][realization] dic = dict(project="CMIP6_extent", experiment=ref_experiment, extent_experiment=scenario, variable=variable, period=period, mip="*", model=model, institute=institute_for_model(model), table=table_for_var_and_experiment(variable, scenario), version=version, extent_version=scenario_version, grid=grid, realization=realization) else: grid, version, _ = data_versions[scenario][variable][table][model][ realization] dic = dict(project="CMIP6", experiment=scenario, variable=variable, period=period, mip="*", model=model, institute=institute_for_model(model), table=table_for_var_and_experiment(variable, scenario), version=version, grid=grid, realization=realization) # # Compute relevant stat if stat == "mean": # Average over years, possibly after selecting season time_operation = "timmean" if season != "anm": time_operation += "-selseason,%s" % season rep = ccdo(ds(**dic), operator=time_operation) if operator is not None: rep = operator(rep, **operator_args) # elif stat == "std": # First compute annual mean or seasonal mean for each year time_operation = "yearmean" if season != "anm": time_operation = "selseason,%s -seasmean" % season cached = ccdo(ds(**dic), operator=time_operation) if operator is not None: cached = operator(cached, **operator_args) if detrend: # Must detrend init_trend() from climaf.operators import ctrend, csubtrend a = ctrend(cached) detrended = csubtrend(cached, a, a.b) else: detrended = cached # Compute standard deviation of seasonal values series rep = ccdo_fast(detrended, operator="timstd1") else: raise ValueError("stat %s cannot yet be processed by mean_or_std for season %s"+\ "and variable %s"%(stat,season,variable)) if compute: ceval(rep) # if house_keeping and stat == 'std': cdrop(cached) cdrop(detrended) return rep
if house_keeping: cdrop(regridded) if test: return masked else: return average test_basin_average = False # if test_basin_average: model = "CNRM-CM6-1" dic = dict(project="CMIP6_extent", experiment="ssp585", model=model, institute=institute_for_model(model), period="2015-2020", variable="mrro", table="Lmon", version="latest", mip="ScenarioMIP", realization=default_variant(model, "ssp585", "mrro")) basin_mean = basin_average(ds(**dic), "", "Amazon", True, False, True) ncview(basin_mean) def land_average(data, model, compute=False, house_keeping=False, test=False): """ """ # Try to access land fraction field sftlf