def StringInDict(string_or_list, dictionary, inspect_stack): """ ################################################################################# Description: Tests if 'string_or_list' is in the given 'dictionary' ################################################################################# :param string_or_list: string or list key or list of keys to look for in 'dictionary' :param dictionary: dict dictionary in which the given keys are looked for :param inspect_stack: array list of information about the program/module/line,... created using inspect.stack() :return: """ # test input parameters if not isinstance(dictionary, dict): EnsoErrorsWarnings.ObjectTypeError('dictionary', 'dictionary', type(dictionary), INSPECTstack()) if isinstance(string_or_list, basestring): # 'string_or_list' is a string if string_or_list not in dictionary.keys(): # key 'string_or_list' is not in 'dictionary' -> raise error list_strings = [ "ERROR" + EnsoErrorsWarnings.MessageFormating(inspect_stack) + ": item not included", str().ljust(5) + " key: " + str(string_or_list) + " is not in the given dictionary", str().ljust(10) + "key(s) in the dictionary: " + str(sorted(dictionary.keys())) ] EnsoErrorsWarnings.MyError(list_strings) elif isinstance(string_or_list, list): # 'string_or_list' is a list key_not_included = list() for key in string_or_list: if key not in dictionary.keys(): # lists keys that are not in 'dictionary' key_not_included.append(key) if key_not_included: # if 'key_not_included' is not empty -> raise error list_strings = [ "ERROR" + EnsoErrorsWarnings.MessageFormating(inspect_stack) + ": item not included", str().ljust(5) + " key(s): " + str(key_not_included) + " are (is) not in the given dictionary", str().ljust(10) + "key(s) in the dictionary: " + str(sorted(dictionary.keys())) ] EnsoErrorsWarnings.MyError(list_strings) else: # 'string_or_list' is neither a string nor a list -> raise error EnsoErrorsWarnings.ObjectTypeError('string_or_list', '[string, list]', type(string_or_list), INSPECTstack()) return
def find_xml_cmip(model, project, experiment, ensemble, frequency, realm, variable): try: pathnc, filenc = find_path_and_files(ens=ensemble, exp=experiment, fre=frequency, mod=model, pro=project, rea=realm, var=variable) except: if realm == 'O': new_realm = 'A' elif realm == 'A': new_realm = 'O' # if var is not in realm 'O' (for ocean), look for it in realm 'A' (for atmosphere), and conversely try: pathnc, filenc = find_path_and_files(ens=ensemble, exp=experiment, fre=frequency, mod=model, pro=project, rea=new_realm, var=variable) except: # given variable is neither in realm 'A' nor 'O' print bcolors.FAIL + '%%%%% ----- %%%%%' print 'ERROR: function: ' + str( INSPECTstack()[0][3]) + ', line: ' + str(INSPECTstack()[0][2]) print 'given variable cannot be found in either realm A or O: ' + str( variable) print 'param: ' + str(model) + ', ' + str(project) + ', ' + str( experiment) + ', ' + str(ensemble) + ', ' + str( frequency) + ', ' + str(realm) print '%%%%% ----- %%%%%' + bcolors.ENDC sys.exit('') file_area, file_land = find_fx(model, project=project, experiment=experiment, ensemble=ensemble, realm=new_realm) else: file_area, file_land = find_fx(model, project=project, experiment=experiment, ensemble=ensemble, realm=realm) file_name = OSpath__join(pathnc, filenc[0]) return file_name, file_area, file_land
def percentage_val_eastward(val_longitude, metric, region, threshold=-140): """ ################################################################################# Description: Computes the percentage of given values (longitude in val_longitude) eastward of 'threshold' ################################################################################# :param val_longitude: list list of longitude :param threshold: float, optional threshold to define the westward boundary of the region :return ep: float percentage of given values eastward of 'threshold' """ keyerror = None pos_lon = 'pos' if all(ii >= 0 for ii in val_longitude) is True else \ ('neg' if all(ii <= 0 for ii in val_longitude) is True else False) if pos_lon is False: keyerror = "longitude in lon_axis are neither all positive nor all negative" list_strings = [ "ERROR " + EnsoErrorsWarnings.MessageFormating(INSPECTstack()) + ": unknown longitude", str().ljust(5) + metric + ": " + str(region) + ": " + keyerror + ": " + str(val_longitude) ] EnsoErrorsWarnings.MyWarning(list_strings) ep_event = None else: if pos_lon == 'pos': ep_event = [1 for val in val_longitude if val > 360 + threshold] else: ep_event = [1 for val in val_longitude if val > threshold] ep_event = sum(ep_event) * 100. / len(val_longitude) return ep_event, keyerror
def read_var(var_to_read, filename_nc, model, reference, metric_variables, dict_metric): if isinstance(var_to_read, str): var_to_read = [var_to_read] if isinstance(filename_nc, str): tab_mod, tab_obs, metval, obs = reader(filename_nc, model, reference, var_to_read, metric_variables, dict_metric) else: tab_mod, metval, obs = list(), list(), list() for ii in range(len(filename_nc)): tmp1, tab_obs, tmp2, tmp3 = reader(filename_nc[ii], model[ii], reference, var_to_read, metric_variables, dict_metric) tab_mod.append(tmp1) metval.append(tmp2) obs.append(tmp3) obs = list(set(obs)) if len(obs) > 1: list_strings = ["ERROR" + EnsoErrorsWarnings.MessageFormating(INSPECTstack()) + ": too many obs", str().ljust(5) + "var_to_read = "+str(var_to_read), str().ljust(5) + "filename_nc = " + str(filename_nc), str().ljust(5) + "model = " + str(model), str().ljust(5) + "reference = " + str(reference)] EnsoErrorsWarnings.MyError(list_strings) else: obs = obs[0] return tab_mod, tab_obs, metval, obs
def statistical_dispersion(tab, method='IQR'): """ ################################################################################# Description: Computes the statistical dispersion of the distribution ################################################################################# :param tab: list or `cdms2` variable A list or a `cdms2` variable containing the data to be analysed :param method: string, optional method to compute the statistical dispersion 'IQR': interquartile range, IQR = Q3 - Q1 'MAD': median absolute deviation, MAD = median([Xi - median(tab)]) Default is 'IQR' :return stat_disp: float statistical_dispersion """ known_methods = sorted(['IQR', 'MAD']) if method not in known_methods: # if 'method' is not defined -> raise error list_strings = [ "ERROR" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": unknown method", str().ljust(5) + "method " + str(method) + " is not defined", str().ljust(10) + "known methods: " + str(known_methods) ] EnsoErrorsWarnings.my_error(list_strings) if method == 'IQR': stat_disp = abs(float(SCIPYstats__scoreatpercentile(tab, 75) - SCIPYstats__scoreatpercentile(tab, 25))) else: med = float(SCIPYstats__scoreatpercentile(tab, 50)) stat_disp = float(SCIPYstats__scoreatpercentile([abs(ii - med) for ii in tab], 50)) return stat_disp
def default_arg_values(arg): default = { 'detrending': False, 'frequency': None, 'metric_computation': 'difference', 'min_time_steps': None, 'normalization': False, 'project_interpreter': 'CMIP', 'regridding': False, 'smoothing': False, 'treshold_ep_ev': -140, 'time_bounds': None, 'time_bounds_mod': None, 'time_bounds_obs': None, } try: default[arg] except: unknown_key_arg(arg, INSPECTstack()) return default[arg]
def find_xml_cmip(experiment, frequency, model, project, realm, ensemble, variable): """ Finds cmip variable file, as well as corresponding areacell and landmask Inputs: ------ :param experiment: string experiment name (e.g., "historical", "piControl") :param frequency: string data frequency: "day" for daily, "mon" for monthly :param model: string model name (e.g., "CNRM-CM5", "IPSL-CM5A-LR") :param project: string project name (e.g., "CMIP5", "CMIP6") :param realm: string data realm: "A" for atmosphere, "O" for ocean :param ensemble: string ensemble name (e.g., "r1i1p1", "r1i1p1f1") :param variable: string variable name (e.g., "pr", "tos") Outputs: ------- :return file_name: string Path and file name corresponding to the given information (e.g., /path/to/file/filename.xml) :return file_area: string Path and areacell file name corresponding to the given information (e.g., /path/to/file/areacell.xml) Set to None if the file cannot be found :return file_land: string Path and landmask file name corresponding to the given information (e.g., /path/to/file/landmask.xml) Set to None if the file cannot be found """ try: pathnc, filenc = find_path_and_files(ens=ensemble, exp=experiment, fre=frequency, mod=model, pro=project, rea=realm, var=variable) except: if realm == "O": new_realm = "A" else: new_realm = "O" # if var is not in realm 'O' (for ocean), look for it in realm 'A' (for atmosphere), and conversely try: pathnc, filenc = find_path_and_files(ens=ensemble, exp=experiment, fre=frequency, mod=model, pro=project, rea=new_realm, var=variable) except: pathnc, filenc = None, [None] # given variable is neither in realm 'A' nor 'O' print(bcolors.FAIL + "%%%%% ----- %%%%%") print("ERROR: function: " + str(INSPECTstack()[0][3]) + ", line: " + str(INSPECTstack()[0][2])) print("given variable cannot be found in either realm A or O: " + str(variable)) print("param: " + str(model) + ", " + str(project) + ", " + str(experiment) + ", " + str(ensemble) + ", " + str(frequency) + ", " + str(realm)) print("%%%%% ----- %%%%%" + bcolors.ENDC) SYSexit("") file_area, file_land =\ find_fx(model, project=project, experiment=experiment, ensemble=ensemble, realm=new_realm) else: file_area, file_land = find_fx(model, project=project, experiment=experiment, ensemble=ensemble, realm=realm) file_name = OSpath__join(pathnc, str(filenc[0])) return file_name, file_area, file_land
def read_var(var_to_read, filename_nc, model, reference, metric_variables, dict_metric, models2=None, member=None, shading=False, met_in_file=False, met_type=None, met_pattern=""): if isinstance(var_to_read, str): var_to_read = [var_to_read] if isinstance(filename_nc, str): tab_mod, tab_obs, metval, obs = reader(filename_nc, model, reference, var_to_read, metric_variables, dict_metric, member=member, met_in_file=met_in_file, met_type=met_type, met_pattern=met_pattern) else: tab_mod, metval, obs = list(), list(), list() if shading is True: for jj in range(len(model)): tmp1, tmp2 = list(), list() for ii in range(len(filename_nc[model[jj]])): ttt1, tab_obs, ttt2, tmp3 = reader( filename_nc[model[jj]][ii], models2[model[jj]][ii], reference, var_to_read, metric_variables, dict_metric[model[jj]], member=member, met_in_file=met_in_file, met_type=met_type, met_pattern=met_pattern) tmp1.append(ttt1) tmp2.append(ttt2) obs.append(tmp3) tab_mod.append(tmp1) metval.append(tmp2) else: for ii in range(len(filename_nc)): tmp1, tab_obs, tmp2, tmp3 = reader(filename_nc[ii], model[ii], reference, var_to_read, metric_variables, dict_metric, member=member) tab_mod.append(tmp1) metval.append(tmp2) obs.append(tmp3) obs = list(set(obs)) if len(obs) > 1: list_strings = [ "ERROR" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": too many obs", str().ljust(5) + "var_to_read = " + str(var_to_read), str().ljust(5) + "filename_nc = " + str(filename_nc), str().ljust(5) + "model = " + str(model), str().ljust(5) + "reference = " + str(reference) ] EnsoErrorsWarnings.my_error(list_strings) else: obs = obs[0] return tab_mod, tab_obs, metval, obs
def minmax_plot(tab, metric=False): # define minimum and maximum mini, maxi = minimaxi(tab) if mini == maxi or abs(maxi - mini) / float(abs(maxi + mini)) < 1e-2: tt = max(abs(mini), abs(maxi)) / 10. tmp = int(str("%.e" % tt)[3:]) if mini == maxi or (mini < 0 and maxi > 0): tmp = 10**-tmp if tt < 1 else 10**tmp elif mini >= 0: tmp = 10**-tmp if tt < 1 else 10**tmp else: tmp = 10**-tmp if tt < 1 else 10**tmp mini = 0 if mini > 0 and (mini - tmp) < 0 else mini - tmp maxi = 0 if maxi < 0 and (maxi + tmp) > 0 else maxi + tmp if mini < 0 and maxi > 0: locmaxi = max([abs(mini), abs(maxi)]) locmini = -deepcopy(locmaxi) else: locmini, locmaxi = deepcopy(mini), deepcopy(maxi) # find the power of ten to get an interval between 1 and 10 mult = pow(10, int(str("%e" % abs(locmaxi - locmini)).split('e')[1])) locmini, locmaxi = int(MATHfloor(float(locmini) / mult)), int( MATHceil(float(locmaxi) / mult)) if locmaxi == 2 and maxi < 15 and mult == 10 and abs(locmini) != locmaxi: locmini, locmaxi = 0, 15 mult = 1. scalmini, scalemaxi = mini / mult, maxi / mult interval = locmaxi - locmini listbase = list( NUMPYaround( [ii * 10**exp for exp in range(-1, 1) for ii in range(1, 6)], decimals=1)) listbase = listbase + listbase listmult = [3] * int(round(len(listbase) / 2.)) + [4] * int( round(len(listbase) / 2.)) list1 = list( NUMPYaround( [listbase[ii] * listmult[ii] for ii in range(len(listbase))], decimals=1)) list2 = list(NUMPYaround([abs(ii - interval) for ii in list1], decimals=1)) interval = list1[list2.index(min(list2))] base = listbase[list1.index(interval)] if base * 4.5 < interval: ii = 1 tmp = sorted(list2) while base * 4.5 < interval: interval = list1[list2.index(tmp[ii])] base = listbase[list1.index(interval)] ii += 1 if abs(locmini) == locmaxi: maxi_out = 2 * base while maxi_out - base > locmaxi: maxi_out -= base if metric is True and maxi_out < scalemaxi + base * 0.4: maxi_out += base mini_out = -maxi_out else: if locmini < 0 and locmaxi <= 0: locmini, locmaxi = abs(locmaxi), abs(locmini) sign = -1 else: sign = 1 half_int = int(round(interval / 2.)) tmp_middle = locmini + half_int mini_out = max([0, tmp_middle - half_int]) while mini_out > locmini: mini_out -= base while mini_out + base < locmini: mini_out += base maxi_out = mini_out + 2 * base while maxi_out < locmaxi: maxi_out += base while maxi_out - base > locmaxi: maxi_out -= base minmax = list( NUMPYaround(NUMPYarray([mini_out, maxi_out]) * sign, decimals=0).astype(int)) mini_out, maxi_out = min(minmax), max(minmax) if metric is True: if maxi_out < scalemaxi + base * 0.4: maxi_out += base tick_labels = NUMPYarange(mini_out, maxi_out + base / 2., base) tick_labels = list(NUMPYaround(tick_labels * mult, decimals=4)) if all([True if int(ii) == float(ii) else False for ii in tick_labels]): tick_labels = list(NUMPYaround(tick_labels, decimals=0).astype(int)) if len(tick_labels) > 7: list_strings = [ "WARNING" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": too many ticks for axis", str().ljust(5) + str(len(tick_labels)) + " ticks: " + str(tick_labels), str().ljust(5) + "there should not be more than 7" ] EnsoErrorsWarnings.my_warning(list_strings) if min(tick_labels) > mini or max(tick_labels) < maxi: list_strings = [ "WARNING" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": wrong bounds in ticks for axis" ] if min(tick_labels) > mini: list_strings += [ str().ljust(5) + "ticks minimum (" + str(min(tick_labels)) + ") > tab minimum (" + str(mini) + ")" ] if max(tick_labels) < maxi: list_strings += [ str().ljust(5) + "ticks maximum (" + str(max(tick_labels)) + ") > tab maximum (" + str(maxi) + ")" ] EnsoErrorsWarnings.my_warning(list_strings) return tick_labels
def math_metric_computation(model, model_err, obs=None, obs_err=None, keyword='difference'): """ ################################################################################# Description: Computes the metric value, i.e., distance between a model and an observational dataset ################################################################################# :param model: float or None scalar value computed with a model :param model_err: float or None error value on the scalar value computed with a model :param obs: float or None, optional scalar value computed with an observational dataset default value is None :param obs_err: float or None, optional error value on the scalar value computed with an observational dataset default value is None :param keyword: string, optional name of a mathematical method to apply to compute the metric value (distance between a model and an observational dataset): 'difference', 'ratio', 'relative_difference', 'abs_relative_difference' default value is 'difference' :return metric: float or None distance between a model and an observational dataset or None if 'model' and/or 'obs' is/are None :return metric_err: float or None error on the distance between a model and an observational dataset or None if 'model' and/or 'obs' and/or 'metric_err' and/or 'obs_err' is/are None :return description_metric: string description of the mathematical method used to compute the metric value """ if keyword not in ['difference', 'ratio', 'relative_difference', 'abs_relative_difference']: metric, metric_err, description_metric = \ None, None, "unknown keyword for the mathematical computation of the metric: " + str(keyword) list_strings = ["ERROR" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": keyword", str().ljust(5) + description_metric] EnsoErrorsWarnings.my_warning(list_strings) else: if model is not None and obs is not None: if keyword == 'difference': description_metric = \ "The metric is the difference between model and observations values (M = model - obs)" metric = model - obs elif keyword == 'ratio': description_metric = "The metric is the ratio between model and observations values (M = model / obs)" metric = model / float(obs) elif keyword == 'relative_difference': description_metric = "The metric is the relative difference between model and observations values " + \ "(M = [model-obs] / obs)" metric = (model - obs) / float(obs) else: description_metric = "The metric is the absolute value of the relative difference between model " + \ "and observations values (M = 100 * abs[[model-obs] / obs])" metric = 100. * abs((model - obs) / float(obs)) else: metric, description_metric = None, '' if model_err is not None or obs_err is not None: if keyword == 'difference': # mathematical definition of the error on addition / subtraction metric_err = model_err + obs_err elif keyword == 'ratio': # mathematical definition of the error on division if model is not None and obs is not None: metric_err = float((obs * model_err + model * obs_err) / NUMPYsquare(obs)) else: metric_err = None else: # mathematical definition of the error on division if model is not None and obs is not None: metric_err = float((obs * (model_err + obs_err) + (model - obs) * obs_err) / NUMPYsquare(obs)) if keyword == 'abs_relative_difference': metric_err = 100. * metric_err else: metric_err = None else: metric_err = None return metric, metric_err, description_metric