def StringInDict(string_or_list, dictionary, inspect_stack):
"""
#################################################################################
Description:
Tests if 'string_or_list' is in the given 'dictionary'
#################################################################################
:param string_or_list: string or list
key or list of keys to look for in 'dictionary'
:param dictionary: dict
dictionary in which the given keys are looked for
:param inspect_stack: array
list of information about the program/module/line,... created using inspect.stack()
:return:
"""
# test input parameters
if not isinstance(dictionary, dict):
EnsoErrorsWarnings.ObjectTypeError('dictionary', 'dictionary',
type(dictionary), INSPECTstack())
if isinstance(string_or_list, basestring):
# 'string_or_list' is a string
if string_or_list not in dictionary.keys():
# key 'string_or_list' is not in 'dictionary' -> raise error
list_strings = [
"ERROR" + EnsoErrorsWarnings.MessageFormating(inspect_stack) +
": item not included",
str().ljust(5) + " key: " + str(string_or_list) +
" is not in the given dictionary",
str().ljust(10) + "key(s) in the dictionary: " +
str(sorted(dictionary.keys()))
]
EnsoErrorsWarnings.MyError(list_strings)
elif isinstance(string_or_list, list):
# 'string_or_list' is a list
key_not_included = list()
for key in string_or_list:
if key not in dictionary.keys():
# lists keys that are not in 'dictionary'
key_not_included.append(key)
if key_not_included:
# if 'key_not_included' is not empty -> raise error
list_strings = [
"ERROR" + EnsoErrorsWarnings.MessageFormating(inspect_stack) +
": item not included",
str().ljust(5) + " key(s): " + str(key_not_included) +
" are (is) not in the given dictionary",
str().ljust(10) + "key(s) in the dictionary: " +
str(sorted(dictionary.keys()))
]
EnsoErrorsWarnings.MyError(list_strings)
else:
# 'string_or_list' is neither a string nor a list -> raise error
EnsoErrorsWarnings.ObjectTypeError('string_or_list', '[string, list]',
type(string_or_list),
INSPECTstack())
return
def find_xml_cmip(model, project, experiment, ensemble, frequency, realm,
variable):
try:
pathnc, filenc = find_path_and_files(ens=ensemble,
exp=experiment,
fre=frequency,
mod=model,
pro=project,
rea=realm,
var=variable)
except:
if realm == 'O':
new_realm = 'A'
elif realm == 'A':
new_realm = 'O'
# if var is not in realm 'O' (for ocean), look for it in realm 'A' (for atmosphere), and conversely
try:
pathnc, filenc = find_path_and_files(ens=ensemble,
exp=experiment,
fre=frequency,
mod=model,
pro=project,
rea=new_realm,
var=variable)
except:
# given variable is neither in realm 'A' nor 'O'
print bcolors.FAIL + '%%%%% ----- %%%%%'
print 'ERROR: function: ' + str(
INSPECTstack()[0][3]) + ', line: ' + str(INSPECTstack()[0][2])
print 'given variable cannot be found in either realm A or O: ' + str(
variable)
print 'param: ' + str(model) + ', ' + str(project) + ', ' + str(
experiment) + ', ' + str(ensemble) + ', ' + str(
frequency) + ', ' + str(realm)
print '%%%%% ----- %%%%%' + bcolors.ENDC
sys.exit('')
file_area, file_land = find_fx(model,
project=project,
experiment=experiment,
ensemble=ensemble,
realm=new_realm)
else:
file_area, file_land = find_fx(model,
project=project,
experiment=experiment,
ensemble=ensemble,
realm=realm)
file_name = OSpath__join(pathnc, filenc[0])
return file_name, file_area, file_land
def percentage_val_eastward(val_longitude, metric, region, threshold=-140):
"""
#################################################################################
Description:
Computes the percentage of given values (longitude in val_longitude) eastward of 'threshold'
#################################################################################
:param val_longitude: list
list of longitude
:param threshold: float, optional
threshold to define the westward boundary of the region
:return ep: float
percentage of given values eastward of 'threshold'
"""
keyerror = None
pos_lon = 'pos' if all(ii >= 0 for ii in val_longitude) is True else \
('neg' if all(ii <= 0 for ii in val_longitude) is True else False)
if pos_lon is False:
keyerror = "longitude in lon_axis are neither all positive nor all negative"
list_strings = [
"ERROR " + EnsoErrorsWarnings.MessageFormating(INSPECTstack()) +
": unknown longitude",
str().ljust(5) + metric + ": " + str(region) + ": " + keyerror +
": " + str(val_longitude)
]
EnsoErrorsWarnings.MyWarning(list_strings)
ep_event = None
else:
if pos_lon == 'pos':
ep_event = [1 for val in val_longitude if val > 360 + threshold]
else:
ep_event = [1 for val in val_longitude if val > threshold]
ep_event = sum(ep_event) * 100. / len(val_longitude)
return ep_event, keyerror
def read_var(var_to_read, filename_nc, model, reference, metric_variables, dict_metric):
if isinstance(var_to_read, str):
var_to_read = [var_to_read]
if isinstance(filename_nc, str):
tab_mod, tab_obs, metval, obs = reader(filename_nc, model, reference, var_to_read, metric_variables,
dict_metric)
else:
tab_mod, metval, obs = list(), list(), list()
for ii in range(len(filename_nc)):
tmp1, tab_obs, tmp2, tmp3 = reader(filename_nc[ii], model[ii], reference, var_to_read, metric_variables,
dict_metric)
tab_mod.append(tmp1)
metval.append(tmp2)
obs.append(tmp3)
obs = list(set(obs))
if len(obs) > 1:
list_strings = ["ERROR" + EnsoErrorsWarnings.MessageFormating(INSPECTstack()) + ": too many obs",
str().ljust(5) + "var_to_read = "+str(var_to_read),
str().ljust(5) + "filename_nc = " + str(filename_nc),
str().ljust(5) + "model = " + str(model),
str().ljust(5) + "reference = " + str(reference)]
EnsoErrorsWarnings.MyError(list_strings)
else:
obs = obs[0]
return tab_mod, tab_obs, metval, obs
def statistical_dispersion(tab, method='IQR'):
"""
#################################################################################
Description:
Computes the statistical dispersion of the distribution
#################################################################################
:param tab: list or `cdms2` variable
A list or a `cdms2` variable containing the data to be analysed
:param method: string, optional
method to compute the statistical dispersion
'IQR': interquartile range, IQR = Q3 - Q1
'MAD': median absolute deviation, MAD = median([Xi - median(tab)])
Default is 'IQR'
:return stat_disp: float
statistical_dispersion
"""
known_methods = sorted(['IQR', 'MAD'])
if method not in known_methods:
# if 'method' is not defined -> raise error
list_strings = [
"ERROR" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": unknown method",
str().ljust(5) + "method " + str(method) + " is not defined",
str().ljust(10) + "known methods: " + str(known_methods)
]
EnsoErrorsWarnings.my_error(list_strings)
if method == 'IQR':
stat_disp = abs(float(SCIPYstats__scoreatpercentile(tab, 75) - SCIPYstats__scoreatpercentile(tab, 25)))
else:
med = float(SCIPYstats__scoreatpercentile(tab, 50))
stat_disp = float(SCIPYstats__scoreatpercentile([abs(ii - med) for ii in tab], 50))
return stat_disp
def default_arg_values(arg):
default = {
'detrending': False, 'frequency': None, 'metric_computation': 'difference', 'min_time_steps': None,
'normalization': False, 'project_interpreter': 'CMIP', 'regridding': False, 'smoothing': False,
'treshold_ep_ev': -140, 'time_bounds': None, 'time_bounds_mod': None, 'time_bounds_obs': None,
}
try:
default[arg]
except:
unknown_key_arg(arg, INSPECTstack())
return default[arg]
def find_xml_cmip(experiment, frequency, model, project, realm, ensemble,
variable):
"""
Finds cmip variable file, as well as corresponding areacell and landmask
Inputs:
------
:param experiment: string
experiment name (e.g., "historical", "piControl")
:param frequency: string
data frequency: "day" for daily, "mon" for monthly
:param model: string
model name (e.g., "CNRM-CM5", "IPSL-CM5A-LR")
:param project: string
project name (e.g., "CMIP5", "CMIP6")
:param realm: string
data realm: "A" for atmosphere, "O" for ocean
:param ensemble: string
ensemble name (e.g., "r1i1p1", "r1i1p1f1")
:param variable: string
variable name (e.g., "pr", "tos")
Outputs:
-------
:return file_name: string
Path and file name corresponding to the given information (e.g., /path/to/file/filename.xml)
:return file_area: string
Path and areacell file name corresponding to the given information (e.g., /path/to/file/areacell.xml)
Set to None if the file cannot be found
:return file_land: string
Path and landmask file name corresponding to the given information (e.g., /path/to/file/landmask.xml)
Set to None if the file cannot be found
"""
try:
pathnc, filenc = find_path_and_files(ens=ensemble,
exp=experiment,
fre=frequency,
mod=model,
pro=project,
rea=realm,
var=variable)
except:
if realm == "O":
new_realm = "A"
else:
new_realm = "O"
# if var is not in realm 'O' (for ocean), look for it in realm 'A' (for atmosphere), and conversely
try:
pathnc, filenc = find_path_and_files(ens=ensemble,
exp=experiment,
fre=frequency,
mod=model,
pro=project,
rea=new_realm,
var=variable)
except:
pathnc, filenc = None, [None]
# given variable is neither in realm 'A' nor 'O'
print(bcolors.FAIL + "%%%%% ----- %%%%%")
print("ERROR: function: " + str(INSPECTstack()[0][3]) +
", line: " + str(INSPECTstack()[0][2]))
print("given variable cannot be found in either realm A or O: " +
str(variable))
print("param: " + str(model) + ", " + str(project) + ", " +
str(experiment) + ", " + str(ensemble) + ", " +
str(frequency) + ", " + str(realm))
print("%%%%% ----- %%%%%" + bcolors.ENDC)
SYSexit("")
file_area, file_land =\
find_fx(model, project=project, experiment=experiment, ensemble=ensemble, realm=new_realm)
else:
file_area, file_land = find_fx(model,
project=project,
experiment=experiment,
ensemble=ensemble,
realm=realm)
file_name = OSpath__join(pathnc, str(filenc[0]))
return file_name, file_area, file_land
def read_var(var_to_read,
filename_nc,
model,
reference,
metric_variables,
dict_metric,
models2=None,
member=None,
shading=False,
met_in_file=False,
met_type=None,
met_pattern=""):
if isinstance(var_to_read, str):
var_to_read = [var_to_read]
if isinstance(filename_nc, str):
tab_mod, tab_obs, metval, obs = reader(filename_nc,
model,
reference,
var_to_read,
metric_variables,
dict_metric,
member=member,
met_in_file=met_in_file,
met_type=met_type,
met_pattern=met_pattern)
else:
tab_mod, metval, obs = list(), list(), list()
if shading is True:
for jj in range(len(model)):
tmp1, tmp2 = list(), list()
for ii in range(len(filename_nc[model[jj]])):
ttt1, tab_obs, ttt2, tmp3 = reader(
filename_nc[model[jj]][ii],
models2[model[jj]][ii],
reference,
var_to_read,
metric_variables,
dict_metric[model[jj]],
member=member,
met_in_file=met_in_file,
met_type=met_type,
met_pattern=met_pattern)
tmp1.append(ttt1)
tmp2.append(ttt2)
obs.append(tmp3)
tab_mod.append(tmp1)
metval.append(tmp2)
else:
for ii in range(len(filename_nc)):
tmp1, tab_obs, tmp2, tmp3 = reader(filename_nc[ii],
model[ii],
reference,
var_to_read,
metric_variables,
dict_metric,
member=member)
tab_mod.append(tmp1)
metval.append(tmp2)
obs.append(tmp3)
obs = list(set(obs))
if len(obs) > 1:
list_strings = [
"ERROR" +
EnsoErrorsWarnings.message_formating(INSPECTstack()) +
": too many obs",
str().ljust(5) + "var_to_read = " + str(var_to_read),
str().ljust(5) + "filename_nc = " + str(filename_nc),
str().ljust(5) + "model = " + str(model),
str().ljust(5) + "reference = " + str(reference)
]
EnsoErrorsWarnings.my_error(list_strings)
else:
obs = obs[0]
return tab_mod, tab_obs, metval, obs
def minmax_plot(tab, metric=False):
# define minimum and maximum
mini, maxi = minimaxi(tab)
if mini == maxi or abs(maxi - mini) / float(abs(maxi + mini)) < 1e-2:
tt = max(abs(mini), abs(maxi)) / 10.
tmp = int(str("%.e" % tt)[3:])
if mini == maxi or (mini < 0 and maxi > 0):
tmp = 10**-tmp if tt < 1 else 10**tmp
elif mini >= 0:
tmp = 10**-tmp if tt < 1 else 10**tmp
else:
tmp = 10**-tmp if tt < 1 else 10**tmp
mini = 0 if mini > 0 and (mini - tmp) < 0 else mini - tmp
maxi = 0 if maxi < 0 and (maxi + tmp) > 0 else maxi + tmp
if mini < 0 and maxi > 0:
locmaxi = max([abs(mini), abs(maxi)])
locmini = -deepcopy(locmaxi)
else:
locmini, locmaxi = deepcopy(mini), deepcopy(maxi)
# find the power of ten to get an interval between 1 and 10
mult = pow(10, int(str("%e" % abs(locmaxi - locmini)).split('e')[1]))
locmini, locmaxi = int(MATHfloor(float(locmini) / mult)), int(
MATHceil(float(locmaxi) / mult))
if locmaxi == 2 and maxi < 15 and mult == 10 and abs(locmini) != locmaxi:
locmini, locmaxi = 0, 15
mult = 1.
scalmini, scalemaxi = mini / mult, maxi / mult
interval = locmaxi - locmini
listbase = list(
NUMPYaround(
[ii * 10**exp for exp in range(-1, 1) for ii in range(1, 6)],
decimals=1))
listbase = listbase + listbase
listmult = [3] * int(round(len(listbase) / 2.)) + [4] * int(
round(len(listbase) / 2.))
list1 = list(
NUMPYaround(
[listbase[ii] * listmult[ii] for ii in range(len(listbase))],
decimals=1))
list2 = list(NUMPYaround([abs(ii - interval) for ii in list1], decimals=1))
interval = list1[list2.index(min(list2))]
base = listbase[list1.index(interval)]
if base * 4.5 < interval:
ii = 1
tmp = sorted(list2)
while base * 4.5 < interval:
interval = list1[list2.index(tmp[ii])]
base = listbase[list1.index(interval)]
ii += 1
if abs(locmini) == locmaxi:
maxi_out = 2 * base
while maxi_out - base > locmaxi:
maxi_out -= base
if metric is True and maxi_out < scalemaxi + base * 0.4:
maxi_out += base
mini_out = -maxi_out
else:
if locmini < 0 and locmaxi <= 0:
locmini, locmaxi = abs(locmaxi), abs(locmini)
sign = -1
else:
sign = 1
half_int = int(round(interval / 2.))
tmp_middle = locmini + half_int
mini_out = max([0, tmp_middle - half_int])
while mini_out > locmini:
mini_out -= base
while mini_out + base < locmini:
mini_out += base
maxi_out = mini_out + 2 * base
while maxi_out < locmaxi:
maxi_out += base
while maxi_out - base > locmaxi:
maxi_out -= base
minmax = list(
NUMPYaround(NUMPYarray([mini_out, maxi_out]) * sign,
decimals=0).astype(int))
mini_out, maxi_out = min(minmax), max(minmax)
if metric is True:
if maxi_out < scalemaxi + base * 0.4:
maxi_out += base
tick_labels = NUMPYarange(mini_out, maxi_out + base / 2., base)
tick_labels = list(NUMPYaround(tick_labels * mult, decimals=4))
if all([True if int(ii) == float(ii) else False for ii in tick_labels]):
tick_labels = list(NUMPYaround(tick_labels, decimals=0).astype(int))
if len(tick_labels) > 7:
list_strings = [
"WARNING" + EnsoErrorsWarnings.message_formating(INSPECTstack()) +
": too many ticks for axis",
str().ljust(5) + str(len(tick_labels)) + " ticks: " +
str(tick_labels),
str().ljust(5) + "there should not be more than 7"
]
EnsoErrorsWarnings.my_warning(list_strings)
if min(tick_labels) > mini or max(tick_labels) < maxi:
list_strings = [
"WARNING" + EnsoErrorsWarnings.message_formating(INSPECTstack()) +
": wrong bounds in ticks for axis"
]
if min(tick_labels) > mini:
list_strings += [
str().ljust(5) + "ticks minimum (" + str(min(tick_labels)) +
") > tab minimum (" + str(mini) + ")"
]
if max(tick_labels) < maxi:
list_strings += [
str().ljust(5) + "ticks maximum (" + str(max(tick_labels)) +
") > tab maximum (" + str(maxi) + ")"
]
EnsoErrorsWarnings.my_warning(list_strings)
return tick_labels
def math_metric_computation(model, model_err, obs=None, obs_err=None, keyword='difference'):
"""
#################################################################################
Description:
Computes the metric value, i.e., distance between a model and an observational dataset
#################################################################################
:param model: float or None
scalar value computed with a model
:param model_err: float or None
error value on the scalar value computed with a model
:param obs: float or None, optional
scalar value computed with an observational dataset
default value is None
:param obs_err: float or None, optional
error value on the scalar value computed with an observational dataset
default value is None
:param keyword: string, optional
name of a mathematical method to apply to compute the metric value (distance between a model and an
observational dataset): 'difference', 'ratio', 'relative_difference', 'abs_relative_difference'
default value is 'difference'
:return metric: float or None
distance between a model and an observational dataset or None if 'model' and/or 'obs' is/are None
:return metric_err: float or None
error on the distance between a model and an observational dataset or None if 'model' and/or 'obs' and/or
'metric_err' and/or 'obs_err' is/are None
:return description_metric: string
description of the mathematical method used to compute the metric value
"""
if keyword not in ['difference', 'ratio', 'relative_difference', 'abs_relative_difference']:
metric, metric_err, description_metric = \
None, None, "unknown keyword for the mathematical computation of the metric: " + str(keyword)
list_strings = ["ERROR" + EnsoErrorsWarnings.message_formating(INSPECTstack()) + ": keyword",
str().ljust(5) + description_metric]
EnsoErrorsWarnings.my_warning(list_strings)
else:
if model is not None and obs is not None:
if keyword == 'difference':
description_metric = \
"The metric is the difference between model and observations values (M = model - obs)"
metric = model - obs
elif keyword == 'ratio':
description_metric = "The metric is the ratio between model and observations values (M = model / obs)"
metric = model / float(obs)
elif keyword == 'relative_difference':
description_metric = "The metric is the relative difference between model and observations values " + \
"(M = [model-obs] / obs)"
metric = (model - obs) / float(obs)
else:
description_metric = "The metric is the absolute value of the relative difference between model " + \
"and observations values (M = 100 * abs[[model-obs] / obs])"
metric = 100. * abs((model - obs) / float(obs))
else:
metric, description_metric = None, ''
if model_err is not None or obs_err is not None:
if keyword == 'difference':
# mathematical definition of the error on addition / subtraction
metric_err = model_err + obs_err
elif keyword == 'ratio':
# mathematical definition of the error on division
if model is not None and obs is not None:
metric_err = float((obs * model_err + model * obs_err) / NUMPYsquare(obs))
else:
metric_err = None
else:
# mathematical definition of the error on division
if model is not None and obs is not None:
metric_err = float((obs * (model_err + obs_err) + (model - obs) * obs_err) / NUMPYsquare(obs))
if keyword == 'abs_relative_difference':
metric_err = 100. * metric_err
else:
metric_err = None
else:
metric_err = None
return metric, metric_err, description_metric
