def lab_grid_policy(cmvarid, grids): # TBD
"""
See doc of lab_adhoc_grid_policy
"""
internal_dict = get_settings_values("internal")
cmvar_uid = get_element_uid(cmvarid)
if cmvar_uid is not None and cmvar_uid.label in ["sos"]:
return [g for g in grids if g in ["", "1deg"]]
elif cmvar_uid is not None and cmvar_uid.label in [
"tos"
] and (cmvar_uid.mipTable not in ["3hr"]
or internal_dict["allow_tos_3hr_1deg"]):
if internal_dict["adhoc_policy_do_add_1deg_grid_for_tos"]:
list_grids = list()
if "" in grids:
list_grids.append("")
list_grids.append("1deg")
return list_grids
else:
return [g for g in grids if g in ["", "1deg"]]
else:
ngrids = [
g for g in grids if g not in ["1deg", "2deg", "100km", "50km"]
]
# if "cfsites" in grids : return ["","cfsites"]
if len(ngrids) == 0:
ngrids = [""] # We should at least provide native grid
return ngrids
def fill_homevar(home_var):
logger = get_logger()
if home_var.spatial_shp in ["XY-perso", "XY-HG"]:
home_var_sdims_info = get_settings_values("internal",
'perso_sdims_description')
if home_var.label in home_var_sdims_info:
home_var_sdims = OrderedDict()
for home_var_dim in home_var_sdims_info[home_var.label]:
home_var_sdim = SimpleDim()
home_var_sdim.label = home_var_dim
for sdim_key in [
"zoom_label", "stdname", "long_name", "positive",
"requested", "value", "out_name", "units",
"is_zoom_of", "bounds", "boundsValue", "axis", "type",
"coords", "title", "is_union_for"
]:
if sdim_key in home_var_sdims_info[
home_var.label][home_var_dim]:
setattr(
home_var_sdim, sdim_key, home_var_sdims_info[
home_var.label][home_var_dim][sdim_key])
home_var_sdims[home_var_dim] = home_var_sdim
home_var.sdims = home_var_sdims
else:
logger.error(
"Could not find custom sdims for {} in simulation settings.".
format(home_var.label))
raise VarsError(
"Could not find custom sdims for %s in simulation settings." %
home_var.label)
actual_mip = home_var.mip
if actual_mip.startswith(r"[") and actual_mip.endswith(r"]"):
home_var.mip = actual_mip.lstrip("[").rstrip("]").split(",")
return home_var
def gather_AllSimpleVars(year=False, select="on_expt_and_year"):
"""
List of mip variables asked
:param year: year when the variables are created
:param select: selection criteria
:return: list of mip variables
"""
logger = get_logger()
internal_dict = get_settings_values("internal")
if select in ["on_expt_and_year", ""]:
mip_vars_list = select_data_request_CMORvars_for_lab(True, year)
elif select in ["on_expt", ]:
mip_vars_list = select_data_request_CMORvars_for_lab(True, None)
elif select in ["no", ]:
mip_vars_list = select_data_request_CMORvars_for_lab(False, None)
else:
logger.error("Choice %s is not allowed for arg 'select'" % select)
raise Dr2xmlError("Choice %s is not allowed for arg 'select'" % select)
#
if internal_dict['listof_home_vars']:
exp = internal_dict['experiment_for_requests']
mip_vars_list = process_home_vars(mip_vars_list, internal_dict["mips"][get_grid_choice()], expid=exp)
else:
logger.info("Info: No HOMEvars list provided.")
return mip_vars_list
def ping_alias(svar, error_on_fail=False):
"""
Dans le pingfile, grace a la gestion des interpolations
verticales, on n'attend pas forcement les alias complets des
variables (CMIP6_<label>), on peut se contenter des alias
reduits (CMIP6_<lwps>)
par ailleurs, si on a defini un label non ambigu alors on l'utilise
comme ping_alias (i.e. le field_ref)
"""
pingvars = get_config_variable("pingvars")
pref = get_settings_values("internal", "ping_variables_prefix")
if svar.label_non_ambiguous:
# print "+++ non ambiguous", svar.label,svar.label_non_ambiguous
alias_ping = pref + svar.label_non_ambiguous # e.g. 'CMIP6_tsn_land' and not 'CMIP6_tsn'
else:
# print "+++ ambiguous", svar.label
# Ping file may provide the variable on the relevant pressure level - e.g. CMIP6_rv850
alias_ping = pref + svar.ref_var
if alias_ping not in pingvars:
# if not, ping_alias is supposed to be without a pressure level suffix
alias_ping = pref + svar.label_without_psuffix # e.g. 'CMIP6_hus' and not 'CMIP6_hus7h'
# print "+++ alias_ping = ", pref, svar.label_without_psuffix, alias_ping
if alias_ping not in pingvars:
if error_on_fail:
raise Dr2xmlError("Cannot find an alias in ping for variable %s" %
svar.label)
else:
return None
return alias_ping
def check_extra_variable(home_var, hv_info):
logger = get_logger()
if home_var.Priority <= get_settings_values("internal", "max_priority"):
logger.debug(
"Info: %s HOMEVar is read in an extra Table with priority %s => Taken into account."
% (hv_info, home_var.Priority))
return home_var
else:
return None
def experiment_end_year_from_sset(exp):
"""
Find the end year of an experiment
:param exp: experiment
:return: end year of the experiment
"""
internal_dict = get_settings_values("internal")
if "end_year" in internal_dict:
return internal_dict["end_year"]
else:
return experiment_end_year(exp)
def process_home_vars(mip_vars_list, mips, expid="False"):
"""
Deal with home variables
:param mip_vars_list:
:param mips:
:param expid:
:return:
"""
logger = get_logger()
internal_dict = get_settings_values("internal")
# Read HOME variables
homevars = internal_dict['listof_home_vars']
path_extra_tables = internal_dict['path_extra_tables']
logger.info("homevars file: %s" % homevars)
home_vars_list = read_home_vars_list(homevars, expid, mips,
path_extra_tables)
logger.info("homevars list: %s" %
" ".join([sv.label for sv in home_vars_list]))
#
for hv in home_vars_list:
hv_info = {
"varname": hv.label,
"realm": hv.modeling_realm,
"freq": hv.frequency,
"table": hv.mipTable
}
logger.debug(hv_info)
if hv.type in [
"cmor",
]:
new_hv = check_cmor_variable(hv, mip_vars_list, hv_info)
elif hv.type in [
"perso",
]:
new_hv = check_perso_variable(hv, hv_info)
elif hv.type in [
"dev",
]:
new_hv = check_dev_variable(hv, hv_info)
elif hv.type in [
"extra",
]:
new_hv = check_extra_variable(hv, hv_info)
else:
logger.error("%s HOMEVar type %s")
raise VarsError(
"Error: %s HOMEVar type %s does not correspond to any known keyword. "
"=> Not taken into account." % (hv_info, hv.type))
if new_hv:
mip_vars_list.append(new_hv)
return mip_vars_list
def initialize_laboratory_settings():
global laboratory_source
if laboratory_source is None:
from dr2xml.settings_interface import get_settings_values
internal_dict = get_settings_values("internal")
institution_id = internal_dict["institution_id"]
if institution_id in ["CNRM-CERFACS", "CNRM", "lfpw"]:
from . import CNRM_CERFACS
laboratory_source = CNRM_CERFACS
else:
laboratory_used = internal_dict["laboratory_used"]
if laboratory_used is not None:
if os.path.isfile(laboratory_used):
laboratory_source = SourceFileLoader(os.path.basename(laboratory_used),
laboratory_used).load_module(os.path.basename(laboratory_used))
if laboratory_source is None:
raise ValueError("Could not find the laboratory source module for %s" % institution_id)
def experiment_start_year(exp):
"""
Find star year of an experiment
:param exp: experiment
:param debug: boolean for debug verbose level activation
:return: start year of an experiment
"""
internal_dict = get_settings_values("internal")
if "branch_year_in_child" in internal_dict:
return internal_dict["branch_year_in_child"]
else:
starty = experiment_start_year_without_sset(exp)
if starty is None:
form = "Cannot guess first year for experiment %s: DR says:'%s' "
if is_sset_not_None():
form += "and 'branch_year_in_child' is not provided in experiment's settings"
raise Dr2xmlError(form % (exp.label, exp.starty))
def decide_for_grids(cmvarid, grids):
"""
Decide which set of grids a given variable should be produced on
CMVARID is uid of the CMORvar
GRIDS is a list of strings for grid as specified in requestLink
LSET is the laboratory settings dictionary. It carries a policy re. grids
Returns a list of grid strings (with some normalization) (see below)
TBD : use Martin's acronyms for grid policy
"""
# Normalize and remove duplicates in grids list
ngrids = map(normalize_grid, grids)
ngrids = sorted(list(set(ngrids)))
#
policy = get_settings_values("internal", "grid_policy")
if policy in [None, "DR"]: # Follow DR spec
return ngrids
elif policy in [
"native",
]: # Follow lab grids choice (gr or gn depending on context - see lset['grids"])
if ngrids == ['cfsites']:
return ngrids
else:
return [
"",
]
elif policy in [
"native+DR",
]: # Produce both in 'native' and DR grid
if ngrids == ['cfsites']:
return ngrids
else:
return sorted(list(set(ngrids) | {''}))
elif policy in [
"adhoc",
]:
return lab_adhoc_grid_policy(cmvarid, ngrids)
else:
Dr2xmlError("Invalid grid policy %s" % policy)
def select_variables_to_be_processed(year, context, select):
"""
Return the list of variables to be processed.
"""
internal_dict = get_settings_values("internal")
logger = get_logger()
#
# --------------------------------------------------------------------
# Extract CMOR variables for the experiment and year and lab settings
# --------------------------------------------------------------------
mip_vars_list = gather_AllSimpleVars(year, select)
# Group vars per realm
svars_per_realm = defaultdict(list)
for svar in mip_vars_list:
realm = svar.modeling_realm
if svar not in svars_per_realm[realm]:
add = not any([test_variables_similar(svar, ovar) for ovar in svars_per_realm[realm]])
# Settings may allow for duplicate var in two tables.
# In DR01.00.21, this actually applies to very few fields (ps-Aermon, tas-ImonAnt, areacellg)
if internal_dict['allow_duplicates'] or add:
svars_per_realm[realm].append(svar)
else:
logger.warning("Not adding duplicate %s (from %s) for realm %s" % (svar.label, svar.mipTable, realm))
else:
logger.warning("Duplicate svar %s %s" % (svar.label, svar.grid))
logger.info("\nRealms for these CMORvars: %s" % " ".join(sorted(list(svars_per_realm))))
#
# --------------------------------------------------------------------
# Select on context realms, grouping by table
# Excluding 'excluded_vars' and 'excluded_spshapes' lists
# --------------------------------------------------------------------
context_realms = internal_dict['realms_per_context']
processed_realms = sorted(list(set(context_realms) & set(list(svars_per_realm))))
non_processed_realms = sorted(list(set(context_realms) - set(list(svars_per_realm))))
for realm in non_processed_realms:
print("Processing realm '%s' of context '%s' -- no variable asked (skip)" % (realm, context))
svars_per_table = defaultdict(list)
for realm in processed_realms:
print("Processing realm '%s' of context '%s'" % (realm, context))
excluded_vars = defaultdict(list)
for svar in svars_per_realm[realm]:
# exclusion de certaines spatial shapes (ex. Polar Stereograpic Antarctic/Groenland)
test, reason = check_exclusion(svar,
("label", internal_dict["excluded_vars_lset"],
"They are in exclusion list"),
("spatial_shp", [None, False], "They have no spatial shape"),
("spatial_shp", internal_dict["excluded_spshapes_lset"],
"They have excluded spatial shape : %s" % svar.spatial_shp))
if test:
excluded_vars[reason].append((svar.label, svar.mipTable))
else:
svars_per_table[svar.mipTable].append(svar)
if len(excluded_vars) > 0:
logger.info("The following pairs (variable,table) have been excluded for these reasons:\n%s" %
"\n".join(["%s: %s" % (reason, print_struct(excluded_vars[reason], skip_sep=True, sort=True))
for reason in sorted(list(excluded_vars))]))
for table in sorted(list(svars_per_table)):
logger.debug("For table %s: %s" % (table, " ".join([v.label for v in svars_per_table[table]])))
#
# --------------------------------------------------------------------
# Add svars belonging to the orphan list
# --------------------------------------------------------------------
orphan_variables = internal_dict["orphan_variables"]
if context in orphan_variables:
orphans = orphan_variables[context]
for svar in [svar for svar in mip_vars_list if svar.label in orphans]:
test, reason = check_exclusion(svar, ("label", internal_dict["excluded_vars_lset"], ""),
("spatial_shp", [None, False], ""),
("spatial_shp", internal_dict["excluded_spshapes_lset"],
""))
if not test:
svars_per_table[svar.mipTable].append(svar)
#
# --------------------------------------------------------------------
# Remove svars belonging to other contexts' orphan lists
# --------------------------------------------------------------------
other_contexts = sorted(list(set(orphan_variables) - set([context, ])))
orphans = list()
for other_context in other_contexts:
orphans.extend(orphan_variables[other_context])
orphans = sorted(list(set(orphans)))
for table in svars_per_table:
svars_per_table[table] = [svar for svar in svars_per_table[table] if svar.label not in orphans]
return svars_per_table
def select_data_request_CMORvars_for_lab(sset=False, year=None):
"""
A function to list CMOR variables relevant for a lab (and also,
optionally for an experiment and a year)
The variables relative to the laboratory settings are get using the dict_interface module:
list of MIPS, max Tier, list of excluded variables names
Args:
sset (boolean): should simulation settings be used
the parameter taken here are: source_type,
max priority (and all for filtering on the simulation)
If sset is False, use union of mips among all grid choices
year (int,optional) : simulation year - used when sset is not None,
to additionally filter on year
Returns:
A list of 'simplified CMOR variables'
"""
logger = get_logger()
internal_settings = get_settings_values("internal")
# From MIPS set to Request links
global global_rls, grid_choice, rls_for_all_experiments, sn_issues
if sset:
tierMax = internal_settings['tierMax']
else:
tierMax = internal_settings['tierMax_lset']
sc = get_scope(tierMax)
# Set sizes for lab settings, if available (or use CNRM-CM6-1 defaults)
if sset:
grid_choice = internal_settings["grid_choice"]
mips_list = set(internal_settings['mips'][grid_choice])
sizes = internal_settings["sizes"]
sc.mcfg = sc.build_mcfg(sizes)
else:
mip_list_by_grid = internal_settings["mips"]
mips_list = set().union(
*[set(mip_list_by_grid[grid]) for grid in mip_list_by_grid])
grid_choice = "LR"
# Sort mip_list for reproducibility
mips_list = sorted(list(mips_list))
#
if rls_for_all_experiments is None:
# Because scope do not accept list types
rls_for_mips = sc.get_request_link_by_mip(mips_list)
logger.info("Number of Request Links which apply to MIPS %s is: %d" %
(print_struct(mips_list), len(rls_for_mips)))
#
excluded_links = internal_settings["excluded_request_links"]
rls_for_mips = [
rl for rl in rls_for_mips if is_elt_applicable(
rl, attribute="label", excluded=excluded_links)
]
logger.info(
"Number of Request Links after filtering by excluded_request_links is: %d"
% len(rls_for_mips))
#
inclinks = internal_settings["included_request_links"]
if len(inclinks) > 0:
excluded_rls = [
rl for rl in rls_for_mips if
not is_elt_applicable(rl, attribute="label", included=inclinks)
]
for rl in excluded_rls:
logger.critical("RequestLink %s is not included" % rl.label)
rls_for_mips.remove(rl)
logger.info(
"Number of Request Links after filtering by included_request_links is: %d"
% len(rls_for_mips))
rls_for_all_experiments = [rl for rl in rls_for_mips]
else:
rls_for_mips = sorted(rls_for_all_experiments, key=lambda x: x.label)
#
if sset:
experiment_id = internal_settings["experiment_for_requests"]
exp = get_element_uid(get_experiment_label(experiment_id))
if exp is not None:
starty = exp.starty
endy = exp.endy
else:
starty = "??"
endy = "??"
logger.info(
"Filtering for experiment %s, covering years [ %s , %s ] in DR" %
(experiment_id, starty, endy))
# print "Request links before filter :"+`[ rl.label for rl in rls_for_mips ]`
filtered_rls = []
for rl in rls_for_mips:
# Access all requesItems ids which refer to this RequestLink
rl_req_items = get_request_by_id_by_sect(rl.uid, 'requestItem')
if any([
RequestItem_applies_for_exp_and_year(
get_element_uid(ri_id), experiment_id, year)[0]
for ri_id in rl_req_items
]):
filtered_rls.append(rl)
rls = filtered_rls
logger.info(
"Number of Request Links which apply to experiment %s member %s and MIPs %s is: %d"
% (experiment_id, internal_settings['realization_index'],
print_struct(mips_list), len(rls)))
# print "Request links that apply :"+`[ rl.label for rl in filtered_rls ]`
else:
rls = rls_for_mips
global_rls = rls
# From Request links to CMOR vars + grid
# miprl_ids=[ rl.uid for rl in rls ]
# miprl_vars=sc.varsByRql(miprl_ids, pmax=lset['max_priority'])
if sset:
pmax = internal_settings['max_priority']
else:
pmax = internal_settings['max_priority_lset']
miprl_vars_grids = set()
for rl in rls:
logger.debug("processing RequestLink %s" % rl.title)
for v in sc.get_vars_by_request_link(request_link=rl.uid, pmax=pmax):
# The requested grid is given by the RequestLink except if spatial shape matches S-*
gr = rl.grid
cmvar = get_element_uid(v)
st = get_element_uid(cmvar.stid)
sp = get_element_uid(st.spid)
if sp.label.startswith("S-"):
gr = 'cfsites'
miprl_vars_grids.add((v, gr))
# if 'ua' in cmvar.label : print "adding %s %s"%(cmvar.label,get_element_uid(cmvar.vid).label)
# else:
# print "Duplicate pair var/grid : ",cmvar.label,cmvar.mipTable,gr
miprl_vars_grids = sorted(list(miprl_vars_grids))
logger.info(
'Number of (CMOR variable, grid) pairs for these requestLinks is: %s' %
len(miprl_vars_grids))
exctab = internal_settings["excluded_tables_lset"]
if not isinstance(exctab, list):
exctab = [
exctab,
]
excvars = internal_settings['excluded_vars_lset']
if not isinstance(excvars, list):
excvars = [
excvars,
]
excpairs = internal_settings['excluded_pairs_lset']
if not isinstance(excpairs, list):
excpairs = [
excpairs,
]
if sset:
inctab = internal_settings["included_tables"]
if not isinstance(inctab, list):
inctab = [
inctab,
]
exctab.extend(internal_settings["excluded_tables_sset"])
incvars = internal_settings["included_vars"]
if not isinstance(incvars, list):
incvars = [
incvars,
]
excvars_sset = internal_settings['excluded_vars_sset']
if not isinstance(excvars_sset, list):
excvars_sset = [
excvars_sset,
]
excvars.extend(excvars_sset)
excvars_config = internal_settings['excluded_vars_per_config']
if not isinstance(excvars_config, list):
excvars_config = [
excvars_config,
]
excvars.extend(excvars_config)
excpairs_sset = internal_settings['excluded_pairs_sset']
if not isinstance(excpairs_sset, list):
excpairs_sset = [
excpairs_sset,
]
excpairs.extend(excpairs_sset)
else:
inctab = internal_settings["included_tables_lset"]
if not isinstance(inctab, list):
inctab = [
inctab,
]
incvars = internal_settings['included_vars_lset']
if not isinstance(incvars, list):
incvars = [
incvars,
]
filtered_vars = list()
for (v, g) in miprl_vars_grids:
cmvar = get_element_uid(v)
ttable = get_element_uid(cmvar.mtid)
mipvar = get_element_uid(cmvar.vid)
if is_elt_applicable(mipvar, attribute="label", excluded=excvars, included=incvars) and \
is_elt_applicable(ttable, attribute="label", excluded=exctab, included=inctab) and \
is_elt_applicable((mipvar, ttable), attribute="label", excluded=excpairs):
filtered_vars.append((v, g))
logger.debug("adding var %s, grid=%s, ttable=%s=" %
(cmvar.label, g, ttable.label)) # ,exctab,excvars
logger.info(
'Number once filtered by excluded/included vars and tables and spatial shapes is: %s'
% len(filtered_vars))
# Filter the list of grids requested for each variable based on lab policy
d = defaultdict(set)
for (v, g) in filtered_vars:
d[v].add(g)
logger.info('Number of distinct CMOR variables (whatever the grid): %d' %
len(d))
multiple_grids = list()
for v in d:
d[v] = decide_for_grids(v, d[v])
if len(d[v]) > 1:
multiple_grids.append(get_element_uid(v).label)
if print_multiple_grids:
logger.info(
"\tVariable %s will be processed with multiple grids: %s" %
(multiple_grids[-1], repr(d[v])))
if not print_multiple_grids and multiple_grids is not None and len(
multiple_grids) > 0:
logger.info(
"\tThese variables will be processed with multiple grids "
"(rerun with print_multiple_grids set to True for details): %s" %
repr(sorted(multiple_grids)))
#
# Print a count of distinct var labels
logger.info('Number of distinct var labels is: %d' %
len(set([get_element_uid(v).label for v in d])))
# Translate CMORvars to a list of simplified CMORvar objects
simplified_vars = []
allow_pseudo = internal_settings['allow_pseudo_standard_names']
for v in d:
svar = SimpleCMORVar()
cmvar = get_element_uid(v)
sn_issues = complement_svar_using_cmorvar(svar, cmvar, sn_issues, [],
allow_pseudo)
svar.Priority = analyze_priority(cmvar, mips_list)
svar.grids = d[v]
if get_element_uid(v).label in [
"tas",
]:
logger.debug("When complementing, tas is included , grids are %s" %
svar.grids)
simplified_vars.append(svar)
logger.info('Number of simplified vars is: %d' % len(simplified_vars))
logger.info("Issues with standard names are: %s" %
print_struct(sorted(list(sn_issues))))
return simplified_vars
def RequestItem_applies_for_exp_and_year(ri, experiment, year=None):
"""
Returns True if requestItem 'ri' in data request is relevant
for a given 'experiment' and 'year'. Toggle 'debug' allow some printouts
"""
# Returns a couple : relevant, endyear.
# RELEVANT is True if requestItem RI applies to EXPERIMENT and
# has a timeslice wich includes YEAR, either implicitly or explicitly
# ENDYEAR is meaningful if RELEVANT is True, and is the
# last year in the timeslice (or None if timeslice ==
# the whole experiment duration)
# Acces experiment or experiment group for the RequestItem
# if (ri.label=='C4mipC4mipLandt2') : debug=True
# if ri.title=='AerChemMIP, AERmon-3d, piControl' : debug=True
# if ri.title=='CFMIP, CFMIP.CFsubhr, amip' : debug=True
internal_dict = get_settings_values("internal")
logger = get_logger()
logger.debug("In RIapplies.. Checking % 15s" % ri.title)
item_exp = get_element_uid(ri.esid)
ri_applies_to_experiment = False
endyear = None
# esid can link to an experiment or an experiment group
if item_exp._h.label in [
'experiment',
]:
logger.debug("%20s %s" % ("Simple Expt case", item_exp.label))
if item_exp.label in [
experiment,
]:
logger.debug(" OK", )
ri_applies_to_experiment = True
elif item_exp._h.label in [
'exptgroup',
]:
logger.debug("%20s %s" % ("Expt Group case ", item_exp.label))
exps_id = get_request_by_id_by_sect(ri.esid, 'experiment')
for e in [get_element_uid(eid) for eid in exps_id]:
if e.label in [
experiment,
]:
logger.debug(" OK for experiment based on group %s" %
item_exp.label)
ri_applies_to_experiment = True
elif item_exp._h.label in [
'mip',
]:
logger.debug("%20s %s" %
("Mip case ", get_element_uid(item_exp.label).label))
exps_id = get_request_by_id_by_sect(ri.esid, 'experiment')
for e in [get_element_uid(eid) for eid in exps_id]:
logger.debug(e.label + ",", )
if e.label == experiment:
logger.debug(" OK for experiment based on mip %s" %
item_exp.label)
ri_applies_to_experiment = True
else:
logger.debug("Error on esid link for ri: %s uid=%s %s" %
(ri.title, ri.uid, item_exp._h.label))
# print "ri=%s"%ri.title,
# if year is not None :
# print "Filtering for year %d"%year
filter_on_realization = internal_dict["filter_on_realization"]
if filter_on_realization:
if ri.nenmax != -1 and (internal_dict["realization_index"] >
ri.nenmax):
ri_applies_to_experiment = False
if ri_applies_to_experiment:
logger.debug("Year considered: %s %s" % (year, type(year)))
if year is None:
rep = True
endyear = None
logger.debug(" ..applies because arg year is None")
else:
year = int(year)
exp = get_element_uid(get_experiment_label(experiment))
rep, endyear = year_in_ri(ri, exp, year)
logger.debug(" ..year in ri returns: %s %s" % (rep, endyear))
# if (ri.label=="AerchemmipAermonthly3d") :
# print "reqItem=%s,experiment=%s,year=%d,rep=%s,"%(ri.label,experiment,year,rep)
# print " rep=",rep
return rep, endyear
else:
# print
return False, None
def get_correspond_cmor_var(homevar):
"""
For a home variable, find the CMOR var which corresponds.
"""
logger = get_logger()
count = 0
empty_table = (homevar.mipTable in [
'NONE',
]) or (homevar.mipTable.startswith("None"))
for cmvarid in get_cmor_var_id_by_label(homevar.ref_var):
cmvar = get_element_uid(cmvarid)
logger.debug("get_corresp, checking %s vs %s in %s" %
(homevar.label, cmvar.label, cmvar.mipTable))
#
# Consider case where no modeling_realm associated to the
# current CMORvar as matching anyway.
# mpmoine_TBD: A mieux gerer avec les orphan_variables ?
match_label = (cmvar.label == homevar.ref_var)
match_freq = (cmvar.frequency == homevar.frequency) or \
("SoilPools" in homevar.label and homevar.frequency in ["mon", ] and
cmvar.frequency in ["monPt", ])
match_table = (cmvar.mipTable == homevar.mipTable)
match_realm = (homevar.modeling_realm in cmvar.modeling_realm.split(' ')) or \
(homevar.modeling_realm == cmvar.modeling_realm)
empty_realm = (cmvar.modeling_realm in [
'',
])
matching = (match_label and (match_freq or empty_table)
and (match_table or empty_table)
and (match_realm or empty_realm))
if matching:
logger.debug("matches")
same_shapes = (get_spatial_and_temporal_shapes(cmvar) == [
homevar.spatial_shp, homevar.temporal_shp
])
if same_shapes:
count += 1
cmvar_found = cmvar
logger.debug("and same shapes !")
else:
if not empty_table:
logger.error(
"(%s %s) HOMEVar: Spatial and Temporal Shapes specified DO NOT match CMORvar ones. "
"-> Provided: (%s, %s) Expected: (%s, %s)" %
(homevar.label, homevar.mipTable, homevar.spatial_shp,
homevar.temporal_shp,
*get_spatial_and_temporal_shapes(cmvar)))
else:
logger.debug(
"doesn't match %s %s %s %s %s %s %s %s" %
(match_label, match_freq, cmvar.frequency, homevar.frequency,
match_table, match_realm, empty_realm, homevar.mipTable))
if count >= 1:
# empty table means that the frequency is changed (but the variable exists in another frequency cmor table
if empty_table:
var_freq_asked = homevar.frequency
allow_pseudo = get_settings_values("internal",
'allow_pseudo_standard_names')
global sn_issues_home
sn_issues_home = complement_svar_using_cmorvar(homevar, cmvar_found,
sn_issues_home, [],
allow_pseudo)
if empty_table:
homevar.frequency = var_freq_asked
homevar.mipTable = "None" + homevar.frequency
return homevar
else:
return False
def year_in_ri_tslice(ri, exp, year):
"""
Returns a couple : relevant, endyear.
RELEVANT is True if requestItem RI applies to
YEAR, either implicitly or explicitly (e.g. timeslice)
ENDYEAR, which is meaningful if RELEVANT is True, and is the
last year in the timeslice (or None if timeslice ==
the whole experiment duration)
"""
logger = get_logger()
internal_dict = get_settings_values("internal")
if 'tslice' not in ri.__dict__:
logger.debug("No tslice for reqItem %s -> OK for any year" % ri.title)
return True, None
if ri.tslice in [
'__unset__',
]:
logger.debug("tslice is unset for reqItem %s " % ri.title)
return True, None
#
relevant = False
endyear = None
tslice = get_element_uid(ri.tslice)
logger.debug("tslice label/type is %s/%s for reqItem %s " %
(tslice.label, tslice.type, ri.title))
if tslice.type in [
"relativeRange",
]: # e.g. _slice_abrupt30
first_year = experiment_start_year(exp)
# first_year = sset["branch_year_in_child"]
relevant = (year >= tslice.start + first_year - 1
and year <= tslice.end + first_year - 1)
endyear = first_year + tslice.end - 1
elif tslice.type in [
"simpleRange",
]: # e.g. _slice_DAMIP20
relevant = (year >= tslice.start and year <= tslice.end)
endyear = tslice.end
elif tslice.type in [
"sliceList",
]: # e.g. _slice_DAMIP40
for start in range(tslice.start, int(tslice.end - tslice.sliceLen + 2),
int(tslice.step)):
if year >= start and year < start + tslice.sliceLen:
relevant = True
endyear = start + tslice.sliceLen - 1
elif tslice.type in [
"dayList",
]: # e.g. _slice_RFMIP2
# e.g. startList[i]: [1980, 1, 1, 1980, 4, 1, 1980, 7, 1, 1980, 10, 1, 1992, 1, 1, 1992, 4, 1]
years = [
tslice.startList[3 * i] for i in range(len(tslice.startList) / 3)
]
if year in years:
relevant = True
endyear = year
elif tslice.type in [
"startRange",
]: # e.g. _slice_VolMIP3
# used only for VolMIP : _slice_VolMIP3
start_year = experiment_start_year(exp)
relevant = (year >= start_year and year < start_year + nyear)
endyear = start_year + nyear - 1
elif tslice.type in [
"monthlyClimatology",
]: # e.g. _slice_clim20
relevant = (year >= tslice.start and year <= tslice.end)
endyear = tslice.end
elif tslice.type in [
"branchedYears",
]: # e.g. _slice_piControl020
branching = internal_dict["branching"]
if tslice.child in branching:
endyear = False
(refyear, starts) = branching[tslice.child]
for start in starts:
if ((year - start >= tslice.start - refyear) and
(year - start < tslice.start - refyear + tslice.nyears)):
relevant = True
lastyear = start + tslice.start - refyear + tslice.nyears - 1
if endyear is False:
endyear = lastyear
else:
endyear = max(endyear, lastyear)
logger.debug(
"slice OK: year=%d, start=%d tslice.start=%d refyear=%d tslice.nyears=%d lastyear=%d"
% (year, start, tslice.start, refyear, tslice.nyears,
lastyear))
else:
raise Dr2xmlError(
"For tslice %s, child %s start year is not documented" %
(tslice.title, tslice.child))
else:
raise Dr2xmlError("type %s for time slice %s is not handled" %
(tslice.type, tslice.title))
logger.debug(
"for year %d and experiment %s, relevant is %s for tslice %s of type %s, endyear=%s"
% (year, exp.label, repr(relevant), ri.title, tslice.type,
repr(endyear)))
return relevant, endyear
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Interface between data request and dr2xml
"""
from __future__ import print_function, division, absolute_import, unicode_literals
from dr2xml.settings_interface import get_settings_values
data_request_version = get_settings_values("internal", "data_request_used")
if data_request_version in [
"CMIP6",
]:
from .CMIP6 import *
elif data_request_version in ["no", "none", "None", None]:
from .no import *
else:
raise ValueError("The data request specified (%s) is not known." %
data_request_version)
def endyear_for_CMORvar(cv, expt, year):
"""
For a CMORvar, returns the largest year in the time slice(s)
of those requestItems which apply for experiment EXPT and which
include YEAR. If no time slice applies, returns None
"""
# 1- Get the RequestItems which apply to CmorVar
# 2- Select those requestItems which include expt,
# and retain their endyear if larger than former one
internal_dict = get_settings_values("internal")
global global_rls
# Some debug material
logger = get_logger()
logger.debug("In end_year for %s %s" % (cv.label, cv.mipTable))
pmax = internal_dict['max_priority']
# 1- Get the RequestItems which apply to CmorVar
rVarsUid = get_request_by_id_by_sect(cv.uid, 'requestVar')
rVars = [
get_element_uid(uid) for uid in rVarsUid
if get_element_uid(uid).priority <= pmax
]
logger.debug("les requestVars: %s" %
" ".join([rVar.title for rVar in rVars]))
VarGroups = [get_element_uid(rv.vgid) for rv in rVars]
logger.debug("les requestVars groups: %s" %
" ".join([rVg.label for rVg in VarGroups]))
RequestLinksId = list()
for vg in VarGroups:
RequestLinksId.extend(get_request_by_id_by_sect(vg.uid, 'requestLink'))
FilteredRequestLinks = list()
for rlid in RequestLinksId:
rl = get_element_uid(rlid)
if rl in global_rls:
FilteredRequestLinks.append(rl)
logger.debug(
"les requestlinks: %s" %
" ".join([get_element_uid(rlid).label for rlid in RequestLinksId]))
logger.debug("les FilteredRequestlinks: %s" %
" ".join([rl.label for rl in FilteredRequestLinks]))
RequestItems = list()
for rl in FilteredRequestLinks:
RequestItems.extend(get_request_by_id_by_sect(rl.uid, 'requestItem'))
logger.debug(
"les requestItems: %s" %
" ".join([get_element_uid(riid).label for riid in RequestItems]))
# 2- Select those request links which include expt and year
larger = None
for riid in RequestItems:
ri = get_element_uid(riid)
applies, endyear = RequestItem_applies_for_exp_and_year(ri, expt, year)
logger.debug(
"For var and freq selected for debug and year %s, for ri %s, applies=%s, endyear=%s"
% (str(year), ri.title, str(applies), str(endyear)))
if applies:
if endyear is None:
return None # One of the timeslices cover the whole expt
if larger is None:
larger = endyear
else:
larger = max(larger, endyear)
return larger