def get_spatial_and_temporal_shapes(cmvar):
"""
Get the spatial et temporal shape of a CMOR variable from the DR.
"""
logger = get_logger()
if cmvar.stid == "__struct_not_found_001__":
if print_DR_errors:
logger.warning(
"Warning: stid for %s in table %s is a broken link to structure in DR: %s"
% (cmvar.label, cmvar.mipTable, cmvar.stid))
spatial_shape = False
temporal_shape = False
else:
struct = get_element_uid(cmvar.stid)
spatial_shape = get_element_uid(
struct.spid,
error_msg=
"Warning: spatial shape for %s in table %s not found in DR." %
(cmvar.label, cmvar.mipTable))
if spatial_shape is not None:
spatial_shape = spatial_shape.label
temporal_shape = get_element_uid(
struct.tmid,
error_msg=
"Warning: temporal shape for %s in table %s not found in DR." %
(cmvar.label, cmvar.mipTable))
if temporal_shape is not None:
temporal_shape = temporal_shape.label
return [spatial_shape, temporal_shape]
def get_simple_dim_from_dim_id(dimid):
"""
Build a simple_Dim object which characteristics fit with dimid.
"""
d = get_element_uid(dimid)
#
stdname = get_element_uid(
d.standardName,
check_print_stdnames_error=True,
check_print_DR_errors=False,
error_msg="Issue with standardname for dimid %s" % dimid)
if stdname is not None:
stdname = stdname.uid
else:
stdname = ""
#
sdim = SimpleDim(label=d.label,
positive=d.positive,
requested=d.requested,
value=d.value,
stdname=stdname,
long_name=d.title,
out_name=d.altLabel,
units=d.units,
bounds=d.bounds,
boundsValues=d.boundsValues,
axis=d.axis,
coords=d.coords,
title=d.title,
type=d.type)
return sdim
def initialize_dim_variables():
global dims2shape, dim2dimid, dr_single_levels
if dims2shape is None:
dims2shape = OrderedDict()
dims2shape["longitude|latitude"] = "XY-na"
for sshp in get_list_of_elements_by_id('spatialShape').items:
dims2shape[sshp.dimensions] = sshp.label
# mpmoine_future_modif:dims2shape: ajout a la main des correpondances dims->shapes Primavera qui ne sont pas
# couvertes par la DR
# mpmoine_note: attention, il faut mettre a jour dim2shape a chaque fois qu'une nouvelle correpondance
# est introduite
# mpmoine_note: attention, dans les extra-Tables
dims2shape['longitude|latitude|height100m'] = 'XY-na'
# mpmoine_note: provisoire, XY-P12 juste pour exemple
dims2shape['longitude|latitude|plev12'] = 'XY-P12'
# mpmoine_zoom_modif:dims2shape:: ajout de XY-P23 qui a disparu de la DR-00.00.04 mais est demande dans les
# tables Primavera
dims2shape['longitude|latitude|plev23'] = 'XY-P23'
# mpmoine_zoom_modif:dims2shape:: ajout de XY-P10 qui n'est pas dans la DR mais demande dans les tables
# Primavera
dims2shape['longitude|latitude|plev10'] = 'XY-P10'
# David : test
dims2shape['longitude|latitude|plev7hm'] = 'XY-P7HM'
# Romain
dims2shape['longitude|latitude|plev19hm'] = 'XY-P19HM'
# By level for CORDEX
dims2shape['longitude|latitude|plev925'] = 'XY-P925HM'
dims2shape['longitude|latitude|plev850'] = 'XY-P850HM'
dims2shape['longitude|latitude|plev700'] = 'XY-P700HM'
dims2shape['longitude|latitude|plev500'] = 'XY-P500HM'
dims2shape['longitude|latitude|plev200'] = 'XY-P200HM'
#
if dim2dimid is None:
dim2dimid = OrderedDict()
for g in get_list_of_elements_by_id('grids').items:
dim2dimid[g.label] = g.uid
#
if dr_single_levels is None:
dr_single_levels = list()
for struct in get_list_of_elements_by_id('structure').items:
spshp = get_element_uid(struct.spid)
if spshp.label in [
"XY-na",
] and 'cids' in struct.__dict__:
if isinstance(struct.cids[0],
six.string_types) and len(struct.cids[0]) > 0:
# this line is needed prior to version 01.00.08.
c = get_element_uid(struct.cids[0])
# if c.axis == 'Z': # mpmoine_note: non car je veux dans dr_single_levels toutes les dimensions
# singletons (ex. 'typenatgr'), par seulement les niveaux
dr_single_levels.append(c.label)
# other single levels in extra Tables, not in DR
# mpmoine: les ajouts ici correspondent aux single levels Primavera.
other_single_levels = ['height50m', 'p100']
dr_single_levels.extend(other_single_levels)
return dims2shape, dim2dimid, dr_single_levels
def analyze_priority(cmvar, lmips):
"""
Returns the max priority of the CMOR variable, for a set of mips
"""
prio = cmvar.defaultPriority
rv_ids = get_request_by_id_by_sect(cmvar.uid, 'requestVar')
for rv_id in rv_ids:
rv = get_element_uid(rv_id)
vg = get_element_uid(rv.vgid)
if vg.mip in lmips:
if rv.priority < prio:
prio = rv.priority
return prio
def lab_grid_policy(cmvarid, grids): # TBD
"""
See doc of lab_adhoc_grid_policy
"""
internal_dict = get_settings_values("internal")
cmvar_uid = get_element_uid(cmvarid)
if cmvar_uid is not None and cmvar_uid.label in ["sos"]:
return [g for g in grids if g in ["", "1deg"]]
elif cmvar_uid is not None and cmvar_uid.label in [
"tos"
] and (cmvar_uid.mipTable not in ["3hr"]
or internal_dict["allow_tos_3hr_1deg"]):
if internal_dict["adhoc_policy_do_add_1deg_grid_for_tos"]:
list_grids = list()
if "" in grids:
list_grids.append("")
list_grids.append("1deg")
return list_grids
else:
return [g for g in grids if g in ["", "1deg"]]
else:
ngrids = [
g for g in grids if g not in ["1deg", "2deg", "100km", "50km"]
]
# if "cfsites" in grids : return ["","cfsites"]
if len(ngrids) == 0:
ngrids = [""] # We should at least provide native grid
return ngrids
def analyze_ambiguous_mip_varnames(debug=[]):
"""
Return the list of MIP varnames whose list of CMORvars for a single realm
show distinct values for the area part of the cell_methods
"""
# Compute a dict which keys are MIP varnames and values = list
# of CMORvars items for the varname
logger = get_logger()
d = OrderedDict()
for v in get_list_of_elements_by_id('var').items:
if v.label not in d:
d[v.label] = []
if v.label in debug:
logger.debug("Adding %s" % v.label)
refs = get_request_by_id_by_sect(v.uid, 'CMORvar')
for r in refs:
d[v.label].append(get_element_uid(r))
if v.label in debug:
logger.debug("Adding CmorVar %s(%s) for %s" %
(v.label, get_element_uid(r).mipTable,
get_element_uid(r).label))
# Replace dic values by dic of area portion of cell_methods
for vlabel in d:
if len(d[vlabel]) > 1:
cvl = d[vlabel]
d[vlabel] = OrderedDict()
for cv in cvl:
st = get_element_uid(cv.stid)
cm = get_element_uid(
st.cmid,
check_print_DR_errors=False,
error_msg="No cell method for %-15s %s(%s)" %
(st.label, cv.label, cv.mipTable))
if cm is not None:
cm = cm.cell_methods
if cm is not None:
area = cellmethod2area(cm)
realm = cv.modeling_realm
if area == 'sea' and realm == 'ocean':
area = None
# realm=""
if vlabel in debug:
logger.debug("for %s 's CMORvar %s(%s), area=%s" %
(vlabel, cv.label, cv.mipTable, area))
if realm not in d[vlabel]:
d[vlabel][realm] = OrderedDict()
if area not in d[vlabel][realm]:
d[vlabel][realm][area] = []
d[vlabel][realm][area].append(cv.mipTable)
if vlabel in debug:
print(vlabel, d[vlabel])
else:
d[vlabel] = None
# Analyze ambiguous cases regarding area part of the cell_method
ambiguous = []
for vlabel in d:
if d[vlabel]:
# print vlabel,d[vlabel]
for realm in d[vlabel]:
if len(d[vlabel][realm]) > 1:
ambiguous.append((vlabel, (realm, d[vlabel][realm])))
if "all" in debug:
for v, p in ambiguous:
logger.debug(v)
b, d = p
for r in d:
logger.debug("\t %s %s" % (r, d[r]))
return ambiguous
def complement_svar_using_cmorvar(svar,
cmvar,
sn_issues,
debug=[],
allow_pseudo=False):
"""
SVAR will have an attribute label_non_ambiguous suffixed by an
area name if the MIPvarname is ambiguous for that
Used by get_corresp_cmor_var and by select_cmor_vars_for_lab
"""
logger = get_logger()
global ambiguous_mipvarnames
if ambiguous_mipvarnames is None:
ambiguous_mipvarnames = analyze_ambiguous_mip_varnames()
# Get information form CMORvar
spatial_shp, temporal_shp = get_spatial_and_temporal_shapes(cmvar)
if cmvar.description:
description = cmvar.description.rstrip(' ')
else:
description = cmvar.title
svar.set_attributes(prec=cmvar.type,
frequency=cmvar.frequency,
mipTable=cmvar.mipTable,
Priority=cmvar.defaultPriority,
positive=cmvar.positive,
modeling_realm=cmvar.modeling_realm,
label=cmvar.label,
spatial_shp=spatial_shp,
temporal_shp=temporal_shp,
cmvar=cmvar,
long_name=cmvar.title,
description=description,
ref_var=cmvar.label)
# Get information from MIPvar
# In case no unit is found with stdname
mipvar = get_element_uid(cmvar.vid)
# see https://github.com/cmip6dr/CMIP6_DataRequest_VariableDefinitions/issues/279
stdname = ''
sn = get_element_uid(mipvar.sn) # None
if sn._h.label in [
'standardname',
]:
stdname = sn.uid
else:
if allow_pseudo:
stdname = sn.uid
if sn_issues is not None:
if stdname not in sn_issues:
sn_issues[svar.label] = set()
sn_issues[svar.label].add(svar.mipTable)
svar.set_attributes(mipVarLabel=mipvar.label,
units=mipvar.units,
stdname=stdname)
#
# Get information form Structure
st = get_element_uid(
cmvar.stid,
error_msg="DR Error: issue with stid for %s in Table %s "
"=> no cell_methods, cell_measures, dimids and sdims derived." %
(svar.label, svar.mipTable))
if st is not None:
svar.set_attributes(struct=st)
methods = get_element_uid(
st.cmid,
error_msg="DR Error: issue with cell_method for %s" % st.label)
if methods is not None:
methods = methods.cell_methods
new_methods = methods.replace("mask=siconc or siconca",
"mask=siconc")
svar.set_attributes(cm=methods, cell_methods=new_methods)
#
measures = get_element_uid(
cmvar.stid,
error_msg="DR Error: Issue with cell_measures for %s" %
repr(cmvar))
if measures is not None:
measures = measures.cell_measures
svar.set_attributes(cell_measures=measures)
#
product_of_other_dims = 1
all_dimids = list()
if spatial_shp not in [
"na-na",
]:
spid = get_element_uid(st.spid)
all_dimids.extend(spid.dimids)
if 'cids' in st.__dict__:
cids = st.cids
# when cids not empty, cids=('dim:p850',) or ('dim:typec4pft', 'dim:typenatgr') for e.g.;
# when empty , cids=('',).
if cids[0] not in [
'',
]: # this line is needed prior to version 01.00.08.
all_dimids.extend(cids)
# if (svar.label=="rv850") : print "rv850 has cids %s"%cids
if 'dids' in st.__dict__:
dids = st.dids
if dids[0] not in [
'',
]:
all_dimids.extend(dids)
sdims_dict = dict()
for dimid in all_dimids:
sdim = get_simple_dim_from_dim_id(dimid)
if sdim.dimsize > 1:
# print "for var % 15s and dim % 15s, size=%3d"%(svar.label,dimid,dimsize)
pass
product_of_other_dims *= sdim.dimsize
sdims_dict[sdim.label] = sdim
svar.update_attributes(sdims=sdims_dict)
if product_of_other_dims > 1:
# print 'for % 20s'%svar.label,' product_of_other_dims=',product_of_other_dims
svar.set_attributes(other_dims_size=product_of_other_dims)
area = cellmethod2area(svar.cell_methods)
if svar.label in debug:
logger.debug("complement_svar ... processing %s, area=%s" %
(svar.label, str(area)))
if area:
ambiguous = any([
svar.label == alabel and svar.modeling_realm == arealm
for (alabel, (arealm, lmethod)) in ambiguous_mipvarnames
])
if svar.label in debug:
logger.debug("complement_svar ... processing %s, ambiguous=%s" %
(svar.label, repr(ambiguous)))
if ambiguous:
# Special case for a set of land variables
if not (svar.modeling_realm == 'land' and svar.label[0] == 'c'):
svar.label_non_ambiguous = svar.label + "_" + area
if svar.label in debug:
logger.debug(
"complement_svar ... processing %s, label_non_ambiguous=%s" %
(svar.label, svar.label_non_ambiguous))
# removing pressure suffix must occur after resolving ambiguities (add of area suffix)
# because this 2 processes cannot be cumulate at this stage.
# this is acceptable since none of the variables requested on pressure levels have ambiguous names.
svar.set_attributes(type="cmor",
mip_era="CMIP6",
ref_var=svar.label,
label_without_psuffix=remove_p_suffix(
svar,
multi_plev_suffixes,
single_plev_suffixes,
realms=["atmos", "aerosol", "atmosChem"]))
#
return sn_issues
def get_correspond_cmor_var(homevar):
"""
For a home variable, find the CMOR var which corresponds.
"""
logger = get_logger()
count = 0
empty_table = (homevar.mipTable in [
'NONE',
]) or (homevar.mipTable.startswith("None"))
for cmvarid in get_cmor_var_id_by_label(homevar.ref_var):
cmvar = get_element_uid(cmvarid)
logger.debug("get_corresp, checking %s vs %s in %s" %
(homevar.label, cmvar.label, cmvar.mipTable))
#
# Consider case where no modeling_realm associated to the
# current CMORvar as matching anyway.
# mpmoine_TBD: A mieux gerer avec les orphan_variables ?
match_label = (cmvar.label == homevar.ref_var)
match_freq = (cmvar.frequency == homevar.frequency) or \
("SoilPools" in homevar.label and homevar.frequency in ["mon", ] and
cmvar.frequency in ["monPt", ])
match_table = (cmvar.mipTable == homevar.mipTable)
match_realm = (homevar.modeling_realm in cmvar.modeling_realm.split(' ')) or \
(homevar.modeling_realm == cmvar.modeling_realm)
empty_realm = (cmvar.modeling_realm in [
'',
])
matching = (match_label and (match_freq or empty_table)
and (match_table or empty_table)
and (match_realm or empty_realm))
if matching:
logger.debug("matches")
same_shapes = (get_spatial_and_temporal_shapes(cmvar) == [
homevar.spatial_shp, homevar.temporal_shp
])
if same_shapes:
count += 1
cmvar_found = cmvar
logger.debug("and same shapes !")
else:
if not empty_table:
logger.error(
"(%s %s) HOMEVar: Spatial and Temporal Shapes specified DO NOT match CMORvar ones. "
"-> Provided: (%s, %s) Expected: (%s, %s)" %
(homevar.label, homevar.mipTable, homevar.spatial_shp,
homevar.temporal_shp,
*get_spatial_and_temporal_shapes(cmvar)))
else:
logger.debug(
"doesn't match %s %s %s %s %s %s %s %s" %
(match_label, match_freq, cmvar.frequency, homevar.frequency,
match_table, match_realm, empty_realm, homevar.mipTable))
if count >= 1:
# empty table means that the frequency is changed (but the variable exists in another frequency cmor table
if empty_table:
var_freq_asked = homevar.frequency
allow_pseudo = get_settings_values("internal",
'allow_pseudo_standard_names')
global sn_issues_home
sn_issues_home = complement_svar_using_cmorvar(homevar, cmvar_found,
sn_issues_home, [],
allow_pseudo)
if empty_table:
homevar.frequency = var_freq_asked
homevar.mipTable = "None" + homevar.frequency
return homevar
else:
return False
def endyear_for_CMORvar(cv, expt, year):
"""
For a CMORvar, returns the largest year in the time slice(s)
of those requestItems which apply for experiment EXPT and which
include YEAR. If no time slice applies, returns None
"""
# 1- Get the RequestItems which apply to CmorVar
# 2- Select those requestItems which include expt,
# and retain their endyear if larger than former one
internal_dict = get_settings_values("internal")
global global_rls
# Some debug material
logger = get_logger()
logger.debug("In end_year for %s %s" % (cv.label, cv.mipTable))
pmax = internal_dict['max_priority']
# 1- Get the RequestItems which apply to CmorVar
rVarsUid = get_request_by_id_by_sect(cv.uid, 'requestVar')
rVars = [
get_element_uid(uid) for uid in rVarsUid
if get_element_uid(uid).priority <= pmax
]
logger.debug("les requestVars: %s" %
" ".join([rVar.title for rVar in rVars]))
VarGroups = [get_element_uid(rv.vgid) for rv in rVars]
logger.debug("les requestVars groups: %s" %
" ".join([rVg.label for rVg in VarGroups]))
RequestLinksId = list()
for vg in VarGroups:
RequestLinksId.extend(get_request_by_id_by_sect(vg.uid, 'requestLink'))
FilteredRequestLinks = list()
for rlid in RequestLinksId:
rl = get_element_uid(rlid)
if rl in global_rls:
FilteredRequestLinks.append(rl)
logger.debug(
"les requestlinks: %s" %
" ".join([get_element_uid(rlid).label for rlid in RequestLinksId]))
logger.debug("les FilteredRequestlinks: %s" %
" ".join([rl.label for rl in FilteredRequestLinks]))
RequestItems = list()
for rl in FilteredRequestLinks:
RequestItems.extend(get_request_by_id_by_sect(rl.uid, 'requestItem'))
logger.debug(
"les requestItems: %s" %
" ".join([get_element_uid(riid).label for riid in RequestItems]))
# 2- Select those request links which include expt and year
larger = None
for riid in RequestItems:
ri = get_element_uid(riid)
applies, endyear = RequestItem_applies_for_exp_and_year(ri, expt, year)
logger.debug(
"For var and freq selected for debug and year %s, for ri %s, applies=%s, endyear=%s"
% (str(year), ri.title, str(applies), str(endyear)))
if applies:
if endyear is None:
return None # One of the timeslices cover the whole expt
if larger is None:
larger = endyear
else:
larger = max(larger, endyear)
return larger
def year_in_ri_tslice(ri, exp, year):
"""
Returns a couple : relevant, endyear.
RELEVANT is True if requestItem RI applies to
YEAR, either implicitly or explicitly (e.g. timeslice)
ENDYEAR, which is meaningful if RELEVANT is True, and is the
last year in the timeslice (or None if timeslice ==
the whole experiment duration)
"""
logger = get_logger()
internal_dict = get_settings_values("internal")
if 'tslice' not in ri.__dict__:
logger.debug("No tslice for reqItem %s -> OK for any year" % ri.title)
return True, None
if ri.tslice in [
'__unset__',
]:
logger.debug("tslice is unset for reqItem %s " % ri.title)
return True, None
#
relevant = False
endyear = None
tslice = get_element_uid(ri.tslice)
logger.debug("tslice label/type is %s/%s for reqItem %s " %
(tslice.label, tslice.type, ri.title))
if tslice.type in [
"relativeRange",
]: # e.g. _slice_abrupt30
first_year = experiment_start_year(exp)
# first_year = sset["branch_year_in_child"]
relevant = (year >= tslice.start + first_year - 1
and year <= tslice.end + first_year - 1)
endyear = first_year + tslice.end - 1
elif tslice.type in [
"simpleRange",
]: # e.g. _slice_DAMIP20
relevant = (year >= tslice.start and year <= tslice.end)
endyear = tslice.end
elif tslice.type in [
"sliceList",
]: # e.g. _slice_DAMIP40
for start in range(tslice.start, int(tslice.end - tslice.sliceLen + 2),
int(tslice.step)):
if year >= start and year < start + tslice.sliceLen:
relevant = True
endyear = start + tslice.sliceLen - 1
elif tslice.type in [
"dayList",
]: # e.g. _slice_RFMIP2
# e.g. startList[i]: [1980, 1, 1, 1980, 4, 1, 1980, 7, 1, 1980, 10, 1, 1992, 1, 1, 1992, 4, 1]
years = [
tslice.startList[3 * i] for i in range(len(tslice.startList) / 3)
]
if year in years:
relevant = True
endyear = year
elif tslice.type in [
"startRange",
]: # e.g. _slice_VolMIP3
# used only for VolMIP : _slice_VolMIP3
start_year = experiment_start_year(exp)
relevant = (year >= start_year and year < start_year + nyear)
endyear = start_year + nyear - 1
elif tslice.type in [
"monthlyClimatology",
]: # e.g. _slice_clim20
relevant = (year >= tslice.start and year <= tslice.end)
endyear = tslice.end
elif tslice.type in [
"branchedYears",
]: # e.g. _slice_piControl020
branching = internal_dict["branching"]
if tslice.child in branching:
endyear = False
(refyear, starts) = branching[tslice.child]
for start in starts:
if ((year - start >= tslice.start - refyear) and
(year - start < tslice.start - refyear + tslice.nyears)):
relevant = True
lastyear = start + tslice.start - refyear + tslice.nyears - 1
if endyear is False:
endyear = lastyear
else:
endyear = max(endyear, lastyear)
logger.debug(
"slice OK: year=%d, start=%d tslice.start=%d refyear=%d tslice.nyears=%d lastyear=%d"
% (year, start, tslice.start, refyear, tslice.nyears,
lastyear))
else:
raise Dr2xmlError(
"For tslice %s, child %s start year is not documented" %
(tslice.title, tslice.child))
else:
raise Dr2xmlError("type %s for time slice %s is not handled" %
(tslice.type, tslice.title))
logger.debug(
"for year %d and experiment %s, relevant is %s for tslice %s of type %s, endyear=%s"
% (year, exp.label, repr(relevant), ri.title, tslice.type,
repr(endyear)))
return relevant, endyear
def RequestItem_applies_for_exp_and_year(ri, experiment, year=None):
"""
Returns True if requestItem 'ri' in data request is relevant
for a given 'experiment' and 'year'. Toggle 'debug' allow some printouts
"""
# Returns a couple : relevant, endyear.
# RELEVANT is True if requestItem RI applies to EXPERIMENT and
# has a timeslice wich includes YEAR, either implicitly or explicitly
# ENDYEAR is meaningful if RELEVANT is True, and is the
# last year in the timeslice (or None if timeslice ==
# the whole experiment duration)
# Acces experiment or experiment group for the RequestItem
# if (ri.label=='C4mipC4mipLandt2') : debug=True
# if ri.title=='AerChemMIP, AERmon-3d, piControl' : debug=True
# if ri.title=='CFMIP, CFMIP.CFsubhr, amip' : debug=True
internal_dict = get_settings_values("internal")
logger = get_logger()
logger.debug("In RIapplies.. Checking % 15s" % ri.title)
item_exp = get_element_uid(ri.esid)
ri_applies_to_experiment = False
endyear = None
# esid can link to an experiment or an experiment group
if item_exp._h.label in [
'experiment',
]:
logger.debug("%20s %s" % ("Simple Expt case", item_exp.label))
if item_exp.label in [
experiment,
]:
logger.debug(" OK", )
ri_applies_to_experiment = True
elif item_exp._h.label in [
'exptgroup',
]:
logger.debug("%20s %s" % ("Expt Group case ", item_exp.label))
exps_id = get_request_by_id_by_sect(ri.esid, 'experiment')
for e in [get_element_uid(eid) for eid in exps_id]:
if e.label in [
experiment,
]:
logger.debug(" OK for experiment based on group %s" %
item_exp.label)
ri_applies_to_experiment = True
elif item_exp._h.label in [
'mip',
]:
logger.debug("%20s %s" %
("Mip case ", get_element_uid(item_exp.label).label))
exps_id = get_request_by_id_by_sect(ri.esid, 'experiment')
for e in [get_element_uid(eid) for eid in exps_id]:
logger.debug(e.label + ",", )
if e.label == experiment:
logger.debug(" OK for experiment based on mip %s" %
item_exp.label)
ri_applies_to_experiment = True
else:
logger.debug("Error on esid link for ri: %s uid=%s %s" %
(ri.title, ri.uid, item_exp._h.label))
# print "ri=%s"%ri.title,
# if year is not None :
# print "Filtering for year %d"%year
filter_on_realization = internal_dict["filter_on_realization"]
if filter_on_realization:
if ri.nenmax != -1 and (internal_dict["realization_index"] >
ri.nenmax):
ri_applies_to_experiment = False
if ri_applies_to_experiment:
logger.debug("Year considered: %s %s" % (year, type(year)))
if year is None:
rep = True
endyear = None
logger.debug(" ..applies because arg year is None")
else:
year = int(year)
exp = get_element_uid(get_experiment_label(experiment))
rep, endyear = year_in_ri(ri, exp, year)
logger.debug(" ..year in ri returns: %s %s" % (rep, endyear))
# if (ri.label=="AerchemmipAermonthly3d") :
# print "reqItem=%s,experiment=%s,year=%d,rep=%s,"%(ri.label,experiment,year,rep)
# print " rep=",rep
return rep, endyear
else:
# print
return False, None
def select_data_request_CMORvars_for_lab(sset=False, year=None):
"""
A function to list CMOR variables relevant for a lab (and also,
optionally for an experiment and a year)
The variables relative to the laboratory settings are get using the dict_interface module:
list of MIPS, max Tier, list of excluded variables names
Args:
sset (boolean): should simulation settings be used
the parameter taken here are: source_type,
max priority (and all for filtering on the simulation)
If sset is False, use union of mips among all grid choices
year (int,optional) : simulation year - used when sset is not None,
to additionally filter on year
Returns:
A list of 'simplified CMOR variables'
"""
logger = get_logger()
internal_settings = get_settings_values("internal")
# From MIPS set to Request links
global global_rls, grid_choice, rls_for_all_experiments, sn_issues
if sset:
tierMax = internal_settings['tierMax']
else:
tierMax = internal_settings['tierMax_lset']
sc = get_scope(tierMax)
# Set sizes for lab settings, if available (or use CNRM-CM6-1 defaults)
if sset:
grid_choice = internal_settings["grid_choice"]
mips_list = set(internal_settings['mips'][grid_choice])
sizes = internal_settings["sizes"]
sc.mcfg = sc.build_mcfg(sizes)
else:
mip_list_by_grid = internal_settings["mips"]
mips_list = set().union(
*[set(mip_list_by_grid[grid]) for grid in mip_list_by_grid])
grid_choice = "LR"
# Sort mip_list for reproducibility
mips_list = sorted(list(mips_list))
#
if rls_for_all_experiments is None:
# Because scope do not accept list types
rls_for_mips = sc.get_request_link_by_mip(mips_list)
logger.info("Number of Request Links which apply to MIPS %s is: %d" %
(print_struct(mips_list), len(rls_for_mips)))
#
excluded_links = internal_settings["excluded_request_links"]
rls_for_mips = [
rl for rl in rls_for_mips if is_elt_applicable(
rl, attribute="label", excluded=excluded_links)
]
logger.info(
"Number of Request Links after filtering by excluded_request_links is: %d"
% len(rls_for_mips))
#
inclinks = internal_settings["included_request_links"]
if len(inclinks) > 0:
excluded_rls = [
rl for rl in rls_for_mips if
not is_elt_applicable(rl, attribute="label", included=inclinks)
]
for rl in excluded_rls:
logger.critical("RequestLink %s is not included" % rl.label)
rls_for_mips.remove(rl)
logger.info(
"Number of Request Links after filtering by included_request_links is: %d"
% len(rls_for_mips))
rls_for_all_experiments = [rl for rl in rls_for_mips]
else:
rls_for_mips = sorted(rls_for_all_experiments, key=lambda x: x.label)
#
if sset:
experiment_id = internal_settings["experiment_for_requests"]
exp = get_element_uid(get_experiment_label(experiment_id))
if exp is not None:
starty = exp.starty
endy = exp.endy
else:
starty = "??"
endy = "??"
logger.info(
"Filtering for experiment %s, covering years [ %s , %s ] in DR" %
(experiment_id, starty, endy))
# print "Request links before filter :"+`[ rl.label for rl in rls_for_mips ]`
filtered_rls = []
for rl in rls_for_mips:
# Access all requesItems ids which refer to this RequestLink
rl_req_items = get_request_by_id_by_sect(rl.uid, 'requestItem')
if any([
RequestItem_applies_for_exp_and_year(
get_element_uid(ri_id), experiment_id, year)[0]
for ri_id in rl_req_items
]):
filtered_rls.append(rl)
rls = filtered_rls
logger.info(
"Number of Request Links which apply to experiment %s member %s and MIPs %s is: %d"
% (experiment_id, internal_settings['realization_index'],
print_struct(mips_list), len(rls)))
# print "Request links that apply :"+`[ rl.label for rl in filtered_rls ]`
else:
rls = rls_for_mips
global_rls = rls
# From Request links to CMOR vars + grid
# miprl_ids=[ rl.uid for rl in rls ]
# miprl_vars=sc.varsByRql(miprl_ids, pmax=lset['max_priority'])
if sset:
pmax = internal_settings['max_priority']
else:
pmax = internal_settings['max_priority_lset']
miprl_vars_grids = set()
for rl in rls:
logger.debug("processing RequestLink %s" % rl.title)
for v in sc.get_vars_by_request_link(request_link=rl.uid, pmax=pmax):
# The requested grid is given by the RequestLink except if spatial shape matches S-*
gr = rl.grid
cmvar = get_element_uid(v)
st = get_element_uid(cmvar.stid)
sp = get_element_uid(st.spid)
if sp.label.startswith("S-"):
gr = 'cfsites'
miprl_vars_grids.add((v, gr))
# if 'ua' in cmvar.label : print "adding %s %s"%(cmvar.label,get_element_uid(cmvar.vid).label)
# else:
# print "Duplicate pair var/grid : ",cmvar.label,cmvar.mipTable,gr
miprl_vars_grids = sorted(list(miprl_vars_grids))
logger.info(
'Number of (CMOR variable, grid) pairs for these requestLinks is: %s' %
len(miprl_vars_grids))
exctab = internal_settings["excluded_tables_lset"]
if not isinstance(exctab, list):
exctab = [
exctab,
]
excvars = internal_settings['excluded_vars_lset']
if not isinstance(excvars, list):
excvars = [
excvars,
]
excpairs = internal_settings['excluded_pairs_lset']
if not isinstance(excpairs, list):
excpairs = [
excpairs,
]
if sset:
inctab = internal_settings["included_tables"]
if not isinstance(inctab, list):
inctab = [
inctab,
]
exctab.extend(internal_settings["excluded_tables_sset"])
incvars = internal_settings["included_vars"]
if not isinstance(incvars, list):
incvars = [
incvars,
]
excvars_sset = internal_settings['excluded_vars_sset']
if not isinstance(excvars_sset, list):
excvars_sset = [
excvars_sset,
]
excvars.extend(excvars_sset)
excvars_config = internal_settings['excluded_vars_per_config']
if not isinstance(excvars_config, list):
excvars_config = [
excvars_config,
]
excvars.extend(excvars_config)
excpairs_sset = internal_settings['excluded_pairs_sset']
if not isinstance(excpairs_sset, list):
excpairs_sset = [
excpairs_sset,
]
excpairs.extend(excpairs_sset)
else:
inctab = internal_settings["included_tables_lset"]
if not isinstance(inctab, list):
inctab = [
inctab,
]
incvars = internal_settings['included_vars_lset']
if not isinstance(incvars, list):
incvars = [
incvars,
]
filtered_vars = list()
for (v, g) in miprl_vars_grids:
cmvar = get_element_uid(v)
ttable = get_element_uid(cmvar.mtid)
mipvar = get_element_uid(cmvar.vid)
if is_elt_applicable(mipvar, attribute="label", excluded=excvars, included=incvars) and \
is_elt_applicable(ttable, attribute="label", excluded=exctab, included=inctab) and \
is_elt_applicable((mipvar, ttable), attribute="label", excluded=excpairs):
filtered_vars.append((v, g))
logger.debug("adding var %s, grid=%s, ttable=%s=" %
(cmvar.label, g, ttable.label)) # ,exctab,excvars
logger.info(
'Number once filtered by excluded/included vars and tables and spatial shapes is: %s'
% len(filtered_vars))
# Filter the list of grids requested for each variable based on lab policy
d = defaultdict(set)
for (v, g) in filtered_vars:
d[v].add(g)
logger.info('Number of distinct CMOR variables (whatever the grid): %d' %
len(d))
multiple_grids = list()
for v in d:
d[v] = decide_for_grids(v, d[v])
if len(d[v]) > 1:
multiple_grids.append(get_element_uid(v).label)
if print_multiple_grids:
logger.info(
"\tVariable %s will be processed with multiple grids: %s" %
(multiple_grids[-1], repr(d[v])))
if not print_multiple_grids and multiple_grids is not None and len(
multiple_grids) > 0:
logger.info(
"\tThese variables will be processed with multiple grids "
"(rerun with print_multiple_grids set to True for details): %s" %
repr(sorted(multiple_grids)))
#
# Print a count of distinct var labels
logger.info('Number of distinct var labels is: %d' %
len(set([get_element_uid(v).label for v in d])))
# Translate CMORvars to a list of simplified CMORvar objects
simplified_vars = []
allow_pseudo = internal_settings['allow_pseudo_standard_names']
for v in d:
svar = SimpleCMORVar()
cmvar = get_element_uid(v)
sn_issues = complement_svar_using_cmorvar(svar, cmvar, sn_issues, [],
allow_pseudo)
svar.Priority = analyze_priority(cmvar, mips_list)
svar.grids = d[v]
if get_element_uid(v).label in [
"tas",
]:
logger.debug("When complementing, tas is included , grids are %s" %
svar.grids)
simplified_vars.append(svar)
logger.info('Number of simplified vars is: %d' % len(simplified_vars))
logger.info("Issues with standard names are: %s" %
print_struct(sorted(list(sn_issues))))
return simplified_vars