def lab_grid_policy(cmvarid, grids): # TBD """ See doc of lab_adhoc_grid_policy """ internal_dict = get_settings_values("internal") cmvar_uid = get_element_uid(cmvarid) if cmvar_uid is not None and cmvar_uid.label in ["sos"]: return [g for g in grids if g in ["", "1deg"]] elif cmvar_uid is not None and cmvar_uid.label in [ "tos" ] and (cmvar_uid.mipTable not in ["3hr"] or internal_dict["allow_tos_3hr_1deg"]): if internal_dict["adhoc_policy_do_add_1deg_grid_for_tos"]: list_grids = list() if "" in grids: list_grids.append("") list_grids.append("1deg") return list_grids else: return [g for g in grids if g in ["", "1deg"]] else: ngrids = [ g for g in grids if g not in ["1deg", "2deg", "100km", "50km"] ] # if "cfsites" in grids : return ["","cfsites"] if len(ngrids) == 0: ngrids = [""] # We should at least provide native grid return ngrids
def fill_homevar(home_var): logger = get_logger() if home_var.spatial_shp in ["XY-perso", "XY-HG"]: home_var_sdims_info = get_settings_values("internal", 'perso_sdims_description') if home_var.label in home_var_sdims_info: home_var_sdims = OrderedDict() for home_var_dim in home_var_sdims_info[home_var.label]: home_var_sdim = SimpleDim() home_var_sdim.label = home_var_dim for sdim_key in [ "zoom_label", "stdname", "long_name", "positive", "requested", "value", "out_name", "units", "is_zoom_of", "bounds", "boundsValue", "axis", "type", "coords", "title", "is_union_for" ]: if sdim_key in home_var_sdims_info[ home_var.label][home_var_dim]: setattr( home_var_sdim, sdim_key, home_var_sdims_info[ home_var.label][home_var_dim][sdim_key]) home_var_sdims[home_var_dim] = home_var_sdim home_var.sdims = home_var_sdims else: logger.error( "Could not find custom sdims for {} in simulation settings.". format(home_var.label)) raise VarsError( "Could not find custom sdims for %s in simulation settings." % home_var.label) actual_mip = home_var.mip if actual_mip.startswith(r"[") and actual_mip.endswith(r"]"): home_var.mip = actual_mip.lstrip("[").rstrip("]").split(",") return home_var
def gather_AllSimpleVars(year=False, select="on_expt_and_year"): """ List of mip variables asked :param year: year when the variables are created :param select: selection criteria :return: list of mip variables """ logger = get_logger() internal_dict = get_settings_values("internal") if select in ["on_expt_and_year", ""]: mip_vars_list = select_data_request_CMORvars_for_lab(True, year) elif select in ["on_expt", ]: mip_vars_list = select_data_request_CMORvars_for_lab(True, None) elif select in ["no", ]: mip_vars_list = select_data_request_CMORvars_for_lab(False, None) else: logger.error("Choice %s is not allowed for arg 'select'" % select) raise Dr2xmlError("Choice %s is not allowed for arg 'select'" % select) # if internal_dict['listof_home_vars']: exp = internal_dict['experiment_for_requests'] mip_vars_list = process_home_vars(mip_vars_list, internal_dict["mips"][get_grid_choice()], expid=exp) else: logger.info("Info: No HOMEvars list provided.") return mip_vars_list
def ping_alias(svar, error_on_fail=False): """ Dans le pingfile, grace a la gestion des interpolations verticales, on n'attend pas forcement les alias complets des variables (CMIP6_<label>), on peut se contenter des alias reduits (CMIP6_<lwps>) par ailleurs, si on a defini un label non ambigu alors on l'utilise comme ping_alias (i.e. le field_ref) """ pingvars = get_config_variable("pingvars") pref = get_settings_values("internal", "ping_variables_prefix") if svar.label_non_ambiguous: # print "+++ non ambiguous", svar.label,svar.label_non_ambiguous alias_ping = pref + svar.label_non_ambiguous # e.g. 'CMIP6_tsn_land' and not 'CMIP6_tsn' else: # print "+++ ambiguous", svar.label # Ping file may provide the variable on the relevant pressure level - e.g. CMIP6_rv850 alias_ping = pref + svar.ref_var if alias_ping not in pingvars: # if not, ping_alias is supposed to be without a pressure level suffix alias_ping = pref + svar.label_without_psuffix # e.g. 'CMIP6_hus' and not 'CMIP6_hus7h' # print "+++ alias_ping = ", pref, svar.label_without_psuffix, alias_ping if alias_ping not in pingvars: if error_on_fail: raise Dr2xmlError("Cannot find an alias in ping for variable %s" % svar.label) else: return None return alias_ping
def check_extra_variable(home_var, hv_info): logger = get_logger() if home_var.Priority <= get_settings_values("internal", "max_priority"): logger.debug( "Info: %s HOMEVar is read in an extra Table with priority %s => Taken into account." % (hv_info, home_var.Priority)) return home_var else: return None
def experiment_end_year_from_sset(exp): """ Find the end year of an experiment :param exp: experiment :return: end year of the experiment """ internal_dict = get_settings_values("internal") if "end_year" in internal_dict: return internal_dict["end_year"] else: return experiment_end_year(exp)
def process_home_vars(mip_vars_list, mips, expid="False"): """ Deal with home variables :param mip_vars_list: :param mips: :param expid: :return: """ logger = get_logger() internal_dict = get_settings_values("internal") # Read HOME variables homevars = internal_dict['listof_home_vars'] path_extra_tables = internal_dict['path_extra_tables'] logger.info("homevars file: %s" % homevars) home_vars_list = read_home_vars_list(homevars, expid, mips, path_extra_tables) logger.info("homevars list: %s" % " ".join([sv.label for sv in home_vars_list])) # for hv in home_vars_list: hv_info = { "varname": hv.label, "realm": hv.modeling_realm, "freq": hv.frequency, "table": hv.mipTable } logger.debug(hv_info) if hv.type in [ "cmor", ]: new_hv = check_cmor_variable(hv, mip_vars_list, hv_info) elif hv.type in [ "perso", ]: new_hv = check_perso_variable(hv, hv_info) elif hv.type in [ "dev", ]: new_hv = check_dev_variable(hv, hv_info) elif hv.type in [ "extra", ]: new_hv = check_extra_variable(hv, hv_info) else: logger.error("%s HOMEVar type %s") raise VarsError( "Error: %s HOMEVar type %s does not correspond to any known keyword. " "=> Not taken into account." % (hv_info, hv.type)) if new_hv: mip_vars_list.append(new_hv) return mip_vars_list
def initialize_laboratory_settings(): global laboratory_source if laboratory_source is None: from dr2xml.settings_interface import get_settings_values internal_dict = get_settings_values("internal") institution_id = internal_dict["institution_id"] if institution_id in ["CNRM-CERFACS", "CNRM", "lfpw"]: from . import CNRM_CERFACS laboratory_source = CNRM_CERFACS else: laboratory_used = internal_dict["laboratory_used"] if laboratory_used is not None: if os.path.isfile(laboratory_used): laboratory_source = SourceFileLoader(os.path.basename(laboratory_used), laboratory_used).load_module(os.path.basename(laboratory_used)) if laboratory_source is None: raise ValueError("Could not find the laboratory source module for %s" % institution_id)
def experiment_start_year(exp): """ Find star year of an experiment :param exp: experiment :param debug: boolean for debug verbose level activation :return: start year of an experiment """ internal_dict = get_settings_values("internal") if "branch_year_in_child" in internal_dict: return internal_dict["branch_year_in_child"] else: starty = experiment_start_year_without_sset(exp) if starty is None: form = "Cannot guess first year for experiment %s: DR says:'%s' " if is_sset_not_None(): form += "and 'branch_year_in_child' is not provided in experiment's settings" raise Dr2xmlError(form % (exp.label, exp.starty))
def decide_for_grids(cmvarid, grids): """ Decide which set of grids a given variable should be produced on CMVARID is uid of the CMORvar GRIDS is a list of strings for grid as specified in requestLink LSET is the laboratory settings dictionary. It carries a policy re. grids Returns a list of grid strings (with some normalization) (see below) TBD : use Martin's acronyms for grid policy """ # Normalize and remove duplicates in grids list ngrids = map(normalize_grid, grids) ngrids = sorted(list(set(ngrids))) # policy = get_settings_values("internal", "grid_policy") if policy in [None, "DR"]: # Follow DR spec return ngrids elif policy in [ "native", ]: # Follow lab grids choice (gr or gn depending on context - see lset['grids"]) if ngrids == ['cfsites']: return ngrids else: return [ "", ] elif policy in [ "native+DR", ]: # Produce both in 'native' and DR grid if ngrids == ['cfsites']: return ngrids else: return sorted(list(set(ngrids) | {''})) elif policy in [ "adhoc", ]: return lab_adhoc_grid_policy(cmvarid, ngrids) else: Dr2xmlError("Invalid grid policy %s" % policy)
def select_variables_to_be_processed(year, context, select): """ Return the list of variables to be processed. """ internal_dict = get_settings_values("internal") logger = get_logger() # # -------------------------------------------------------------------- # Extract CMOR variables for the experiment and year and lab settings # -------------------------------------------------------------------- mip_vars_list = gather_AllSimpleVars(year, select) # Group vars per realm svars_per_realm = defaultdict(list) for svar in mip_vars_list: realm = svar.modeling_realm if svar not in svars_per_realm[realm]: add = not any([test_variables_similar(svar, ovar) for ovar in svars_per_realm[realm]]) # Settings may allow for duplicate var in two tables. # In DR01.00.21, this actually applies to very few fields (ps-Aermon, tas-ImonAnt, areacellg) if internal_dict['allow_duplicates'] or add: svars_per_realm[realm].append(svar) else: logger.warning("Not adding duplicate %s (from %s) for realm %s" % (svar.label, svar.mipTable, realm)) else: logger.warning("Duplicate svar %s %s" % (svar.label, svar.grid)) logger.info("\nRealms for these CMORvars: %s" % " ".join(sorted(list(svars_per_realm)))) # # -------------------------------------------------------------------- # Select on context realms, grouping by table # Excluding 'excluded_vars' and 'excluded_spshapes' lists # -------------------------------------------------------------------- context_realms = internal_dict['realms_per_context'] processed_realms = sorted(list(set(context_realms) & set(list(svars_per_realm)))) non_processed_realms = sorted(list(set(context_realms) - set(list(svars_per_realm)))) for realm in non_processed_realms: print("Processing realm '%s' of context '%s' -- no variable asked (skip)" % (realm, context)) svars_per_table = defaultdict(list) for realm in processed_realms: print("Processing realm '%s' of context '%s'" % (realm, context)) excluded_vars = defaultdict(list) for svar in svars_per_realm[realm]: # exclusion de certaines spatial shapes (ex. Polar Stereograpic Antarctic/Groenland) test, reason = check_exclusion(svar, ("label", internal_dict["excluded_vars_lset"], "They are in exclusion list"), ("spatial_shp", [None, False], "They have no spatial shape"), ("spatial_shp", internal_dict["excluded_spshapes_lset"], "They have excluded spatial shape : %s" % svar.spatial_shp)) if test: excluded_vars[reason].append((svar.label, svar.mipTable)) else: svars_per_table[svar.mipTable].append(svar) if len(excluded_vars) > 0: logger.info("The following pairs (variable,table) have been excluded for these reasons:\n%s" % "\n".join(["%s: %s" % (reason, print_struct(excluded_vars[reason], skip_sep=True, sort=True)) for reason in sorted(list(excluded_vars))])) for table in sorted(list(svars_per_table)): logger.debug("For table %s: %s" % (table, " ".join([v.label for v in svars_per_table[table]]))) # # -------------------------------------------------------------------- # Add svars belonging to the orphan list # -------------------------------------------------------------------- orphan_variables = internal_dict["orphan_variables"] if context in orphan_variables: orphans = orphan_variables[context] for svar in [svar for svar in mip_vars_list if svar.label in orphans]: test, reason = check_exclusion(svar, ("label", internal_dict["excluded_vars_lset"], ""), ("spatial_shp", [None, False], ""), ("spatial_shp", internal_dict["excluded_spshapes_lset"], "")) if not test: svars_per_table[svar.mipTable].append(svar) # # -------------------------------------------------------------------- # Remove svars belonging to other contexts' orphan lists # -------------------------------------------------------------------- other_contexts = sorted(list(set(orphan_variables) - set([context, ]))) orphans = list() for other_context in other_contexts: orphans.extend(orphan_variables[other_context]) orphans = sorted(list(set(orphans))) for table in svars_per_table: svars_per_table[table] = [svar for svar in svars_per_table[table] if svar.label not in orphans] return svars_per_table
def select_data_request_CMORvars_for_lab(sset=False, year=None): """ A function to list CMOR variables relevant for a lab (and also, optionally for an experiment and a year) The variables relative to the laboratory settings are get using the dict_interface module: list of MIPS, max Tier, list of excluded variables names Args: sset (boolean): should simulation settings be used the parameter taken here are: source_type, max priority (and all for filtering on the simulation) If sset is False, use union of mips among all grid choices year (int,optional) : simulation year - used when sset is not None, to additionally filter on year Returns: A list of 'simplified CMOR variables' """ logger = get_logger() internal_settings = get_settings_values("internal") # From MIPS set to Request links global global_rls, grid_choice, rls_for_all_experiments, sn_issues if sset: tierMax = internal_settings['tierMax'] else: tierMax = internal_settings['tierMax_lset'] sc = get_scope(tierMax) # Set sizes for lab settings, if available (or use CNRM-CM6-1 defaults) if sset: grid_choice = internal_settings["grid_choice"] mips_list = set(internal_settings['mips'][grid_choice]) sizes = internal_settings["sizes"] sc.mcfg = sc.build_mcfg(sizes) else: mip_list_by_grid = internal_settings["mips"] mips_list = set().union( *[set(mip_list_by_grid[grid]) for grid in mip_list_by_grid]) grid_choice = "LR" # Sort mip_list for reproducibility mips_list = sorted(list(mips_list)) # if rls_for_all_experiments is None: # Because scope do not accept list types rls_for_mips = sc.get_request_link_by_mip(mips_list) logger.info("Number of Request Links which apply to MIPS %s is: %d" % (print_struct(mips_list), len(rls_for_mips))) # excluded_links = internal_settings["excluded_request_links"] rls_for_mips = [ rl for rl in rls_for_mips if is_elt_applicable( rl, attribute="label", excluded=excluded_links) ] logger.info( "Number of Request Links after filtering by excluded_request_links is: %d" % len(rls_for_mips)) # inclinks = internal_settings["included_request_links"] if len(inclinks) > 0: excluded_rls = [ rl for rl in rls_for_mips if not is_elt_applicable(rl, attribute="label", included=inclinks) ] for rl in excluded_rls: logger.critical("RequestLink %s is not included" % rl.label) rls_for_mips.remove(rl) logger.info( "Number of Request Links after filtering by included_request_links is: %d" % len(rls_for_mips)) rls_for_all_experiments = [rl for rl in rls_for_mips] else: rls_for_mips = sorted(rls_for_all_experiments, key=lambda x: x.label) # if sset: experiment_id = internal_settings["experiment_for_requests"] exp = get_element_uid(get_experiment_label(experiment_id)) if exp is not None: starty = exp.starty endy = exp.endy else: starty = "??" endy = "??" logger.info( "Filtering for experiment %s, covering years [ %s , %s ] in DR" % (experiment_id, starty, endy)) # print "Request links before filter :"+`[ rl.label for rl in rls_for_mips ]` filtered_rls = [] for rl in rls_for_mips: # Access all requesItems ids which refer to this RequestLink rl_req_items = get_request_by_id_by_sect(rl.uid, 'requestItem') if any([ RequestItem_applies_for_exp_and_year( get_element_uid(ri_id), experiment_id, year)[0] for ri_id in rl_req_items ]): filtered_rls.append(rl) rls = filtered_rls logger.info( "Number of Request Links which apply to experiment %s member %s and MIPs %s is: %d" % (experiment_id, internal_settings['realization_index'], print_struct(mips_list), len(rls))) # print "Request links that apply :"+`[ rl.label for rl in filtered_rls ]` else: rls = rls_for_mips global_rls = rls # From Request links to CMOR vars + grid # miprl_ids=[ rl.uid for rl in rls ] # miprl_vars=sc.varsByRql(miprl_ids, pmax=lset['max_priority']) if sset: pmax = internal_settings['max_priority'] else: pmax = internal_settings['max_priority_lset'] miprl_vars_grids = set() for rl in rls: logger.debug("processing RequestLink %s" % rl.title) for v in sc.get_vars_by_request_link(request_link=rl.uid, pmax=pmax): # The requested grid is given by the RequestLink except if spatial shape matches S-* gr = rl.grid cmvar = get_element_uid(v) st = get_element_uid(cmvar.stid) sp = get_element_uid(st.spid) if sp.label.startswith("S-"): gr = 'cfsites' miprl_vars_grids.add((v, gr)) # if 'ua' in cmvar.label : print "adding %s %s"%(cmvar.label,get_element_uid(cmvar.vid).label) # else: # print "Duplicate pair var/grid : ",cmvar.label,cmvar.mipTable,gr miprl_vars_grids = sorted(list(miprl_vars_grids)) logger.info( 'Number of (CMOR variable, grid) pairs for these requestLinks is: %s' % len(miprl_vars_grids)) exctab = internal_settings["excluded_tables_lset"] if not isinstance(exctab, list): exctab = [ exctab, ] excvars = internal_settings['excluded_vars_lset'] if not isinstance(excvars, list): excvars = [ excvars, ] excpairs = internal_settings['excluded_pairs_lset'] if not isinstance(excpairs, list): excpairs = [ excpairs, ] if sset: inctab = internal_settings["included_tables"] if not isinstance(inctab, list): inctab = [ inctab, ] exctab.extend(internal_settings["excluded_tables_sset"]) incvars = internal_settings["included_vars"] if not isinstance(incvars, list): incvars = [ incvars, ] excvars_sset = internal_settings['excluded_vars_sset'] if not isinstance(excvars_sset, list): excvars_sset = [ excvars_sset, ] excvars.extend(excvars_sset) excvars_config = internal_settings['excluded_vars_per_config'] if not isinstance(excvars_config, list): excvars_config = [ excvars_config, ] excvars.extend(excvars_config) excpairs_sset = internal_settings['excluded_pairs_sset'] if not isinstance(excpairs_sset, list): excpairs_sset = [ excpairs_sset, ] excpairs.extend(excpairs_sset) else: inctab = internal_settings["included_tables_lset"] if not isinstance(inctab, list): inctab = [ inctab, ] incvars = internal_settings['included_vars_lset'] if not isinstance(incvars, list): incvars = [ incvars, ] filtered_vars = list() for (v, g) in miprl_vars_grids: cmvar = get_element_uid(v) ttable = get_element_uid(cmvar.mtid) mipvar = get_element_uid(cmvar.vid) if is_elt_applicable(mipvar, attribute="label", excluded=excvars, included=incvars) and \ is_elt_applicable(ttable, attribute="label", excluded=exctab, included=inctab) and \ is_elt_applicable((mipvar, ttable), attribute="label", excluded=excpairs): filtered_vars.append((v, g)) logger.debug("adding var %s, grid=%s, ttable=%s=" % (cmvar.label, g, ttable.label)) # ,exctab,excvars logger.info( 'Number once filtered by excluded/included vars and tables and spatial shapes is: %s' % len(filtered_vars)) # Filter the list of grids requested for each variable based on lab policy d = defaultdict(set) for (v, g) in filtered_vars: d[v].add(g) logger.info('Number of distinct CMOR variables (whatever the grid): %d' % len(d)) multiple_grids = list() for v in d: d[v] = decide_for_grids(v, d[v]) if len(d[v]) > 1: multiple_grids.append(get_element_uid(v).label) if print_multiple_grids: logger.info( "\tVariable %s will be processed with multiple grids: %s" % (multiple_grids[-1], repr(d[v]))) if not print_multiple_grids and multiple_grids is not None and len( multiple_grids) > 0: logger.info( "\tThese variables will be processed with multiple grids " "(rerun with print_multiple_grids set to True for details): %s" % repr(sorted(multiple_grids))) # # Print a count of distinct var labels logger.info('Number of distinct var labels is: %d' % len(set([get_element_uid(v).label for v in d]))) # Translate CMORvars to a list of simplified CMORvar objects simplified_vars = [] allow_pseudo = internal_settings['allow_pseudo_standard_names'] for v in d: svar = SimpleCMORVar() cmvar = get_element_uid(v) sn_issues = complement_svar_using_cmorvar(svar, cmvar, sn_issues, [], allow_pseudo) svar.Priority = analyze_priority(cmvar, mips_list) svar.grids = d[v] if get_element_uid(v).label in [ "tas", ]: logger.debug("When complementing, tas is included , grids are %s" % svar.grids) simplified_vars.append(svar) logger.info('Number of simplified vars is: %d' % len(simplified_vars)) logger.info("Issues with standard names are: %s" % print_struct(sorted(list(sn_issues)))) return simplified_vars
def RequestItem_applies_for_exp_and_year(ri, experiment, year=None): """ Returns True if requestItem 'ri' in data request is relevant for a given 'experiment' and 'year'. Toggle 'debug' allow some printouts """ # Returns a couple : relevant, endyear. # RELEVANT is True if requestItem RI applies to EXPERIMENT and # has a timeslice wich includes YEAR, either implicitly or explicitly # ENDYEAR is meaningful if RELEVANT is True, and is the # last year in the timeslice (or None if timeslice == # the whole experiment duration) # Acces experiment or experiment group for the RequestItem # if (ri.label=='C4mipC4mipLandt2') : debug=True # if ri.title=='AerChemMIP, AERmon-3d, piControl' : debug=True # if ri.title=='CFMIP, CFMIP.CFsubhr, amip' : debug=True internal_dict = get_settings_values("internal") logger = get_logger() logger.debug("In RIapplies.. Checking % 15s" % ri.title) item_exp = get_element_uid(ri.esid) ri_applies_to_experiment = False endyear = None # esid can link to an experiment or an experiment group if item_exp._h.label in [ 'experiment', ]: logger.debug("%20s %s" % ("Simple Expt case", item_exp.label)) if item_exp.label in [ experiment, ]: logger.debug(" OK", ) ri_applies_to_experiment = True elif item_exp._h.label in [ 'exptgroup', ]: logger.debug("%20s %s" % ("Expt Group case ", item_exp.label)) exps_id = get_request_by_id_by_sect(ri.esid, 'experiment') for e in [get_element_uid(eid) for eid in exps_id]: if e.label in [ experiment, ]: logger.debug(" OK for experiment based on group %s" % item_exp.label) ri_applies_to_experiment = True elif item_exp._h.label in [ 'mip', ]: logger.debug("%20s %s" % ("Mip case ", get_element_uid(item_exp.label).label)) exps_id = get_request_by_id_by_sect(ri.esid, 'experiment') for e in [get_element_uid(eid) for eid in exps_id]: logger.debug(e.label + ",", ) if e.label == experiment: logger.debug(" OK for experiment based on mip %s" % item_exp.label) ri_applies_to_experiment = True else: logger.debug("Error on esid link for ri: %s uid=%s %s" % (ri.title, ri.uid, item_exp._h.label)) # print "ri=%s"%ri.title, # if year is not None : # print "Filtering for year %d"%year filter_on_realization = internal_dict["filter_on_realization"] if filter_on_realization: if ri.nenmax != -1 and (internal_dict["realization_index"] > ri.nenmax): ri_applies_to_experiment = False if ri_applies_to_experiment: logger.debug("Year considered: %s %s" % (year, type(year))) if year is None: rep = True endyear = None logger.debug(" ..applies because arg year is None") else: year = int(year) exp = get_element_uid(get_experiment_label(experiment)) rep, endyear = year_in_ri(ri, exp, year) logger.debug(" ..year in ri returns: %s %s" % (rep, endyear)) # if (ri.label=="AerchemmipAermonthly3d") : # print "reqItem=%s,experiment=%s,year=%d,rep=%s,"%(ri.label,experiment,year,rep) # print " rep=",rep return rep, endyear else: # print return False, None
def get_correspond_cmor_var(homevar): """ For a home variable, find the CMOR var which corresponds. """ logger = get_logger() count = 0 empty_table = (homevar.mipTable in [ 'NONE', ]) or (homevar.mipTable.startswith("None")) for cmvarid in get_cmor_var_id_by_label(homevar.ref_var): cmvar = get_element_uid(cmvarid) logger.debug("get_corresp, checking %s vs %s in %s" % (homevar.label, cmvar.label, cmvar.mipTable)) # # Consider case where no modeling_realm associated to the # current CMORvar as matching anyway. # mpmoine_TBD: A mieux gerer avec les orphan_variables ? match_label = (cmvar.label == homevar.ref_var) match_freq = (cmvar.frequency == homevar.frequency) or \ ("SoilPools" in homevar.label and homevar.frequency in ["mon", ] and cmvar.frequency in ["monPt", ]) match_table = (cmvar.mipTable == homevar.mipTable) match_realm = (homevar.modeling_realm in cmvar.modeling_realm.split(' ')) or \ (homevar.modeling_realm == cmvar.modeling_realm) empty_realm = (cmvar.modeling_realm in [ '', ]) matching = (match_label and (match_freq or empty_table) and (match_table or empty_table) and (match_realm or empty_realm)) if matching: logger.debug("matches") same_shapes = (get_spatial_and_temporal_shapes(cmvar) == [ homevar.spatial_shp, homevar.temporal_shp ]) if same_shapes: count += 1 cmvar_found = cmvar logger.debug("and same shapes !") else: if not empty_table: logger.error( "(%s %s) HOMEVar: Spatial and Temporal Shapes specified DO NOT match CMORvar ones. " "-> Provided: (%s, %s) Expected: (%s, %s)" % (homevar.label, homevar.mipTable, homevar.spatial_shp, homevar.temporal_shp, *get_spatial_and_temporal_shapes(cmvar))) else: logger.debug( "doesn't match %s %s %s %s %s %s %s %s" % (match_label, match_freq, cmvar.frequency, homevar.frequency, match_table, match_realm, empty_realm, homevar.mipTable)) if count >= 1: # empty table means that the frequency is changed (but the variable exists in another frequency cmor table if empty_table: var_freq_asked = homevar.frequency allow_pseudo = get_settings_values("internal", 'allow_pseudo_standard_names') global sn_issues_home sn_issues_home = complement_svar_using_cmorvar(homevar, cmvar_found, sn_issues_home, [], allow_pseudo) if empty_table: homevar.frequency = var_freq_asked homevar.mipTable = "None" + homevar.frequency return homevar else: return False
def year_in_ri_tslice(ri, exp, year): """ Returns a couple : relevant, endyear. RELEVANT is True if requestItem RI applies to YEAR, either implicitly or explicitly (e.g. timeslice) ENDYEAR, which is meaningful if RELEVANT is True, and is the last year in the timeslice (or None if timeslice == the whole experiment duration) """ logger = get_logger() internal_dict = get_settings_values("internal") if 'tslice' not in ri.__dict__: logger.debug("No tslice for reqItem %s -> OK for any year" % ri.title) return True, None if ri.tslice in [ '__unset__', ]: logger.debug("tslice is unset for reqItem %s " % ri.title) return True, None # relevant = False endyear = None tslice = get_element_uid(ri.tslice) logger.debug("tslice label/type is %s/%s for reqItem %s " % (tslice.label, tslice.type, ri.title)) if tslice.type in [ "relativeRange", ]: # e.g. _slice_abrupt30 first_year = experiment_start_year(exp) # first_year = sset["branch_year_in_child"] relevant = (year >= tslice.start + first_year - 1 and year <= tslice.end + first_year - 1) endyear = first_year + tslice.end - 1 elif tslice.type in [ "simpleRange", ]: # e.g. _slice_DAMIP20 relevant = (year >= tslice.start and year <= tslice.end) endyear = tslice.end elif tslice.type in [ "sliceList", ]: # e.g. _slice_DAMIP40 for start in range(tslice.start, int(tslice.end - tslice.sliceLen + 2), int(tslice.step)): if year >= start and year < start + tslice.sliceLen: relevant = True endyear = start + tslice.sliceLen - 1 elif tslice.type in [ "dayList", ]: # e.g. _slice_RFMIP2 # e.g. startList[i]: [1980, 1, 1, 1980, 4, 1, 1980, 7, 1, 1980, 10, 1, 1992, 1, 1, 1992, 4, 1] years = [ tslice.startList[3 * i] for i in range(len(tslice.startList) / 3) ] if year in years: relevant = True endyear = year elif tslice.type in [ "startRange", ]: # e.g. _slice_VolMIP3 # used only for VolMIP : _slice_VolMIP3 start_year = experiment_start_year(exp) relevant = (year >= start_year and year < start_year + nyear) endyear = start_year + nyear - 1 elif tslice.type in [ "monthlyClimatology", ]: # e.g. _slice_clim20 relevant = (year >= tslice.start and year <= tslice.end) endyear = tslice.end elif tslice.type in [ "branchedYears", ]: # e.g. _slice_piControl020 branching = internal_dict["branching"] if tslice.child in branching: endyear = False (refyear, starts) = branching[tslice.child] for start in starts: if ((year - start >= tslice.start - refyear) and (year - start < tslice.start - refyear + tslice.nyears)): relevant = True lastyear = start + tslice.start - refyear + tslice.nyears - 1 if endyear is False: endyear = lastyear else: endyear = max(endyear, lastyear) logger.debug( "slice OK: year=%d, start=%d tslice.start=%d refyear=%d tslice.nyears=%d lastyear=%d" % (year, start, tslice.start, refyear, tslice.nyears, lastyear)) else: raise Dr2xmlError( "For tslice %s, child %s start year is not documented" % (tslice.title, tslice.child)) else: raise Dr2xmlError("type %s for time slice %s is not handled" % (tslice.type, tslice.title)) logger.debug( "for year %d and experiment %s, relevant is %s for tslice %s of type %s, endyear=%s" % (year, exp.label, repr(relevant), ri.title, tslice.type, repr(endyear))) return relevant, endyear
#!/usr/bin/env python # -*- coding: utf-8 -*- """ Interface between data request and dr2xml """ from __future__ import print_function, division, absolute_import, unicode_literals from dr2xml.settings_interface import get_settings_values data_request_version = get_settings_values("internal", "data_request_used") if data_request_version in [ "CMIP6", ]: from .CMIP6 import * elif data_request_version in ["no", "none", "None", None]: from .no import * else: raise ValueError("The data request specified (%s) is not known." % data_request_version)
def endyear_for_CMORvar(cv, expt, year): """ For a CMORvar, returns the largest year in the time slice(s) of those requestItems which apply for experiment EXPT and which include YEAR. If no time slice applies, returns None """ # 1- Get the RequestItems which apply to CmorVar # 2- Select those requestItems which include expt, # and retain their endyear if larger than former one internal_dict = get_settings_values("internal") global global_rls # Some debug material logger = get_logger() logger.debug("In end_year for %s %s" % (cv.label, cv.mipTable)) pmax = internal_dict['max_priority'] # 1- Get the RequestItems which apply to CmorVar rVarsUid = get_request_by_id_by_sect(cv.uid, 'requestVar') rVars = [ get_element_uid(uid) for uid in rVarsUid if get_element_uid(uid).priority <= pmax ] logger.debug("les requestVars: %s" % " ".join([rVar.title for rVar in rVars])) VarGroups = [get_element_uid(rv.vgid) for rv in rVars] logger.debug("les requestVars groups: %s" % " ".join([rVg.label for rVg in VarGroups])) RequestLinksId = list() for vg in VarGroups: RequestLinksId.extend(get_request_by_id_by_sect(vg.uid, 'requestLink')) FilteredRequestLinks = list() for rlid in RequestLinksId: rl = get_element_uid(rlid) if rl in global_rls: FilteredRequestLinks.append(rl) logger.debug( "les requestlinks: %s" % " ".join([get_element_uid(rlid).label for rlid in RequestLinksId])) logger.debug("les FilteredRequestlinks: %s" % " ".join([rl.label for rl in FilteredRequestLinks])) RequestItems = list() for rl in FilteredRequestLinks: RequestItems.extend(get_request_by_id_by_sect(rl.uid, 'requestItem')) logger.debug( "les requestItems: %s" % " ".join([get_element_uid(riid).label for riid in RequestItems])) # 2- Select those request links which include expt and year larger = None for riid in RequestItems: ri = get_element_uid(riid) applies, endyear = RequestItem_applies_for_exp_and_year(ri, expt, year) logger.debug( "For var and freq selected for debug and year %s, for ri %s, applies=%s, endyear=%s" % (str(year), ri.title, str(applies), str(endyear))) if applies: if endyear is None: return None # One of the timeslices cover the whole expt if larger is None: larger = endyear else: larger = max(larger, endyear) return larger