def test_create_task():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
ece2cmorlib.add_task(tsk)
assert tsk in ece2cmorlib.tasks
ece2cmorlib.finalize()
def test_create_task(self):
ece2cmorlib_tests.init()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
ece2cmorlib.add_task(tsk)
ok_(tsk in ece2cmorlib.tasks)
ece2cmorlib.finalize()
def test_duplicate_task():
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("49.128")
tsk1 = cmor_task.cmor_task(src1, tgt)
ece2cmorlib.add_task(tsk1)
src2 = cmor_source.ifs_source.read("79.128")
tsk2 = cmor_task.cmor_task(src2, tgt)
ece2cmorlib.add_task(tsk2)
assert len(ece2cmorlib.tasks) == 1
assert tsk2 in ece2cmorlib.tasks
ece2cmorlib.finalize()
def test_duplicate_task(self):
ece2cmorlib_tests.init()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("49.128")
tsk1 = cmor_task.cmor_task(src1, tgt)
ece2cmorlib.add_task(tsk1)
src2 = cmor_source.ifs_source.read("79.128")
tsk2 = cmor_task.cmor_task(src2, tgt)
ece2cmorlib.add_task(tsk2)
eq_(len(ece2cmorlib.tasks), 1)
ok_(tsk2 in ece2cmorlib.tasks)
ece2cmorlib.finalize()
def create_tasks(matches, active_components):
global log, ignored_vars_file, json_table_key, skip_tables
result = []
model_vars = load_model_vars()
for model, targets in matches.items():
if isinstance(active_components, list) and model not in active_components:
continue
if isinstance(active_components, basestring) and model != active_components:
continue
parblocks = model_vars[model]
for target in targets:
parmatches = [b for b in parblocks if matchvarpar(target, b)]
if not any(parmatches):
log.error("Variable %s in table %s is not supported by %s in ece2cmor3; if you do expect an ec-earth "
"output variable here, please create an issue or pull request on our github page"
% (target.variable, target.table, model))
continue
parmatch = parmatches[0]
if len(parmatches) > 1:
log.warning("Multiple matching parameters for %s found for variable %s in table %s: proceeding with "
"first match %s" % (model, target.variable, target.table, parmatch.get("source", None)))
if parmatch.get("table_override", {}).get("table", "") == target.table:
parmatch = parmatch["table_override"]
task = create_cmor_task(parmatch, target, model)
if ece2cmorlib.add_task(task):
result.append(task)
log.info("Created %d ece2cmor tasks from input variable list." % len(result))
return result
def create_tasks(targets, load_atm_tasks=True, load_oce_tasks=True, load_tm5_tasks=True, silent=False):
global log, ignored_vars_file, json_table_key, models, skip_tables
modelflags = {"ifs": load_atm_tasks, "nemo": load_oce_tasks, "tm5": load_tm5_tasks}
realmflags = {}
for m in models:
flag = modelflags.get(m, True)
for r in models[m]["realms"]:
curval = realmflags.get(r, False)
realmflags[r] = flag or curval # True if any model can produce the realm
params = {}
for model in models:
paramfile = models.get(model, {}).get(parfile_key, "")
if os.path.isfile(paramfile):
with open(paramfile) as f:
params[model] = json.loads(f.read())
else:
params[model] = []
omitvarlist_01 = load_checkvars_excel(omit_vars_file_01)
omitvarlist_02 = load_checkvars_excel(omit_vars_file_02)
omitvarlist_03 = load_checkvars_excel(omit_vars_file_03)
omitvarlist_04 = load_checkvars_excel(omit_vars_file_04)
omitvarlist_05 = load_checkvars_excel(omit_vars_file_05)
ignoredvarlist = load_checkvars_excel(ignored_vars_file)
identifiedmissingvarlist = load_checkvars_excel(identified_missing_vars_file)
loadedtargets, ignoredtargets, identifiedmissingtargets, missingtargets = [], [], [], []
for target in targets:
realms = getattr(target, cmor_target.realm_key, None).split()
if not any([realmflags.get(r, True) for r in realms]):
continue # If all variable's realms are flagged false, skip
matchpars = {}
for model in models:
if modelflags.get(model, True): # Only consider models that are 'enabled'
matches = [p for p in params.get(model, []) if
matchvarpar(target.variable, p) and target.table == p.get(json_table_key, target.table)]
if any(matches):
matchpars[model] = matches
if not any(matchpars):
key = target.variable if skip_tables else (target.table, target.variable)
if key in ignoredvarlist:
target.ecearth_comment, target.comment_author = ignoredvarlist[key]
ignoredtargets.append(target)
varword = "ignored"
elif key in identifiedmissingvarlist:
target.ecearth_comment, target.comment_author = identifiedmissingvarlist[key]
identifiedmissingtargets.append(target)
varword = "identified missing"
elif key in omitvarlist_01:
varword = "omit 01"
elif key in omitvarlist_02:
varword = "omit 02"
elif key in omitvarlist_03:
varword = "omit 03"
elif key in omitvarlist_04:
varword = "omit 04"
elif key in omitvarlist_05:
varword = "omit 05"
else:
missingtargets.append(target)
varword = "missing"
if not silent:
log.error(
"Could not find parameter table entry for %s in table %s...skipping variable. "
"This variable is %s" % (target.variable, target.table, varword))
continue
modelmatch = None
for model in matchpars:
modelmatch = model
if len(matchpars) == 1:
break
modelrealms = set(models.get(model, {}).get("realms", []))
shared_realms = set(realms).intersection(modelrealms)
if any(shared_realms):
log.info("Multiple models %s found for variable %s, model %s matched by shared realms %s" % (
matchpars.keys(), target.variable, model, shared_realms))
break
pars = matchpars[modelmatch]
table_pars = [p for p in pars if json_table_key in p]
notable_pars = [p for p in pars if json_table_key not in p]
if len(table_pars) > 1 or len(notable_pars) > 1:
log.warning("Multiple entries found for variable %s, table %s in file %s...choosing first." % (
target.variable, target.table, models[modelmatch]["parfile"]))
parmatch = table_pars[0] if any(table_pars) else pars[0]
task = create_cmor_task(parmatch, target, modelmatch)
ece2cmorlib.add_task(task)
if parmatch.get(cmor_source.expression_key, None) is None:
target.ecearth_comment = task.source.model_component() + ' code name = ' + parmatch.get(json_source_key,
None)
else:
target.ecearth_comment = task.source.model_component() + ' code name = ' \
+ parmatch.get(json_source_key, None) + ', expression = ' \
+ parmatch.get(cmor_source.expression_key, None)
target.comment_author = 'automatic'
loadedtargets.append(target)
log.info("Created %d ece2cmor tasks from input variable list." % len(loadedtargets))
for par in params["ifs"]:
if json_mask_key in par:
name = par[json_mask_key]
expr = par.get(cmor_source.expression_key, None)
if not expr:
log.error("No expression given for mask %s, ignoring mask definition" % name)
else:
srcstr, func, val = parse_maskexpr(expr)
if srcstr:
src = create_cmor_source({json_source_key: srcstr}, "ifs")
ece2cmorlib.add_mask(name, src, func, val)
return loadedtargets, ignoredtargets, identifiedmissingtargets, missingtargets