def test_load_cfc12_Omon_prefs():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"Omon": ["cfc12"]},
config="EC-EARTH-AOGCM")
assert not any(matches["nemo"])
matches, omitted = taskloader.load_drq({"Omon": ["cfc12"]},
config="EC-EARTH-CC")
assert len(matches["nemo"]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_use_zg_preferences():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["zg7h", "zg27"]}, check_prefs=False)
assert len(matches["ifs"]) == 2
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["zg7h", "zg27"]}, check_prefs=True)
assert len(matches["ifs"]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_use_level_preferences():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["ua", "ua7h"]}, check_prefs=False)
eq_(len(matches["ifs"]), 2)
matches, omitted = taskloader.load_drq(
{"6hrPlevPt": ["ua", "ua7h"]}, check_prefs=True)
eq_(len(matches["ifs"]), 1)
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_ps_AERmon_prefs():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"AERmon": ["ps"]},
check_prefs=False)
assert len(matches["ifs"]) == 1
assert len(matches["tm5"]) == 1
matches, omitted = taskloader.load_drq({"AERmon": ["ps"]},
check_prefs=True)
assert len(matches["ifs"]) == 0
assert len(matches["tm5"]) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_load_oclim_variable():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"Oclim": ["difvho"]},
check_prefs=False)
assert len(matches["nemo"]) == 1
assert not any(omitted)
finally:
ece2cmorlib.finalize_without_cmor()
def test_ignored_variables():
ece2cmorlib.initialize_without_cmor()
try:
matches, omissions = taskloader.load_drq({
"Emon": ["rsdsdiff"],
"Amon": ["clivi"]
})
ignored, identified_missing, missing, dismissed = taskloader.split_targets(
omissions)
assert len(matches["nemo"]) == 0
assert len(matches["ifs"]) == 1
assert len(ignored) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def test_dismiss_duplicates():
ece2cmorlib.initialize_without_cmor()
try:
matches, omitted = taskloader.load_drq({"AERmon": ["ps"]},
check_prefs=False)
assert len(matches["ifs"]) == 1
assert len(matches["tm5"]) == 1
tasks = taskloader.load_tasks(matches,
active_components=["ifs"],
target_filters=None,
check_duplicates=True)
assert not any(tasks)
tasks = taskloader.load_tasks(matches,
active_components=["ifs"],
target_filters=None,
check_duplicates=False)
assert len(tasks) == 1
finally:
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(
description="Validate input variable list against CMIP tables")
parser.add_argument(
"--drq",
metavar="FILE",
type=str,
required=True,
help=
"File (json|f90 namelist|xlsx) containing cmor variables (Required)")
cmor_utils.ScriptUtils.add_model_exclusive_options(parser, "checkvars")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
parser.add_argument("--output",
metavar="FILE",
type=str,
default=None,
help="Output path to write variables to")
parser.add_argument(
"--withouttablescheck",
action="store_true",
default=False,
help="Ignore variable tables when performing var checking")
parser.add_argument("--withping",
action="store_true",
default=False,
help="Read and write addition ping file fields")
parser.add_argument(
"-v",
"--verbose",
action="store_true",
default=False,
help=
"Write xlsx and ASCII files with verbose output (suppress the related terminal messages "
"as the content of these files contain this information)")
parser.add_argument("--asciionly",
action="store_true",
default=False,
help="Write the ascii file(s) only")
cmor_utils.ScriptUtils.add_model_tabfile_options(parser)
args = parser.parse_args()
print()
print('Running checkvars with:')
print(' checkvars ' + cmor_utils.ScriptUtils.get_drq_vars_options(args))
print()
if not os.path.isfile(args.drq):
log.fatal("Error: Your data request file %s cannot be found." %
args.drq)
sys.exit(' Exiting checkvars.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,
mode=ece2cmorlib.PRESERVE,
tabledir=args.tabdir,
tableprefix=args.tabid)
active_components = cmor_utils.ScriptUtils.get_active_components(args)
# Configure task loader:
taskloader.skip_tables = args.withouttablescheck
taskloader.with_pingfile = args.withping
# Load the variables as task targets:
try:
matches, omitted_targets = taskloader.load_drq(args.drq,
check_prefs=False)
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting checkvars.')
loaded = [t for m in active_components for t in matches[m]]
ignored, identified_missing, missing, dismissed = taskloader.split_targets(
omitted_targets)
loaded_targets = sorted(list(set(loaded)),
key=lambda tgt: (tgt.table, tgt.variable))
ignored_targets = sorted(list(set(ignored)),
key=lambda tgt: (tgt.table, tgt.variable))
identified_missing_targets = sorted(identified_missing,
key=lambda tgt:
(tgt.table, tgt.variable))
missing_targets = sorted(missing,
key=lambda tgt: (tgt.table, tgt.variable))
if args.output:
output_dir = os.path.dirname(args.output)
if not os.path.isdir(output_dir):
if output_dir != '':
os.makedirs(output_dir)
if not args.asciionly:
write_varlist(loaded, args.output + ".available.json")
if args.verbose:
if not args.asciionly:
write_varlist_excel(loaded_targets,
args.output + ".available.xlsx",
args.withping)
write_varlist_excel(ignored_targets,
args.output + ".ignored.xlsx",
args.withping)
write_varlist_excel(identified_missing_targets,
args.output + ".identifiedmissing.xlsx",
args.withping)
write_varlist_excel(missing_targets,
args.output + ".missing.xlsx",
args.withping)
if args.drq[-4:] == 'xlsx':
write_varlist_ascii(loaded_targets,
args.output + ".available.txt", True)
write_varlist_ascii(ignored_targets,
args.output + ".ignored.txt", True)
write_varlist_ascii(identified_missing_targets,
args.output + ".identifiedmissing.txt",
True)
write_varlist_ascii(missing_targets,
args.output + ".missing.txt", True)
else:
write_varlist_ascii(loaded_targets,
args.output + ".available.txt", False)
if False:
# Add writting of a json data request formatted file which includes all available variables in order to provide a
# single test which covers all identified & available variables. If this block is activated and the following is run:
# ./determine-missing-variables.sh CMIP,AerChemMIP,CDRMIP,C4MIP,DCPP,HighResMIP,ISMIP6,LS3MIP,LUMIP,OMIP,PAMIP,PMIP,RFMIP,ScenarioMIP,VolMIP,CORDEX,DynVarMIP,SIMIP,VIACSAB CMIP 1 1
# At least an equivalent json data request which covers all Core MIP requests is produced. However this does not
# necessarily include all specific MIP requests. In fact it would be better to create a json data request equivalent
# based on the ifspar.json.
result = {}
for model, targetlist in matches.items():
result[model] = {}
for target in targetlist:
table = target.table
if table in result[model]:
result[model][table].append(target.variable)
else:
result[model][table] = [target.variable]
#with open(args.output + "-varlist-all-available.json", 'w') as ofile:
with open("varlist-all.json", 'w') as ofile:
json.dump(result,
ofile,
indent=4,
separators=(',', ': '),
sort_keys=True)
ofile.write(
'\n'
) # Add newline at the end of the json file because the python json package doesn't do this.
ofile.close()
# Finishing up
ece2cmorlib.finalize_without_cmor()
def main():
parser = argparse.ArgumentParser(
description=
"Create component-specified varlist json for given data request")
required = parser.add_argument_group("required arguments")
varsarg = required.add_mutually_exclusive_group(required=True)
varsarg.add_argument(
"--drq",
metavar="FILE",
type=str,
help="File (xlsx|json) containing requested cmor variables (Required)")
varsarg.add_argument("--allvars",
action="store_true",
default=False,
help="Read all possible variables from CMOR tables")
parser.add_argument("--varlist",
"-o",
metavar="FILE.json",
type=str,
default="ece-cmip6-data-request-varlist.json",
help="Output file name")
parser.add_argument(
"--ececonf",
metavar='|'.join(components.ece_configs.keys()),
type=str,
help="EC-Earth configuration (only used with --drq option)")
parser.add_argument("--tabdir",
metavar="DIR",
type=str,
default=ece2cmorlib.table_dir_default,
help="Cmorization table directory")
parser.add_argument("--tabid",
metavar="PREFIX",
type=str,
default=ece2cmorlib.prefix_default,
help="Cmorization table prefix string")
args = parser.parse_args()
print ""
print "Running drq2varlist.py with:"
print "./drq2varlist.py " + cmor_utils.ScriptUtils.get_drq_vars_options(
args)
print ""
if not args.allvars and not os.path.isfile(args.drq):
log.fatal("Your data request file %s cannot be found." % args.drq)
sys.exit(' Exiting drq2varlist.')
# Initialize ece2cmor:
ece2cmorlib.initialize_without_cmor(tabledir=args.tabdir,
tableprefix=args.tabid)
try:
if getattr(args, "allvars", False):
matches, omitted = taskloader.load_drq("allvars",
config=args.ececonf,
check_prefs=True)
else:
matches, omitted = taskloader.load_drq(args.drq,
config=args.ececonf,
check_prefs=True)
# Here we load extra permanent tasks for LPJ-GUESS because the LPJ_GUESS community likes to output these variables at any time independent wheter they are requested by the data request:
if args.ececonf in [
"EC-EARTH-CC", "EC-EARTH-Veg", "EC-EARTH-Veg-LR"
]:
matches_permanent, omitted_permanent = taskloader.load_drq(
os.path.join(os.path.dirname(__file__), "..", "resources",
"permanent-tasks.json"),
config=args.ececonf,
check_prefs=True)
for model, targetlist in matches_permanent.items():
if model in matches:
for target in targetlist:
if target not in matches[model]:
matches[model].append(target)
else:
matches[model] = targetlist
except taskloader.SwapDrqAndVarListException as e:
log.error(e.message)
opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars"
log.error(
"It seems you are using the --%s option where you should use the --%s option for this file"
% (opt1, opt2))
sys.exit(' Exiting drq2varlist.')
result = {}
for model, targetlist in matches.items():
result[model] = {}
for target in targetlist:
table = target.table
# Taking off several variables from the json data request files:
skip_case = False
if target.variable in ['intdoc']:
# See issue #521:
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/521"
% (target.table, target.variable))
skip_case = True
if target.variable in [
'rlntds', 'hfibthermds', 'hflso', 'agessc', 'ficeberg',
'hfsso', 'hfcorr', 'wfcorr', 'nwdFracLut'
]:
# See issue #498 & #469:
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/498 & https://github.com/EC-Earth/ece2cmor3/issues/469"
% (target.table, target.variable))
skip_case = True
if target.variable in ['hfibthermds2d', 'ficeberg2d', 'fgcfc12']:
# See issue #516 and #609-36 & #609-37 at ece-portal:
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/516 & https://dev.ec-earth.org/issues/609#note-36"
% (target.table, target.variable))
skip_case = True
if target.variable in ['cfc11', 'fgsf6']:
# See issue #504:
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/504"
% (target.table, target.variable))
skip_case = True
if table in ['Oyr'] and target.variable in [
'cfc11', 'ocontempdiff', 'ocontemppadvect',
'ocontemppmdiff', 'ocontemprmadvect', 'ocontemptend',
'osaltdiff', 'osaltpadvect', 'osaltpmdiff',
'osaltrmadvect', 'osalttend'
]:
# See issue #493 & #542:
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/493 & https://github.com/EC-Earth/ece2cmor3/issues/542"
% (target.table, target.variable))
skip_case = True
if getattr(args, "allvars", True):
if table in ['6hrPlevPt'
] and target.variable in ['ta27', 'hus27']:
# See issue #542:
# Conflicting combinations (skip the 2nd one, an arbitrary choice):
# 6hrPlevPt: ta7h, ta27
# 6hrPlevPt: hus7h, hus27
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/542"
% (target.table, target.variable))
skip_case = True
if table in ['Emon'] and target.variable in [
'hus27', 'va27', 'ua27'
]:
# See issue #542:
# Emon: hus, hus27
# Emon: va, va27
# Emon: ua, ua27
log.info(
" Variable %s %s is listed in the omit list of drq2varlist and therefore skipped. See https://github.com/EC-Earth/ece2cmor3/issues/542"
% (target.table, target.variable))
skip_case = True
if skip_case is False:
if table in result[model]:
result[model][table].append(target.variable)
else:
result[model][table] = [target.variable]
with open(args.varlist, 'w') as ofile:
json.dump(result,
ofile,
indent=4,
separators=(',', ': '),
sort_keys=True)
ofile.write(
'\n'
) # Add newline at the end of the json file because the python json package doesn't do this.
ofile.close()