def error(self, message):
self.print_usage(sys.stderr)
# missing argument
# TODO: assumes comp_interface='mct'
if "expected one argument" in message:
if "compset" in message:
components = get_compsets(Files(comp_interface="mct"))
self.exit(
2,
"{}: error: {}\nValid input arguments are {}\n".format(
self.prog, message, components),
)
elif "component" in message:
files = Files(comp_interface="mct")
components = get_components(files)
# Loop through the elements for each component class (in config_files.xml)
valid_components = []
for comp in components:
string = "CONFIG_{}_FILE".format(comp)
# determine all components in string
components = files.get_components(string)
for item in components:
valid_components.append(item)
self.exit(
2,
"{}: error: {}\nValid input arguments are {}\n".format(
self.prog, message, valid_components),
)
# for all other errors
self.exit(2, "{}: error: {}\n".format(self.prog, message))
def list_compsets(self):
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
for comp in components:
infile = files.get_value("COMPSETS_SPEC_FILE", {"component": comp})
compsetobj = Compsets(infile=infile, files=files)
_, compsets = compsetobj.return_all_values()
self.CompsetList.addItems(sorted(compsets.keys()))
self.CompsetList.insertSeparator(999)
def get_tests_from_xml(xml_machine=None,
xml_category=None,
xml_compiler=None,
xml_testlist=None,
machine=None,
compiler=None):
"""
Parse testlists for a list of tests
"""
listoftests = []
testlistfiles = []
if (machine is not None):
thismach = machine
if (compiler is not None):
thiscompiler = compiler
if (xml_testlist is not None):
expect(os.path.isfile(xml_testlist),
"Testlist not found or not readable " + xml_testlist)
testlistfiles.append(xml_testlist)
else:
files = Files()
comps = files.get_components("TESTS_SPEC_FILE")
for comp in comps:
test_spec_file = files.get_value("TESTS_SPEC_FILE",
{"component": comp})
if (os.path.isfile(test_spec_file)):
testlistfiles.append(test_spec_file)
for testlistfile in testlistfiles:
thistestlistfile = Testlist(testlistfile)
logger.debug("Testlist file is " + testlistfile)
logger.debug("xml_machine %s xml_category %s xml_compiler %s" %
(xml_machine, xml_category, xml_compiler))
newtests = thistestlistfile.get_tests(xml_machine, xml_category,
xml_compiler)
for test in newtests:
if (machine is None):
thismach = test["machine"]
if (compiler is None):
thiscompiler = test["compiler"]
test["name"] = CIME.utils.get_full_test_name(
test["testname"],
grid=test["grid"],
compset=test["compset"],
machine=thismach,
compiler=thiscompiler,
testmod=None if "testmods" not in test else test["testmods"])
logger.debug("Adding test %s with compiler %s" %
(test["name"], test["compiler"]))
listoftests += newtests
logger.debug("Found %d tests" % len(listoftests))
return listoftests
def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None):
"""
Loop through all the compset files and find the compset
specifation file that matches either the input 'compset_name'.
Note that the input compset name (i.e. compset_name) can be
either a longname or an alias. This will also set the
compsets and pes specfication files.
"""
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components)
# Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
# that has a match for either the alias or the longname in that order
for component in components:
# Determine the compsets file for this component
compsets_filename = files.get_value("COMPSETS_SPEC_FILE", {"component":component})
# If the file exists, read it and see if there is a match for the compset alias or longname
if (os.path.isfile(compsets_filename)):
compsets = Compsets(compsets_filename)
match = compsets.get_compset_match(name=compset_name)
pesfile = files.get_value("PES_SPEC_FILE" , {"component":component})
if match is not None:
self._pesfile = pesfile
self._compsetsfile = compsets_filename
self._compsetname = match
tests_filename = files.get_value("TESTS_SPEC_FILE" , {"component":component}, resolved=False)
tests_mods_dir = files.get_value("TESTS_MODS_DIR" , {"component":component}, resolved=False)
user_mods_dir = files.get_value("USER_MODS_DIR" , {"component":component}, resolved=False)
self.set_lookup_value("COMPSETS_SPEC_FILE" ,
files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False))
self.set_lookup_value("TESTS_SPEC_FILE" , tests_filename)
self.set_lookup_value("TESTS_MODS_DIR" , tests_mods_dir)
self.set_lookup_value("USER_MODS_DIR" , user_mods_dir)
self.set_lookup_value("PES_SPEC_FILE" ,
files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False))
logger.info("Compset longname is %s " %(match))
logger.info("Compset specification file is %s" %(compsets_filename))
logger.info("Pes specification file is %s" %(pesfile))
return
if user_compset is True:
#Do not error out for user_compset
logger.warn("Could not find a compset match for either alias or longname in %s" %(compset_name))
self._compsetname = compset_name
self._pesfile = pesfile
self.set_lookup_value("PES_SPEC_FILE", pesfile)
else:
expect(False,
"Could not find a compset match for either alias or longname in %s" %(compset_name))
def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None):
"""
Loop through all the compset files and find the compset
specifation file that matches either the input 'compset_name'.
Note that the input compset name (i.e. compset_name) can be
either a longname or an alias. This will also set the
compsets and pes specfication files.
"""
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components)
# Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
# that has a match for either the alias or the longname in that order
for component in components:
# Determine the compsets file for this component
compsets_filename = files.get_value("COMPSETS_SPEC_FILE", {"component":component})
# If the file exists, read it and see if there is a match for the compset alias or longname
if (os.path.isfile(compsets_filename)):
compsets = Compsets(compsets_filename)
match = compsets.get_compset_match(name=compset_name)
pesfile = files.get_value("PES_SPEC_FILE" , {"component":component})
if match is not None:
self._pesfile = pesfile
self._compsetsfile = compsets_filename
self._compsetname = match
tests_filename = files.get_value("TESTS_SPEC_FILE" , {"component":component}, resolved=False)
tests_mods_dir = files.get_value("TESTS_MODS_DIR" , {"component":component}, resolved=False)
user_mods_dir = files.get_value("USER_MODS_DIR" , {"component":component}, resolved=False)
self.set_lookup_value("COMPSETS_SPEC_FILE" ,
files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False))
self.set_lookup_value("TESTS_SPEC_FILE" , tests_filename)
self.set_lookup_value("TESTS_MODS_DIR" , tests_mods_dir)
self.set_lookup_value("USER_MODS_DIR" , user_mods_dir)
self.set_lookup_value("PES_SPEC_FILE" ,
files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False))
logger.info("Compset longname is %s " %(match))
logger.info("Compset specification file is %s" %(compsets_filename))
logger.info("Pes specification file is %s" %(pesfile))
return
if user_compset is True:
#Do not error out for user_compset
logger.warn("Could not find a compset match for either alias or longname in %s" %(compset_name))
self._compsetname = compset_name
self._pesfile = pesfile
self.set_lookup_value("PES_SPEC_FILE", pesfile)
else:
expect(False,
"Could not find a compset match for either alias or longname in %s" %(compset_name))
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
# append any component specific config_tests.xml files
for comp in files.get_components("CONFIG_TESTS_FILE"):
if comp is None:
continue
infile = files.get_value("CONFIG_TESTS_FILE", attribute={"component":comp})
if os.path.isfile(infile):
self.read(infile)
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
# append any component specific config_tests.xml files
for comp in files.get_components("CONFIG_TESTS_FILE"):
if comp is None:
continue
infile = files.get_value("CONFIG_TESTS_FILE", attribute={"component":comp})
if os.path.isfile(infile):
self.read(infile)
def CompsetSelect(self, text):
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
for comp in components:
infile = files.get_value("COMPSETS_SPEC_FILE", {"component": comp})
compsetobj = Compsets(infile=infile, files=files)
longname, _, _ = compsetobj.get_compset_match(text)
if longname is not None:
self.CompsetLongName.setEnabled(True)
self.CompsetLongName.setText(longname)
valid_grids = self.ResList.gridsobj.find_valid_alias_list(
longname)
self.ResList.clear()
for grid in valid_grids:
self.ResList.addItem(grid[0])
self.CreateNewcaseArgs["compset"] = longname
self.check_ready_to_apply()
def get_tests_from_xml(xml_machine=None,xml_category=None,xml_compiler=None, xml_testlist=None,
machine=None, compiler=None):
"""
Parse testlists for a list of tests
"""
listoftests = []
testlistfiles = []
if(machine is not None):
thismach=machine
if(compiler is not None):
thiscompiler = compiler
if(xml_testlist is not None):
expect(os.path.isfile(xml_testlist), "Testlist not found or not readable "+xml_testlist)
testlistfiles.append(xml_testlist)
else:
files = Files()
comps = files.get_components("TESTS_SPEC_FILE")
for comp in comps:
test_spec_file = files.get_value("TESTS_SPEC_FILE", {"component":comp})
if(os.path.isfile(test_spec_file)):
testlistfiles.append(test_spec_file)
for testlistfile in testlistfiles:
thistestlistfile = Testlist(testlistfile)
logger.debug("Testlist file is "+testlistfile)
logger.debug("xml_machine {} xml_category {} xml_compiler {}".format(xml_machine, xml_category, xml_compiler))
newtests = thistestlistfile.get_tests(xml_machine, xml_category, xml_compiler)
for test in newtests:
if(machine is None):
thismach = test["machine"]
if(compiler is None):
thiscompiler = test["compiler"]
test["name"] = CIME.utils.get_full_test_name(test["testname"], grid=test["grid"], compset=test["compset"],
machine=thismach, compiler=thiscompiler,
testmod=None if "testmods" not in test else test["testmods"])
logger.debug("Adding test {} with compiler {}".format(test["name"], test["compiler"]))
listoftests += newtests
logger.debug("Found {:d} tests".format(len(listoftests)))
return listoftests
def _main_func(options, work_dir):
###############################################################################
files = Files()
model_version = options.version[0]
comp_classes = ("CPL", "ATM", "LND", "ICE", "OCN", "ROF", "GLC", "WAV",
"ESP")
components = ["allactive"]
for comp in comp_classes:
components.extend(files.get_components(
"COMP_ROOT_DIR_{}".format(comp)))
compset_files = []
compset_dict = {}
for comp in components:
if comp != "nemo":
compset_file = files.get_value("COMPSETS_SPEC_FILE",
attribute={"component": comp})
if compset_file not in compset_files:
expect(os.path.isfile(compset_file),
"Could not find file {}".format(compset_file))
compset_files.append(compset_file)
compset = Compsets(infile=compset_file, files=files)
longnames = compset.get_compset_longnames()
for longname in longnames:
_, alias, science_support = compset.get_compset_match(
name=longname)
elements = longname.split("_")
numelems = len(elements)
expect(numelems > 7,
"This longname not supported {}".format(longname))
compset_dict[longname] = {
"alias": alias,
"science_support_grids": science_support,
"defined_by": comp,
"init_opt": elements[0],
"atm_opt": elements[1],
"lnd_opt": elements[2],
"seaice_opt": elements[3],
"ocn_opt": elements[4],
"rof_opt": elements[5],
"glc_opt": elements[6],
"wav_opt": elements[7]
}
for i in range(8, numelems):
if elements[i].startswith("BGC"):
compset_dict[longname].update(
{"bgc_opt": elements[i]})
elif 'ESP' in elements[i]:
compset_dict[longname].update(
{"esp_opt": elements[i]})
elif elements[i] == 'TEST':
logger.info("Longname is {}".format(longname))
else:
logger.warn(
"Unrecognized longname: {} {} {} ".format(
longname, i, elements[i]))
components = []
for element in elements:
if element.startswith("BGC%") or element.startswith(
"TEST"):
continue
else:
element_component = element.split('%')[0].lower()
if "ww" not in element_component:
element_component = re.sub(
r'[0-9]*', "", element_component)
components.append(element_component)
for i in range(1, len(components)):
comp_class = comp_classes[i]
comp_config_file = files.get_value(
"CONFIG_{}_FILE".format(comp_class),
{"component": components[i]})
compobj = Component(comp_config_file,
comp_class)
compset_dict[longname].update({
"{}_desc".format(comp_class):
compobj.get_description(longname)
})
## print ("compset_dict = {}".format(compset_dict))
# load up jinja template
templateLoader = jinja2.FileSystemLoader(
searchpath='{0}/templates'.format(work_dir))
templateEnv = jinja2.Environment(loader=templateLoader)
tmplFile = 'compsetdef2html.tmpl'
template = templateEnv.get_template(tmplFile)
#TODO change the template to just loop through the html_dict
templateVars = {
'compset_dict': compset_dict,
'today': _now,
'model_version': model_version
}
# render the template
comp_tmpl = template.render(templateVars)
# write the output file
with open(options.htmlfile[0], 'w') as html:
html.write(comp_tmpl)
return 0
def get_tests_from_xml(xml_machine=None,
xml_category=None,
xml_compiler=None,
xml_testlist=None,
machine=None,
compiler=None,
driver=None):
"""
Parse testlists for a list of tests
"""
listoftests = []
testlistfiles = []
if (machine is not None):
thismach = machine
if (compiler is not None):
thiscompiler = compiler
if (xml_testlist is not None):
expect(os.path.isfile(xml_testlist),
"Testlist not found or not readable " + xml_testlist)
testlistfiles.append(xml_testlist)
else:
files = Files()
comps = files.get_components("TESTS_SPEC_FILE")
for comp in comps:
test_spec_file = files.get_value("TESTS_SPEC_FILE",
{"component": comp})
if (os.path.isfile(test_spec_file)):
testlistfiles.append(test_spec_file)
for testlistfile in testlistfiles:
thistestlistfile = Testlist(testlistfile)
logger.debug("Testlist file is " + testlistfile)
logger.debug("xml_machine {} xml_category {} xml_compiler {}".format(
xml_machine, xml_category, xml_compiler))
newtests = thistestlistfile.get_tests(xml_machine, xml_category,
xml_compiler)
for test in newtests:
if (machine is None):
thismach = test["machine"]
if (compiler is None):
thiscompiler = test["compiler"]
test["name"] = CIME.utils.get_full_test_name(
test["testname"],
grid=test["grid"],
compset=test["compset"],
machine=thismach,
compiler=thiscompiler,
testmod=None if "testmods" not in test else test["testmods"])
if driver:
# override default or specified driver
founddriver = False
for specdriver in ("Vnuopc", "Vmct", "Vmoab"):
if specdriver in test["name"]:
test["name"] = test["name"].replace(
specdriver, "V{}".format(driver))
founddriver = True
if not founddriver:
name = test["name"]
index = name.find('.')
test["name"] = name[:index] + "_V{}".format(
driver) + name[index:]
logger.debug("Adding test {} with compiler {}".format(
test["name"], test["compiler"]))
listoftests += newtests
logger.debug("Found {:d} tests".format(len(listoftests)))
return listoftests
