def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = env_archive.make_child("components", attributes={"version":"2.0"})
model = get_model()
if 'drv' not in components:
components.append('drv')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component {}".format(comp))
else:
logger.debug("Archive file {} for component {} not found".format(infile,comp))
specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component {}".format(comp))
else:
logger.debug("adding archive spec for {}".format(comp))
env_archive.add_child(specs, root=components_node)
def __init__(self,
compiler=None,
machine=None,
os_=None,
mpilib=None,
infile=None,
files=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile)
self.machine = machine
self.os = os_
self.mpilib = mpilib
self.compiler_nodes = None # Listed from last to first
self.compiler = compiler
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"), ".cime",
"config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = env_archive.make_child("components", attributes={"version":"2.0"})
model = get_model()
if 'drv' not in components:
components.append('drv')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_child(name="comp_archive_spec", attributes={"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component {}".format(comp))
else:
logger.debug("Archive file {} for component {} not found".format(infile,comp))
specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component {}".format(comp))
else:
logger.debug("adding archive spec for {}".format(comp))
env_archive.add_child(specs, root=components_node)
def _create_caseroot_tools(self):
machines_dir = os.path.abspath(self.get_value("MACHDIR"))
toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
# setup executable files in caseroot/
exefiles = (os.path.join(toolsdir, "case.setup"),
os.path.join(toolsdir, "case.build"),
os.path.join(toolsdir, "case.submit"),
os.path.join(toolsdir, "preview_namelists"),
os.path.join(toolsdir, "check_input_data"),
os.path.join(toolsdir, "check_case"),
os.path.join(toolsdir, "archive_metadata.sh"),
os.path.join(toolsdir, "xmlchange"),
os.path.join(toolsdir, "xmlquery"))
try:
for exefile in exefiles:
destfile = os.path.join(self._caseroot,os.path.basename(exefile))
os.symlink(exefile, destfile)
except Exception as e:
logger.warning("FAILED to set up exefiles: %s" % str(e))
# set up utility files in caseroot/Tools/
toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
os.path.join(toolsdir, "lt_archive.sh"),
os.path.join(toolsdir, "getTiming"),
os.path.join(toolsdir, "save_provenance"),
os.path.join(machines_dir,"Makefile"),
os.path.join(machines_dir,"mkSrcfiles"),
os.path.join(machines_dir,"mkDepends"))
for toolfile in toolfiles:
destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile))
expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
try:
os.symlink(toolfile, destfile)
except Exception as e:
logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))
# Create Macros file.
machine = self.get_value("MACH")
files = Files()
# Use config_build if the environment variable is set, or if there is no
# config_compilers file.
if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \
files.get_value("COMPILERS_SPEC_FILE") is None:
build_file = files.get_value("BUILD_SPEC_FILE")
machobj = Machines(machine=machine, files=files)
macro_maker = Build(machobj)
macros_path = os.path.join(self._caseroot, "Macros")
with open(macros_path, "w") as macros_file:
macro_maker.write_macros('Makefile', build_file, macros_file)
# Copy any system or compiler Depends files to the case.
compiler = self.get_value("COMPILER")
for dep in (machine, compiler):
dfile = "Depends.%s"%dep
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile))
dfile = "Depends.%s.%s"%(machine,compiler)
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
def __init__(self,
batch_system=None,
machine=None,
infile=None,
files=None):
"""
initialize an object
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("BATCH_SPEC_FILE")
schema = files.get_schema("BATCH_SPEC_FILE")
GenericXML.__init__(self, infile, schema=schema)
self.batch_system_node = None
self.machine_node = None
self.batch_system = batch_system
self.machine = machine
#Append the contents of $HOME/.cime/config_batch.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"), ".cime",
"config_batch.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.batch_system is not None:
self.set_batch_system(self.batch_system, machine=machine)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("PIO_SPEC_FILE")
EntryID.__init__(self, infile)
def __init__(self, infile=None, files=None, machine=None):
"""
initialize an object
if a filename is provided it will be used,
otherwise if a files object is provided it will be used
otherwise create a files object from default values
"""
self.machine_node = None
self.machine = None
self.machines_dir = None
if infile is None:
if files is None:
files = Files()
infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
infile = files.get_resolved_value(infile)
self.machines_dir = os.path.dirname(infile)
GenericXML.__init__(self, infile)
# Append the contents of $HOME/.cime/config_machines.xml if it exists
# This could cause problems if node matchs are repeated when only one is expected
local_infile = os.path.join(os.environ.get("HOME"), ".cime", "config_machines.xml")
logger.debug("Infile: %s", local_infile)
if os.path.exists(local_infile):
GenericXML.read(self, local_infile)
if machine is None:
machine = self.probe_machine_name()
expect(machine is not None, "Could not initialize machine object from %s or %s" % (infile, local_infile))
self.set_machine(machine)
def __init__(self, compiler=None, machine=None, os_= None, mpilib=None, infile=None, files=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile)
self.machine = machine
self.os = os_
self.mpilib = mpilib
self.compiler_nodes = None # Listed from last to first
self.compiler = compiler
if self.compiler is not None:
self.set_compiler(compiler)
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
test_case, case_opts, grid, compset, machine, compiler, test_mods = CIME.utils.parse_test_name(test)
if (compiler != self._compiler):
raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test, self._compiler))
if (self._parallel_jobs == 1):
scratch_dir = self._machobj.get_value("CESMSCRATCHROOT")
if (self._project is not None):
scratch_dir = scratch_dir.replace("$PROJECT", self._project)
sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
else:
# Parallelizing builds introduces potential sync problems with sharedlibroot
# Just let every case build it's own
sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id)
create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -sharedlibroot %s" % \
(os.path.join(self._cime_root,"scripts", "create_newcase"),
self._cime_model,test_dir, grid, machine, compiler, compset, test_case, self._project,
sharedlibroot)
if (test_case != 'PFS'):
create_newcase_cmd += " -nosavetiming "
if (case_opts is not None):
create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
if (test_mods is not None):
files = Files()
(component, mods) = test_mods.split('/')
testmods_dir = files.get_value("TESTS_MODS_DIR",{"component": component})
test_mod_file = os.path.join(testmods_dir, component, mods)
if (not os.path.exists(test_mod_file)):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
logging.info("Calling create_newcase: "+create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def error(self, message):
self.print_usage(sys.stderr)
# missing argument
# TODO: assumes comp_interface='mct'
if "expected one argument" in message:
if "compset" in message:
components = get_compsets(Files(comp_interface="mct"))
self.exit(
2,
"{}: error: {}\nValid input arguments are {}\n".format(
self.prog, message, components),
)
elif "component" in message:
files = Files(comp_interface="mct")
components = get_components(files)
# Loop through the elements for each component class (in config_files.xml)
valid_components = []
for comp in components:
string = "CONFIG_{}_FILE".format(comp)
# determine all components in string
components = files.get_components(string)
for item in components:
valid_components.append(item)
self.exit(
2,
"{}: error: {}\nValid input arguments are {}\n".format(
self.prog, message, valid_components),
)
# for all other errors
self.exit(2, "{}: error: {}\n".format(self.prog, message))
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = ET.Element("components")
components_node.set("version", "2.0")
model = get_model()
if 'cpl' not in components:
components.append('cpl')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_node("comp_archive_spec", {"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component %s"%comp)
else:
logger.debug("Archive file %s for component %s not found"%(infile,comp))
specs = self.get_optional_node("comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component %s"%comp)
else:
logger.debug("adding archive spec for %s"%comp)
components_node.append(specs)
env_archive.add_child(components_node)
def __init__(self, batch_system=None, machine=None, infile=None, files=None):
"""
initialize an object
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("BATCH_SPEC_FILE")
schema = files.get_schema("BATCH_SPEC_FILE")
GenericXML.__init__(self, infile, schema=schema)
self.batch_system_node = None
self.machine_node = None
self.batch_system = batch_system
self.machine = machine
#Append the contents of $HOME/.cime/config_batch.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.batch_system is not None:
self.set_batch_system(self.batch_system, machine=machine)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("PIO_SPEC_FILE")
EntryID.__init__(self, infile)
def _create_caseroot_tools(self):
machines_dir = os.path.abspath(self.get_value("MACHDIR"))
toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
# setup executable files in caseroot/
exefiles = (os.path.join(toolsdir, "case.setup"),
os.path.join(toolsdir, "case.build"),
os.path.join(toolsdir, "case.submit"),
os.path.join(toolsdir, "preview_namelists"),
os.path.join(toolsdir, "check_input_data"),
os.path.join(toolsdir, "check_case"),
os.path.join(toolsdir, "archive_metadata.sh"),
os.path.join(toolsdir, "xmlchange"),
os.path.join(toolsdir, "xmlquery"))
try:
for exefile in exefiles:
destfile = os.path.join(self._caseroot,os.path.basename(exefile))
os.symlink(exefile, destfile)
except Exception as e:
logger.warning("FAILED to set up exefiles: %s" % str(e))
# set up utility files in caseroot/Tools/
toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
os.path.join(toolsdir, "lt_archive.sh"),
os.path.join(toolsdir, "getTiming"),
os.path.join(toolsdir, "save_provenance"),
os.path.join(machines_dir,"Makefile"),
os.path.join(machines_dir,"mkSrcfiles"),
os.path.join(machines_dir,"mkDepends"))
for toolfile in toolfiles:
destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile))
expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
try:
os.symlink(toolfile, destfile)
except Exception as e:
logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))
# Create Macros file.
machine = self.get_value("MACH")
files = Files()
# Use config_build if the environment variable is set, or if there is no
# config_compilers file.
if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \
files.get_value("COMPILERS_SPEC_FILE") is None:
build_file = files.get_value("BUILD_SPEC_FILE")
machobj = Machines(machine=machine, files=files)
macro_maker = Build(machobj)
macros_path = os.path.join(self._caseroot, "Macros")
with open(macros_path, "w") as macros_file:
macro_maker.write_macros('Makefile', build_file, macros_file)
# Copy any system or compiler Depends files to the case.
compiler = self.get_value("COMPILER")
for dep in (machine, compiler):
dfile = "Depends.%s"%dep
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile))
dfile = "Depends.%s.%s"%(machine,compiler)
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
def __init__(self, infile=None, files=None, machine=None):
"""
initialize an object
if a filename is provided it will be used,
otherwise if a files object is provided it will be used
otherwise create a files object from default values
"""
self.machine_node = None
self.machine = None
self.machines_dir = None
if infile is None:
if files is None:
files = Files()
infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
infile = files.get_resolved_value(infile)
self.machines_dir = os.path.dirname(infile)
GenericXML.__init__(self, infile)
# Append the contents of $HOME/.cime/config_machines.xml if it exists
# This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"), ".cime",
"config_machines.xml")
logger.debug("Infile: %s", infile)
if os.path.exists(infile):
GenericXML.read(self, infile)
if machine is None:
machine = self.probe_machine_name()
expect(machine is not None, "Could not initialize machine object")
self.set_machine(machine)
def saveLogs(case, lid):
###############################################################################
logdir = case.get_value("LOGDIR")
if logdir is not None and len(logdir) > 0:
if not os.path.isdir(logdir):
os.makedirs(logdir)
caseroot = case.get_value("CASEROOT")
rundir = case.get_value("RUNDIR")
# get components
files = Files()
config_file = files.get_value("CONFIG_DRV_FILE")
component = Component(config_file)
comps = [x.lower() for x in component.get_valid_model_components()]
comps = [x.replace('drv', 'cpl') for x in comps]
model = [case.get_value("MODEL")]
comps = comps + model
# for each component, compress log files and copy to logdir
for comp in comps:
logfile = os.path.join(rundir, comp + '.log.' + lid)
if os.path.isfile(logfile):
f_in = open(logfile)
f_out = gzip.open(logfile + '.gz', 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove(logfile)
logfile_copy = logfile + '.gz'
shutil.copy(logfile_copy,
os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))
def print_values(self, machine_name="all"): # pylint: disable=arguments-differ
# set flag to look for single machine
if "all" not in machine_name:
single_machine = True
if machine_name == "current":
machine_name = self.probe_machine_name(warn=False)
else:
single_machine = False
# if we can't find the specified machine
if single_machine and machine_name is None:
files = Files()
config_file = files.get_value("MACHINES_SPEC_FILE")
print(
"Machine is not listed in config file: {}".format(config_file))
else: # write out machines
if single_machine:
machine_names = [machine_name]
else:
machine_names = self.list_available_machines()
print("Machine(s)\n")
for name in machine_names:
self.set_machine(name)
desc = self.text(self.get_child("DESC"))
os_ = self.text(self.get_child("OS"))
compilers = self.text(self.get_child("COMPILERS"))
mpilibnodes = self.get_children("MPILIBS",
root=self.machine_node)
mpilibs = []
for node in mpilibnodes:
mpilibs.extend(self.text(node).split(","))
# This does not include the possible depedancy of mpilib on compiler
# it simply provides a list of mpilibs available on the machine
mpilibs = list(set(mpilibs))
max_tasks_per_node = self.text(
self.get_child("MAX_TASKS_PER_NODE"))
mpitasks_node = self.get_optional_child(
"MAX_MPITASKS_PER_NODE", root=self.machine_node)
max_mpitasks_per_node = (self.text(mpitasks_node) if
mpitasks_node else max_tasks_per_node)
max_gpus_node = self.get_optional_child("MAX_GPUS_PER_NODE",
root=self.machine_node)
max_gpus_per_node = self.text(
max_gpus_node) if max_gpus_node else 0
current_machine = self.probe_machine_name(warn=False)
name += (" (current)" if current_machine
and current_machine in name else "")
print(" {} : {} ".format(name, desc))
print(" os ", os_)
print(" compilers ", compilers)
print(" mpilibs ", mpilibs)
if max_mpitasks_per_node is not None:
print(" pes/node ", max_mpitasks_per_node)
if max_tasks_per_node is not None:
print(" max_tasks/node ", max_tasks_per_node)
if max_gpus_per_node is not None:
print(" max_gpus/node ", max_gpus_per_node)
print("")
def _run_and_assert_query_testlist(self, extra_args=""):
"""Ensure that query_testlist runs successfully with the given extra arguments"""
files = Files()
testlist_drv = files.get_value("TESTS_SPEC_FILE", {"component": "drv"})
self.run_cmd_assert_result(
"{}/query_testlists --xml-testlist {} {}".format(
self.SCRIPT_DIR, testlist_drv, extra_args))
def __init__(self, infile=None, files=None):
"""
initialize an object
"""
if files is None:
files = Files()
schema = files.get_schema("ARCHIVE_SPEC_FILE")
super(Archive, self).__init__(infile, schema)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("GRIDS_SPEC_FILE")
logger.debug(" Grid specification file is %s" % infile)
GenericXML.__init__(self, infile)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("GRIDS_SPEC_FILE")
logger.debug(" Grid specification file is %s" % infile)
GenericXML.__init__(self, infile)
def _main_func(options, work_dir):
###############################################################################
"""Construct machines html from an XML file."""
# Initialize a variables for the html template
mach_dict = dict()
model_version = options.version[0]
# get the machine config file
files = Files()
config_file = files.get_value("MACHINES_SPEC_FILE")
expect(os.path.isfile(config_file),
"Cannot find config_file {} on disk".format(config_file))
# instantiate a machines object and read XML values into a dictionary
machines = Machines(config_file, machine="Query")
mach_list = machines.list_available_machines()
# get all the machine values loaded into the mach_dict
mach_dict = machines.return_values()
# intialize the support keys
for machine in mach_list:
mach_dict[(machine, 'support')] = "Unsupported"
# loop through the list of supported machines and flag in the dictionary
supported = options.supported[0].split(',')
for machine in supported:
mach_dict[(machine, 'support')] = "Scientific"
# loop through the list of tested machines and flag in the dictionary
tested = options.tested[0].split(',')
for machine in tested:
mach_dict[(machine, 'support')] = "Tested"
# load up jinja template
templateLoader = jinja2.FileSystemLoader(
searchpath='{0}/templates'.format(work_dir))
templateEnv = jinja2.Environment(loader=templateLoader)
# TODO - get the cesm_version for the CIME root
tmplFile = 'machdef2html.tmpl'
template = templateEnv.get_template(tmplFile)
templateVars = {
'mach_list': mach_list,
'mach_dict': mach_dict,
'today': _now,
'model_version': model_version
}
# render the template
mach_tmpl = template.render(templateVars)
# write the output file
with open(options.htmlfile[0], 'w') as html:
html.write(mach_tmpl)
return 0
def list_compsets(self):
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
for comp in components:
infile = files.get_value("COMPSETS_SPEC_FILE", {"component": comp})
compsetobj = Compsets(infile=infile, files=files)
_, compsets = compsetobj.return_all_values()
self.CompsetList.addItems(sorted(compsets.keys()))
self.CompsetList.insertSeparator(999)
def __init__(self, infile=None, files=None):
"""
initialize an object
"""
if files is None:
files = Files()
schema = files.get_schema("ARCHIVE_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
def __init__(self, infile, files=None):
"""
initialize a files object given input pes specification file
"""
if files is None:
files = Files()
schema = files.get_schema("PES_SPEC_FILE")
logger.debug("DEBUG: infile is {}".format(infile))
GenericXML.__init__(self, infile, schema=schema)
def __init__(self, infile, files=None):
"""
initialize a files object given input pes specification file
"""
if files is None:
files = Files()
schema = files.get_schema("PES_SPEC_FILE")
logger.debug("DEBUG: infile is {}".format(infile))
GenericXML.__init__(self, infile, schema=schema)
def __init__(self, comp_classes, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("PIO_SPEC_FILE")
EntryID.__init__(self, infile)
self._components = list(comp_classes)
def __init__(self, infile=None):
"""
initialize an object
"""
if infile is None:
files = Files()
infile = files.get_value("CONFIG_DRV_FILE")
EntryID.__init__(self,infile)
def __init__(self,infile, files=None):
"""
initialize an object
"""
schema = None
if files is None:
files = Files()
schema = files.get_schema("TESTS_SPEC_FILE")
GenericXML.__init__(self, infile, schema=schema)
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
_, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
" --test" % \
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset)
if self._project is not None:
create_newcase_cmd += " --project %s " % self._project
if self._output_root is not None:
create_newcase_cmd += " --output-root %s " % self._output_root
if test_mods is not None:
files = Files()
(component, modspath) = test_mods.split('/', 1)
testmods_dir = files.get_value("TESTS_MODS_DIR",
{"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
self._log_output(test,
"Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " --user-mods-dir %s" % test_mod_file
if case_opts is not None:
for case_opt in case_opts: # pylint: disable=not-an-iterable
if case_opt.startswith('M'):
mpilib = case_opt[1:]
create_newcase_cmd += " --mpilib %s" % mpilib
logger.debug(" MPILIB set to %s" % mpilib)
if case_opt.startswith('N'):
ninst = case_opt[1:]
create_newcase_cmd += " --ninst %s" % ninst
logger.debug(" NINST set to %s" % ninst)
if case_opt.startswith('P'):
pesize = case_opt[1:]
create_newcase_cmd += " --pecount %s" % pesize
if self._queue is not None:
create_newcase_cmd += " --queue=%s" % self._queue
if self._walltime is not None:
create_newcase_cmd += " --walltime %s" % self._walltime
elif test in self._test_data and "options" in self._test_data[test] and \
"wallclock" in self._test_data[test]['options']:
create_newcase_cmd += " --walltime %s" % self._test_data[test][
'options']['wallclock']
logger.debug("Calling create_newcase: " + create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd,
CREATE_NEWCASE_PHASE)
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
def __init__(self, infile):
"""
initialize an object
"""
files = Files()
schema = None
# not checking schema on external components yet
cimeroot = get_cime_root()
if cimeroot in os.path.abspath(infile):
schema = files.get_schema("CONFIG_CPL_FILE")
EntryID.__init__(self, infile, schema=schema)
def get_tests_from_xml(xml_machine=None,
xml_category=None,
xml_compiler=None,
xml_testlist=None,
machine=None,
compiler=None):
"""
Parse testlists for a list of tests
"""
listoftests = []
testlistfiles = []
if (machine is not None):
thismach = machine
if (compiler is not None):
thiscompiler = compiler
if (xml_testlist is not None):
expect(os.path.isfile(xml_testlist),
"Testlist not found or not readable " + xml_testlist)
testlistfiles.append(xml_testlist)
else:
files = Files()
comps = files.get_components("TESTS_SPEC_FILE")
for comp in comps:
test_spec_file = files.get_value("TESTS_SPEC_FILE",
{"component": comp})
if (os.path.isfile(test_spec_file)):
testlistfiles.append(test_spec_file)
for testlistfile in testlistfiles:
thistestlistfile = Testlist(testlistfile)
logger.debug("Testlist file is " + testlistfile)
logger.debug("xml_machine %s xml_category %s xml_compiler %s" %
(xml_machine, xml_category, xml_compiler))
newtests = thistestlistfile.get_tests(xml_machine, xml_category,
xml_compiler)
for test in newtests:
if (machine is None):
thismach = test["machine"]
if (compiler is None):
thiscompiler = test["compiler"]
test["name"] = CIME.utils.get_full_test_name(
test["testname"],
grid=test["grid"],
compset=test["compset"],
machine=thismach,
compiler=thiscompiler,
testmod=None if "testmods" not in test else test["testmods"])
logger.debug("Adding test %s with compiler %s" %
(test["name"], test["compiler"]))
listoftests += newtests
logger.debug("Found %d tests" % len(listoftests))
return listoftests
def __init__(self,infile, files=None):
"""
initialize an object
"""
schema = None
if files is None:
files = Files()
schema = files.get_schema("TESTS_SPEC_FILE")
GenericXML.__init__(self, infile, schema=schema)
expect(self.get_version() >= 2.0,
"{} is an unsupported version of the testfile format and will be ignored".format(infile))
def __init__(self, infile=None, files=None):
if files is None:
files = Files()
if infile is None:
infile = files.get_value("GRIDS_SPEC_FILE")
logger.debug(" Grid specification file is %s" % infile)
schema = files.get_schema("GRIDS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._version = self.get_version()
self._comp_gridnames = self._get_grid_names()
def __init__(self, infile):
"""
initialize an object
"""
files = Files()
schema = None
# not checking schema on external components yet
cimeroot = get_cime_root()
if cimeroot in os.path.abspath(infile):
schema = files.get_schema("CONFIG_DRV_FILE")
EntryID.__init__(self, infile, schema=schema)
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
_, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
" --test" % \
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset)
if self._project is not None:
create_newcase_cmd += " --project %s " % self._project
if self._output_root is not None:
create_newcase_cmd += " --output-root %s " % self._output_root
if test_mods is not None:
files = Files()
(component,modspath) = test_mods.split('/',1)
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " --user-mods-dir %s" % test_mod_file
if case_opts is not None:
for case_opt in case_opts: # pylint: disable=not-an-iterable
if case_opt.startswith('M'):
mpilib = case_opt[1:]
create_newcase_cmd += " --mpilib %s" % mpilib
logger.debug (" MPILIB set to %s" % mpilib)
if case_opt.startswith('N'):
ninst = case_opt[1:]
create_newcase_cmd += " --ninst %s" %ninst
logger.debug (" NINST set to %s" % ninst)
if case_opt.startswith('P'):
pesize = case_opt[1:]
create_newcase_cmd += " --pecount %s"%pesize
if self._queue is not None:
create_newcase_cmd += " --queue=%s" % self._queue
if self._walltime is not None:
create_newcase_cmd += " --walltime %s" % self._walltime
elif test in self._test_data and "options" in self._test_data[test] and \
"wallclock" in self._test_data[test]['options']:
create_newcase_cmd += " --walltime %s" % self._test_data[test]['options']['wallclock']
logger.debug("Calling create_newcase: " + create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self, infile=None):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/CIME/data/config/config_headers.xml'
"""
if infile is None:
files = Files()
infile = files.get_value("CASEFILE_HEADERS", resolved=True)
super(Headers, self).__init__(infile)
def __init__(self, infile=None):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/cime_config/config_headers.xml'
"""
if infile is None:
files = Files()
infile = files.get_value('CASEFILE_HEADERS', resolved=True)
EntryID.__init__(self, infile)
def __init__(self,infile=None):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/config/config_headers.xml'
"""
if infile is None:
files = Files()
infile = files.get_value('CASEFILE_HEADERS', resolved=True)
super(Headers, self).__init__(infile)
def __init__(self, infile=None, files=None):
if files is None:
files = Files()
if infile is None:
infile = files.get_value("GRIDS_SPEC_FILE")
logger.debug(" Grid specification file is %s" % infile)
schema = files.get_schema("GRIDS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._version = self.get_version()
self._comp_gridnames = self._get_grid_names()
def __init__(self, machine, files=None, infile=None):
"""
initialize an object
"""
if (infile is None):
if files is None:
files = Files()
infile = files.get_value("LTARCHIVE_SPEC_FILE", resolved=False)
infile = files.get_resolved_value(infile)
GenericXML.__init__(self, infile)
self.machine = machine
def __init__(self, infile=None, files=None):
"""
initialize a files object given input pes specification file
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("INPUTDATA_SPEC_FILE")
schema = files.get_schema("INPUTDATA_SPEC_FILE")
logger.debug("DEBUG: infile is {}".format(infile))
GenericXML.__init__(self, infile, schema=schema)
self._servernode = None
def __init__(self, infile):
"""
initialize an object
"""
files = Files()
schema = files.get_schema("CONFIG_DRV_FILE")
if schema is not None:
# not checking schema on external components yet
cimeroot = get_cime_root()
if cimeroot in os.path.abspath(infile):
self.validate_xml_file(infile, schema)
EntryID.__init__(self, infile)
def __init__(self, machine, files=None, infile=None):
"""
initialize an object
"""
if (infile is None):
if files is None:
files = Files()
infile = files.get_value("LTARCHIVE_SPEC_FILE", resolved=False)
infile = files.get_resolved_value(infile)
GenericXML.__init__(self, infile)
self.machine = machine
def __init__(self, infile=None, files=None):
"""
initialize a files object given input pes specification file
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("INPUTDATA_SPEC_FILE")
schema = files.get_schema("INPUTDATA_SPEC_FILE")
logger.debug("DEBUG: infile is {}".format(infile))
GenericXML.__init__(self, infile, schema=schema)
self._servernode = None
def test_st_archive(self, testdir="st_archive_test"):
archive = Archive()
files = Files()
components = []
# expect(not self.get_value("MULTI_DRIVER"),"Test not configured for multi-driver cases")
config_archive_files = archive.get_all_config_archive_files(files)
# create the run directory testdir and populate it with rest_file and hist_file from
# config_archive.xml test_file_names
if os.path.exists(testdir):
logger.info("Removing existing test directory {}".format(testdir))
shutil.rmtree(testdir)
dout_s_root=os.path.join(testdir,"archive")
archive = Archive()
schema = files.get_schema("ARCHIVE_SPEC_FILE")
for config_archive_file in config_archive_files:
archive.read(config_archive_file, schema)
comp_archive_specs = archive.get_children("comp_archive_spec")
for comp_archive_spec in comp_archive_specs:
components.append(archive.get(comp_archive_spec, 'compname'))
test_file_names = archive.get_optional_child("test_file_names", root=comp_archive_spec)
if test_file_names is not None:
if not os.path.exists(testdir):
os.makedirs(os.path.join(testdir,"archive"))
for file_node in archive.get_children("tfile", root=test_file_names):
fname = os.path.join(testdir,archive.text(file_node))
disposition = archive.get(file_node, "disposition")
logger.info("Create file {} with disposition {}".
format(fname, disposition))
with open(fname, 'w') as fd:
fd.write(disposition+"\n")
logger.info("testing components: {} ".format(list(set(components))))
_archive_process(self, archive, None, False, False,components=list(set(components)),
dout_s_root=dout_s_root,
casename="casename", rundir=testdir, testonly=True)
_check_disposition(testdir)
# Now test the restore capability
testdir2 = os.path.join(testdir,"run2")
os.makedirs(testdir2)
restore_from_archive(self, rundir=testdir2, dout_s_root=dout_s_root)
restfiles = [f for f in os.listdir(os.path.join(testdir,"archive","rest","1976-01-01-00000"))]
for _file in restfiles:
expect(os.path.isfile(os.path.join(testdir2,_file)), "Expected file {} to be restored from rest dir".format(_file))
return True
def __init__(self,
machobj,
infile=None,
compiler=None,
mpilib=None,
files=None,
version=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
schema = files.get_schema("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._machobj = machobj
if version is not None:
# this is used in scripts_regression_tests to force version 2, it should not be used otherwise
self._version = version
else:
self._version = self.get_version()
self.machine = machobj.get_machine_name()
self.os = machobj.get_value("OS")
if mpilib is None:
mpilib = machobj.get_default_MPIlib()
self.mpilib = mpilib
if compiler is None:
compiler = machobj.get_default_compiler()
self.compiler = compiler
self.compiler_nodes = None # Listed from last to first
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"), ".cime",
"config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
if self._version != "1.0":
# Run an XPath query to extract the list of flag variable names.
ns = {"xs": "http://www.w3.org/2001/XMLSchema"}
flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']"
flag_elems = ET.parse(schema).getroot().findall(flag_xpath, ns)
self.flag_vars = set(elem.get('name') for elem in flag_elems)
def test_st_archive(self, testdir="st_archive_test"):
files = Files()
archive = Archive(files=files)
components = []
# expect(not self.get_value("MULTI_DRIVER"),"Test not configured for multi-driver cases")
config_archive_files = archive.get_all_config_archive_files(files)
# create the run directory testdir and populate it with rest_file and hist_file from
# config_archive.xml test_file_names
if os.path.exists(testdir):
logger.info("Removing existing test directory {}".format(testdir))
shutil.rmtree(testdir)
dout_s_root=os.path.join(testdir,"archive")
archive = Archive()
schema = files.get_schema("ARCHIVE_SPEC_FILE")
for config_archive_file in config_archive_files:
archive.read(config_archive_file, schema)
comp_archive_specs = archive.get_children("comp_archive_spec")
for comp_archive_spec in comp_archive_specs:
components.append(archive.get(comp_archive_spec, 'compname'))
test_file_names = archive.get_optional_child("test_file_names", root=comp_archive_spec)
if test_file_names is not None:
if not os.path.exists(testdir):
os.makedirs(os.path.join(testdir,"archive"))
for file_node in archive.get_children("tfile", root=test_file_names):
fname = os.path.join(testdir,archive.text(file_node))
disposition = archive.get(file_node, "disposition")
logger.info("Create file {} with disposition {}".
format(fname, disposition))
with open(fname, 'w') as fd:
fd.write(disposition+"\n")
logger.info("testing components: {} ".format(list(set(components))))
_archive_process(self, archive, None, False, False,components=list(set(components)),
dout_s_root=dout_s_root,
casename="casename", rundir=testdir, testonly=True)
_check_disposition(testdir)
# Now test the restore capability
testdir2 = os.path.join(testdir,"run2")
os.makedirs(testdir2)
restore_from_archive(self, rundir=testdir2, dout_s_root=dout_s_root, test=True)
restfiles = [f for f in os.listdir(os.path.join(testdir,"archive","rest","1976-01-01-00000"))]
for _file in restfiles:
expect(os.path.isfile(os.path.join(testdir2,_file)), "Expected file {} to be restored from rest dir".format(_file))
return True
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
# append any component specific config_tests.xml files
for comp in files.get_components("CONFIG_TESTS_FILE"):
if comp is None:
continue
infile = files.get_value("CONFIG_TESTS_FILE", attribute={"component":comp})
if os.path.isfile(infile):
self.read(infile)
def __init__(self, infile, comp_class):
"""
initialize a Component obect from the component xml file in infile
associate the component class with comp_class if provided.
"""
self._comp_class = comp_class
if infile == 'testingonly':
self.filename = infile
return
files = Files()
schema = None
EntryID.__init__(self, infile)
schema = files.get_schema("CONFIG_{}_FILE".format(comp_class), attributes={"version":"{}".format(self.get_version())})
if schema is not None:
self.validate_xml_file(infile, schema)
def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
schema = files.get_schema("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._machobj = machobj
if version is not None:
# this is used in scripts_regression_tests to force version 2, it should not be used otherwise
self._version = version
else:
self._version = self.get_version()
self.machine = machobj.get_machine_name()
self.os = machobj.get_value("OS")
if compiler is None:
compiler = machobj.get_default_compiler()
self.compiler = compiler
if mpilib is None:
if compiler is None:
mpilib = machobj.get_default_MPIlib()
else:
mpilib = machobj.get_default_MPIlib(attributes={'compiler':compiler})
self.mpilib = mpilib
self.compiler_nodes = None # Listed from last to first
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
if self._version > 1.0:
schema_db = GenericXML(infile=schema)
compiler_vars = schema_db.get_child("{http://www.w3.org/2001/XMLSchema}group", attributes={"name":"compilerVars"})
choice = schema_db.get_child(name="{http://www.w3.org/2001/XMLSchema}choice", root=compiler_vars)
self.flag_vars = set(schema_db.get(elem, "name") for elem in schema_db.get_children(root=choice, attributes={"type":"flagsVar"}))
def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
schema = files.get_schema("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._machobj = machobj
if version is not None:
# this is used in scripts_regression_tests to force version 2, it should not be used otherwise
self._version = version
else:
self._version = self.get_version()
self.machine = machobj.get_machine_name()
self.os = machobj.get_value("OS")
if mpilib is None:
mpilib = machobj.get_default_MPIlib()
self.mpilib = mpilib
if compiler is None:
compiler = machobj.get_default_compiler()
self.compiler = compiler
self.compiler_nodes = None # Listed from last to first
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
if self._version > 1.0:
# Run an XPath query to extract the list of flag variable names.
ns = {"xs": "http://www.w3.org/2001/XMLSchema"}
flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']"
flag_elems = ET.parse(schema).getroot().findall(flag_xpath, ns)
self.flag_vars = set(elem.get('name') for elem in flag_elems)
def __init__(self, infile, files=None):
"""Construct a `NamelistDefinition` from an XML file."""
# if the file is invalid we may not be able to check the version
# but we need to do it this way until we remove the version 1 files
schema = None
if files is None:
files = Files()
schema = files.get_schema("NAMELIST_DEFINITION_FILE")
expect(os.path.isfile(infile), "File {} does not exist".format(infile))
super(NamelistDefinition, self).__init__(infile, schema=schema)
self._attributes = {}
self._entry_nodes = []
self._entry_ids = []
self._valid_values = {}
self._entry_types = {}
self._group_names = {}
self._nodes = {}
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
_, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
if compiler != self._compiler:
raise StandardError("Test '%s' has compiler that does"
" not match instance compliler '%s'" % (test, self._compiler))
create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
" --project %s --test"%\
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset, self._project)
if test_mods is not None:
files = Files()
(component,modspath) = test_mods.split('/',1)
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " --user-mods-dir %s" % test_mod_file
if case_opts is not None:
for case_opt in case_opts:
if case_opt.startswith('M'):
mpilib = case_opt[1:]
create_newcase_cmd += " --mpilib %s" % mpilib
logger.debug (" MPILIB set to %s" % mpilib)
if case_opt.startswith('N'):
ninst = case_opt[1:]
create_newcase_cmd += " --ninst %s" %ninst
logger.debug (" NINST set to %s" % ninst)
pesize = re.match('P([SMLX][12]?)', case_opt)
if pesize:
create_newcase_cmd += " --pecount %s"%pesize.group(1)
logger.debug("Calling create_newcase: " + create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self, infile=None, files=None, machine=None):
"""
initialize an object
if a filename is provided it will be used,
otherwise if a files object is provided it will be used
otherwise create a files object from default values
"""
self.machine_node = None
self.machine = None
self.machines_dir = None
schema = None
if files is None:
files = Files()
if infile is None:
infile = files.get_value("MACHINES_SPEC_FILE")
schema = files.get_schema("MACHINES_SPEC_FILE")
logger.debug("Verifying using schema {}".format(schema))
self.machines_dir = os.path.dirname(infile)
GenericXML.__init__(self, infile, schema)
# Append the contents of $HOME/.cime/config_machines.xml if it exists
# This could cause problems if node matchs are repeated when only one is expected
local_infile = os.path.join(os.environ.get("HOME"),".cime","config_machines.xml")
logger.debug("Infile: {}".format(local_infile))
if os.path.exists(local_infile):
GenericXML.read(self, local_infile, schema)
if machine is None:
if "CIME_MACHINE" in os.environ:
machine = os.environ["CIME_MACHINE"]
else:
cime_config = get_cime_config()
if cime_config.has_option("main", "machine"):
machine = cime_config.get("main", "machine")
if machine is None:
machine = self.probe_machine_name()
expect(machine is not None, "Could not initialize machine object from {} or {}".format(infile, local_infile))
self.set_machine(machine)
def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None):
"""
Loop through all the compset files and find the compset
specifation file that matches either the input 'compset_name'.
Note that the input compset name (i.e. compset_name) can be
either a longname or an alias. This will also set the
compsets and pes specfication files.
"""
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components)
# Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
# that has a match for either the alias or the longname in that order
for component in components:
# Determine the compsets file for this component
compsets_filename = files.get_value("COMPSETS_SPEC_FILE", {"component":component})
# If the file exists, read it and see if there is a match for the compset alias or longname
if (os.path.isfile(compsets_filename)):
compsets = Compsets(compsets_filename)
match = compsets.get_compset_match(name=compset_name)
pesfile = files.get_value("PES_SPEC_FILE" , {"component":component})
if match is not None:
self._pesfile = pesfile
self._compsetsfile = compsets_filename
self._compsetname = match
tests_filename = files.get_value("TESTS_SPEC_FILE" , {"component":component}, resolved=False)
tests_mods_dir = files.get_value("TESTS_MODS_DIR" , {"component":component}, resolved=False)
user_mods_dir = files.get_value("USER_MODS_DIR" , {"component":component}, resolved=False)
self.set_lookup_value("COMPSETS_SPEC_FILE" ,
files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False))
self.set_lookup_value("TESTS_SPEC_FILE" , tests_filename)
self.set_lookup_value("TESTS_MODS_DIR" , tests_mods_dir)
self.set_lookup_value("USER_MODS_DIR" , user_mods_dir)
self.set_lookup_value("PES_SPEC_FILE" ,
files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False))
logger.info("Compset longname is %s " %(match))
logger.info("Compset specification file is %s" %(compsets_filename))
logger.info("Pes specification file is %s" %(pesfile))
return
if user_compset is True:
#Do not error out for user_compset
logger.warn("Could not find a compset match for either alias or longname in %s" %(compset_name))
self._compsetname = compset_name
self._pesfile = pesfile
self.set_lookup_value("PES_SPEC_FILE", pesfile)
else:
expect(False,
"Could not find a compset match for either alias or longname in %s" %(compset_name))
def get_tests_from_xml(xml_machine=None,xml_category=None,xml_compiler=None, xml_testlist=None,
machine=None, compiler=None):
"""
Parse testlists for a list of tests
"""
listoftests = []
testlistfiles = []
if(machine is not None):
thismach=machine
if(compiler is not None):
thiscompiler = compiler
if(xml_testlist is not None):
expect(os.path.isfile(xml_testlist), "Testlist not found or not readable "+xml_testlist)
testlistfiles.append(xml_testlist)
else:
files = Files()
comps = files.get_components("TESTS_SPEC_FILE")
for comp in comps:
test_spec_file = files.get_value("TESTS_SPEC_FILE", {"component":comp})
if(os.path.isfile(test_spec_file)):
testlistfiles.append(test_spec_file)
for testlistfile in testlistfiles:
thistestlistfile = Testlist(testlistfile)
logger.debug("Testlist file is "+testlistfile)
logger.debug("xml_machine {} xml_category {} xml_compiler {}".format(xml_machine, xml_category, xml_compiler))
newtests = thistestlistfile.get_tests(xml_machine, xml_category, xml_compiler)
for test in newtests:
if(machine is None):
thismach = test["machine"]
if(compiler is None):
thiscompiler = test["compiler"]
test["name"] = CIME.utils.get_full_test_name(test["testname"], grid=test["grid"], compset=test["compset"],
machine=thismach, compiler=thiscompiler,
testmod=None if "testmods" not in test else test["testmods"])
logger.debug("Adding test {} with compiler {}".format(test["name"], test["compiler"]))
listoftests += newtests
logger.debug("Found {:d} tests".format(len(listoftests)))
return listoftests