def _create_caseroot_tools(self):
machines_dir = os.path.abspath(self.get_value("MACHDIR"))
toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
# setup executable files in caseroot/
exefiles = (os.path.join(toolsdir, "case.setup"),
os.path.join(toolsdir, "case.build"),
os.path.join(toolsdir, "case.submit"),
os.path.join(toolsdir, "preview_namelists"),
os.path.join(toolsdir, "check_input_data"),
os.path.join(toolsdir, "check_case"),
os.path.join(toolsdir, "archive_metadata.sh"),
os.path.join(toolsdir, "xmlchange"),
os.path.join(toolsdir, "xmlquery"))
try:
for exefile in exefiles:
destfile = os.path.join(self._caseroot,os.path.basename(exefile))
os.symlink(exefile, destfile)
except Exception as e:
logger.warning("FAILED to set up exefiles: %s" % str(e))
# set up utility files in caseroot/Tools/
toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
os.path.join(toolsdir, "lt_archive.sh"),
os.path.join(toolsdir, "getTiming"),
os.path.join(toolsdir, "save_provenance"),
os.path.join(machines_dir,"Makefile"),
os.path.join(machines_dir,"mkSrcfiles"),
os.path.join(machines_dir,"mkDepends"))
for toolfile in toolfiles:
destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile))
expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
try:
os.symlink(toolfile, destfile)
except Exception as e:
logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))
# Create Macros file.
machine = self.get_value("MACH")
files = Files()
# Use config_build if the environment variable is set, or if there is no
# config_compilers file.
if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \
files.get_value("COMPILERS_SPEC_FILE") is None:
build_file = files.get_value("BUILD_SPEC_FILE")
machobj = Machines(machine=machine, files=files)
macro_maker = Build(machobj)
macros_path = os.path.join(self._caseroot, "Macros")
with open(macros_path, "w") as macros_file:
macro_maker.write_macros('Makefile', build_file, macros_file)
# Copy any system or compiler Depends files to the case.
compiler = self.get_value("COMPILER")
for dep in (machine, compiler):
dfile = "Depends.%s"%dep
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile))
dfile = "Depends.%s.%s"%(machine,compiler)
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None):
"""
Loop through all the compset files and find the compset
specifation file that matches either the input 'compset_name'.
Note that the input compset name (i.e. compset_name) can be
either a longname or an alias. This will also set the
compsets and pes specfication files.
"""
files = Files()
components = files.get_components("COMPSETS_SPEC_FILE")
logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components)
# Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
# that has a match for either the alias or the longname in that order
for component in components:
# Determine the compsets file for this component
compsets_filename = files.get_value("COMPSETS_SPEC_FILE", {"component":component})
pes_filename = files.get_value("PES_SPEC_FILE" , {"component":component})
tests_filename = files.get_value("TESTS_SPEC_FILE" , {"component":component}, resolved=False)
tests_mods_dir = files.get_value("TESTS_MODS_DIR" , {"component":component}, resolved=False)
user_mods_dir = files.get_value("USER_MODS_DIR" , {"component":component}, resolved=False)
# If the file exists, read it and see if there is a match for the compset alias or longname
if (os.path.isfile(compsets_filename)):
compsets = Compsets(compsets_filename)
match = compsets.get_compset_match(name=compset_name)
if match is not None:
self._pesfile = pes_filename
self._compsetsfile = compsets_filename
self._compsetname = match
self.set_value("COMPSETS_SPEC_FILE" ,
files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False))
self.set_value("TESTS_SPEC_FILE" , tests_filename)
self.set_value("TESTS_MODS_DIR" , tests_mods_dir)
self.set_value("USER_MODS_DIR" , user_mods_dir)
self.set_value("PES_SPEC_FILE" ,
files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False))
logger.info("Compset longname is %s " %(match))
logger.info("Compset specification file is %s" %(compsets_filename))
logger.info("Pes specification file is %s" %(pes_filename))
return
if user_compset is True:
#Do not error out for user_compset
logger.warn("Could not find a compset match for either alias or longname in %s" %(compset_name))
self._compsetname = compset_name
self._pesfile = pesfile
self.set_value("PES_SPEC_FILE", pesfile)
else:
expect(False,
"Could not find a compset match for either alias or longname in %s" %(compset_name))
def __init__(self, batch_system=None, machine=None, infile=None, files=None):
"""
initialize an object
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("BATCH_SPEC_FILE")
schema = files.get_schema("BATCH_SPEC_FILE")
GenericXML.__init__(self, infile, schema=schema)
self.batch_system_node = None
self.machine_node = None
self.batch_system = batch_system
self.machine = machine
#Append the contents of $HOME/.cime/config_batch.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.batch_system is not None:
self.set_batch_system(self.batch_system, machine=machine)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("PIO_SPEC_FILE")
EntryID.__init__(self, infile)
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = ET.Element("components")
components_node.set("version", "2.0")
model = get_model()
if 'cpl' not in components:
components.append('cpl')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_node("comp_archive_spec", {"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component %s"%comp)
else:
logger.debug("Archive file %s for component %s not found"%(infile,comp))
specs = self.get_optional_node("comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component %s"%comp)
else:
logger.debug("adding archive spec for %s"%comp)
components_node.append(specs)
env_archive.add_child(components_node)
def saveLogs(case, lid):
###############################################################################
logdir = case.get_value("LOGDIR")
if logdir is not None and len(logdir) > 0:
if not os.path.isdir(logdir):
os.makedirs(logdir)
caseroot = case.get_value("CASEROOT")
rundir = case.get_value("RUNDIR")
# get components
files = Files()
config_file = files.get_value("CONFIG_DRV_FILE")
component = Component(config_file)
comps = [x.lower() for x in component.get_valid_model_components()]
comps = [x.replace('drv', 'cpl') for x in comps]
model = [case.get_value("MODEL")]
comps = comps + model
# for each component, compress log files and copy to logdir
for comp in comps:
logfile = os.path.join(rundir, comp + '.log.' + lid)
if os.path.isfile(logfile):
f_in = open(logfile)
f_out = gzip.open(logfile + '.gz', 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove(logfile)
logfile_copy = logfile + '.gz'
shutil.copy(logfile_copy,
os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))
def __init__(self, infile=None, files=None, machine=None):
"""
initialize an object
if a filename is provided it will be used,
otherwise if a files object is provided it will be used
otherwise create a files object from default values
"""
self.machine_node = None
self.machine = None
self.machines_dir = None
if infile is None:
if files is None:
files = Files()
infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
infile = files.get_resolved_value(infile)
self.machines_dir = os.path.dirname(infile)
GenericXML.__init__(self, infile)
# Append the contents of $HOME/.cime/config_machines.xml if it exists
# This could cause problems if node matchs are repeated when only one is expected
local_infile = os.path.join(os.environ.get("HOME"), ".cime", "config_machines.xml")
logger.debug("Infile: %s", local_infile)
if os.path.exists(local_infile):
GenericXML.read(self, local_infile)
if machine is None:
machine = self.probe_machine_name()
expect(machine is not None, "Could not initialize machine object from %s or %s" % (infile, local_infile))
self.set_machine(machine)
def __init__(self, compiler=None, machine=None, os_= None, mpilib=None, infile=None, files=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile)
self.machine = machine
self.os = os_
self.mpilib = mpilib
self.compiler_nodes = None # Listed from last to first
self.compiler = compiler
if self.compiler is not None:
self.set_compiler(compiler)
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = env_archive.make_child("components", attributes={"version":"2.0"})
model = get_model()
if 'drv' not in components:
components.append('drv')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component {}".format(comp))
else:
logger.debug("Archive file {} for component {} not found".format(infile,comp))
specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component {}".format(comp))
else:
logger.debug("adding archive spec for {}".format(comp))
env_archive.add_child(specs, root=components_node)
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
test_case, case_opts, grid, compset, machine, compiler, test_mods = CIME.utils.parse_test_name(test)
if (compiler != self._compiler):
raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test, self._compiler))
if (self._parallel_jobs == 1):
scratch_dir = self._machobj.get_value("CESMSCRATCHROOT")
if (self._project is not None):
scratch_dir = scratch_dir.replace("$PROJECT", self._project)
sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
else:
# Parallelizing builds introduces potential sync problems with sharedlibroot
# Just let every case build it's own
sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id)
create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -sharedlibroot %s" % \
(os.path.join(self._cime_root,"scripts", "create_newcase"),
self._cime_model,test_dir, grid, machine, compiler, compset, test_case, self._project,
sharedlibroot)
if (test_case != 'PFS'):
create_newcase_cmd += " -nosavetiming "
if (case_opts is not None):
create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
if (test_mods is not None):
files = Files()
(component, mods) = test_mods.split('/')
testmods_dir = files.get_value("TESTS_MODS_DIR",{"component": component})
test_mod_file = os.path.join(testmods_dir, component, mods)
if (not os.path.exists(test_mod_file)):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
logging.info("Calling create_newcase: "+create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = env_archive.make_child("components", attributes={"version":"2.0"})
model = get_model()
if 'drv' not in components:
components.append('drv')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component {}".format(comp))
else:
logger.debug("Archive file {} for component {} not found".format(infile,comp))
specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component {}".format(comp))
else:
logger.debug("adding archive spec for {}".format(comp))
env_archive.add_child(specs, root=components_node)
def __init__(self,
batch_system=None,
machine=None,
infile=None,
files=None):
"""
initialize an object
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("BATCH_SPEC_FILE")
schema = files.get_schema("BATCH_SPEC_FILE")
GenericXML.__init__(self, infile, schema=schema)
self.batch_system_node = None
self.machine_node = None
self.batch_system = batch_system
self.machine = machine
#Append the contents of $HOME/.cime/config_batch.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"), ".cime",
"config_batch.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.batch_system is not None:
self.set_batch_system(self.batch_system, machine=machine)
def __init__(self,
compiler=None,
machine=None,
os_=None,
mpilib=None,
infile=None,
files=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile)
self.machine = machine
self.os = os_
self.mpilib = mpilib
self.compiler_nodes = None # Listed from last to first
self.compiler = compiler
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"), ".cime",
"config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
# append any component specific config_tests.xml files
for comp in files.get_components("CONFIG_TESTS_FILE"):
if comp is None:
continue
infile = files.get_value("CONFIG_TESTS_FILE", attribute={"component":comp})
if os.path.isfile(infile):
self.read(infile)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("PIO_SPEC_FILE")
EntryID.__init__(self, infile)
def __init__(self, infile=None, files=None):
if infile is None:
if files is None:
files = Files()
infile = files.get_value("GRIDS_SPEC_FILE")
logger.debug(" Grid specification file is %s" % infile)
GenericXML.__init__(self, infile)
def __init__(self, infile=None):
"""
initialize an object
"""
if infile is None:
files = Files()
infile = files.get_value("CONFIG_DRV_FILE")
EntryID.__init__(self,infile)
def __init__(self, infile=None, files=None):
"""
initialize an object interface to file config_tests.xml
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("CONFIG_TESTS_FILE")
GenericXML.__init__(self, infile)
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
_, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
" --test" % \
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset)
if self._project is not None:
create_newcase_cmd += " --project %s " % self._project
if self._output_root is not None:
create_newcase_cmd += " --output-root %s " % self._output_root
if test_mods is not None:
files = Files()
(component,modspath) = test_mods.split('/',1)
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " --user-mods-dir %s" % test_mod_file
if case_opts is not None:
for case_opt in case_opts: # pylint: disable=not-an-iterable
if case_opt.startswith('M'):
mpilib = case_opt[1:]
create_newcase_cmd += " --mpilib %s" % mpilib
logger.debug (" MPILIB set to %s" % mpilib)
if case_opt.startswith('N'):
ninst = case_opt[1:]
create_newcase_cmd += " --ninst %s" %ninst
logger.debug (" NINST set to %s" % ninst)
if case_opt.startswith('P'):
pesize = case_opt[1:]
create_newcase_cmd += " --pecount %s"%pesize
if self._queue is not None:
create_newcase_cmd += " --queue=%s" % self._queue
if self._walltime is not None:
create_newcase_cmd += " --walltime %s" % self._walltime
elif test in self._test_data and "options" in self._test_data[test] and \
"wallclock" in self._test_data[test]['options']:
create_newcase_cmd += " --walltime %s" % self._test_data[test]['options']['wallclock']
logger.debug("Calling create_newcase: " + create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self,infile=None):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/config/config_headers.xml'
"""
if infile is None:
files = Files()
infile = files.get_value('CASEFILE_HEADERS', resolved=True)
super(Headers, self).__init__(infile)
def __init__(self, infile=None, files=None):
if files is None:
files = Files()
if infile is None:
infile = files.get_value("GRIDS_SPEC_FILE")
logger.debug(" Grid specification file is %s" % infile)
schema = files.get_schema("GRIDS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._version = self.get_version()
self._comp_gridnames = self._get_grid_names()
def _get_component_config_data(self):
# attributes used for multi valued defaults ($attlist is a hash reference)
attlist = {"compset":self._compsetname, "grid":self._gridname}
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
# Add the group and elements for the config_files.xml
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(files, attlist)
drv_config_file = files.get_value("CONFIG_DRV_FILE")
drv_comp = Component(drv_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(drv_comp, attributes=attlist)
# loop over all elements of both component_classes and components - and get config_component_file for
# for each component
self._component_classes =drv_comp.get_valid_model_components()
if len(self._component_classes) > len(self._components):
self._components.append('sesp')
for i in xrange(1,len(self._component_classes)):
comp_class = self._component_classes[i]
comp_name = self._components[i-1]
node_name = 'CONFIG_' + comp_class + '_FILE'
# Add the group and elements for the config_files.xml
comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=False)
self.set_value(node_name, comp_config_file)
comp_config_file = self.get_resolved_value(comp_config_file)
expect(comp_config_file is not None,"No config file for component %s"%comp_name)
compobj = Component(comp_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(compobj, attributes=attlist)
for key,value in self.lookups.items():
result = self.set_value(key,value)
if result is not None:
del self.lookups[key]
def __init__(self, machine, files=None, infile=None):
"""
initialize an object
"""
if (infile is None):
if files is None:
files = Files()
infile = files.get_value("LTARCHIVE_SPEC_FILE", resolved=False)
infile = files.get_resolved_value(infile)
GenericXML.__init__(self, infile)
self.machine = machine
def __init__(self, infile=None, files=None):
"""
initialize a files object given input pes specification file
"""
if files is None:
files = Files()
if infile is None:
infile = files.get_value("INPUTDATA_SPEC_FILE")
schema = files.get_schema("INPUTDATA_SPEC_FILE")
logger.debug("DEBUG: infile is {}".format(infile))
GenericXML.__init__(self, infile, schema=schema)
self._servernode = None
def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
schema = files.get_schema("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._machobj = machobj
if version is not None:
# this is used in scripts_regression_tests to force version 2, it should not be used otherwise
self._version = version
else:
self._version = self.get_version()
self.machine = machobj.get_machine_name()
self.os = machobj.get_value("OS")
if compiler is None:
compiler = machobj.get_default_compiler()
self.compiler = compiler
if mpilib is None:
if compiler is None:
mpilib = machobj.get_default_MPIlib()
else:
mpilib = machobj.get_default_MPIlib(attributes={'compiler':compiler})
self.mpilib = mpilib
self.compiler_nodes = None # Listed from last to first
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
if self._version > 1.0:
schema_db = GenericXML(infile=schema)
compiler_vars = schema_db.get_child("{http://www.w3.org/2001/XMLSchema}group", attributes={"name":"compilerVars"})
choice = schema_db.get_child(name="{http://www.w3.org/2001/XMLSchema}choice", root=compiler_vars)
self.flag_vars = set(schema_db.get(elem, "name") for elem in schema_db.get_children(root=choice, attributes={"type":"flagsVar"}))
def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None):
"""
initialize an object
"""
if infile is None:
if files is None:
files = Files()
infile = files.get_value("COMPILERS_SPEC_FILE")
schema = files.get_schema("COMPILERS_SPEC_FILE")
GenericXML.__init__(self, infile, schema)
self._machobj = machobj
if version is not None:
# this is used in scripts_regression_tests to force version 2, it should not be used otherwise
self._version = version
else:
self._version = self.get_version()
self.machine = machobj.get_machine_name()
self.os = machobj.get_value("OS")
if mpilib is None:
mpilib = machobj.get_default_MPIlib()
self.mpilib = mpilib
if compiler is None:
compiler = machobj.get_default_compiler()
self.compiler = compiler
self.compiler_nodes = None # Listed from last to first
#Append the contents of $HOME/.cime/config_compilers.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.compiler is not None:
self.set_compiler(compiler)
if self._version > 1.0:
# Run an XPath query to extract the list of flag variable names.
ns = {"xs": "http://www.w3.org/2001/XMLSchema"}
flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']"
flag_elems = ET.parse(schema).getroot().findall(flag_xpath, ns)
self.flag_vars = set(elem.get('name') for elem in flag_elems)
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
_, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
if compiler != self._compiler:
raise StandardError("Test '%s' has compiler that does"
" not match instance compliler '%s'" % (test, self._compiler))
create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
" --project %s --test"%\
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset, self._project)
if test_mods is not None:
files = Files()
(component,modspath) = test_mods.split('/',1)
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " --user-mods-dir %s" % test_mod_file
if case_opts is not None:
for case_opt in case_opts:
if case_opt.startswith('M'):
mpilib = case_opt[1:]
create_newcase_cmd += " --mpilib %s" % mpilib
logger.debug (" MPILIB set to %s" % mpilib)
if case_opt.startswith('N'):
ninst = case_opt[1:]
create_newcase_cmd += " --ninst %s" %ninst
logger.debug (" NINST set to %s" % ninst)
pesize = re.match('P([SMLX][12]?)', case_opt)
if pesize:
create_newcase_cmd += " --pecount %s"%pesize.group(1)
logger.debug("Calling create_newcase: " + create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self, infile=None, files=None, machine=None):
"""
initialize an object
if a filename is provided it will be used,
otherwise if a files object is provided it will be used
otherwise create a files object from default values
"""
self.machine_node = None
self.machine = None
self.machines_dir = None
schema = None
if files is None:
files = Files()
if infile is None:
infile = files.get_value("MACHINES_SPEC_FILE")
schema = files.get_schema("MACHINES_SPEC_FILE")
logger.debug("Verifying using schema {}".format(schema))
self.machines_dir = os.path.dirname(infile)
GenericXML.__init__(self, infile, schema)
# Append the contents of $HOME/.cime/config_machines.xml if it exists
# This could cause problems if node matchs are repeated when only one is expected
local_infile = os.path.join(os.environ.get("HOME"),".cime","config_machines.xml")
logger.debug("Infile: {}".format(local_infile))
if os.path.exists(local_infile):
GenericXML.read(self, local_infile, schema)
if machine is None:
if "CIME_MACHINE" in os.environ:
machine = os.environ["CIME_MACHINE"]
else:
cime_config = get_cime_config()
if cime_config.has_option("main", "machine"):
machine = cime_config.get("main", "machine")
if machine is None:
machine = self.probe_machine_name()
expect(machine is not None, "Could not initialize machine object from {} or {}".format(infile, local_infile))
self.set_machine(machine)
def get_tests_from_xml(xml_machine=None,xml_category=None,xml_compiler=None, xml_testlist=None,
machine=None, compiler=None):
"""
Parse testlists for a list of tests
"""
listoftests = []
testlistfiles = []
if(machine is not None):
thismach=machine
if(compiler is not None):
thiscompiler = compiler
if(xml_testlist is not None):
expect(os.path.isfile(xml_testlist), "Testlist not found or not readable "+xml_testlist)
testlistfiles.append(xml_testlist)
else:
files = Files()
comps = files.get_components("TESTS_SPEC_FILE")
for comp in comps:
test_spec_file = files.get_value("TESTS_SPEC_FILE", {"component":comp})
if(os.path.isfile(test_spec_file)):
testlistfiles.append(test_spec_file)
for testlistfile in testlistfiles:
thistestlistfile = Testlist(testlistfile)
logger.debug("Testlist file is "+testlistfile)
logger.debug("xml_machine {} xml_category {} xml_compiler {}".format(xml_machine, xml_category, xml_compiler))
newtests = thistestlistfile.get_tests(xml_machine, xml_category, xml_compiler)
for test in newtests:
if(machine is None):
thismach = test["machine"]
if(compiler is None):
thiscompiler = test["compiler"]
test["name"] = CIME.utils.get_full_test_name(test["testname"], grid=test["grid"], compset=test["compset"],
machine=thismach, compiler=thiscompiler,
testmod=None if "testmods" not in test else test["testmods"])
logger.debug("Adding test {} with compiler {}".format(test["name"], test["compiler"]))
listoftests += newtests
logger.debug("Found {:d} tests".format(len(listoftests)))
return listoftests
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
test_case, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
if compiler != self._compiler:
raise StandardError("Test '%s' has compiler that does"
" not match instance compliler '%s'" % (test, self._compiler))
scratch_dir = self._machobj.get_value("CESMSCRATCHROOT")
if self._project is not None:
scratch_dir = scratch_dir.replace("$PROJECT", self._project)
sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset"\
" %s -testname %s -project %s -sharedlibroot %s" % \
(os.path.join(self._cime_root, "scripts", "create_newcase"),
self._cime_model, test_dir, grid, machine, compiler, compset,
test_case, self._project, sharedlibroot)
if test_case != 'PFS':
create_newcase_cmd += " -nosavetiming "
if case_opts is not None:
create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
if test_mods is not None:
files = Files()
(component,modspath) = test_mods.split('/',1)
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
logger.debug("Calling create_newcase: "+create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def get_components(self):
"""
return dictionary of the form [component_class:component],
e.g. [atm:cam], for all compset components
"""
files = Files()
drv_comp = Component(files.get_value("CONFIG_DRV_FILE"))
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
component_classes = drv_comp.get_valid_model_components()
components = self.get_compset_components()
# Note that component classes can have a bigger range than
# compents since stub esp (sesp) is an optional component - so
# need to take the min of the two below
comp_dict = {}
for i in xrange(0,len(components)):
comp_name = components[i]
comp_class = component_classes[i+1]
comp_dict[comp_class] = comp_name
return comp_dict
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
envtest = EnvTest(self._get_test_dir(test))
files = Files()
drv_config_file = files.get_value("CONFIG_DRV_FILE")
logging.info("Found drv_config_file %s" % drv_config_file)
drv_comp = Component(drv_config_file)
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
test_argv = "-testname %s -testroot %s" % (test, self._test_root)
if (self._generate):
test_argv += " -generate %s" % self._baseline_gen_name
envtest.set_value("BASELINE_NAME_GEN",self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE",os.path.join(self._baseline_gen_name,test))
if (self._compare):
test_argv += " -compare %s" % self._baseline_cmp_name
envtest.set_value("BASELINE_NAME_CMP",self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE",os.path.join(self._baseline_cmp_name,test))
envtest.set_value("TEST_ARGV",test_argv)
envtest.set_value("CLEANUP", ("TRUE" if self._clean else "FALSE"))
if (self._generate or self._compare):
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", "TRUE" if self._generate else "FALSE")
envtest.set_value("COMPARE_BASELINE", "TRUE" if self._compare else "FALSE")
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC",resolved=False))
envtest.write()
return True
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file, "CPL")
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname {} -testroot {}".format(test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate {}".format(self._baseline_gen_name)
basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test)
logger.debug("basegen_case is {}".format(basegen_case_fullpath))
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare {}".format(self._baseline_cmp_name)
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
tput_tolerance = self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False)
envtest.set_value("TEST_TPUT_TOLERANCE", 0.25 if tput_tolerance is None else tput_tolerance)
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are {} ".format(case_opts))
for opt in case_opts: # pylint: disable=not-an-iterable
logger.debug("case_opt is {}".format(opt))
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
logger.debug (" USE_ESMF_LIB set to TRUE")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to {}".format(opt))
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to {}".format(stop_option[opt]))
logger.debug (" STOP_N set to {}".format(opti))
elif opt.startswith('R'):
# R option is for testing in PTS_MODE or Single Column Model
# (SCM) mode
envtest.set_test_parameter("PTS_MODE", "TRUE")
# For PTS_MODE, compile with mpi-serial
envtest.set_test_parameter("MPILIB", "mpi-serial")
elif (opt.startswith('I') or # Marker to distinguish tests with same name - ignored
opt.startswith('M') or # handled in create_newcase
opt.startswith('P') or # handled in create_newcase
opt.startswith('N') or # handled in create_newcase
opt.startswith('C') or # handled in create_newcase
opt.startswith('V')): # handled in create_newcase
pass
elif opt.startswith('IOP'):
logger.warning("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '{}' ".format(opt))
envtest.write()
lock_file("env_run.xml", caseroot=test_dir, newname="env_run.orig.xml")
with Case(test_dir, read_only=False) as case:
if self._output_root is None:
self._output_root = case.get_value("CIME_OUTPUT_ROOT")
# if we are running a single test we don't need sharedlibroot
if len(self._tests) > 1 and self._cime_model != "e3sm":
case.set_value("SHAREDLIBROOT",
os.path.join(self._output_root,
"sharedlibroot.{}".format(self._test_id)))
envtest.set_initial_values(case)
case.set_value("TEST", True)
case.set_value("SAVE_TIMING", self._save_timing)
# Scale back build parallelism on systems with few cores
if self._model_build_cost > self._proc_pool:
case.set_value("GMAKE_J", self._proc_pool)
self._model_build_cost = self._proc_pool
return True, ""
def _create_newcase_phase(self, test):
###########################################################################
test_dir = self._get_test_dir(test)
_, case_opts, grid, compset,\
machine, compiler, test_mods = CIME.utils.parse_test_name(test)
create_newcase_cmd = "{} --case {} --res {} --compset {}"\
" --test".format(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, compset)
if machine is not None:
create_newcase_cmd += " --machine {}".format(machine)
if compiler is not None:
create_newcase_cmd += " --compiler {}".format(compiler)
if self._project is not None:
create_newcase_cmd += " --project {} ".format(self._project)
if self._output_root is not None:
create_newcase_cmd += " --output-root {} ".format(self._output_root)
if self._input_dir is not None:
create_newcase_cmd += " --input-dir {} ".format(self._input_dir)
if self._pesfile is not None:
create_newcase_cmd += " --pesfile {} ".format(self._pesfile)
if test_mods is not None:
files = Files()
(component, modspath) = test_mods.split('/',1)
testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
test_mod_file = os.path.join(testmods_dir, component, modspath)
if not os.path.exists(test_mod_file):
error = "Missing testmod file '{}'".format(test_mod_file)
self._log_output(test, error)
return False, error
create_newcase_cmd += " --user-mods-dir {}".format(test_mod_file)
mpilib = None
ninst = 1
ncpl = 1
if case_opts is not None:
for case_opt in case_opts: # pylint: disable=not-an-iterable
if case_opt.startswith('M'):
mpilib = case_opt[1:]
create_newcase_cmd += " --mpilib {}".format(mpilib)
logger.debug (" MPILIB set to {}".format(mpilib))
elif case_opt.startswith('N'):
expect(ncpl == 1,"Cannot combine _C and _N options")
ninst = case_opt[1:]
create_newcase_cmd += " --ninst {}".format(ninst)
logger.debug (" NINST set to {}".format(ninst))
elif case_opt.startswith('C'):
expect(ninst == 1,"Cannot combine _C and _N options")
ncpl = case_opt[1:]
create_newcase_cmd += " --ninst {} --multi-driver" .format(ncpl)
logger.debug (" NCPL set to {}" .format(ncpl))
elif case_opt.startswith('P'):
pesize = case_opt[1:]
create_newcase_cmd += " --pecount {}".format(pesize)
elif case_opt.startswith('V'):
driver = case_opt[1:]
create_newcase_cmd += " --driver {}".format(driver)
# create_test mpilib option overrides default but not explicitly set case_opt mpilib
if mpilib is None and self._mpilib is not None:
create_newcase_cmd += " --mpilib {}".format(self._mpilib)
logger.debug (" MPILIB set to {}".format(self._mpilib))
if self._queue is not None:
create_newcase_cmd += " --queue={}".format(self._queue)
if self._walltime is not None:
create_newcase_cmd += " --walltime {}".format(self._walltime)
else:
# model specific ways of setting time
if self._cime_model == "e3sm":
recommended_time = _get_time_est(test, self._baseline_root)
if recommended_time is not None:
create_newcase_cmd += " --walltime {}".format(recommended_time)
else:
if test in self._test_data and "options" in self._test_data[test] and \
"wallclock" in self._test_data[test]['options']:
create_newcase_cmd += " --walltime {}".format(self._test_data[test]['options']['wallclock'])
logger.debug("Calling create_newcase: " + create_newcase_cmd)
return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_DRV_FILE")
drv_comp = Component(drv_config_file)
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname %s -testroot %s" % (test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate %s" % self._baseline_gen_name
basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test)
logger.debug("basegen_case is %s"%basegen_case_fullpath)
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare %s" % self._baseline_cmp_name
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
if self._baseline_gen_name or self._baseline_cmp_name:
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are %s " %case_opts)
for opt in case_opts:
logger.debug("case_opt is %s" %opt)
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
envtest.set_test_parameter("COMP_INTERFACE", "ESMF")
logger.debug (" USE_ESMF_LIB set to TRUE")
logger.debug (" COMP_INTERFACE set to ESMF")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to %s" %opt)
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to %s" %stop_option[opt])
logger.debug (" STOP_N set to %s" %opti)
elif opt.startswith('M'):
# M option handled by create newcase
continue
elif opt.startswith('P'):
# P option handled by create newcase
continue
elif opt.startswith('N'):
# handled in create_newcase
continue
elif opt.startswith('IOP'):
logger.warn("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '%s' " %opt)
envtest.write()
lockedfiles = os.path.join(test_dir, "LockedFiles")
if not os.path.exists(lockedfiles):
os.mkdir(lockedfiles)
shutil.copy(os.path.join(test_dir,"env_run.xml"),
os.path.join(lockedfiles, "env_run.orig.xml"))
with Case(test_dir, read_only=False) as case:
case.set_value("SHAREDLIBROOT",
os.path.join(self._test_root,
"sharedlibroot.%s"%self._test_id))
envtest.set_initial_values(case)
if self._save_timing:
case.set_value("SAVE_TIMING", True)
return True
