def __init__(self, case, remove_dead_tasks=False):
self.case = case
self.remove_dead_tasks = remove_dead_tasks
drv_comp = Component()
models = drv_comp.get_valid_model_components()
for item in ["COMP", "NTASKS", "NTHRDS", "ROOTPE", "NINST", "PSTRID"]:
values = []
for model in models:
model = "CPL" if model == "DRV" else model
if item == "NINST" and model == "CPL":
values.append(0) # no NINST for CPL, set to zero so lists are some length for all items
else:
vid = "_".join([item, model])
value = case.get_value(vid)
setattr(self, vid, value)
values.append(value)
setattr(self, item, values)
for item in [
"MAX_TASKS_PER_NODE",
"PES_PER_NODE",
"PIO_NUMTASKS",
"PIO_ASYNC_INTERFACE",
"EXEROOT",
"COMPILER",
]:
setattr(self, item, case.get_value(item))
self.DEFAULT_RUN_EXE_TEMPLATE_STR = "__DEFAULT_RUN_EXE__"
self.MAX_TASKS_PER_NODE = 1 if self.MAX_TASKS_PER_NODE < 1 else self.MAX_TASKS_PER_NODE
self._compute_values()
def query_component(name, files, all_components=False, xml=False):
"""
query a component by name
"""
# Determine the valid component classes (e.g. atm) for the driver/cpl
# These are then stored in comps_array
components = get_components(files)
# Loop through the elements for each component class (in config_files.xml)
# and see if there is a match for the the target component in the component attribute
match_found = False
valid_components = []
config_exists = False
for comp in components:
string = "CONFIG_{}_FILE".format(comp)
config_file = None
# determine all components in string
root_dir_node_name = "COMP_ROOT_DIR_{}".format(comp)
components = files.get_components(root_dir_node_name)
if components is None:
components = files.get_components(string)
for item in components:
valid_components.append(item)
logger.debug("{}: valid_components {}".format(comp, valid_components))
# determine if config_file is on disk
if name is None:
config_file = files.get_value(string)
elif name in valid_components:
config_file = files.get_value(string,
attribute={"component": name})
logger.debug("query {}".format(config_file))
if config_file is not None:
match_found = True
config_exists = os.path.isfile(config_file)
break
if not all_components and not config_exists:
expect(config_exists,
"Cannot find config_file {} on disk".format(config_file))
elif all_components and not config_exists:
print("WARNING: Couldn't find config_file {} on disk".format(
config_file))
return
# If name is not a valid argument - exit with error
expect(
match_found,
"Invalid input argument {}, valid input arguments are {}".format(
name, valid_components),
)
# Check that file exists on disk, if not exit with error
expect((config_file),
"Cannot find any config_component.xml file for {}".format(name))
# determine component xml content
component = Component(config_file, "CPL")
if xml:
print("{}".format(component.get_raw_record().decode("UTF-8")))
else:
component.print_values()
def saveLogs(case, lid):
###############################################################################
logdir = case.get_value("LOGDIR")
if logdir is not None and len(logdir) > 0:
if not os.path.isdir(logdir):
os.makedirs(logdir)
caseroot = case.get_value("CASEROOT")
rundir = case.get_value("RUNDIR")
# get components
files = Files()
config_file = files.get_value("CONFIG_DRV_FILE")
component = Component(config_file)
comps = [x.lower() for x in component.get_valid_model_components()]
comps = [x.replace('drv', 'cpl') for x in comps]
model = [case.get_value("MODEL")]
comps = comps + model
# for each component, compress log files and copy to logdir
for comp in comps:
logfile = os.path.join(rundir, comp + '.log.' + lid)
if os.path.isfile(logfile):
f_in = open(logfile)
f_out = gzip.open(logfile + '.gz', 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove(logfile)
logfile_copy = logfile + '.gz'
shutil.copy(logfile_copy,
os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))
def get_components(files):
"""
Determine the valid component classes (e.g. atm) for the driver/cpl
These are then stored in comps_array
"""
infile = files.get_value("CONFIG_CPL_FILE")
config_drv = Component(infile, "CPL")
return config_drv.get_valid_model_components()
def _main_func(options, work_dir):
###############################################################################
# Initialize variables for the html template
html_dict = dict()
cesm_version = 'CESM2'
comp = ''
if options.comp:
comp = options.comp[0]
# Create a component object from the xml file
filename = options.inputfile[0]
expect(os.path.isfile(filename), "File %s does not exist" % filename)
component = Component(filename, comp)
helptext, html_dict = component.return_values()
# get the component tag from the command line args
comptag = ''
if options.comptag:
comptag = options.comptag[0]
# get the component version from the command line args
compversion = ''
if options.compversion:
compversion = options.compversion[0]
# load up jinja template
templateLoader = jinja2.FileSystemLoader(
searchpath='{0}/templates'.format(work_dir))
templateEnv = jinja2.Environment(loader=templateLoader)
# populate the template variables
tmplFile = 'compinputdef2html.tmpl'
template = templateEnv.get_template(tmplFile)
templateVars = {
'html_dict': html_dict,
'today': _now,
'cesm_version': cesm_version,
'comp': comp,
'comptag': comptag,
'compversion': compversion,
'hilight': hilight,
'closehilight': closehilight,
'helptext': helptext
}
# render the template
comp_tmpl = template.render(templateVars)
# write the output file
with open(options.htmlfile[0], 'w') as html:
html.write(comp_tmpl)
return 0
def _get_component_config_data(self):
# attributes used for multi valued defaults ($attlist is a hash reference)
attlist = {"compset": self._compsetname, "grid": self._gridname}
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
# Add the group and elements for the config_files.xml
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(files, attlist)
drv_config_file = files.get_value("CONFIG_DRV_FILE")
drv_comp = Component(drv_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(drv_comp, attributes=attlist)
# loop over all elements of both component_classes and components - and get config_component_file for
# for each component
self._component_classes = drv_comp.get_valid_model_components()
if len(self._component_classes) > len(self._components):
self._components.append('sesp')
for i in xrange(1, len(self._component_classes)):
comp_class = self._component_classes[i]
comp_name = self._components[i - 1]
node_name = 'CONFIG_' + comp_class + '_FILE'
# Add the group and elements for the config_files.xml
comp_config_file = files.get_value(node_name,
{"component": comp_name},
resolved=False)
self.set_value(node_name, comp_config_file)
comp_config_file = self.get_resolved_value(comp_config_file)
expect(comp_config_file is not None,
"No config file for component %s" % comp_name)
compobj = Component(comp_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(compobj, attributes=attlist)
for key, value in self.lookups.items():
result = self.set_value(key, value)
if result is not None:
del self.lookups[key]
def _get_component_config_data(self):
# attributes used for multi valued defaults ($attlist is a hash reference)
attlist = {"compset":self._compsetname, "grid":self._gridname}
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
# Add the group and elements for the config_files.xml
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(files, attlist)
drv_config_file = files.get_value("CONFIG_DRV_FILE")
drv_comp = Component(drv_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(drv_comp, attributes=attlist)
# loop over all elements of both component_classes and components - and get config_component_file for
# for each component
self._component_classes =drv_comp.get_valid_model_components()
if len(self._component_classes) > len(self._components):
self._components.append('sesp')
for i in xrange(1,len(self._component_classes)):
comp_class = self._component_classes[i]
comp_name = self._components[i-1]
node_name = 'CONFIG_' + comp_class + '_FILE'
# Add the group and elements for the config_files.xml
comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=False)
self.set_value(node_name, comp_config_file)
comp_config_file = self.get_resolved_value(comp_config_file)
expect(comp_config_file is not None,"No config file for component %s"%comp_name)
compobj = Component(comp_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(compobj, attributes=attlist)
for key,value in self.lookups.items():
result = self.set_value(key,value)
if result is not None:
del self.lookups[key]
def get_components(self):
"""
return dictionary of the form [component_class:component],
e.g. [atm:cam], for all compset components
"""
files = Files()
drv_comp = Component(files.get_value("CONFIG_DRV_FILE"))
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
component_classes = drv_comp.get_valid_model_components()
components = self.get_compset_components()
# Note that component classes can have a bigger range than
# compents since stub esp (sesp) is an optional component - so
# need to take the min of the two below
comp_dict = {}
for i in xrange(0,len(components)):
comp_name = components[i]
comp_class = component_classes[i+1]
comp_dict[comp_class] = comp_name
return comp_dict
def get_components(self):
"""
return dictionary of the form [component_class:component],
e.g. [atm:cam], for all compset components
"""
files = Files()
drv_comp = Component(files.get_value("CONFIG_DRV_FILE"))
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
component_classes = drv_comp.get_valid_model_components()
components = self.get_compset_components()
# Note that component classes can have a bigger range than
# compents since stub esp (sesp) is an optional component - so
# need to take the min of the two below
comp_dict = {}
for i in xrange(0, len(components)):
comp_name = components[i]
comp_class = component_classes[i + 1]
comp_dict[comp_class] = comp_name
return comp_dict
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file)
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value(
"TEST_MEMLEAK_TOLERANCE",
self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname %s -testroot %s" % (test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate %s" % self._baseline_gen_name
basegen_case_fullpath = os.path.join(self._baseline_root,
self._baseline_gen_name, test)
logger.debug("basegen_case is %s" % basegen_case_fullpath)
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE",
os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare %s" % self._baseline_cmp_name
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE",
os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
if self._baseline_gen_name or self._baseline_cmp_name:
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name
is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name
is not None)
envtest.set_value(
"CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC",
resolved=False))
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are %s " % case_opts)
for opt in case_opts:
logger.debug("case_opt is %s" % opt)
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug(" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
envtest.set_test_parameter("COMP_INTERFACE", "ESMF")
logger.debug(" USE_ESMF_LIB set to TRUE")
logger.debug(" COMP_INTERFACE set to ESMF")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug(" CALENDAR set to %s" % opt)
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {
"y": "nyears",
"m": "nmonths",
"d": "ndays",
"h": "nhours",
"s": "nseconds",
"n": "nsteps"
}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION", stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug(" STOP_OPTION set to %s" % stop_option[opt])
logger.debug(" STOP_N set to %s" % opti)
elif opt.startswith('M'):
# M option handled by create newcase
continue
elif opt.startswith('P'):
# P option handled by create newcase
continue
elif opt.startswith('N'):
# handled in create_newcase
continue
elif opt.startswith('IOP'):
logger.warn("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '%s' " % opt)
envtest.write()
lockedfiles = os.path.join(test_dir, "LockedFiles")
if not os.path.exists(lockedfiles):
os.mkdir(lockedfiles)
shutil.copy(os.path.join(test_dir, "env_run.xml"),
os.path.join(lockedfiles, "env_run.orig.xml"))
with Case(test_dir, read_only=False) as case:
if self._output_root is None:
self._output_root = case.get_value("CIME_OUTPUT_ROOT")
case.set_value(
"SHAREDLIBROOT",
os.path.join(self._output_root,
"sharedlibroot.%s" % self._test_id))
envtest.set_initial_values(case)
case.set_value("TEST", True)
if self._save_timing:
case.set_value("SAVE_TIMING", True)
def _main_func(options, work_dir):
###############################################################################
files = Files()
model_version = options.version[0]
comp_classes = ("CPL", "ATM", "LND", "ICE", "OCN", "ROF", "GLC", "WAV",
"ESP")
components = ["allactive"]
for comp in comp_classes:
components.extend(files.get_components(
"COMP_ROOT_DIR_{}".format(comp)))
compset_files = []
compset_dict = {}
for comp in components:
if comp != "nemo":
compset_file = files.get_value("COMPSETS_SPEC_FILE",
attribute={"component": comp})
if compset_file not in compset_files:
expect(os.path.isfile(compset_file),
"Could not find file {}".format(compset_file))
compset_files.append(compset_file)
compset = Compsets(infile=compset_file, files=files)
longnames = compset.get_compset_longnames()
for longname in longnames:
_, alias, science_support = compset.get_compset_match(
name=longname)
elements = longname.split("_")
numelems = len(elements)
expect(numelems > 7,
"This longname not supported {}".format(longname))
compset_dict[longname] = {
"alias": alias,
"science_support_grids": science_support,
"defined_by": comp,
"init_opt": elements[0],
"atm_opt": elements[1],
"lnd_opt": elements[2],
"seaice_opt": elements[3],
"ocn_opt": elements[4],
"rof_opt": elements[5],
"glc_opt": elements[6],
"wav_opt": elements[7]
}
for i in range(8, numelems):
if elements[i].startswith("BGC"):
compset_dict[longname].update(
{"bgc_opt": elements[i]})
elif 'ESP' in elements[i]:
compset_dict[longname].update(
{"esp_opt": elements[i]})
elif elements[i] == 'TEST':
logger.info("Longname is {}".format(longname))
else:
logger.warn(
"Unrecognized longname: {} {} {} ".format(
longname, i, elements[i]))
components = []
for element in elements:
if element.startswith("BGC%") or element.startswith(
"TEST"):
continue
else:
element_component = element.split('%')[0].lower()
if "ww" not in element_component:
element_component = re.sub(
r'[0-9]*', "", element_component)
components.append(element_component)
for i in range(1, len(components)):
comp_class = comp_classes[i]
comp_config_file = files.get_value(
"CONFIG_{}_FILE".format(comp_class),
{"component": components[i]})
compobj = Component(comp_config_file,
comp_class)
compset_dict[longname].update({
"{}_desc".format(comp_class):
compobj.get_description(longname)
})
## print ("compset_dict = {}".format(compset_dict))
# load up jinja template
templateLoader = jinja2.FileSystemLoader(
searchpath='{0}/templates'.format(work_dir))
templateEnv = jinja2.Environment(loader=templateLoader)
tmplFile = 'compsetdef2html.tmpl'
template = templateEnv.get_template(tmplFile)
#TODO change the template to just loop through the html_dict
templateVars = {
'compset_dict': compset_dict,
'today': _now,
'model_version': model_version
}
# render the template
comp_tmpl = template.render(templateVars)
# write the output file
with open(options.htmlfile[0], 'w') as html:
html.write(comp_tmpl)
return 0
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file, "CPL")
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname {} -testroot {}".format(test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate {}".format(self._baseline_gen_name)
basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test)
logger.debug("basegen_case is {}".format(basegen_case_fullpath))
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare {}".format(self._baseline_cmp_name)
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
tput_tolerance = self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False)
envtest.set_value("TEST_TPUT_TOLERANCE", 0.25 if tput_tolerance is None else tput_tolerance)
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are {} ".format(case_opts))
for opt in case_opts: # pylint: disable=not-an-iterable
logger.debug("case_opt is {}".format(opt))
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
logger.debug (" USE_ESMF_LIB set to TRUE")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to {}".format(opt))
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to {}".format(stop_option[opt]))
logger.debug (" STOP_N set to {}".format(opti))
elif opt.startswith('R'):
# R option is for testing in PTS_MODE or Single Column Model
# (SCM) mode
envtest.set_test_parameter("PTS_MODE", "TRUE")
# For PTS_MODE, compile with mpi-serial
envtest.set_test_parameter("MPILIB", "mpi-serial")
elif (opt.startswith('I') or # Marker to distinguish tests with same name - ignored
opt.startswith('M') or # handled in create_newcase
opt.startswith('P') or # handled in create_newcase
opt.startswith('N') or # handled in create_newcase
opt.startswith('C') or # handled in create_newcase
opt.startswith('V')): # handled in create_newcase
pass
elif opt.startswith('IOP'):
logger.warning("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '{}' ".format(opt))
envtest.write()
lock_file("env_run.xml", caseroot=test_dir, newname="env_run.orig.xml")
with Case(test_dir, read_only=False) as case:
if self._output_root is None:
self._output_root = case.get_value("CIME_OUTPUT_ROOT")
# if we are running a single test we don't need sharedlibroot
if len(self._tests) > 1 and self._cime_model != "e3sm":
case.set_value("SHAREDLIBROOT",
os.path.join(self._output_root,
"sharedlibroot.{}".format(self._test_id)))
envtest.set_initial_values(case)
case.set_value("TEST", True)
case.set_value("SAVE_TIMING", self._save_timing)
# Scale back build parallelism on systems with few cores
if self._model_build_cost > self._proc_pool:
case.set_value("GMAKE_J", self._proc_pool)
self._model_build_cost = self._proc_pool
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file)
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname %s -testroot %s" % (test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate %s" % self._baseline_gen_name
basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test)
logger.debug("basegen_case is %s"%basegen_case_fullpath)
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare %s" % self._baseline_cmp_name
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
if self._baseline_gen_name or self._baseline_cmp_name:
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
tput_tolerance = self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False)
envtest.set_value("TEST_TPUT_TOLERANCE", 0.25 if tput_tolerance is None else tput_tolerance)
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are %s " %case_opts)
for opt in case_opts:
logger.debug("case_opt is %s" %opt)
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
envtest.set_test_parameter("COMP_INTERFACE", "ESMF")
logger.debug (" USE_ESMF_LIB set to TRUE")
logger.debug (" COMP_INTERFACE set to ESMF")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to %s" %opt)
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to %s" %stop_option[opt])
logger.debug (" STOP_N set to %s" %opti)
elif opt.startswith('R'):
# R option is for testing in PTS_MODE or Single Column Model
# (SCM) mode
envtest.set_test_parameter("PTS_MODE", "TRUE")
# For PTS_MODE, compile with mpi-serial
envtest.set_test_parameter("MPILIB", "mpi-serial")
comps=["ATM","LND","ICE","OCN","CPL","GLC","ROF","WAV"]
for comp in comps:
envtest.set_test_parameter("NTASKS_"+comp,"1")
# Set latitude and longitude for the appropriate case
# Below for ARM97, default SCM test case
if 'A97' in test:
envtest.set_test_parameter("PTS_LAT", "36.6")
envtest.set_test_parameter("PTS_LON", "262.5")
elif opt.startswith('M'):
# M option handled by create newcase
continue
elif opt.startswith('P'):
# P option handled by create newcase
continue
elif opt.startswith('N'):
# handled in create_newcase
continue
elif opt.startswith('IOP'):
logger.warn("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '%s' " %opt)
envtest.write()
lock_file("env_run.xml", caseroot=test_dir, newname="env_run.orig.xml")
with Case(test_dir, read_only=False) as case:
if self._output_root is None:
self._output_root = case.get_value("CIME_OUTPUT_ROOT")
# if we are running a single test we don't need sharedlibroot
if len(self._tests) > 1 and get_model() != "acme":
case.set_value("SHAREDLIBROOT",
os.path.join(self._output_root,
"sharedlibroot.%s"%self._test_id))
envtest.set_initial_values(case)
case.set_value("TEST", True)
case.set_value("SAVE_TIMING", self._save_timing)
def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, reset=False):
###############################################################################
os.chdir(caseroot)
msg = "case.setup starting"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
cimeroot = os.environ["CIMEROOT"]
# Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
din_loc_root = case.get_value("DIN_LOC_ROOT")
testcase = case.get_value("TESTCASE")
expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
"inputdata root is not a directory: \"$din_loc_root\" ")
# Check that userdefine settings are specified before expanding variable
for vid, value in case:
expect(not (type(value) is str and "USERDEFINED_required_build" in value),
"Parameter '%s' must be defined" % vid)
# Create batch script
if reset or clean:
# Clean batch script
backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S")
if not os.path.isdir(backup_dir):
os.makedirs(backup_dir)
# back up relevant files
for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]:
for filename in glob.glob(fileglob):
shutil.copy(filename, backup_dir)
if os.path.exists("case.run"):
os.remove("case.run")
# only do the following if are NOT in testmode
if not test_mode:
# rebuild the models (even on restart)
case.set_value("BUILD_COMPLETE", False)
# backup and then clean test script
if os.path.exists("case.test"):
shutil.copy("case.test", backup_dir)
os.remove("case.test")
logger.info("Successfully cleaned test script case.test")
if os.path.exists("case.testdriver"):
shutil.copy("case.testdriver", backup_dir)
os.remove("case.testdriver")
logger.info("Successfully cleaned test script case.testdriver")
logger.info("Successfully cleaned batch script case.run")
logger.info("Successfully cleaned batch script case.run")
logger.info("Some files have been saved to %s" % backup_dir)
msg = "case.setup clean complete"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
if not clean:
drv_comp = Component()
models = drv_comp.get_valid_model_components()
models.remove("DRV")
mach, compiler, debug, mpilib = \
case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB")
expect(mach is not None, "xml variable MACH is not set")
# Create Macros file only if it does not exist
if not os.path.exists("Macros"):
logger.debug("Creating Macros file for %s" % mach)
compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib)
compilers.write_macros_file()
else:
logger.debug("Macros script already created ...skipping")
# Set tasks to 1 if mpi-serial library
if mpilib == "mpi-serial":
for vid, value in case:
if vid.startswith("NTASKS_") and value != 1:
case.set_value(vid, 1)
# Check ninst.
# In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
# Save ninst in a dict to use later in apply_user_mods
ninst = dict()
for comp in models:
comp_model = case.get_value("COMP_%s" % comp)
ninst[comp_model] = case.get_value("NINST_%s" % comp)
ntasks = case.get_value("NTASKS_%s" % comp)
if ninst[comp_model] > ntasks:
if ntasks == 1:
case.set_value("NTASKS_%s" % comp, ninst[comp_model])
else:
expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks))
expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"),
"it is not possible to run with OpenMP if using the NAG Fortran compiler")
if os.path.exists("case.run"):
logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
else:
_check_pelayouts_require_rebuild(case, models)
if os.path.exists("LockedFiles/env_build.xml"):
os.remove("LockedFiles/env_build.xml")
case.flush()
check_lockedfiles()
tm = TaskMaker(case)
mtpn = case.get_value("MAX_TASKS_PER_NODE")
pespn = case.get_value("PES_PER_NODE")
# This is hardcoded because on yellowstone by default we
# run with 15 pes per node
# but pay for 16 pes per node. See github issue #518
if case.get_value("MACH") == "yellowstone":
pespn = 16
pestot = tm.totaltasks
if mtpn > pespn:
pestot = pestot * (mtpn // pespn)
case.set_value("COST_PES", tm.num_nodes*pespn)
else:
# reset cost_pes to totalpes
case.set_value("COST_PES", 0)
case.set_value("TOTALPES", pestot)
# Compute cost based on PE count
pval = 1
pcnt = 0
while pval < pestot:
pval *= 2
pcnt += 6 # (scaling like sqrt(6/10))
pcost = 3 - pcnt / 10 # (3 is 64 with 6)
# Compute cost based on DEBUG
dcost = 3 if debug else 0
# Compute cost based on run length
# For simplicity, we use a heuristic just based on STOP_OPTION (not considering
# STOP_N), and only deal with options longer than ndays
lcost = 0
if "nmonth" in case.get_value("STOP_OPTION"):
# N months costs 30x as much as N days; since cost is based on log-base-2, add 5
lcost = 5
elif "nyear" in case.get_value("STOP_OPTION"):
# N years costs 365x as much as N days; since cost is based on log-base-2, add 9
lcost = 9
estcost = pcost + dcost + lcost
for cost in ["CCSM_CCOST", "CCSM_GCOST", "CCSM_TCOST", "CCSM_CCOST"]:
estcost += case.get_value(cost)
case.set_value("CCSM_PCOST", pcost)
case.set_value("CCSM_ESTCOST", estcost)
# create batch file
logger.info("Creating batch script case.run")
# Use BatchFactory to get the appropriate instance of a BatchMaker,
# use it to create our batch scripts
env_batch = case.get_env("batch")
for job in env_batch.get_jobs():
input_batch_script = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job))
if job == "case.test" and testcase is not None and not test_mode:
logger.info("Writing %s script" % job)
testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase)
# Short term fix to be removed when csh tests are removed
if not os.path.exists(testscript):
env_batch.make_batch_script(input_batch_script, job, case)
elif job != "case.test":
logger.info("Writing %s script" % job)
env_batch.make_batch_script(input_batch_script, job, case)
# Make a copy of env_mach_pes.xml in order to be able
# to check that it does not change once case.setup is invoked
logger.info("Locking file env_mach_pes.xml")
case.flush()
shutil.copy("env_mach_pes.xml", "LockedFiles")
# Create user_nl files for the required number of instances
if not os.path.exists("user_nl_cpl"):
logger.info("Creating user_nl_xxx files for components and cpl")
# loop over models
for model in models:
comp = case.get_value("COMP_%s" % model)
logger.info("Building %s usernl files"%model)
_build_usernl_files(case, model, comp)
if comp == "cism":
run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot))
_build_usernl_files(case, "drv", "cpl")
user_mods_path = case.get_value("USER_MODS_FULLPATH")
if user_mods_path is not None:
apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)
elif case.get_value("TEST"):
test_mods = parse_test_name(casebaseid)[6]
if test_mods is not None:
user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"), test_mods)
apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)
# Run preview namelists for scripts
logger.info("preview_namelists")
preview_namelists(case)
logger.info("See ./CaseDoc for component namelists")
logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")
# Create test script if appropriate
# Short term fix to be removed when csh tests are removed
if os.path.exists("env_test.xml"):
if not os.path.exists("case.test"):
logger.info("Starting testcase.setup")
run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot)
logger.info("Finished testcase.setup")
msg = "case.setup complete"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
# Record env information
env_module = case.get_env("mach_specific")
env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
with open("software_environment.txt", "w") as f:
f.write(env_module.list_modules())
run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
env >> software_environment.txt")
def _get_component_config_data(self):
# attributes used for multi valued defaults
# attlist is a dictionary used to determine the value element that has the most matches
attlist = {
"compset": self._compsetname,
"grid": self._gridname,
"cime_model": self._cime_model
}
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
# Add the group and elements for the config_files.xml
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(files, attlist)
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(drv_comp, attributes=attlist)
# Add the group and elements for env_batch
env_batch = self.get_env("batch")
env_batch.add_elements_by_group(drv_comp, attributes=attlist)
# loop over all elements of both component_classes and components - and get config_component_file for
# for each component
self._set_comp_classes(drv_comp.get_valid_model_components())
if len(self._component_classes) > len(self._components):
self._components.append('sesp')
# put anything in the lookups table into env objects
for key, value in self.lookups.items():
result = self.set_value(key, value)
if result is not None:
del self.lookups[key]
for i in xrange(1, len(self._component_classes)):
comp_class = self._component_classes[i]
comp_name = self._components[i - 1]
node_name = 'CONFIG_' + comp_class + '_FILE'
# Add the group and elements for the config_files.xml
comp_config_file = files.get_value(node_name,
{"component": comp_name},
resolved=False)
self.set_value(node_name, comp_config_file)
comp_config_file = self.get_resolved_value(comp_config_file)
expect(
comp_config_file is not None
and os.path.isfile(comp_config_file),
"Config file %s for component %s not found." %
(comp_config_file, comp_name))
compobj = Component(comp_config_file)
for env_file in self._env_entryid_files:
env_file.add_elements_by_group(compobj, attributes=attlist)
# final cleanup of lookups table
for key, value in self.lookups.items():
result = self.set_value(key, value)
if result is not None:
del self.lookups[key]
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_DRV_FILE")
drv_comp = Component(drv_config_file)
component_classes = drv_comp.get_valid_model_components()
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
test_argv = "-testname %s -testroot %s" % (test, self._test_root)
if self._generate:
test_argv += " -generate %s" % self._baseline_gen_name
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._compare:
test_argv += " -compare %s" % self._baseline_cmp_name
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
if self._generate or self._compare:
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._generate)
envtest.set_value("COMPARE_BASELINE", self._compare)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
"""
Add the test instructions from config_test to env_test in the case
"""
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are %s " %case_opts)
for opt in case_opts:
logger.debug("case_opt is %s" %opt)
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
envtest.set_test_parameter("COMP_INTERFACE", "ESMF")
logger.debug (" USE_ESMF_LIB set to TRUE")
logger.debug (" COMP_INTERFACE set to ESMF")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to %s" %opt)
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to %s" %stop_option[opt])
logger.debug (" STOP_N set to %s" %opti)
elif opt.startswith('M'):
# M option handled by create newcase
continue
elif opt.startswith('P'):
match1 = re.match('P([0-9]+)', opt)
match2 = re.match('P([0-9]+)x([0-9]+)', opt)
match3 = re.match('P[SMLX][12]?', opt)
opti_tasks = None
if match1:
opti_tasks = match1.group(1)
for component_class in component_classes:
if component_class == "DRV":
component_class = "CPL"
string = "NTASKS_" + component_class
envtest.set_test_parameter(string, opti_tasks)
string = "NTHRDS_" + component_class
envtest.set_test_parameter(string, str(1))
string = "ROOTPE_" + component_class
envtest.set_test_parameter(string, str(0))
opti_thrds = 1
elif match2:
opti_tasks = match2.group(1)
opti_thrds = match2.group(2)
for component_class in component_classes:
if component_class == "DRV":
component_class = "CPL"
string = "NTASKS_" + component_class
envtest.set_test_parameter(string, opti_tasks)
string = "NTHRDS_" + component_class
envtest.set_test_parameter(string, opti_thrds)
string = "ROOTPE_" + component_class
envtest.set_test_parameter(string, str(0))
elif match3:
# handled by create_newcase
continue
if not match3:
expect(opti_tasks is not None, "No match found for PE option %s"%opt)
logger.debug (" NTASKS_xxx set to %s" %opti_tasks)
logger.debug (" NTHRDS_xxx set to %s" %opti_thrds)
logger.debug (" ROOTPE_xxx set to %s 0")
elif opt.startswith('N'):
# handled in create_newcase
continue
elif opt.startswith('IOP'):
logger.warn("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '%s' " %opt)
envtest.write()
lockedfiles = os.path.join(test_dir, "LockedFiles")
try:
os.stat(lockedfiles)
except:
os.mkdir(lockedfiles)
shutil.copy(os.path.join(test_dir,"env_run.xml"),
os.path.join(lockedfiles, "env_run.orig.xml"))
case = Case(test_dir)
case.set_value("SHAREDLIBROOT",
os.path.join(self._test_root,
"sharedlibroot.%s"%self._test_id))
envtest.set_initial_values(case)
return True
