def run(self, skip_pnl=False):
"""
Do NOT override this method, this method is the framework that controls
the run phase. run_phase is the extension point that subclasses should use.
"""
success = True
start_time = time.time()
self._skip_pnl = skip_pnl
try:
self._resetup_case(RUN_PHASE)
with self._test_status:
self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS)
self.run_phase()
if self._case.get_value("GENERATE_BASELINE"):
self._phase_modifying_call(GENERATE_PHASE, self._generate_baseline)
if self._case.get_value("COMPARE_BASELINE"):
self._phase_modifying_call(BASELINE_PHASE, self._compare_baseline)
self._phase_modifying_call(MEMCOMP_PHASE, self._compare_memory)
self._phase_modifying_call(THROUGHPUT_PHASE, self._compare_throughput)
self._phase_modifying_call(MEMLEAK_PHASE, self._check_for_memleak)
self._phase_modifying_call(STARCHIVE_PHASE, self._st_archive_case_test)
except BaseException as e: # We want KeyboardInterrupts to generate FAIL status
success = False
if isinstance(e, CIMEError):
# Don't want to print stacktrace for a model failure since that
# is not a CIME/infrastructure problem.
excmsg = str(e)
else:
excmsg = "Exception during run:\n{}\n{}".format(str(e), traceback.format_exc())
append_testlog(excmsg)
raise
finally:
# Writing the run status should be the very last thing due to wait_for_tests
time_taken = time.time() - start_time
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
with self._test_status:
self._test_status.set_status(RUN_PHASE, status, comments=("time={:d}".format(int(time_taken))))
if success and get_model() == "e3sm":
save_test_time(self._case.get_value("BASELINE_ROOT"), self._casebaseid, time_taken)
if get_model() == "cesm" and self._case.get_value("GENERATE_BASELINE"):
baseline_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"))
generate_teststatus(self._caseroot, baseline_dir)
# We return success if the run phase worked; memleaks, diffs will not be taken into account
# with this return value.
return success
def test_f_createnewcase_with_user_compset(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testcreatenewcase_with_user_compset")
if os.path.exists(testdir):
shutil.rmtree(testdir)
cls._testdirs.append(testdir)
if utils.get_model() == "cesm":
if utils.get_cime_default_driver() == "nuopc":
pesfile = os.path.join(
utils.get_src_root(),
"components",
"cmeps",
"cime_config",
"config_pes.xml",
)
else:
pesfile = os.path.join(
utils.get_src_root(),
"components",
"cpl7",
"driver",
"cime_config",
"config_pes.xml",
)
else:
pesfile = os.path.join(
utils.get_src_root(), "driver-mct", "cime_config", "config_pes.xml"
)
args = (
"--case %s --compset 2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --pesfile %s --res f19_g16 --output-root %s --handle-preexisting-dirs=r"
% (testdir, pesfile, cls._testroot)
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args += " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"%s/create_newcase %s" % (self.SCRIPT_DIR, args), from_dir=self.SCRIPT_DIR
)
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
cls._do_teardown.append(testdir)
def bless_history(test_name, testcase_dir_for_test, baseline_name,
baseline_root, compiler, report_only, force):
###############################################################################
with Case(testcase_dir_for_test) as case:
if get_model() == "acme":
baseline_full_dir = os.path.join(baseline_root, compiler,
baseline_name,
case.get_value("CASEBASEID"))
else:
baseline_full_dir = os.path.join(baseline_root, baseline_name,
case.get_value("CASEBASEID"))
result, comments = compare_baseline(case,
baseline_dir=baseline_full_dir,
outfile_suffix=None)
if result:
logging.info("Diff appears to have been already resolved.")
return True, None
else:
print comments
if (not report_only and
(force or raw_input("Update this diff (y/n)? ").upper()
in ["Y", "YES"])):
result, comments = generate_baseline(
case, baseline_dir=baseline_full_dir)
if not result:
logging.warning("Hist file bless FAILED for test %s" %
test_name)
return False, "Generate baseline failed: %s" % comments
else:
print comments
return True, None
else:
return True, None
def test_cime_case_test_walltime_mgmt_3(self):
if utils.get_model() != "e3sm":
self.skipTest(
"Skipping walltime test. Depends on E3SM batch settings")
test_name = "ERS_P64.f19_g16_rx1.A"
casedir = self._create_test(
[
"--no-setup",
"--machine=blues",
"--non-local",
"--walltime=0:10:00",
test_name,
],
test_id=self._baseline_name,
env_changes="unset CIME_GLOBAL_WALLTIME &&",
)
result = self.run_cmd_assert_result(
"./xmlquery JOB_WALLCLOCK_TIME -N --subgroup=case.test --value",
from_dir=casedir,
)
self.assertEqual(result, "00:10:00")
result = self.run_cmd_assert_result(
"./xmlquery JOB_QUEUE -N --subgroup=case.test --value",
from_dir=casedir)
self.assertEqual(result,
"biggpu") # Not smart enough to select faster queue
def build_phase(self, sharedlib_only=False, model_only=False):
# Subtle issue: case1 is already in a writeable state since it tends to be opened
# with a with statement in all the API entrances in CIME. case2 was created via clone,
# not a with statement, so it's not in a writeable state, so we need to use a with
# statement here to put it in a writeable state.
with self._case2:
if self._separate_builds:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
self._activate_case2()
# Although we're doing separate builds, it still makes sense
# to share the sharedlibroot area with case1 so we can reuse
# pieces of the build from there.
if get_model() != "e3sm":
# We need to turn off this change for E3SM because it breaks
# the MPAS build system
self._case2.set_value("SHAREDLIBROOT",
self._case1.get_value("SHAREDLIBROOT"))
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
else:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
# pio_typename may be changed during the build if the default is not a
# valid value for this build, update case2 to reflect this change
for comp in self._case1.get_values("COMP_CLASSES"):
comp_pio_typename = "{}_PIO_TYPENAME".format(comp)
self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename))
# The following is needed when _case_two_setup has a case_setup call
# despite sharing the build (e.g., to change NTHRDS)
self._case2.set_value("BUILD_COMPLETE",True)
def build_phase(self, sharedlib_only=False, model_only=False):
# Subtle issue: case1 is already in a writeable state since it tends to be opened
# with a with statement in all the API entrances in CIME. case2 was created via clone,
# not a with statement, so it's not in a writeable state, so we need to use a with
# statement here to put it in a writeable state.
with self._case2:
if self._separate_builds:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
self._activate_case2()
# Although we're doing separate builds, it still makes sense
# to share the sharedlibroot area with case1 so we can reuse
# pieces of the build from there.
if get_model() != "e3sm":
# We need to turn off this change for E3SM because it breaks
# the MPAS build system
self._case2.set_value("SHAREDLIBROOT",
self._case1.get_value("SHAREDLIBROOT"))
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
else:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
# pio_typename may be changed during the build if the default is not a
# valid value for this build, update case2 to reflect this change
for comp in self._case1.get_values("COMP_CLASSES"):
comp_pio_typename = "{}_PIO_TYPENAME".format(comp)
self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename))
# The following is needed when _case_two_setup has a case_setup call
# despite sharing the build (e.g., to change NTHRDS)
self._case2.set_value("BUILD_COMPLETE",True)
def uses_kokkos(case):
###############################################################################
cam_target = case.get_value("CAM_TARGET")
# atm_comp = case.get_value("COMP_ATM") # scream does not use the shared kokkoslib for now
return get_model() == "e3sm" and cam_target in ("preqx_kokkos", "theta-l",
"theta-l_kokkos")
def _create_caseroot_sourcemods(self):
components = self.get_compset_components()
for component in components:
directory = os.path.join(self._caseroot,"SourceMods","src.%s"%component)
if not os.path.exists(directory):
os.makedirs(directory)
directory = os.path.join(self._caseroot, "SourceMods", "src.share")
if not os.path.exists(directory):
os.makedirs(directory)
directory = os.path.join(self._caseroot,"SourceMods","src.drv")
if not os.path.exists(directory):
os.makedirs(directory)
if get_model() == "cesm":
# Note: this is CESM specific, given that we are referencing cism explitly
if "cism" in components:
directory = os.path.join(self._caseroot, "SourceMods", "src.cism", "glimmer-cism")
if not os.path.exists(directory):
os.makedirs(directory)
readme_file = os.path.join(directory, "README")
str_to_write = """
Put source mods for the glimmer-cism library in the glimmer-cism subdirectory
This includes any files that are in the glimmer-cism subdirectory of $cimeroot/../components/cism
Anything else (e.g., mods to source_glc or drivers) goes in this directory, NOT in glimmer-cism/"""
with open(readme_file, "w") as fd:
fd.write(str_to_write)
def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False):
###########################################################################
if phase == RUN_PHASE and (self._no_batch or no_batch):
test_dir = self._get_test_dir(test)
total_pes = int(run_cmd_no_fail("./xmlquery TOTALPES --value", from_dir=test_dir))
threads = eval(run_cmd_no_fail("./xmlquery NTHRDS --value", from_dir=test_dir))
max_threads = 0
for item in threads:
_, comp_threads = item.split(":")
comp_threads = int(comp_threads)
if comp_threads > max_threads:
max_threads = comp_threads
max_cores = total_pes * max_threads
return max_cores
elif (phase == SHAREDLIB_BUILD_PHASE):
if get_model() == "cesm":
# Will force serialization of sharedlib builds
# TODO - instead of serializing, compute all library configs needed and build
# them all in parallel
for _, _, running_phase in threads_in_flight.values():
if (running_phase == SHAREDLIB_BUILD_PHASE):
return self._proc_pool + 1
return 1
elif (phase == MODEL_BUILD_PHASE):
# Model builds now happen in parallel
return self._model_build_cost
else:
return 1
def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False):
###########################################################################
if phase == RUN_PHASE and (self._no_batch or no_batch):
test_dir = self._get_test_dir(test)
total_pes = int(run_cmd_no_fail("./xmlquery TOTALPES --value", from_dir=test_dir))
threads = eval(run_cmd_no_fail("./xmlquery NTHRDS --value", from_dir=test_dir))
max_threads = 0
for item in threads:
_, comp_threads = item.split(":")
comp_threads = int(comp_threads)
if comp_threads > max_threads:
max_threads = comp_threads
max_cores = total_pes * max_threads
return max_cores
elif (phase == SHAREDLIB_BUILD_PHASE):
if get_model() == "cesm":
# Will force serialization of sharedlib builds
# TODO - instead of serializing, compute all library configs needed and build
# them all in parallel
for _, _, running_phase in threads_in_flight.values():
if (running_phase == SHAREDLIB_BUILD_PHASE):
return self._proc_pool + 1
return 1
elif (phase == MODEL_BUILD_PHASE):
# Model builds now happen in parallel
return self._model_build_cost
else:
return 1
def bless_namelists(test_name, test_dir, report_only, force, baseline_name, baseline_root):
###############################################################################
# Be aware that restart test will overwrite the original namelist files
# with versions of the files that should not be blessed. This forces us to
# re-run create_test.
# Update namelist files
logger.info("Test '{}' had namelist diff".format(test_name))
if (not report_only and
(force or six.moves.input("Update namelists (y/n)? ").upper() in ["Y", "YES"])):
if baseline_name is None:
stat, baseline_name, _ = run_cmd("./xmlquery --value BASELINE_NAME_CMP", from_dir=test_dir)
if stat != 0 or not baseline_name:
baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
if baseline_root is None:
stat, baseline_root, _ = run_cmd("./xmlquery --value BASELINE_ROOT", from_dir=test_dir)
if stat != 0 or not baseline_root:
return False, "Could not determine baseline root"
create_test_gen_args = " -g {} ".format(baseline_name if get_model() == "cesm" else " -g -b {} ".format(baseline_name))
stat, out, _ = run_cmd("{}/create_test {} -n {} --baseline-root {} -o".format(get_scripts_root(), test_name, create_test_gen_args, baseline_root), combine_output=True)
if stat != 0:
return False, "Namelist regen failed: '{}'".format(out)
else:
return True, None
else:
return True, None
def build_phase(self, sharedlib_only=False, model_only=False):
if self._separate_builds:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
self._activate_case2()
# Although we're doing separate builds, it still makes sense
# to share the sharedlibroot area with case1 so we can reuse
# pieces of the build from there.
if get_model() != "e3sm":
# We need to turn off this change for E3SM because it breaks
# the MPAS build system
self._case2.set_value("SHAREDLIBROOT",
self._case1.get_value("SHAREDLIBROOT"))
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
else:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
# pio_typename may be changed during the build if the default is not a
# valid value for this build, update case2 to reflect this change
for comp in self._case1.get_values("COMP_CLASSES"):
comp_pio_typename = "{}_PIO_TYPENAME".format(comp)
self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename))
# The following is needed when _case_two_setup has a case_setup call
# despite sharing the build (e.g., to change NTHRDS)
self._case2.set_value("BUILD_COMPLETE",True)
self._case2.flush()
def _build_model_thread(
config_dir,
compclass,
compname,
caseroot,
libroot,
bldroot,
incroot,
file_build,
thread_bad_results,
smp,
compiler,
):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning(
"WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd),
"Could not find buildlib for {}".format(compname))
compile_cmd = "COMP_CLASS={compclass} COMP_NAME={compname} {cmd} {caseroot} {libroot} {bldroot} ".format(
compclass=compclass,
compname=compname,
cmd=cmd,
caseroot=caseroot,
libroot=libroot,
bldroot=bldroot,
)
if get_model() != "ufs":
compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd)
if is_python_executable(cmd):
logging_options = get_logging_options()
if logging_options != "":
compile_cmd = compile_cmd + logging_options
with open(file_build, "w") as fd:
stat = run_cmd(compile_cmd,
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
if stat != 0:
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compname, file_build))
analyze_build_log(compclass, file_build, compiler)
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def build_phase(self, sharedlib_only=False, model_only=False):
if self._separate_builds:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
self._activate_case2()
# Although we're doing separate builds, it still makes sense
# to share the sharedlibroot area with case1 so we can reuse
# pieces of the build from there.
if get_model() != "e3sm":
# We need to turn off this change for E3SM because it breaks
# the MPAS build system
self._case2.set_value("SHAREDLIBROOT",
self._case1.get_value("SHAREDLIBROOT"))
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
else:
self._activate_case1()
self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
# pio_typename may be changed during the build if the default is not a
# valid value for this build, update case2 to reflect this change
for comp in self._case1.get_values("COMP_CLASSES"):
comp_pio_typename = "{}_PIO_TYPENAME".format(comp)
self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename))
# The following is needed when _case_two_setup has a case_setup call
# despite sharing the build (e.g., to change NTHRDS)
self._case2.set_value("BUILD_COMPLETE",True)
self._case2.flush()
def test_cime_case_test_custom_project(self):
test_name = "ERS_P1.f19_g16_rx1.A"
# have to use a machine both models know and one that doesn't put PROJECT in any key paths
if utils.get_model() == "e3sm":
machine = "mappy"
else:
machine = "melvin"
compiler = "gnu"
casedir = self._create_test(
[
"--no-setup",
"--machine={}".format(machine),
"--compiler={}".format(compiler),
"--project=testproj",
test_name,
"--mpilib=mpi-serial",
],
test_id=self._baseline_name,
env_changes="unset CIME_GLOBAL_WALLTIME &&",
)
result = self.run_cmd_assert_result(
"./xmlquery --value PROJECT --subgroup=case.test",
from_dir=casedir)
self.assertEqual(result, "testproj")
def compare_baseline(case, baseline_dir=None, outfile_suffix=""):
"""
compare the current test output to a baseline result
case - The case containing the hist files to be compared against baselines
baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config
outfile_suffix - if non-blank, then the cprnc output file name ends with
this suffix (with a '.' added before the given suffix). if None, no output file saved.
returns (SUCCESS, comments)
SUCCESS means all hist files matched their corresponding baseline
"""
rundir = case.get_value("RUNDIR")
if baseline_dir is None:
baselineroot = case.get_value("BASELINE_ROOT")
basecmp_dir = os.path.join(baselineroot, case.get_value("BASECMP_CASE"))
dirs_to_check = (baselineroot, basecmp_dir)
else:
basecmp_dir = baseline_dir
dirs_to_check = (basecmp_dir,)
for bdir in dirs_to_check:
if not os.path.isdir(bdir):
return False, "ERROR {} baseline directory '{}' does not exist".format(TEST_NO_BASELINES_COMMENT,bdir)
success, comments = _compare_hists(case, rundir, basecmp_dir, outfile_suffix=outfile_suffix)
if get_model() == "e3sm":
bless_log = os.path.join(basecmp_dir, BLESS_LOG_NAME)
if os.path.exists(bless_log):
last_line = open(bless_log, "r").readlines()[-1]
comments += "\n Most recent bless: {}".format(last_line)
return success, comments
def _setup_cs_files(self):
###########################################################################
try:
python_libs_root = CIME.utils.get_python_libs_root()
template_file = os.path.join(python_libs_root,
"cs.status.template")
template = open(template_file, "r").read()
template = template.replace("<PATH>",
os.path.join(self._cime_root,"scripts","Tools")).replace\
("<TESTID>", self._test_id).replace\
("<TESTROOT>", self._test_root)
if not os.path.exists(self._test_root):
os.makedirs(self._test_root)
cs_status_file = os.path.join(self._test_root,
"cs.status.{}".format(self._test_id))
with open(cs_status_file, "w") as fd:
fd.write(template)
os.chmod(
cs_status_file,
os.stat(cs_status_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
template_file = os.path.join(python_libs_root,
"cs.submit.template")
template = open(template_file, "r").read()
setup_cmd = "./case.setup" if self._no_setup else ":"
build_cmd = "./case.build" if self._no_build else ":"
test_cmd = "./case.submit"
template = template.replace("<SETUP_CMD>", setup_cmd).\
replace("<BUILD_CMD>", build_cmd).\
replace("<RUN_CMD>", test_cmd).\
replace("<TESTID>", self._test_id)
if self._no_run:
cs_submit_file = os.path.join(
self._test_root, "cs.submit.{}".format(self._test_id))
with open(cs_submit_file, "w") as fd:
fd.write(template)
os.chmod(
cs_submit_file,
os.stat(cs_submit_file).st_mode | stat.S_IXUSR
| stat.S_IXGRP)
if get_model() == "cesm":
template_file = os.path.join(python_libs_root,
"testreporter.template")
template = open(template_file, "r").read()
template = template.replace(
"<PATH>", os.path.join(self._cime_root, "scripts",
"Tools"))
testreporter_file = os.path.join(self._test_root,
"testreporter")
with open(testreporter_file, "w") as fd:
fd.write(template)
os.chmod(
testreporter_file,
os.stat(testreporter_file).st_mode
| stat.S_IXUSR | stat.S_IXGRP)
except Exception as e:
logger.warning("FAILED to set up cs files: {}".format(str(e)))
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = env_archive.make_child("components", attributes={"version":"2.0"})
model = get_model()
if 'drv' not in components:
components.append('drv')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component {}".format(comp))
else:
logger.debug("Archive file {} for component {} not found".format(infile,comp))
specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component {}".format(comp))
else:
logger.debug("adding archive spec for {}".format(comp))
env_archive.add_child(specs, root=components_node)
def _create_caseroot_sourcemods(self):
components = self.get_compset_components()
for component in components:
directory = os.path.join(self._caseroot, "SourceMods",
"src.%s" % component)
if not os.path.exists(directory):
os.makedirs(directory)
directory = os.path.join(self._caseroot, "SourceMods", "src.share")
if not os.path.exists(directory):
os.makedirs(directory)
directory = os.path.join(self._caseroot, "SourceMods", "src.drv")
if not os.path.exists(directory):
os.makedirs(directory)
if get_model() == "cesm":
# Note: this is CESM specific, given that we are referencing cism explitly
if "cism" in components:
directory = os.path.join(self._caseroot, "SourceMods",
"src.cism", "glimmer-cism")
if not os.path.exists(directory):
os.makedirs(directory)
readme_file = os.path.join(directory, "README")
str_to_write = """
Put source mods for the glimmer-cism library in the glimmer-cism subdirectory
This includes any files that are in the glimmer-cism subdirectory of $cimeroot/../components/cism
Anything else (e.g., mods to source_glc or drivers) goes in this directory, NOT in glimmer-cism/"""
with open(readme_file, "w") as fd:
fd.write(str_to_write)
def simple_test(self,
testdir,
expected_results,
extra_args="",
build_name=None):
# Need these flags to test dashboard if e3sm
if utils.get_model() == "e3sm" and build_name is not None:
extra_args += " -b %s" % build_name
expected_stat = 0
for expected_result in expected_results:
if not (expected_result == "PASS" or
(expected_result == "PEND" and "-n" in extra_args)):
expected_stat = utils.TESTS_FAILED_ERR_CODE
output = self.run_cmd_assert_result(
"%s/wait_for_tests -p ACME_test */TestStatus %s" %
(self.TOOLS_DIR, extra_args),
from_dir=testdir,
expected_stat=expected_stat,
)
lines = [
line for line in output.splitlines()
if (line.startswith("PASS") or line.startswith("FAIL")
or line.startswith("PEND"))
]
self.assertEqual(len(lines), len(expected_results))
for idx, line in enumerate(lines):
testname, status = test_utils.parse_test_status(line)
self.assertEqual(status, expected_results[idx])
self.assertEqual(testname, "Test_%d" % idx)
def test_cime_case_test_walltime_mgmt_8(self):
if utils.get_model() != "e3sm":
self.skipTest(
"Skipping walltime test. Depends on E3SM batch settings")
test_name = "SMS_P25600.f19_g16_rx1.A"
machine, compiler = "theta", "gnu"
casedir = self._create_test(
[
"--no-setup",
"--machine={}".format(machine),
"--compiler={}".format(compiler),
"--project e3sm",
test_name,
],
test_id=self._baseline_name,
env_changes="unset CIME_GLOBAL_WALLTIME &&",
)
result = self.run_cmd_assert_result(
"./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value",
from_dir=casedir,
)
self.assertEqual(result, "09:00:00")
result = self.run_cmd_assert_result(
"./xmlquery JOB_QUEUE --subgroup=case.test --value",
from_dir=casedir)
self.assertEqual(result, "default")
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = ET.Element("components")
components_node.set("version", "2.0")
model = get_model()
if 'cpl' not in components:
components.append('cpl')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_node("comp_archive_spec", {"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component %s"%comp)
else:
logger.debug("Archive file %s for component %s not found"%(infile,comp))
specs = self.get_optional_node("comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component %s"%comp)
else:
logger.debug("adding archive spec for %s"%comp)
components_node.append(specs)
env_archive.add_child(components_node)
def __init__(self, comp_interface="mct"):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/config/config_headers.xml'
"""
cimeroot = get_cime_root()
infile = os.path.join(cimeroot, "config", get_model(),
"config_files.xml")
expect(os.path.isfile(infile),
"Could not find or open file {}".format(infile))
schema = os.path.join(cimeroot, "config", "xml_schemas",
"entry_id.xsd")
EntryID.__init__(self, infile, schema=schema)
config_files_override = os.path.join(os.path.dirname(cimeroot),
".config_files.xml")
# variables COMP_ROOT_DIR_{} are mutable, all other variables are read only
self.COMP_ROOT_DIR = {}
self._comp_interface = comp_interface
# .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files
if os.path.isfile(config_files_override):
self.read(config_files_override)
self.overwrite_existing_entries()
def test_cime_case_test_walltime_mgmt_5(self):
if utils.get_model() != "e3sm":
self.skipTest(
"Skipping walltime test. Depends on E3SM batch settings")
test_name = "ERS_P1.f19_g16_rx1.A"
casedir = self._create_test(
["--no-setup", "--machine=blues", test_name],
test_id=self._baseline_name,
env_changes="unset CIME_GLOBAL_WALLTIME &&",
)
self.run_cmd_assert_result(
"./xmlchange JOB_QUEUE=slartibartfast --subgroup=case.test",
from_dir=casedir,
expected_stat=1,
)
self.run_cmd_assert_result(
"./xmlchange JOB_QUEUE=slartibartfast --force --subgroup=case.test",
from_dir=casedir,
)
result = self.run_cmd_assert_result(
"./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value",
from_dir=casedir,
)
self.assertEqual(result, "01:00:00")
result = self.run_cmd_assert_result(
"./xmlquery JOB_QUEUE --subgroup=case.test --value",
from_dir=casedir)
self.assertEqual(result, "slartibartfast")
def bless_namelists(test_name, report_only, force, baseline_name,
baseline_root):
###############################################################################
# Be aware that restart test will overwrite the original namelist files
# with versions of the files that should not be blessed. This forces us to
# re-run create_test.
# Update namelist files
logger.info("Test '{}' had namelist diff".format(test_name))
if (not report_only
and (force or six.moves.input("Update namelists (y/n)? ").upper()
in ["Y", "YES"])):
create_test_gen_args = " -g {} ".format(baseline_name if get_model(
) == "cesm" else " -g -b {} ".format(baseline_name))
stat, out, _ = run_cmd(
"{}/create_test {} -n {} --baseline-root {} -o".format(
get_scripts_root(), test_name, create_test_gen_args,
baseline_root),
combine_output=True)
if stat != 0:
return False, "Namelist regen failed: '{}'".format(out)
else:
return True, None
else:
return True, None
def setup(self, env_archive, components, files=None):
if files is None:
files = Files()
components_node = env_archive.make_child("components", attributes={"version":"2.0"})
model = get_model()
if 'drv' not in components:
components.append('drv')
if 'dart' not in components and model == 'cesm':
components.append('dart')
for comp in components:
infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})
if infile is not None and os.path.isfile(infile):
arch = Archive(infile=infile, files=files)
specs = arch.get_child(name="comp_archive_spec", attributes={"compname":comp})
else:
if infile is None:
logger.debug("No archive file defined for component {}".format(comp))
else:
logger.debug("Archive file {} for component {} not found".format(infile,comp))
specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
if specs is None:
logger.debug("No archive specs found for component {}".format(comp))
else:
logger.debug("adding archive spec for {}".format(comp))
env_archive.add_child(specs, root=components_node)
def test_n_createnewcase_bad_compset(self):
cls = self.__class__
model = utils.get_model()
testdir = os.path.join(cls._testroot, "testcreatenewcase_bad_compset")
if os.path.exists(testdir):
shutil.rmtree(testdir)
args = (
" --case %s --compset InvalidCompsetName --output-root %s --handle-preexisting-dirs=r "
% (testdir, cls._testroot)
)
if model == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"./create_newcase %s" % (args), from_dir=self.SCRIPT_DIR, expected_stat=1
)
self.assertFalse(os.path.exists(testdir))
def __init__(self, infile=None):
"""
initialize an object
"""
if infile is None:
infile = os.path.join(get_cime_root(), "cime_config", get_model(), "archive.xml")
GenericXML.__init__(self, infile)
def _restart_fake_phase(self):
# Swap out model.exe for one that emits node failures
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
driver = self._case.get_value("COMP_INTERFACE")
if driver == "nuopc":
logname = "drv"
else:
logname = "cpl"
fake_exe = """#!/bin/bash
fail_sentinel={0}
cpl_log={1}/{4}.log.$LID
model_log={1}/{2}.log.$LID
touch $cpl_log
touch $fail_sentinel
declare -i num_fails=$(cat $fail_sentinel | wc -l)
declare -i times_to_fail=${{NODEFAIL_NUM_FAILS:-3}}
if ((num_fails < times_to_fail)); then
echo FAKE FAIL >> $cpl_log
echo FAIL >> $fail_sentinel
echo '{3}' >> $model_log
sleep 1
exit -1
else
echo Insta pass
echo SUCCESSFUL TERMINATION > $cpl_log
fi
""".format(
self._fail_sentinel, rundir, get_model(), self._fail_str, logname
)
fake_exe_file = os.path.join(exeroot, "fake.sh")
with open(fake_exe_file, "w") as fd:
fd.write(fake_exe)
os.chmod(fake_exe_file, 0o755)
prev_run_exe = self._case.get_value("run_exe")
env_mach_specific = self._case.get_env("mach_specific")
env_mach_specific.set_value("run_exe", fake_exe_file)
self._case.flush(flushall=True)
# This flag is needed by mpt to run a script under mpiexec
mpilib = self._case.get_value("MPILIB")
if mpilib == "mpt":
os.environ["MPI_SHEPHERD"] = "true"
self.run_indv(suffix=None)
if mpilib == "mpt":
del os.environ["MPI_SHEPHERD"]
env_mach_specific = self._case.get_env("mach_specific")
env_mach_specific.set_value("run_exe", prev_run_exe)
self._case.flush(flushall=True)
def __init__(self):
"""
initialize an object
"""
expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database." )
self.root = None
GenericXML.__init__(self)
def __init__(self):
"""
initialize an object
"""
expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database." )
self.root = None
GenericXML.__init__(self, root_name_override="testrecord", read_only=False, infile="TestRecord.xml")
def __init__(self, infile=None):
"""
initialize an object
"""
if infile is None:
infile = os.path.join(get_cime_root(), "cime_config", get_model(),
"config_archive.xml")
GenericXML.__init__(self, infile)
def __init__(self):
"""
initialize an object
"""
expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database." )
self.root = None
GenericXML.__init__(self)
def test_success_recording(self):
if utils.get_model() != "e3sm":
self.skipTest("Skipping success recording tests. E3SM feature")
fake_test1 = "faketest1"
fake_test2 = "faketest2"
baseline_dir = os.path.join(self._baseline_area, self._baseline_name)
# Test initial state
was_success, last_pass, trans_fail = provenance.get_test_success(
baseline_dir, None, fake_test1, testing=True)
self.assertFalse(was_success, msg="Broken initial was_success")
self.assertEqual(last_pass, None, msg="Broken initial last_pass")
self.assertEqual(trans_fail, None, msg="Broken initial trans_fail")
# Test first result (test1 fails, test2 passes)
# test_name , success, commit , expP , expTF, baseline)
self._record_success(fake_test1, False, "AAA", None, "AAA",
baseline_dir)
self._record_success(fake_test2, True, "AAA", "AAA", None,
baseline_dir)
# Test second result matches first (no transition) (test1 fails, test2 passes)
# test_name , success, commit , expP , expTF, baseline)
self._record_success(fake_test1, False, "BBB", None, "AAA",
baseline_dir)
self._record_success(fake_test2, True, "BBB", "BBB", None,
baseline_dir)
# Test transition to new state (first real transition) (test1 passes, test2 fails)
# test_name , success, commit , expP , expTF, baseline)
self._record_success(fake_test1, True, "CCC", "CCC", "AAA",
baseline_dir)
self._record_success(fake_test2, False, "CCC", "BBB", "CCC",
baseline_dir)
# Test transition to new state (second real transition) (test1 fails, test2 passes)
# test_name , success, commit , expP , expTF, baseline)
self._record_success(fake_test1, False, "DDD", "CCC", "DDD",
baseline_dir)
self._record_success(fake_test2, True, "DDD", "DDD", "CCC",
baseline_dir)
# Test final repeat (test1 fails, test2 passes)
# test_name , success, commit , expP , expTF, baseline)
self._record_success(fake_test1, False, "EEE", "CCC", "DDD",
baseline_dir)
self._record_success(fake_test2, True, "EEE", "EEE", "CCC",
baseline_dir)
# Test final transition (test1 passes, test2 fails)
# test_name , success, commit , expP , expTF, baseline)
self._record_success(fake_test1, True, "FFF", "FFF", "DDD",
baseline_dir)
self._record_success(fake_test2, False, "FFF", "EEE", "FFF",
baseline_dir)
def __init__(self, machine, infile=None):
"""
initialize an object
"""
if (infile is None):
infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_lt_archive.xml")
GenericXML.__init__(self, infile)
self.machine = machine
def __init__(self):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/cime_config/config_headers.xml'
"""
infile = os.path.join(get_cime_root(), "cime_config", get_model(), "config_files.xml")
EntryID.__init__(self, infile)
def compare_namelists(case, baseline_name, baseline_root, logfile_name, compiler):
###############################################################################
if get_model() == "acme":
baseline_name = os.path.join(compiler, baseline_name)
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
success = case_cmpgen_namelists(case, compare=True, compare_name=baseline_name, baseline_root=baseline_root, logfile_name=logfile_name)
logging.getLogger().setLevel(log_lvl)
return success
def build_phase(self, sharedlib_only=False, model_only=False):
if "TESTBUILDFAIL_PASS" in os.environ:
TESTRUNPASS.build_phase(self, sharedlib_only, model_only)
else:
if (not sharedlib_only):
blddir = self._case.get_value("EXEROOT")
bldlog = os.path.join(blddir, "{}.bldlog.{}".format(get_model(), get_timestamp("%y%m%d-%H%M%S")))
with open(bldlog, "w") as fd:
fd.write("BUILD FAIL: Intentional fail for testing infrastructure")
expect(False, "BUILD FAIL: Intentional fail for testing infrastructure")
def __init__(self):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/cime_config/config_headers.xml'
"""
infile = os.path.join(get_cime_root(), "cime_config", get_model(),
"config_files.xml")
EntryID.__init__(self, infile)
def _create_caseroot_tools(self):
cime_model = get_model()
machines_dir = os.path.abspath(self.get_value("MACHDIR"))
toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
caseroot = self.get_value("CASEROOT")
# setup executable files in caseroot/
exefiles = (os.path.join(toolsdir, "case.setup"),
os.path.join(toolsdir, "case.build"),
os.path.join(toolsdir, "case.submit"),
os.path.join(toolsdir, "preview_namelists"),
os.path.join(toolsdir, "testcase.setup"),
os.path.join(toolsdir, "check_input_data"),
os.path.join(toolsdir, "check_case"),
os.path.join(toolsdir, "archive_metadata.sh"),
os.path.join(toolsdir, "create_production_test"),
os.path.join(toolsdir, "xmlchange"),
os.path.join(toolsdir, "xmlquery"))
try:
for exefile in exefiles:
destfile = os.path.join(caseroot,os.path.basename(exefile))
os.symlink(exefile, destfile)
except Exception as e:
logger.warning("FAILED to set up exefiles: %s" % str(e))
# set up utility files in caseroot/Tools/
toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
os.path.join(toolsdir, "lt_archive.sh"),
os.path.join(toolsdir, "st_archive"),
os.path.join(toolsdir, "getTiming"),
os.path.join(toolsdir, "compare_namelists.pl"),
os.path.join(machines_dir,"taskmaker.pl"),
os.path.join(machines_dir,"Makefile"),
os.path.join(machines_dir,"mkSrcfiles"),
os.path.join(machines_dir,"mkDepends"))
for toolfile in toolfiles:
destfile = os.path.join(caseroot,"Tools",os.path.basename(toolfile))
expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
try:
os.symlink(toolfile, destfile)
except Exception as e:
logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))
# Copy any system or compiler Depends files to the case
machine = self.get_value("MACH")
compiler = self.get_value("COMPILER")
for dep in (machine, compiler):
dfile = "Depends.%s"%dep
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(caseroot,dfile))
dfile = "Depends.%s.%s"%(machine,compiler)
if os.path.isfile(os.path.join(machines_dir,dfile)):
shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(caseroot, dfile))
def simple_test(self, manual_timing=False):
if self.NO_FORTRAN_RUN:
self.skipTest("Skipping fortran test")
timing_flag = "" if manual_timing else "--save-timing"
driver = utils.get_cime_default_driver()
if driver == "mct":
walltime = "00:15:00"
else:
walltime = "00:30:00"
self._create_test(
[
"SMS_Ln9_P1.f19_g16_rx1.A", timing_flag,
"--walltime=" + walltime
],
test_id=self._baseline_name,
)
statuses = glob.glob("%s/*%s/TestStatus" %
(self._testroot, self._baseline_name))
self.assertEqual(
len(statuses),
1,
msg="Should have had exactly one match, found %s" % statuses,
)
casedir = os.path.dirname(statuses[0])
with Case(casedir, read_only=True) as case:
lids = utils.get_lids(case)
timing_dir = case.get_value("SAVE_TIMING_DIR")
casename = case.get_value("CASE")
self.assertEqual(len(lids), 1, msg="Expected one LID, found %s" % lids)
if manual_timing:
self.run_cmd_assert_result("cd %s && %s/save_provenance postrun" %
(casedir, self.TOOLS_DIR))
if utils.get_model() == "e3sm":
provenance_glob = os.path.join(
timing_dir,
"performance_archive",
getpass.getuser(),
casename,
lids[0] + "*",
)
provenance_dirs = glob.glob(provenance_glob)
self.assertEqual(
len(provenance_dirs),
1,
msg=
"wrong number of provenance dirs, expected 1, got {}, looked for {}"
.format(provenance_dirs, provenance_glob),
)
self.verify_perms("".join(provenance_dirs))
def __init__(self):
"""
initialize an object
"""
expect(get_model() == 'cesm',
"testreport is only meant to populate the CESM test database.")
self.root = None
GenericXML.__init__(self,
root_name_override="testrecord",
read_only=False,
infile="TestRecord.xml")
def generate_teststatus(testdir, baseline_dir):
"""
CESM stores it's TestStatus file in baselines. Do not let exceptions
escape from this function.
"""
if get_model() == "cesm":
try:
if not os.path.isdir(baseline_dir):
os.makedirs(baseline_dir)
safe_copy(os.path.join(testdir, TEST_STATUS_FILENAME), baseline_dir)
except Exception as e:
logger.warning("Could not copy {} to baselines, {}".format(os.path.join(testdir, TEST_STATUS_FILENAME), str(e)))
def __init__(self):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/config/config_headers.xml'
"""
cimeroot = get_cime_root()
infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml")
expect(os.path.isfile(infile), "Could not find or open file {}".format(infile))
schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd")
EntryID.__init__(self, infile, schema=schema)
def generate_teststatus(testdir, baseline_dir):
"""
CESM stores it's TestStatus file in baselines. Do not let exceptions
escape from this function.
"""
if get_model() == "cesm":
try:
if not os.path.isdir(baseline_dir):
os.makedirs(baseline_dir)
safe_copy(os.path.join(testdir, TEST_STATUS_FILENAME), baseline_dir)
except Exception as e:
logger.warning("Could not copy {} to baselines, {}".format(os.path.join(testdir, TEST_STATUS_FILENAME), str(e)))
def setUp(self):
if utils.get_model() != "e3sm":
self.skipTest("Skipping Jenkins tests. E3SM feature")
super().setUp()
# Need to run in a subdir in order to not have CTest clash. Name it
# such that it should be cleaned up by the parent tearDown
self._testdir = os.path.join(self._testroot,
"jenkins_test_%s" % self._baseline_name)
os.makedirs(self._testdir)
# Change root to avoid clashing with other jenkins_generic_jobs
self._jenkins_root = os.path.join(self._testdir, "J")
def write_provenance_info(machine, test_compiler, test_mpilib, test_root):
curr_commit = get_current_commit(repo=CIMEROOT)
logging.info("Testing commit %s" % curr_commit)
cime_model = get_model()
logging.info("Using cime_model = %s" % cime_model)
logging.info("Testing machine = %s" % machine.get_machine_name())
if test_compiler is not None:
logging.info("Testing compiler = %s" % test_compiler)
if test_mpilib is not None:
logging.info("Testing mpilib = %s" % test_mpilib)
logging.info("Test root: %s" % test_root)
logging.info("Test driver: %s" % CIME.utils.get_cime_default_driver())
logging.info("Python version {}\n".format(sys.version))
def _restart_fake_phase(self):
# Swap out model.exe for one that emits node failures
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
fake_exe = \
"""#!/bin/bash
fail_sentinel={0}
cpl_log={1}/cpl.log.$LID
model_log={1}/{2}.log.$LID
touch $cpl_log
touch $fail_sentinel
declare -i num_fails=$(cat $fail_sentinel | wc -l)
declare -i times_to_fail=${{NODEFAIL_NUM_FAILS:-3}}
if ((num_fails < times_to_fail)); then
echo FAKE FAIL >> $cpl_log
echo FAIL >> $fail_sentinel
echo '{3}' >> $model_log
sleep 1
exit -1
else
echo Insta pass
echo SUCCESSFUL TERMINATION > $cpl_log
fi
""".format(self._fail_sentinel, rundir, get_model(), self._fail_str)
fake_exe_file = os.path.join(exeroot, "fake.sh")
with open(fake_exe_file, "w") as fd:
fd.write(fake_exe)
os.chmod(fake_exe_file, 0o755)
prev_run_exe = self._case.get_value("run_exe")
env_mach_specific = self._case.get_env("mach_specific")
env_mach_specific.set_value("run_exe", fake_exe_file)
self._case.flush(flushall=True)
# This flag is needed by mpt to run a script under mpiexec
mpilib = self._case.get_value("MPILIB")
if mpilib == "mpt":
os.environ["MPI_SHEPHERD"] = "true"
self.run_indv(suffix=None)
if mpilib == "mpt":
del os.environ["MPI_SHEPHERD"]
env_mach_specific = self._case.get_env("mach_specific")
env_mach_specific.set_value("run_exe", prev_run_exe)
self._case.flush(flushall=True)
def __init__(self, batch_system=None, machine=None, infile=None):
"""
initialize an object
"""
if infile is None:
infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_batch.xml")
GenericXML.__init__(self, infile)
self.batch_system_node = None
self.machine_node = None
self.batch_system = batch_system
self.machine = machine
if self.batch_system is not None:
self.set_batch_system(self.batch_system, machine=machine)
def compare_history(case, baseline_name, baseline_root, log_id, compiler):
###############################################################################
if get_model() == "acme":
baseline_full_dir = os.path.join(baseline_root, compiler, baseline_name, case.get_value("CASEBASEID"))
else:
baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
outfile_suffix = "{}.{}".format(baseline_name, log_id)
try:
result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
outfile_suffix=outfile_suffix)
except IOError:
result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
outfile_suffix=None)
return result, comments
def run(self, skip_pnl=False):
"""
Do NOT override this method, this method is the framework that controls
the run phase. run_phase is the extension point that subclasses should use.
"""
success = True
start_time = time.time()
self._skip_pnl = skip_pnl
try:
self._resetup_case(RUN_PHASE)
with self._test_status:
self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS)
self.run_phase()
if self._case.get_value("GENERATE_BASELINE"):
self._generate_baseline()
if self._case.get_value("COMPARE_BASELINE"):
self._compare_baseline()
self._check_for_memleak()
except BaseException as e:
success = False
msg = e.__str__()
if "RUN FAIL" in msg:
# Don't want to print stacktrace for a model failure since that
# is not a CIME/infrastructure problem.
excmsg = msg
else:
excmsg = "Exception during run:\n{}\n{}".format(msg, traceback.format_exc())
logger.warning(excmsg)
append_testlog(excmsg)
# Writing the run status should be the very last thing due to wait_for_tests
time_taken = time.time() - start_time
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
with self._test_status:
self._test_status.set_status(RUN_PHASE, status, comments=("time={:d}".format(int(time_taken))))
if success and get_model() == "e3sm":
save_test_time(self._case.get_value("BASELINE_ROOT"), self._casebaseid, time_taken)
# We return success if the run phase worked; memleaks, diffs will not be taken into account
# with this return value.
return success
def _setup_cs_files(self):
###########################################################################
try:
python_libs_root = CIME.utils.get_python_libs_root()
template_file = os.path.join(python_libs_root, "cs.status.template")
template = open(template_file, "r").read()
template = template.replace("<PATH>",
os.path.join(self._cime_root,"scripts","Tools")).replace\
("<TESTID>", self._test_id).replace\
("<TESTROOT>", self._test_root)
if not os.path.exists(self._test_root):
os.makedirs(self._test_root)
cs_status_file = os.path.join(self._test_root, "cs.status.{}".format(self._test_id))
with open(cs_status_file, "w") as fd:
fd.write(template)
os.chmod(cs_status_file, os.stat(cs_status_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
template_file = os.path.join(python_libs_root, "cs.submit.template")
template = open(template_file, "r").read()
setup_cmd = "./case.setup" if self._no_setup else ":"
build_cmd = "./case.build" if self._no_build else ":"
test_cmd = "./case.submit"
template = template.replace("<SETUP_CMD>", setup_cmd).\
replace("<BUILD_CMD>", build_cmd).\
replace("<RUN_CMD>", test_cmd).\
replace("<TESTID>", self._test_id)
if self._no_run:
cs_submit_file = os.path.join(self._test_root, "cs.submit.{}".format(self._test_id))
with open(cs_submit_file, "w") as fd:
fd.write(template)
os.chmod(cs_submit_file,
os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
if get_model() == "cesm":
template_file = os.path.join(python_libs_root, "testreporter.template")
template = open(template_file, "r").read()
template = template.replace("<PATH>",
os.path.join(self._cime_root, "scripts", "Tools"))
testreporter_file = os.path.join(self._test_root, "testreporter")
with open(testreporter_file, "w") as fd:
fd.write(template)
os.chmod(testreporter_file, os.stat(testreporter_file).st_mode
| stat.S_IXUSR | stat.S_IXGRP)
except Exception as e:
logger.warning("FAILED to set up cs files: {}".format(str(e)))
def bless_namelists(test_name, report_only, force, baseline_name, baseline_root):
###############################################################################
# Be aware that restart test will overwrite the original namelist files
# with versions of the files that should not be blessed. This forces us to
# re-run create_test.
# Update namelist files
print("Test '{}' had namelist diff".format(test_name))
if (not report_only and
(force or six.moves.input("Update namelists (y/n)? ").upper() in ["Y", "YES"])):
create_test_gen_args = " -g {} ".format(baseline_name if get_model() == "cesm" else " -g -b {} ".format(baseline_name))
stat, _, err = run_cmd("{}/create_test {} -n {} --baseline-root {} -o".format(get_scripts_root(), test_name, create_test_gen_args, baseline_root))
if stat != 0:
return False, "Namelist regen failed: '{}'".format(err)
else:
return True, None
else:
return True, None
def __init__(self, batch_system=None, machine=None, infile=None):
"""
initialize an object
"""
if infile is None:
infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_batch.xml")
GenericXML.__init__(self, infile)
self.batch_system_node = None
self.machine_node = None
self.batch_system = batch_system
self.machine = machine
#Append the contents of $HOME/.cime/config_batch.xml if it exists
#This could cause problems if node matchs are repeated when only one is expected
infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml")
if os.path.exists(infile):
GenericXML.read(self, infile)
if self.batch_system is not None:
self.set_batch_system(self.batch_system, machine=machine)
def __init__(self):
"""
initialize an object
>>> files = Files()
>>> files.get_value('CASEFILE_HEADERS',resolved=False)
'$CIMEROOT/config/config_headers.xml'
"""
cimeroot = get_cime_root()
infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml")
expect(os.path.isfile(infile), "Could not find or open file {}".format(infile))
schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd")
EntryID.__init__(self, infile, schema=schema)
config_files_override = os.path.join(os.path.dirname(cimeroot),".config_files.xml")
# variables COMP_ROOT_DIR_{} are mutable, all other variables are read only
self.COMP_ROOT_DIR = {}
# .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files
if os.path.isfile(config_files_override):
self.read(config_files_override)
self.overwrite_existing_entries()
def bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, compiler, report_only, force):
###############################################################################
with Case(testcase_dir_for_test) as case:
if get_model() == "acme":
baseline_full_dir = os.path.join(baseline_root, compiler, baseline_name, case.get_value("CASEBASEID"))
else:
baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
result, comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=None)
if result:
return True, None
else:
print(comments)
if (not report_only and
(force or six.moves.input("Update this diff (y/n)? ").upper() in ["Y", "YES"])):
result, comments = generate_baseline(case, baseline_dir=baseline_full_dir)
if not result:
logging.warning("Hist file bless FAILED for test {}".format(test_name))
return False, "Generate baseline failed: {}".format(comments)
else:
print(comments)
return True, None
else:
return True, None
import CIME.utils
from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root, get_model
from CIME.XML.machines import Machines
import six, sys, os
# Expect that, if a model wants to use python-based test lists, they will have a file
# config/$model/tests.py , containing a test dictionary called _TESTS
sys.path.insert(0, os.path.join(get_cime_root(), "config", get_model()))
_ALL_TESTS = {}
try:
from tests import _TESTS # pylint: disable=import-error
_ALL_TESTS.update(_TESTS)
except:
pass
# Here are the tests belonging to e3sm suites. Format is
# <test>.<grid>.<compset>.
# suite_name -> (inherits_from, timelimit, [test [, mods[, machines]]])
# To elaborate, if no mods are needed, a string representing the testname is all that is needed.
# If testmods are needed, a 2-ple must be provided (test, mods)
# If you want to restrict the test mods to certain machines, than a 3-ple is needed (test, mods, [machines])
_CIME_TESTS = {
"cime_tiny" : (None, "0:10:00",
("ERS.f19_g16_rx1.A",
"NCK.f19_g16_rx1.A")
),
"cime_test_only_pass" : (None, "0:10:00",
def __init__(self, test_names, test_data=None,
no_run=False, no_build=False, no_setup=False, no_batch=None,
test_root=None, test_id=None,
machine_name=None, compiler=None,
baseline_root=None, baseline_cmp_name=None, baseline_gen_name=None,
clean=False, namelists_only=False,
project=None, parallel_jobs=None,
walltime=None, proc_pool=None,
use_existing=False, save_timing=False, queue=None,
allow_baseline_overwrite=False, output_root=None,
force_procs=None, force_threads=None, mpilib=None,
input_dir=None, pesfile=None, mail_user=None, mail_type=None):
###########################################################################
self._cime_root = CIME.utils.get_cime_root()
self._cime_model = get_model()
self._save_timing = save_timing
self._queue = queue
self._test_data = {} if test_data is None else test_data # Format: {test_name -> {data_name -> data}}
self._mpilib = mpilib # allow override of default mpilib
self._completed_tests = 0
self._input_dir = input_dir
self._pesfile = pesfile
self._allow_baseline_overwrite = allow_baseline_overwrite
self._mail_user = mail_user
self._mail_type = mail_type
self._machobj = Machines(machine=machine_name)
self._model_build_cost = 4
# If user is forcing procs or threads, re-write test names to reflect this.
if force_procs or force_threads:
test_names = _translate_test_names_for_new_pecount(test_names, force_procs, force_threads)
self._no_setup = no_setup
self._no_build = no_build or no_setup or namelists_only
self._no_run = no_run or self._no_build
self._output_root = output_root
# Figure out what project to use
if project is None:
self._project = CIME.utils.get_project()
if self._project is None:
self._project = self._machobj.get_value("PROJECT")
else:
self._project = project
# We will not use batch system if user asked for no_batch or if current
# machine is not a batch machine
self._no_batch = no_batch or not self._machobj.has_batch_system()
expect(not (self._no_batch and self._queue is not None),
"Does not make sense to request a queue without batch system")
# Determine and resolve test_root
if test_root is not None:
self._test_root = test_root
elif self._output_root is not None:
self._test_root = self._output_root
else:
self._test_root = self._machobj.get_value("CIME_OUTPUT_ROOT")
if self._project is not None:
self._test_root = self._test_root.replace("$PROJECT", self._project)
self._test_root = os.path.abspath(self._test_root)
self._test_id = test_id if test_id is not None else CIME.utils.get_timestamp()
self._compiler = self._machobj.get_default_compiler() if compiler is None else compiler
self._clean = clean
self._namelists_only = namelists_only
self._walltime = walltime
if parallel_jobs is None:
self._parallel_jobs = min(len(test_names),
self._machobj.get_value("MAX_MPITASKS_PER_NODE"))
else:
self._parallel_jobs = parallel_jobs
self._baseline_cmp_name = baseline_cmp_name # Implies comparison should be done if not None
self._baseline_gen_name = baseline_gen_name # Implies generation should be done if not None
# Compute baseline_root
self._baseline_root = baseline_root if baseline_root is not None \
else self._machobj.get_value("BASELINE_ROOT")
if self._project is not None:
self._baseline_root = self._baseline_root.replace("$PROJECT", self._project)
self._baseline_root = os.path.abspath(self._baseline_root)
if baseline_cmp_name or baseline_gen_name:
if self._baseline_cmp_name:
full_baseline_dir = os.path.join(self._baseline_root, self._baseline_cmp_name)
expect(os.path.isdir(full_baseline_dir),
"Missing baseline comparison directory {}".format(full_baseline_dir))
# the following is to assure that the existing generate directory is not overwritten
if self._baseline_gen_name:
full_baseline_dir = os.path.join(self._baseline_root, self._baseline_gen_name)
existing_baselines = []
for test_name in test_names:
test_baseline = os.path.join(full_baseline_dir, test_name)
if os.path.isdir(test_baseline):
existing_baselines.append(test_baseline)
expect(allow_baseline_overwrite or len(existing_baselines) == 0,
"Baseline directories already exists {}\n" \
"Use -o to avoid this error".format(existing_baselines))
if self._cime_model == "e3sm":
_order_tests_by_runtime(test_names, self._baseline_root)
# This is the only data that multiple threads will simultaneously access
# Each test has it's own value and setting/retrieving items from a dict
# is atomic, so this should be fine to use without mutex.
# name -> (phase, status)
self._tests = OrderedDict()
for test_name in test_names:
self._tests[test_name] = (TEST_START, TEST_PASS_STATUS)
# Oversubscribe by 1/4
if proc_pool is None:
pes = int(self._machobj.get_value("MAX_TASKS_PER_NODE"))
self._proc_pool = int(pes * 1.25)
else:
self._proc_pool = int(proc_pool)
self._procs_avail = self._proc_pool
# Setup phases
self._phases = list(PHASES)
if self._no_setup:
self._phases.remove(SETUP_PHASE)
if self._no_build:
self._phases.remove(SHAREDLIB_BUILD_PHASE)
self._phases.remove(MODEL_BUILD_PHASE)
if self._no_run:
self._phases.remove(RUN_PHASE)
if use_existing:
for test in self._tests:
with TestStatus(self._get_test_dir(test)) as ts:
for phase, status in ts:
if phase in CORE_PHASES:
if status in [TEST_PEND_STATUS, TEST_FAIL_STATUS]:
if status == TEST_FAIL_STATUS:
# Import for potential subsequent waits
ts.set_status(phase, TEST_PEND_STATUS)
# We need to pick up here
break
else:
if phase != SUBMIT_PHASE:
# Somewhat subtle. Create_test considers submit/run to be the run phase,
# so don't try to update test status for a passed submit phase
self._update_test_status(test, phase, TEST_PEND_STATUS)
self._update_test_status(test, phase, status)
if phase == RUN_PHASE:
logger.info("Test {} passed and will not be re-run".format(test))
logger.info("Using existing test directory {}".format(self._get_test_dir(test)))
else:
# None of the test directories should already exist.
for test in self._tests:
expect(not os.path.exists(self._get_test_dir(test)),
"Cannot create new case in directory '{}', it already exists."
" Pick a different test-id".format(self._get_test_dir(test)))
logger.info("Creating test directory {}".format(self._get_test_dir(test)))
def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler):
###############################################################################
logs = []
thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
logger.debug("Skip aquap ocn build here")
continue
# coupler handled seperately
if model == "cpl":
continue
# special case for clm
# clm 4_0 is not a shared library and must be built here
# clm 4_5 and newer is a shared library and should be built in build_libraries
if comp == "clm":
if "clm4_0" in clm_config_opts:
logger.info(" - Building clm4_0 Library ")
else:
continue
smp = nthrds > 1 or build_threaded
bldroot = os.path.join(exeroot, model, "obj")
libroot = os.path.join(exeroot, "lib")
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (model, lid))
logger.debug("bldroot is %s" % bldroot)
logger.debug("libroot is %s" % libroot)
# make sure bldroot and libroot exist
for build_dir in [bldroot, libroot]:
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# build the component library
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
t = threading.Thread(target=_build_model_thread,
args=(config_dir, model, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler))
t.start()
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
logs.append(file_build)
# Wait for threads to finish
while(threading.active_count() > 1):
time.sleep(1)
# aquap has a dependancy on atm so we build it after the threaded loop
for model, comp, nthrds, _, config_dir in complist:
smp = nthrds > 1 or build_threaded
if comp == "aquap":
logger.debug("Now build aquap ocn component")
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
_build_model_thread(config_dir, comp, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler)
logs.append(file_build)
expect(not thread_bad_results, "\n".join(thread_bad_results))
#
# Now build the executable
#
cime_model = get_model()
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (cime_model, lid))
config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
stat = run_cmd("%s/buildexe %s %s %s" %
(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=True, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("%s exe"%cime_model, file_build, compiler)
expect(stat == 0, "ERROR: buildexe failed, cat %s" % file_build)
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("%s/%s.exe" % (exeroot, cime_model), "%s/%s.exe.%s" % (exeroot, cime_model, lid))
logs.append(file_build)
return logs
