def case_test(self, testname=None, reset=False, skip_pnl=False):
if testname is None:
testname = self.get_value("TESTCASE")
expect(testname is not None, "testname argument not resolved")
logging.warning("Running test for {}".format(testname))
_set_up_signal_handlers()
try:
# The following line can throw exceptions if the testname is
# not found or the test constructor throws. We need to be
# sure to leave TestStatus in the appropriate state if that
# happens.
test = find_system_test(testname, self)(self)
except BaseException:
caseroot = self.get_value("CASEROOT")
with TestStatus(test_dir=caseroot) as ts:
ts.set_status(RUN_PHASE,
TEST_FAIL_STATUS,
comments="failed to initialize")
append_testlog(str(sys.exc_info()[1]))
raise
if reset:
logger.info("Reset test to initial conditions and exit")
# pylint: disable=protected-access
test._resetup_case(RUN_PHASE)
return True
success = test.run(skip_pnl=skip_pnl)
return success
def run_phase(self):
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
mach = self._case.get_value("MACH")
log = os.path.join(rundir, "funit.log")
if os.path.exists(log):
os.remove(log)
test_spec_dir = self.get_test_spec_dir()
unit_test_tool = os.path.abspath(
os.path.join(get_cime_root(), "scripts", "fortran_unit_testing",
"run_tests.py"))
args = "--build-dir {} --test-spec-dir {} --machine {}".format(
exeroot, test_spec_dir, mach)
# BUG(wjs, 2022-01-07, ESMCI/CIME#4136) For now, these Fortran unit tests only
# work with the old config_compilers.xml-based configuration
my_env = os.environ.copy()
my_env["CIME_NO_CMAKE_MACRO"] = "ON"
stat = run_cmd(
"{} {} >& funit.log".format(unit_test_tool, args),
from_dir=rundir,
env=my_env,
)[0]
append_testlog(open(os.path.join(rundir, "funit.log"), "r").read())
expect(stat == 0, "RUN FAIL for FUNIT")
def case_test(case, testname=None, reset=False):
if testname is None:
testname = case.get_value('TESTCASE')
expect(testname is not None, "testname argument not resolved")
logging.warning("Running test for {}".format(testname))
_set_up_signal_handlers()
try:
# The following line can throw exceptions if the testname is
# not found or the test constructor throws. We need to be
# sure to leave TestStatus in the appropriate state if that
# happens.
test = find_system_test(testname, case)(case)
except:
caseroot = case.get_value("CASEROOT")
with TestStatus(test_dir=caseroot) as ts:
ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize")
append_testlog(str(sys.exc_info()[1]))
return False
if reset:
logger.info("Reset test to initial conditions and exit")
# pylint: disable=protected-access
test._resetup_case(RUN_PHASE)
return True
success = test.run()
return success
def _check_for_memleak(self):
"""
Examine memory usage as recorded in the cpl log file and look for unexpected
increases.
"""
with self._test_status:
latestcpllogs = self._get_latest_cpl_logs()
for cpllog in latestcpllogs:
memlist = self._get_mem_usage(cpllog)
if len(memlist)<3:
self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test")
else:
finaldate = int(memlist[-1][0])
originaldate = int(memlist[0][0])
finalmem = float(memlist[-1][1])
originalmem = float(memlist[0][1])
memdiff = -1
if originalmem > 0:
memdiff = (finalmem - originalmem)/originalmem
tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE")
if tolerance is None:
tolerance = 0.1
expect(tolerance > 0.0, "Bad value for memleak tolerance in test")
if memdiff < 0:
self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test")
elif memdiff < tolerance:
self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS)
else:
comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format(originalmem, finalmem, finaldate-originaldate)
append_testlog(comment)
self._test_status.set_status(MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment)
def run_phase(self):
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
baseline = self._case.get_value("BASELINE_ROOT")
compare = self._case.get_value("COMPARE_BASELINE")
generate = self._case.get_value("GENERATE_BASELINE")
basegen = self._case.get_value("BASEGEN_CASE")
gmake = self._case.get_value("GMAKE")
log = os.path.join(rundir, "homme.log")
if os.path.exists(log):
os.remove(log)
if generate:
full_baseline_dir = os.path.join(baseline, basegen, "tests", "baseline")
stat = run_cmd("{} -j 4 baseline".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0]
if stat == 0:
if os.path.isdir(full_baseline_dir):
shutil.rmtree(full_baseline_dir)
shutil.copytree(os.path.join(exeroot, "tests", "baseline"), full_baseline_dir)
elif compare:
stat = run_cmd("{} -j 4 check".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0]
else:
stat = run_cmd("{} -j 4 baseline".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0]
# Add homme.log output to TestStatus.log so that it can
# appear on the dashboard. Otherwise, the TestStatus.log
# is pretty useless for this test.
append_testlog(open(log, "r").read())
expect(stat == 0, "RUN FAIL for HOMME")
def run_phase(self):
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
baseline = self._case.get_value("BASELINE_ROOT")
compare = self._case.get_value("COMPARE_BASELINE")
generate = self._case.get_value("GENERATE_BASELINE")
basegen = self._case.get_value("BASEGEN_CASE")
gmake = self._case.get_value("GMAKE")
log = os.path.join(rundir, "homme.log")
if os.path.exists(log):
os.remove(log)
if generate:
full_baseline_dir = os.path.join(baseline, basegen, "tests", "baseline")
stat = run_cmd("%s -j 4 baseline" % gmake, arg_stdout=log, combine_output=True, from_dir=exeroot)[0]
if stat == 0:
if os.path.isdir(full_baseline_dir):
shutil.rmtree(full_baseline_dir)
shutil.copytree(os.path.join(exeroot, "tests", "baseline"), full_baseline_dir)
elif compare:
stat = run_cmd("%s -j 4 check" % gmake, arg_stdout=log, combine_output=True, from_dir=exeroot)[0]
else:
stat = run_cmd("%s -j 4 baseline" % gmake, arg_stdout=log, combine_output=True, from_dir=exeroot)[0]
# Add homme.log output to TestStatus.log so that it can
# appear on the dashboard. Otherwise, the TestStatus.log
# is pretty useless for this test.
append_testlog(open(log, "r").read())
expect(stat == 0, "RUN FAIL for HOMME")
def _compare_memory(self):
with self._test_status:
# compare memory usage to baseline
baseline_name = self._case.get_value("BASECMP_CASE")
basecmp_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), baseline_name)
newestcpllogfiles = self._get_latest_cpl_logs()
if len(newestcpllogfiles) > 0:
memlist = self._get_mem_usage(newestcpllogfiles[0])
for cpllog in newestcpllogfiles:
m = re.search(r"/({}.*.log).*.gz".format(self._cpllog),cpllog)
if m is not None:
baselog = os.path.join(basecmp_dir, m.group(1))+".gz"
if baselog is None or not os.path.isfile(baselog):
# for backward compatibility
baselog = os.path.join(basecmp_dir, self._cpllog+".log")
if os.path.isfile(baselog) and len(memlist) > 3:
blmem = self._get_mem_usage(baselog)
blmem = 0 if blmem == [] else blmem[-1][1]
curmem = memlist[-1][1]
diff = (curmem-blmem)/blmem
if diff < 0.1 and self._test_status.get_status(MEMCOMP_PHASE) is None:
self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS)
elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS:
comment = "Error: Memory usage increase > 10% from baseline"
self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment)
append_testlog(comment)
def _compare_throughput(self):
with self._test_status:
# compare memory usage to baseline
baseline_name = self._case.get_value("BASECMP_CASE")
basecmp_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), baseline_name)
newestcpllogfiles = self._get_latest_cpl_logs()
for cpllog in newestcpllogfiles:
m = re.search(r"/({}.*.log).*.gz".format(self._cpllog), cpllog)
if m is not None:
baselog = os.path.join(basecmp_dir, m.group(1))+".gz"
if baselog is None or not os.path.isfile(baselog):
# for backward compatibility
baselog = os.path.join(basecmp_dir, self._cpllog)
if os.path.isfile(baselog):
# compare throughput to baseline
current = self._get_throughput(cpllog)
baseline = self._get_throughput(baselog)
#comparing ypd so bigger is better
if baseline is not None and current is not None:
diff = (baseline - current)/baseline
tolerance = self._case.get_value("TEST_TPUT_TOLERANCE")
if tolerance is None:
tolerance = 0.25
expect(tolerance > 0.0, "Bad value for throughput tolerance in test")
if diff < tolerance and self._test_status.get_status(THROUGHPUT_PHASE) is None:
self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS)
elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS:
comment = "Error: Computation time increase > {:d} pct from baseline".format(int(tolerance*100))
self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment)
append_testlog(comment)
def _log_output(self, test, output):
###########################################################################
test_dir = self._get_test_dir(test)
if not os.path.isdir(test_dir):
# Note: making this directory could cause create_newcase to fail
# if this is run before.
os.makedirs(test_dir)
append_testlog(output, caseroot=test_dir)
def run(self, skip_pnl=False):
"""
Do NOT override this method, this method is the framework that controls
the run phase. run_phase is the extension point that subclasses should use.
"""
success = True
start_time = time.time()
self._skip_pnl = skip_pnl
try:
self._resetup_case(RUN_PHASE)
with self._test_status:
self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS)
self.run_phase()
if self._case.get_value("GENERATE_BASELINE"):
self._phase_modifying_call(GENERATE_PHASE, self._generate_baseline)
if self._case.get_value("COMPARE_BASELINE"):
self._phase_modifying_call(BASELINE_PHASE, self._compare_baseline)
self._phase_modifying_call(MEMCOMP_PHASE, self._compare_memory)
self._phase_modifying_call(THROUGHPUT_PHASE, self._compare_throughput)
self._phase_modifying_call(MEMLEAK_PHASE, self._check_for_memleak)
self._phase_modifying_call(STARCHIVE_PHASE, self._st_archive_case_test)
except BaseException as e: # We want KeyboardInterrupts to generate FAIL status
success = False
if isinstance(e, CIMEError):
# Don't want to print stacktrace for a model failure since that
# is not a CIME/infrastructure problem.
excmsg = str(e)
else:
excmsg = "Exception during run:\n{}\n{}".format(str(e), traceback.format_exc())
append_testlog(excmsg)
raise
finally:
# Writing the run status should be the very last thing due to wait_for_tests
time_taken = time.time() - start_time
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
with self._test_status:
self._test_status.set_status(RUN_PHASE, status, comments=("time={:d}".format(int(time_taken))))
if success and get_model() == "e3sm":
save_test_time(self._case.get_value("BASELINE_ROOT"), self._casebaseid, time_taken)
if get_model() == "cesm" and self._case.get_value("GENERATE_BASELINE"):
baseline_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"))
generate_teststatus(self._caseroot, baseline_dir)
# We return success if the run phase worked; memleaks, diffs will not be taken into account
# with this return value.
return success
def _compare_baseline(self):
"""
compare the current test output to a baseline result
"""
with self._test_status:
# compare baseline
success, comments = compare_baseline(self._case)
append_testlog(comments)
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
baseline_name = self._case.get_value("BASECMP_CASE")
ts_comments = os.path.dirname(baseline_name) + ": " + get_ts_synopsis(comments)
self._test_status.set_status(BASELINE_PHASE, status, comments=ts_comments)
def _run_build_cmd(cmd, exeroot, logfile):
"""
Runs the given build command, with output to the given logfile
Args:
cmd: str (command to run)
exeroot: str (path to exeroot)
logfile: str (path to logfile)
"""
append_testlog(cmd)
run_cmd_no_fail(cmd, arg_stdout=logfile, combine_output=True, from_dir=exeroot)
with open(os.path.join(exeroot, logfile)) as lf:
append_testlog(lf.read())
def _component_compare_test(self, suffix1, suffix2, success_change=False):
"""
Return value is not generally checked, but is provided in case a custom
run case needs indirection based on success.
If success_change is True, success requires some files to be different
"""
success, comments = self._do_compare_test(suffix1, suffix2)
if success_change:
success = not success
append_testlog(comments)
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
with self._test_status:
self._test_status.set_status("{}_{}_{}".format(COMPARE_PHASE, suffix1, suffix2), status)
return success
def _phase_modifying_call(self, phase, function):
"""
Ensures that unexpected exceptions from phases will result in a FAIL result
in the TestStatus file for that phase.
"""
try:
function()
except BaseException as e:
msg = e.__str__()
excmsg = "Exception during {}:\n{}\n{}".format(phase, msg, traceback.format_exc())
logger.warning(excmsg)
append_testlog(excmsg)
with self._test_status:
self._test_status.set_status(phase, TEST_FAIL_STATUS, comments="exception")
def run(self, skip_pnl=False):
"""
Do NOT override this method, this method is the framework that controls
the run phase. run_phase is the extension point that subclasses should use.
"""
success = True
start_time = time.time()
self._skip_pnl = skip_pnl
try:
self._resetup_case(RUN_PHASE)
with self._test_status:
self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS)
self.run_phase()
if self._case.get_value("GENERATE_BASELINE"):
self._generate_baseline()
if self._case.get_value("COMPARE_BASELINE"):
self._compare_baseline()
self._check_for_memleak()
except BaseException as e:
success = False
msg = e.__str__()
if "RUN FAIL" in msg:
# Don't want to print stacktrace for a model failure since that
# is not a CIME/infrastructure problem.
excmsg = msg
else:
excmsg = "Exception during run:\n{}\n{}".format(msg, traceback.format_exc())
logger.warning(excmsg)
append_testlog(excmsg)
# Writing the run status should be the very last thing due to wait_for_tests
time_taken = time.time() - start_time
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
with self._test_status:
self._test_status.set_status(RUN_PHASE, status, comments=("time={:d}".format(int(time_taken))))
if success and get_model() == "e3sm":
save_test_time(self._case.get_value("BASELINE_ROOT"), self._casebaseid, time_taken)
# We return success if the run phase worked; memleaks, diffs will not be taken into account
# with this return value.
return success
def run_phase(self):
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
mach = self._case.get_value("MACH")
log = os.path.join(rundir, "funit.log")
if os.path.exists(log):
os.remove(log)
test_spec_dir = self.get_test_spec_dir()
unit_test_tool = os.path.abspath(os.path.join(get_cime_root(),"scripts","fortran_unit_testing","run_tests.py"))
args = "--build-dir {} --test-spec-dir {} --machine {}".format(exeroot, test_spec_dir, mach)
stat = run_cmd("{} {} >& funit.log".format(unit_test_tool, args), from_dir=rundir)[0]
append_testlog(open(os.path.join(rundir, "funit.log"), "r").read())
expect(stat == 0, "RUN FAIL for FUNIT")
def run_phase(self):
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
mach = self._case.get_value("MACH")
log = os.path.join(rundir, "funit.log")
if os.path.exists(log):
os.remove(log)
test_spec_dir = self.get_test_spec_dir()
unit_test_tool = os.path.abspath(os.path.join(test_spec_dir,"scripts","fortran_unit_testing","run_tests.py"))
args = "--build-dir {} --test-spec-dir {} --machine {}".format(exeroot, test_spec_dir, mach)
stat = run_cmd("{} {} >& funit.log".format(unit_test_tool, args), from_dir=rundir)[0]
append_testlog(open(os.path.join(rundir, "funit.log"), "r").read())
expect(stat == 0, "RUN FAIL for FUNIT")
def _generate_baseline(self):
"""
generate a new baseline case based on the current test
"""
with self._test_status:
# generate baseline
success, comments = generate_baseline(self._case)
append_testlog(comments)
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
baseline_name = self._case.get_value("BASEGEN_CASE")
self._test_status.set_status(GENERATE_PHASE, status, comments=os.path.dirname(baseline_name))
basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"))
# copy latest cpl log to baseline
# drop the date so that the name is generic
newestcpllogfiles = self._get_latest_cpl_logs()
for cpllog in newestcpllogfiles:
m = re.search(r"/({}.*.log).*.gz".format(self._cpllog),cpllog)
if m is not None:
baselog = os.path.join(basegen_dir, m.group(1))+".gz"
safe_copy(cpllog,
os.path.join(basegen_dir,baselog))
def build(self, sharedlib_only=False, model_only=False):
"""
Do NOT override this method, this method is the framework that
controls the build phase. build_phase is the extension point
that subclasses should use.
"""
success = True
for phase_name, phase_bool in [(SHAREDLIB_BUILD_PHASE, not model_only),
(MODEL_BUILD_PHASE, not sharedlib_only)]:
if phase_bool:
self._resetup_case(phase_name)
with self._test_status:
self._test_status.set_status(phase_name, TEST_PEND_STATUS)
start_time = time.time()
try:
self.build_phase(sharedlib_only=(phase_name==SHAREDLIB_BUILD_PHASE),
model_only=(phase_name==MODEL_BUILD_PHASE))
except BaseException as e:
success = False
msg = e.__str__()
if "FAILED, cat" in msg or "BUILD FAIL" in msg:
# Don't want to print stacktrace for a build failure since that
# is not a CIME/infrastructure problem.
excmsg = msg
else:
excmsg = "Exception during build:\n{}\n{}".format(msg, traceback.format_exc())
logger.warning(excmsg)
append_testlog(excmsg)
time_taken = time.time() - start_time
with self._test_status:
self._test_status.set_status(phase_name, TEST_PASS_STATUS if success else TEST_FAIL_STATUS, comments=("time={:d}".format(int(time_taken))))
if not success:
break
return success
def build(self, sharedlib_only=False, model_only=False):
"""
Do NOT override this method, this method is the framework that
controls the build phase. build_phase is the extension point
that subclasses should use.
"""
success = True
for phase_name, phase_bool in [(SHAREDLIB_BUILD_PHASE, not model_only),
(MODEL_BUILD_PHASE, not sharedlib_only)]:
if phase_bool:
self._resetup_case(phase_name)
with self._test_status:
self._test_status.set_status(phase_name, TEST_PEND_STATUS)
start_time = time.time()
try:
self.build_phase(sharedlib_only=(phase_name==SHAREDLIB_BUILD_PHASE),
model_only=(phase_name==MODEL_BUILD_PHASE))
except BaseException as e: # We want KeyboardInterrupts to generate FAIL status
success = False
if isinstance(e, CIMEError):
# Don't want to print stacktrace for a build failure since that
# is not a CIME/infrastructure problem.
excmsg = str(e)
else:
excmsg = "Exception during build:\n{}\n{}".format(str(e), traceback.format_exc())
append_testlog(excmsg)
raise
finally:
time_taken = time.time() - start_time
with self._test_status:
self._test_status.set_status(phase_name, TEST_PASS_STATUS if success else TEST_FAIL_STATUS, comments=("time={:d}".format(int(time_taken))))
return success
def _compare_baseline(self):
"""
compare the current test output to a baseline result
"""
with self._test_status:
# compare baseline
success, comments = compare_baseline(self._case)
append_testlog(comments)
status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
baseline_name = self._case.get_value("BASECMP_CASE")
ts_comments = (os.path.dirname(baseline_name) + ": " + comments) if "\n" not in comments else os.path.dirname(baseline_name)
self._test_status.set_status(BASELINE_PHASE, status, comments=ts_comments)
basecmp_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), baseline_name)
# compare memory usage to baseline
newestcpllogfiles = self._get_latest_cpl_logs()
if len(newestcpllogfiles) > 0:
memlist = self._get_mem_usage(newestcpllogfiles[0])
for cpllog in newestcpllogfiles:
m = re.search(r"/(cpl.*.log).*.gz",cpllog)
if m is not None:
baselog = os.path.join(basecmp_dir, m.group(1))+".gz"
if baselog is None or not os.path.isfile(baselog):
# for backward compatibility
baselog = os.path.join(basecmp_dir, "cpl.log")
if os.path.isfile(baselog) and len(memlist) > 3:
blmem = self._get_mem_usage(baselog)
blmem = 0 if blmem == [] else blmem[-1][1]
curmem = memlist[-1][1]
diff = (curmem-blmem)/blmem
if diff < 0.1 and self._test_status.get_status(MEMCOMP_PHASE) is None:
self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS)
elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS:
comment = "Error: Memory usage increase > 10% from baseline"
self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment)
append_testlog(comment)
# compare throughput to baseline
current = self._get_throughput(cpllog)
baseline = self._get_throughput(baselog)
#comparing ypd so bigger is better
if baseline is not None and current is not None:
diff = (baseline - current)/baseline
tolerance = self._case.get_value("TEST_TPUT_TOLERANCE")
if tolerance is None:
tolerance = 0.25
expect(tolerance > 0.0, "Bad value for throughput tolerance in test")
if diff < tolerance and self._test_status.get_status(THROUGHPUT_PHASE) is None:
self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS)
elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS:
comment = "Error: Computation time increase > {:d} pct from baseline".format(int(tolerance*100))
self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment)
append_testlog(comment)
expect(testname is not None, "testname argument not resolved")
logging.warn("Running test for %s" % testname)
_set_up_signal_handlers()
try:
# The following line can throw exceptions if the testname is
# not found or the test constructor throws. We need to be
# sure to leave TestStatus in the appropriate state if that
# happens.
test = find_system_test(testname, case)(case)
except:
caseroot = case.get_value("CASEROOT")
with TestStatus(test_dir=caseroot) as ts:
ts.set_status(RUN_PHASE,
TEST_FAIL_STATUS,
comments="failed to initialize")
append_testlog(str(sys.exc_info()[1]))
return False
if reset:
logger.info("Reset test to initial conditions and exit")
# pylint: disable=protected-access
test._resetup_case(RUN_PHASE)
return True
success = test.run()
case.set_value("RUN_WITH_SUBMIT", False)
return success
def _generate_baseline(self):
"""
generate a new baseline case based on the current test
"""
with self._test_status:
# generate baseline
# BEGIN: modified CIME.hist_utils.generate_baseline
rundir = self._case.get_value("RUNDIR")
basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"),
self._case.get_value("BASEGEN_CASE"))
testcase = self._case.get_value("CASE")
if not os.path.isdir(basegen_dir):
os.makedirs(basegen_dir)
if os.path.isdir(os.path.join(basegen_dir, testcase)):
expect(
False,
" Cowardly refusing to overwrite existing baseline directory"
)
comments = "Generating baselines into '{}'\n".format(basegen_dir)
num_gen = 0
model = 'cam'
comments += " generating for model '{}'\n".format(model)
hists = _get_all_hist_files(testcase, model, rundir)
logger.debug("mvk_hist_files: {}".format(hists))
num_gen += len(hists)
for hist in hists:
basename = hist[hist.rfind(model):]
baseline = os.path.join(basegen_dir, basename)
if os.path.exists(baseline):
os.remove(baseline)
shutil.copy(hist, baseline)
comments += " generating baseline '{}' from file {}\n".format(
baseline, hist)
newestcpllogfile = self._case.get_latest_cpl_log(
coupler_log_path=self._case.get_value("LOGDIR"))
if newestcpllogfile is None:
logger.warning(
"No cpl.log file found in log directory {}".format(
self._case.get_value("LOGDIR")))
else:
shutil.copyfile(newestcpllogfile,
os.path.join(basegen_dir, "cpl.log.gz"))
expect(
num_gen > 0,
"Could not generate any hist files for case '{}', something is seriously wrong"
.format(os.path.join(rundir, testcase)))
# make sure permissions are open in baseline directory
for root, _, files in os.walk(basegen_dir):
for name in files:
try:
os.chmod(
os.path.join(root,
name), stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
except OSError:
# We tried. Not worth hard failure here.
pass
if get_model() == "e3sm":
bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME)
with open(bless_log, "a") as fd:
fd.write("sha:{} date:{}\n".format(
get_current_commit(
repo=self._case.get_value("CIMEROOT")),
get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S")))
# END: modified CIME.hist_utils.generate_baseline
append_testlog(comments)
status = CIME.test_status.TEST_PASS_STATUS
baseline_name = self._case.get_value("BASEGEN_CASE")
self._test_status.set_status(
"{}".format(CIME.test_status.GENERATE_PHASE),
status,
comments=os.path.dirname(baseline_name))
basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"),
self._case.get_value("BASEGEN_CASE"))
# copy latest cpl log to baseline
# drop the date so that the name is generic
newestcpllogfiles = self._get_latest_cpl_logs()
for cpllog in newestcpllogfiles:
m = re.search(r"/(cpl.*.log).*.gz", cpllog)
if m is not None:
baselog = os.path.join(basegen_dir, m.group(1)) + ".gz"
shutil.copyfile(cpllog, os.path.join(basegen_dir, baselog))
def _generate_baseline(self):
"""
generate a new baseline case based on the current test
"""
with self._test_status:
# generate baseline
# BEGIN: modified CIME.hist_utils.generate_baseline
rundir = self._case.get_value("RUNDIR")
basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"),
self._case.get_value("BASEGEN_CASE"))
testcase = self._case.get_value("CASE")
if not os.path.isdir(basegen_dir):
os.makedirs(basegen_dir)
if os.path.isdir(os.path.join(basegen_dir, testcase)):
expect(False, " Cowardly refusing to overwrite existing baseline directory")
comments = "Generating baselines into '{}'\n".format(basegen_dir)
num_gen = 0
model = 'cam'
comments += " generating for model '{}'\n".format(model)
hists = _get_all_hist_files(testcase, model, rundir)
logger.debug("mvk_hist_files: {}".format(hists))
num_gen += len(hists)
for hist in hists:
basename = hist[hist.rfind(model):]
baseline = os.path.join(basegen_dir, basename)
if os.path.exists(baseline):
os.remove(baseline)
shutil.copy(hist, baseline)
comments += " generating baseline '{}' from file {}\n".format(baseline, hist)
newestcpllogfile = self._case.get_latest_cpl_log(coupler_log_path=self._case.get_value("LOGDIR"))
if newestcpllogfile is None:
logger.warning("No cpl.log file found in log directory {}".format(self._case.get_value("LOGDIR")))
else:
shutil.copyfile(newestcpllogfile,
os.path.join(basegen_dir, "cpl.log.gz"))
expect(num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong".format(
os.path.join(rundir, testcase)))
# make sure permissions are open in baseline directory
for root, _, files in os.walk(basegen_dir):
for name in files:
try:
os.chmod(os.path.join(root, name),
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
except OSError:
# We tried. Not worth hard failure here.
pass
if get_model() == "e3sm":
bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME)
with open(bless_log, "a") as fd:
fd.write("sha:{} date:{}\n".format(get_current_commit(repo=self._case.get_value("CIMEROOT")),
get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S")))
# END: modified CIME.hist_utils.generate_baseline
append_testlog(comments)
status = CIME.test_status.TEST_PASS_STATUS
baseline_name = self._case.get_value("BASEGEN_CASE")
self._test_status.set_status("{}".format(CIME.test_status.GENERATE_PHASE), status,
comments=os.path.dirname(baseline_name))
basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"))
# copy latest cpl log to baseline
# drop the date so that the name is generic
newestcpllogfiles = self._get_latest_cpl_logs()
for cpllog in newestcpllogfiles:
m = re.search(r"/(cpl.*.log).*.gz", cpllog)
if m is not None:
baselog = os.path.join(basegen_dir, m.group(1)) + ".gz"
shutil.copyfile(cpllog,
os.path.join(basegen_dir, baselog))
def _component_compare_copy(self, suffix):
comments = copy(self._case, suffix)
append_testlog(comments)
