def setUp(self):
self._ts = test_status.TestStatus(
test_dir=os.path.join("nonexistent", "path"),
test_name=self._TESTNAME,
no_io=True,
)
self._set_core_phases_to_pass()
def test_wait_for_test_wait_for_missing_run_phase(self):
run_thread = threading.Thread(target=self.threaded_test,
args=(self._testdir_unfinished2,
["PASS"] * 10))
run_thread.daemon = True
run_thread.start()
time.sleep(5) # Kinda hacky
self.assertTrue(run_thread.is_alive(),
msg="wait_for_tests should have waited")
with test_status.TestStatus(
test_dir=os.path.join(self._testdir_unfinished2, "5")) as ts:
ts.set_status(test_status.RUN_PHASE, test_status.TEST_PASS_STATUS)
run_thread.join(timeout=10)
self.assertFalse(run_thread.is_alive(),
msg="wait_for_tests should have finished")
self.assertTrue(
self._thread_error is None,
msg="Thread had failure: %s" % self._thread_error,
)
def create_test_status_core_passes(test_dir_path, test_name):
"""Creates a TestStatus file in the given path, with PASS status
for all core phases"""
with test_status.TestStatus(test_dir=test_dir_path,
test_name=test_name) as ts:
for phase in test_status.CORE_PHASES:
ts.set_status(phase, test_status.TEST_PASS_STATUS)
def make_fake_teststatus(path, testname, status, phase):
expect(phase in test_status.CORE_PHASES, "Bad phase '%s'" % phase)
with test_status.TestStatus(test_dir=path, test_name=testname) as ts:
for core_phase in test_status.CORE_PHASES:
if core_phase == phase:
ts.set_status(
core_phase,
status,
comments=("time=42"
if phase == test_status.RUN_PHASE else ""),
)
break
else:
ts.set_status(
core_phase,
test_status.TEST_PASS_STATUS,
comments=("time=42"
if phase == test_status.RUN_PHASE else ""),
)
def live_test_impl(self, testdir, expected_results, last_phase,
last_status):
run_thread = threading.Thread(target=self.threaded_test,
args=(testdir, expected_results))
run_thread.daemon = True
run_thread.start()
time.sleep(5)
self.assertTrue(run_thread.is_alive(),
msg="wait_for_tests should have waited")
for core_phase in test_status.CORE_PHASES[1:]:
with test_status.TestStatus(test_dir=os.path.join(
self._testdir_teststatus1, "0")) as ts:
ts.set_status(
core_phase,
last_status if core_phase == last_phase else
test_status.TEST_PASS_STATUS,
)
time.sleep(5)
if core_phase != last_phase:
self.assertTrue(
run_thread.is_alive(),
msg=
"wait_for_tests should have waited after passing phase {}".
format(core_phase),
)
else:
run_thread.join(timeout=10)
self.assertFalse(
run_thread.is_alive(),
msg="wait_for_tests should have finished after phase {}".
format(core_phase),
)
break
self.assertTrue(
self._thread_error is None,
msg="Thread had failure: %s" % self._thread_error,
)
def test_c_use_existing(self):
tests = get_tests.get_full_test_names(
[
"TESTBUILDFAIL_P1.f19_g16_rx1.A",
"TESTRUNFAIL_P1.f19_g16_rx1.A",
"TESTRUNPASS_P1.f19_g16_rx1.A",
],
self._machine,
self._compiler,
)
test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp())
ct = test_scheduler.TestScheduler(
tests,
test_id=test_id,
no_batch=self.NO_BATCH,
test_root=self._testroot,
output_root=self._testroot,
compiler=self._compiler,
mpilib=self.TEST_MPILIB,
machine_name=self.MACHINE.get_machine_name(),
)
build_fail_test = [item for item in tests
if "TESTBUILDFAIL" in item][0]
run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0]
pass_test = [item for item in tests if "TESTRUNPASS" in item][0]
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
try:
ct.run_tests()
finally:
logging.getLogger().setLevel(log_lvl)
test_statuses = glob.glob("%s/*%s/TestStatus" %
(self._testroot, test_id))
self.assertEqual(len(tests), len(test_statuses))
self._wait_for_tests(test_id, expect_works=False)
for x in test_statuses:
casedir = os.path.dirname(x)
ts = test_status.TestStatus(test_dir=casedir)
test_name = ts.get_name()
if test_name == build_fail_test:
self.assert_test_status(
test_name,
ts,
test_status.MODEL_BUILD_PHASE,
test_status.TEST_FAIL_STATUS,
)
with test_status.TestStatus(test_dir=casedir) as ts:
ts.set_status(test_status.MODEL_BUILD_PHASE,
test_status.TEST_PEND_STATUS)
elif test_name == run_fail_test:
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_FAIL_STATUS)
with test_status.TestStatus(test_dir=casedir) as ts:
ts.set_status(test_status.SUBMIT_PHASE,
test_status.TEST_PEND_STATUS)
else:
self.assertTrue(test_name == pass_test)
self.assert_test_status(
test_name,
ts,
test_status.MODEL_BUILD_PHASE,
test_status.TEST_PASS_STATUS,
)
self.assert_test_status(
test_name,
ts,
test_status.SUBMIT_PHASE,
test_status.TEST_PASS_STATUS,
)
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
os.environ["TESTBUILDFAIL_PASS"] = "******"
os.environ["TESTRUNFAIL_PASS"] = "******"
ct2 = test_scheduler.TestScheduler(
tests,
test_id=test_id,
no_batch=self.NO_BATCH,
use_existing=True,
test_root=self._testroot,
output_root=self._testroot,
compiler=self._compiler,
mpilib=self.TEST_MPILIB,
machine_name=self.MACHINE.get_machine_name(),
)
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
try:
ct2.run_tests()
finally:
logging.getLogger().setLevel(log_lvl)
self._wait_for_tests(test_id)
for x in test_statuses:
ts = test_status.TestStatus(test_dir=os.path.dirname(x))
test_name = ts.get_name()
self.assert_test_status(
test_name,
ts,
test_status.MODEL_BUILD_PHASE,
test_status.TEST_PASS_STATUS,
)
self.assert_test_status(test_name, ts, test_status.SUBMIT_PHASE,
test_status.TEST_PASS_STATUS)
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
del os.environ["TESTBUILDFAIL_PASS"]
del os.environ["TESTRUNFAIL_PASS"]
# test that passed tests are not re-run
ct2 = test_scheduler.TestScheduler(
tests,
test_id=test_id,
no_batch=self.NO_BATCH,
use_existing=True,
test_root=self._testroot,
output_root=self._testroot,
compiler=self._compiler,
mpilib=self.TEST_MPILIB,
machine_name=self.MACHINE.get_machine_name(),
)
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
try:
ct2.run_tests()
finally:
logging.getLogger().setLevel(log_lvl)
self._wait_for_tests(test_id)
for x in test_statuses:
ts = test_status.TestStatus(test_dir=os.path.dirname(x))
test_name = ts.get_name()
self.assert_test_status(
test_name,
ts,
test_status.MODEL_BUILD_PHASE,
test_status.TEST_PASS_STATUS,
)
self.assert_test_status(test_name, ts, test_status.SUBMIT_PHASE,
test_status.TEST_PASS_STATUS)
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
def test_b_full(self):
tests = get_tests.get_full_test_names(["cime_test_only"],
self._machine, self._compiler)
test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp())
ct = test_scheduler.TestScheduler(
tests,
test_id=test_id,
no_batch=self.NO_BATCH,
test_root=self._testroot,
output_root=self._testroot,
compiler=self._compiler,
mpilib=self.TEST_MPILIB,
machine_name=self.MACHINE.get_machine_name(),
)
build_fail_test = [item for item in tests
if "TESTBUILDFAIL_" in item][0]
build_fail_exc_test = [
item for item in tests if "TESTBUILDFAILEXC" in item
][0]
run_fail_test = [item for item in tests if "TESTRUNFAIL_" in item][0]
run_fail_exc_test = [
item for item in tests if "TESTRUNFAILEXC" in item
][0]
pass_test = [item for item in tests if "TESTRUNPASS" in item][0]
test_diff_test = [item for item in tests if "TESTTESTDIFF" in item][0]
mem_fail_test = [item for item in tests
if "TESTMEMLEAKFAIL" in item][0]
mem_pass_test = [item for item in tests
if "TESTMEMLEAKPASS" in item][0]
st_arch_fail_test = [
item for item in tests if "TESTRUNSTARCFAIL" in item
][0]
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
try:
ct.run_tests()
finally:
logging.getLogger().setLevel(log_lvl)
self._wait_for_tests(test_id, expect_works=False)
test_statuses = glob.glob("%s/*%s/TestStatus" %
(self._testroot, test_id))
self.assertEqual(len(tests), len(test_statuses))
for x in test_statuses:
ts = test_status.TestStatus(test_dir=os.path.dirname(x))
test_name = ts.get_name()
log_files = glob.glob("%s/%s*%s/TestStatus.log" %
(self._testroot, test_name, test_id))
self.assertEqual(
len(log_files),
1,
"Expected exactly one test_status.TestStatus.log file, found %d"
% len(log_files),
)
log_file = log_files[0]
if test_name == build_fail_test:
self.assert_test_status(
test_name,
ts,
test_status.MODEL_BUILD_PHASE,
test_status.TEST_FAIL_STATUS,
)
data = open(log_file, "r").read()
self.assertTrue(
"Intentional fail for testing infrastructure" in data,
"Broken test did not report build error:\n%s" % data,
)
elif test_name == build_fail_exc_test:
data = open(log_file, "r").read()
self.assert_test_status(
test_name,
ts,
test_status.SHAREDLIB_BUILD_PHASE,
test_status.TEST_FAIL_STATUS,
)
self.assertTrue(
"Exception from init" in data,
"Broken test did not report build error:\n%s" % data,
)
elif test_name == run_fail_test:
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_FAIL_STATUS)
elif test_name == run_fail_exc_test:
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_FAIL_STATUS)
data = open(log_file, "r").read()
self.assertTrue(
"Exception from run_phase" in data,
"Broken test did not report run error:\n%s" % data,
)
elif test_name == mem_fail_test:
self.assert_test_status(
test_name,
ts,
test_status.MEMLEAK_PHASE,
test_status.TEST_FAIL_STATUS,
)
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
elif test_name == test_diff_test:
self.assert_test_status(test_name, ts, "COMPARE_base_rest",
test_status.TEST_FAIL_STATUS)
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
elif test_name == st_arch_fail_test:
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
self.assert_test_status(
test_name,
ts,
test_status.STARCHIVE_PHASE,
test_status.TEST_FAIL_STATUS,
)
else:
self.assertTrue(test_name in [pass_test, mem_pass_test])
self.assert_test_status(test_name, ts, test_status.RUN_PHASE,
test_status.TEST_PASS_STATUS)
if test_name == mem_pass_test:
self.assert_test_status(
test_name,
ts,
test_status.MEMLEAK_PHASE,
test_status.TEST_PASS_STATUS,
)
def test_full_system(self):
# Put this inside any test that's slow
if self.FAST_ONLY:
self.skipTest("Skipping slow test")
driver = utils.get_cime_default_driver()
if driver == "mct":
cases = self._create_test(["--walltime=0:15:00", "cime_developer"],
test_id=self._baseline_name)
else:
cases = self._create_test(["--walltime=0:30:00", "cime_developer"],
test_id=self._baseline_name)
self.run_cmd_assert_result(
"%s/cs.status.%s" % (self._testroot, self._baseline_name),
from_dir=self._testroot,
)
# Ensure that we can get test times
for case_dir in cases:
tstatus = os.path.join(case_dir, "TestStatus")
test_time = wait_for_tests.get_test_time(os.path.dirname(tstatus))
self.assertIs(type(test_time),
int,
msg="get time did not return int for %s" % tstatus)
self.assertTrue(test_time > 0,
msg="test time was zero for %s" % tstatus)
# Test that re-running works
skip_tests = None
if utils.get_cime_default_driver() == "nuopc":
skip_tests = [
"SMS_Ln3.T42_T42.S",
"PRE.f19_f19.ADESP_TEST",
"PRE.f19_f19.ADESP",
"DAE.ww3a.ADWAV",
]
tests = get_tests.get_test_suite(
"cime_developer",
machine=self._machine,
compiler=self._compiler,
skip_tests=skip_tests,
)
for test in tests:
casedir = self.get_casedir(test, cases)
# Subtle issue: The run phases of these tests will be in the PASS state until
# the submitted case.test script is run, which could take a while if the system is
# busy. This potentially leaves a window where the wait_for_tests command below will
# not wait for the re-submitted jobs to run because it sees the original PASS.
# The code below forces things back to PEND to avoid this race condition. Note
# that we must use the MEMLEAK phase, not the RUN phase, because RUN being in a non-PEND
# state is how system tests know they are being re-run and must reset certain
# case settings.
if self._hasbatch:
with test_status.TestStatus(test_dir=casedir) as ts:
ts.set_status(test_status.MEMLEAK_PHASE,
test_status.TEST_PEND_STATUS)
self.run_cmd_assert_result("./case.submit --skip-preview-namelist",
from_dir=casedir)
self._wait_for_tests(self._baseline_name)
def set_phase_to_status(test_dir_path, test_name, phase, status):
"""Sets the given phase to the given status for this test"""
with test_status.TestStatus(test_dir=test_dir_path,
test_name=test_name) as ts:
ts.set_status(phase, status)