def test_aa_no_flush_on_instantiate(self):
testdir = os.path.join(self.__class__._testroot, "testcreatenewcase")
with Case(testdir, read_only=False) as case:
for env_file in case._files:
self.assertFalse(
env_file.needsrewrite,
msg="Instantiating a case should not trigger a flush call",
)
with Case(testdir, read_only=False) as case:
case.set_value("HIST_OPTION", "nyears")
runfile = case.get_env("run")
self.assertTrue(
runfile.needsrewrite, msg="Expected flush call not triggered"
)
for env_file in case._files:
if env_file != runfile:
self.assertFalse(
env_file.needsrewrite,
msg="Unexpected flush triggered for file {}".format(
env_file.filename
),
)
# Flush the file
runfile.write()
# set it again to the same value
case.set_value("HIST_OPTION", "nyears")
# now the file should not need to be flushed
for env_file in case._files:
self.assertFalse(
env_file.needsrewrite,
msg="Unexpected flush triggered for file {}".format(
env_file.filename
),
)
def test_cime_case_allow_failed_prereq(self):
testcase_name = "allow_failed_prereq_test"
testdir = self._batch_test_fixture(testcase_name)
with Case(testdir, read_only=False) as case:
depend_allow = case.get_value("depend_allow_string")
if depend_allow is None:
self.skipTest(
"Skipping allow_failed_prereq test, depend_allow_string was not provided for this batch system"
)
job_name = "case.run"
prereq_name = "prereq_allow_fail_test"
depend_allow = depend_allow.replace("jobid", prereq_name)
batch_commands = case.submit_jobs(
prereq=prereq_name,
allow_fail=True,
job=job_name,
skip_pnl=True,
dry_run=True,
)
self.assertTrue(
isinstance(batch_commands, collections.Sequence),
"case.submit_jobs did not return a sequence for a dry run",
)
num_submissions = 1
if case.get_value("DOUT_S"):
num_submissions = 2
self.assertTrue(
len(batch_commands) == num_submissions,
"case.submit_jobs did not return any job submission strings",
)
self.assertTrue(depend_allow in batch_commands[0][1])
def test_cime_case_resubmit_immediate(self):
testcase_name = "resubmit_immediate_test"
testdir = self._batch_test_fixture(testcase_name)
with Case(testdir, read_only=False) as case:
depend_string = case.get_value("depend_string")
if depend_string is None:
self.skipTest(
"Skipping resubmit_immediate test, depend_string was not provided for this batch system"
)
depend_string = re.sub("jobid.*$", "", depend_string)
job_name = "case.run"
num_submissions = 6
case.set_value("RESUBMIT", num_submissions - 1)
batch_commands = case.submit_jobs(job=job_name,
skip_pnl=True,
dry_run=True,
resubmit_immediate=True)
self.assertTrue(
isinstance(batch_commands, collections.Sequence),
"case.submit_jobs did not return a sequence for a dry run",
)
if case.get_value("DOUT_S"):
num_submissions = 12
self.assertTrue(
len(batch_commands) == num_submissions,
"case.submit_jobs did not return {} submitted jobs".format(
num_submissions),
)
for i, cmd in enumerate(batch_commands):
if i > 0:
self.assertTrue(depend_string in cmd[1])
def test_cime_case_test_walltime_mgmt_7(self):
if not self._hasbatch:
self.skipTest("Skipping walltime test. Depends on batch system")
test_name = "ERS_P1.f19_g16_rx1.A"
casedir = self._create_test(
["--no-build", "--walltime=01:00:00", test_name],
test_id=self._baseline_name,
env_changes="unset CIME_GLOBAL_WALLTIME &&",
)
self.run_cmd_assert_result(
"./xmlchange JOB_WALLCLOCK_TIME=421:32:11 --subgroup=case.test",
from_dir=casedir,
)
self.run_cmd_assert_result("./case.setup --reset", from_dir=casedir)
result = self.run_cmd_assert_result(
"./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value",
from_dir=casedir,
)
with Case(casedir) as case:
walltime_format = case.get_value("walltime_format", subgroup=None)
if walltime_format is not None and walltime_format.count(":") == 1:
self.assertEqual(result, "421:32")
else:
self.assertEqual(result, "421:32:11")
def test_a_createnewcase(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testcreatenewcase")
if os.path.exists(testdir):
shutil.rmtree(testdir)
args = " --case %s --compset X --output-root %s --handle-preexisting-dirs=r" % (
testdir,
cls._testroot,
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._testdirs.append(testdir)
self.run_cmd_assert_result("./create_newcase %s" % (args),
from_dir=self.SCRIPT_DIR)
self.assertTrue(os.path.exists(testdir))
self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
with Case(testdir, read_only=False) as case:
ntasks = case.get_value("NTASKS_ATM")
case.set_value("NTASKS_ATM", ntasks + 1)
# this should fail with a locked file issue
self.run_cmd_assert_result("./case.build",
from_dir=testdir,
expected_stat=1)
self.run_cmd_assert_result("./case.setup --reset", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
with Case(testdir, read_only=False) as case:
case.set_value("CHARGE_ACCOUNT", "fred")
def test_e_xmlquery(self):
# Set script and script path
xmlquery = "./xmlquery"
cls = self.__class__
casedir = cls._testdirs[0]
# Check for environment
self.assertTrue(os.path.isdir(self.SCRIPT_DIR))
self.assertTrue(os.path.isdir(self.TOOLS_DIR))
self.assertTrue(os.path.isfile(os.path.join(casedir, xmlquery)))
# Test command line options
with Case(casedir, read_only=True) as case:
STOP_N = case.get_value("STOP_N")
COMP_CLASSES = case.get_values("COMP_CLASSES")
BUILD_COMPLETE = case.get_value("BUILD_COMPLETE")
cmd = xmlquery + " STOP_N --value"
output = utils.run_cmd_no_fail(cmd, from_dir=casedir)
self.assertTrue(output == str(STOP_N), msg="%s != %s" % (output, STOP_N))
cmd = xmlquery + " BUILD_COMPLETE --value"
output = utils.run_cmd_no_fail(cmd, from_dir=casedir)
self.assertTrue(output == "TRUE", msg="%s != %s" % (output, BUILD_COMPLETE))
# we expect DOCN_MODE to be undefined in this X compset
# this test assures that we do not try to resolve this as a compvar
cmd = xmlquery + " DOCN_MODE --value"
_, output, error = utils.run_cmd(cmd, from_dir=casedir)
self.assertTrue(
error == "ERROR: No results found for variable DOCN_MODE",
msg="unexpected result for DOCN_MODE, output {}, error {}".format(
output, error
),
)
for comp in COMP_CLASSES:
caseresult = case.get_value("NTASKS_%s" % comp)
cmd = xmlquery + " NTASKS_%s --value" % comp
output = utils.run_cmd_no_fail(cmd, from_dir=casedir)
self.assertTrue(
output == str(caseresult), msg="%s != %s" % (output, caseresult)
)
cmd = xmlquery + " NTASKS --subgroup %s --value" % comp
output = utils.run_cmd_no_fail(cmd, from_dir=casedir)
self.assertTrue(
output == str(caseresult), msg="%s != %s" % (output, caseresult)
)
if self.MACHINE.has_batch_system():
JOB_QUEUE = case.get_value("JOB_QUEUE", subgroup="case.run")
cmd = xmlquery + " JOB_QUEUE --subgroup case.run --value"
output = utils.run_cmd_no_fail(cmd, from_dir=casedir)
self.assertTrue(
output == JOB_QUEUE, msg="%s != %s" % (output, JOB_QUEUE)
)
cmd = xmlquery + " --listall"
utils.run_cmd_no_fail(cmd, from_dir=casedir)
cls._do_teardown.append(cls._testroot)
def test_cime_case_build_threaded_2(self):
casedir = self._create_test(
["--no-build", "TESTRUNPASS_P1x2.f19_g16_rx1.A"],
test_id=self._baseline_name,
)
with Case(casedir, read_only=False) as case:
build_threaded = case.get_value("SMP_PRESENT")
self.assertTrue(build_threaded)
build_threaded = case.get_build_threaded()
self.assertTrue(build_threaded)
def simple_test(self, manual_timing=False):
if self.NO_FORTRAN_RUN:
self.skipTest("Skipping fortran test")
timing_flag = "" if manual_timing else "--save-timing"
driver = utils.get_cime_default_driver()
if driver == "mct":
walltime = "00:15:00"
else:
walltime = "00:30:00"
self._create_test(
[
"SMS_Ln9_P1.f19_g16_rx1.A", timing_flag,
"--walltime=" + walltime
],
test_id=self._baseline_name,
)
statuses = glob.glob("%s/*%s/TestStatus" %
(self._testroot, self._baseline_name))
self.assertEqual(
len(statuses),
1,
msg="Should have had exactly one match, found %s" % statuses,
)
casedir = os.path.dirname(statuses[0])
with Case(casedir, read_only=True) as case:
lids = utils.get_lids(case)
timing_dir = case.get_value("SAVE_TIMING_DIR")
casename = case.get_value("CASE")
self.assertEqual(len(lids), 1, msg="Expected one LID, found %s" % lids)
if manual_timing:
self.run_cmd_assert_result("cd %s && %s/save_provenance postrun" %
(casedir, self.TOOLS_DIR))
if utils.get_model() == "e3sm":
provenance_glob = os.path.join(
timing_dir,
"performance_archive",
getpass.getuser(),
casename,
lids[0] + "*",
)
provenance_dirs = glob.glob(provenance_glob)
self.assertEqual(
len(provenance_dirs),
1,
msg=
"wrong number of provenance dirs, expected 1, got {}, looked for {}"
.format(provenance_dirs, provenance_glob),
)
self.verify_perms("".join(provenance_dirs))
def cime_case(self):
""" Returns a CIME case object. Must provide the CIME source root
in case_config dict when instantiating this class. Any CIME xml variable,
e.g., OCN_GRID, may be retrieved from the returned object using get_value
method."""
if not self._cime_case:
caseroot = self.get_value('CASEROOT')
cimeroot = self.get_value('CIMEROOT')
if caseroot and cimeroot:
sys.path.append(os.path.join(cimeroot, "scripts", "lib"))
from CIME.case.case import Case
self._cime_case = Case(caseroot)
return self._cime_case
def test_xml_caching(self):
casedir = self._create_test(
["--no-build", "TESTRUNPASS.f19_g16_rx1.A"],
test_id=self._baseline_name)
active = os.path.join(casedir, "env_run.xml")
backup = os.path.join(casedir, "env_run.xml.bak")
utils.safe_copy(active, backup)
with Case(casedir, read_only=False) as case:
env_run = EnvRun(casedir, read_only=True)
self.assertEqual(case.get_value("RUN_TYPE"), "startup")
case.set_value("RUN_TYPE", "branch")
self.assertEqual(case.get_value("RUN_TYPE"), "branch")
self.assertEqual(env_run.get_value("RUN_TYPE"), "branch")
with Case(casedir) as case:
self.assertEqual(case.get_value("RUN_TYPE"), "branch")
time.sleep(0.2)
utils.safe_copy(backup, active)
with Case(casedir, read_only=False) as case:
self.assertEqual(case.get_value("RUN_TYPE"), "startup")
case.set_value("RUN_TYPE", "branch")
with Case(casedir, read_only=False) as case:
self.assertEqual(case.get_value("RUN_TYPE"), "branch")
time.sleep(0.2)
utils.safe_copy(backup, active)
case.read_xml() # Manual re-sync
self.assertEqual(case.get_value("RUN_TYPE"), "startup")
case.set_value("RUN_TYPE", "branch")
self.assertEqual(case.get_value("RUN_TYPE"), "branch")
with Case(casedir) as case:
self.assertEqual(case.get_value("RUN_TYPE"), "branch")
time.sleep(0.2)
utils.safe_copy(backup, active)
env_run = EnvRun(casedir, read_only=True)
self.assertEqual(env_run.get_value("RUN_TYPE"), "startup")
with Case(casedir, read_only=False) as case:
self.assertEqual(case.get_value("RUN_TYPE"), "startup")
case.set_value("RUN_TYPE", "branch")
# behind the back detection.
with self.assertRaises(utils.CIMEError):
with Case(casedir, read_only=False) as case:
case.set_value("RUN_TYPE", "startup")
time.sleep(0.2)
utils.safe_copy(backup, active)
def test_dd_create_clone_not_writable(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "test_create_clone_not_writable")
if os.path.exists(testdir):
shutil.rmtree(testdir)
prevtestdir = cls._testdirs[0]
cls._testdirs.append(testdir)
with Case(prevtestdir, read_only=False) as case1:
case2 = case1.create_clone(testdir)
with self.assertRaises(utils.CIMEError):
case2.set_value("CHARGE_ACCOUNT", "fouc")
cls._do_teardown.append(testdir)
def test_cime_case_force_pecount(self):
casedir = self._create_test(
[
"--no-build",
"--force-procs=16",
"--force-threads=8",
"TESTRUNPASS.f19_g16_rx1.A",
],
test_id=self._baseline_name,
)
with Case(casedir, read_only=True) as case:
self.assertEqual(case.get_value("NTASKS_CPL"), 16)
self.assertEqual(case.get_value("NTHRDS_CPL"), 8)
def test_cime_case_mpi_serial(self):
casedir = self._create_test(
["--no-build", "TESTRUNPASS_Mmpi-serial_P10.f19_g16_rx1.A"],
test_id=self._baseline_name,
)
with Case(casedir, read_only=True) as case:
# Serial cases should not be using pnetcdf
self.assertEqual(case.get_value("CPL_PIO_TYPENAME"), "netcdf")
# Serial cases should be using 1 task
self.assertEqual(case.get_value("TOTALPES"), 1)
self.assertEqual(case.get_value("NTASKS_CPL"), 1)
def wait_for_tests(test_paths,
no_wait=False,
check_throughput=False,
check_memory=False,
ignore_namelists=False,
ignore_memleak=False,
cdash_build_name=None,
cdash_project=E3SM_MAIN_CDASH,
cdash_build_group=CDASH_DEFAULT_BUILD_GROUP,
timeout=None,
force_log_upload=False,
no_run=False,
update_success=False):
###############################################################################
# Set up signal handling, we want to print results before the program
# is terminated
set_up_signal_handlers()
with Timeout(timeout, action=signal_handler):
test_results = wait_for_tests_impl(test_paths, no_wait,
check_throughput, check_memory,
ignore_namelists, ignore_memleak,
no_run)
all_pass = True
for test_name, test_data in sorted(test_results.items()):
test_path, test_status = test_data
logging.info("Test '{}' finished with status '{}'".format(
test_name, test_status))
logging.info(" Path: {}".format(test_path))
all_pass &= test_status == TEST_PASS_STATUS
if update_success:
caseroot = os.path.dirname(test_data[0])
with Case(caseroot, read_only=True) as case:
srcroot = case.get_value("CIMEROOT")
baseline_root = case.get_value("BASELINE_ROOT")
save_test_success(
baseline_root, srcroot, test_name, test_status
in [TEST_PASS_STATUS, NAMELIST_FAIL_STATUS])
if cdash_build_name:
create_cdash_xml(test_results, cdash_build_name, cdash_project,
cdash_build_group, force_log_upload)
return all_pass
def test_cime_case_st_archive_resubmit(self):
testcase_name = "st_archive_resubmit_test"
testdir = self._batch_test_fixture(testcase_name)
with Case(testdir, read_only=False) as case:
case.case_setup(clean=False, test_mode=False, reset=True)
orig_resubmit = 2
case.set_value("RESUBMIT", orig_resubmit)
case.case_st_archive(resubmit=False)
new_resubmit = case.get_value("RESUBMIT")
self.assertTrue(orig_resubmit == new_resubmit,
"st_archive resubmitted when told not to")
case.case_st_archive(resubmit=True)
new_resubmit = case.get_value("RESUBMIT")
self.assertTrue(
(orig_resubmit - 1) == new_resubmit,
"st_archive did not resubmit when told to",
)
def test_m_createnewcase_alternate_drivers(self):
# Test that case.setup runs for nuopc and moab drivers
cls = self.__class__
model = utils.get_model()
for driver in ("nuopc", "moab"):
if not os.path.exists(
os.path.join(utils.get_cime_root(), "src", "drivers", driver)
):
self.skipTest(
"Skipping driver test for {}, driver not found".format(driver)
)
if (model == "cesm" and driver == "moab") or (
model == "e3sm" and driver == "nuopc"
):
continue
testdir = os.path.join(cls._testroot, "testcreatenewcase.{}".format(driver))
if os.path.exists(testdir):
shutil.rmtree(testdir)
args = " --driver {} --case {} --compset X --res f19_g16 --output-root {} --handle-preexisting-dirs=r".format(
driver, testdir, cls._testroot
)
if model == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._testdirs.append(testdir)
self.run_cmd_assert_result(
"./create_newcase %s" % (args), from_dir=self.SCRIPT_DIR
)
self.assertTrue(os.path.exists(testdir))
self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
with Case(testdir, read_only=False) as case:
comp_interface = case.get_value("COMP_INTERFACE")
self.assertTrue(
driver == comp_interface, msg="%s != %s" % (driver, comp_interface)
)
cls._do_teardown.append(testdir)
def test_env_loading(self):
if self._machine != "mappy":
self.skipTest("Skipping env load test - Only works on mappy")
casedir = self._create_test(
["--no-build", "TESTRUNPASS.f19_g16_rx1.A"],
test_id=self._baseline_name)
with Case(casedir, read_only=True) as case:
env_mach = case.get_env("mach_specific")
orig_env = dict(os.environ)
env_mach.load_env(case)
module_env = dict(os.environ)
os.environ.clear()
os.environ.update(orig_env)
env_mach.load_env(case, force_method="generic")
generic_env = dict(os.environ)
os.environ.clear()
os.environ.update(orig_env)
problems = ""
for mkey, mval in module_env.items():
if mkey not in generic_env:
if not mkey.startswith("PS") and mkey != "OLDPWD":
problems += "Generic missing key: {}\n".format(mkey)
elif (mval != generic_env[mkey]
and mkey not in ["_", "SHLVL", "PWD"]
and not mkey.endswith("()")):
problems += "Value mismatch for key {}: {} != {}\n".format(
mkey, repr(mval), repr(generic_env[mkey]))
for gkey in generic_env.keys():
if gkey not in module_env:
problems += "Modules missing key: {}\n".format(gkey)
self.assertEqual(problems, "", msg=problems)
def test_cime_case(self):
casedir = self._create_test(
["--no-build", "TESTRUNPASS_P1.f19_g16_rx1.A"],
test_id=self._baseline_name)
self.assertEqual(type(self.MACHINE.get_value("MAX_TASKS_PER_NODE")),
int)
self.assertTrue(
type(self.MACHINE.get_value("PROJECT_REQUIRED")) in
[type(None), bool])
with Case(casedir, read_only=False) as case:
build_complete = case.get_value("BUILD_COMPLETE")
self.assertFalse(
build_complete,
msg="Build complete had wrong value '%s'" % build_complete,
)
case.set_value("BUILD_COMPLETE", True)
build_complete = case.get_value("BUILD_COMPLETE")
self.assertTrue(
build_complete,
msg="Build complete had wrong value '%s'" % build_complete,
)
case.flush()
build_complete = utils.run_cmd_no_fail(
"./xmlquery BUILD_COMPLETE --value", from_dir=casedir)
self.assertEqual(
build_complete,
"TRUE",
msg="Build complete had wrong value '%s'" % build_complete,
)
# Test some test properties
self.assertEqual(case.get_value("TESTCASE"), "TESTRUNPASS")
def wait_for_tests(
test_paths,
no_wait=False,
check_throughput=False,
check_memory=False,
ignore_namelists=False,
ignore_memleak=False,
cdash_build_name=None,
cdash_project=E3SM_MAIN_CDASH,
cdash_build_group=CDASH_DEFAULT_BUILD_GROUP,
timeout=None,
force_log_upload=False,
no_run=False,
update_success=False,
expect_test_complete=True,
):
###############################################################################
# Set up signal handling, we want to print results before the program
# is terminated
set_up_signal_handlers()
with Timeout(timeout, action=signal_handler):
test_results = wait_for_tests_impl(
test_paths,
no_wait,
check_throughput,
check_memory,
ignore_namelists,
ignore_memleak,
no_run,
)
all_pass = True
env_loaded = False
for test_name, test_data in sorted(test_results.items()):
test_path, test_status, phase = test_data
case_dir = os.path.dirname(test_path)
if test_status not in [
TEST_PASS_STATUS,
TEST_PEND_STATUS,
NAMELIST_FAIL_STATUS,
]:
# Report failed phases
logging.info("{} {} (phase {})".format(test_status, test_name,
phase))
all_pass = False
else:
# Be cautious about telling the user that the test passed since we might
# not know that the test passed yet.
if test_status == TEST_PEND_STATUS:
if expect_test_complete:
logging.info(
"{} {} (phase {} unexpectedly left in PEND)".format(
TEST_PEND_STATUS, test_name, phase))
all_pass = False
else:
logging.info(
"{} {} (phase {} has not yet completed)".format(
TEST_PEND_STATUS, test_name, phase))
elif test_status == NAMELIST_FAIL_STATUS:
logging.info("{} {} (but otherwise OK) {}".format(
NAMELIST_FAIL_STATUS, test_name, phase))
all_pass = False
else:
expect(
test_status == TEST_PASS_STATUS,
"Expected pass if we made it here, instead: {}".format(
test_status),
)
logging.info("{} {} {}".format(test_status, test_name, phase))
logging.info(" Case dir: {}".format(case_dir))
if update_success or (cdash_build_name and not env_loaded):
try:
# This can fail if the case crashed before setup completed
with Case(case_dir, read_only=True) as case:
srcroot = case.get_value("SRCROOT")
baseline_root = case.get_value("BASELINE_ROOT")
# Submitting to cdash requires availability of cmake. We can't guarantee
# that without loading the env for a case
if cdash_build_name and not env_loaded:
case.load_env()
env_loaded = True
if update_success:
save_test_success(
baseline_root,
srcroot,
test_name,
test_status
in [TEST_PASS_STATUS, NAMELIST_FAIL_STATUS],
)
except CIMEError as e:
logging.warning(
"Failed to update success / load_env for Case {}: {}".
format(case_dir, e))
if cdash_build_name:
create_cdash_xml(
test_results,
cdash_build_name,
cdash_project,
cdash_build_group,
force_log_upload,
)
return all_pass
def test_cime_case_prereq(self):
testcase_name = "prereq_test"
testdir = self._batch_test_fixture(testcase_name)
with Case(testdir, read_only=False) as case:
if case.get_value("depend_string") is None:
self.skipTest(
"Skipping prereq test, depend_string was not provided for this batch system"
)
job_name = "case.run"
prereq_name = "prereq_test"
batch_commands = case.submit_jobs(prereq=prereq_name,
job=job_name,
skip_pnl=True,
dry_run=True)
self.assertTrue(
isinstance(batch_commands, collections.Sequence),
"case.submit_jobs did not return a sequence for a dry run",
)
self.assertTrue(
len(batch_commands) > 0,
"case.submit_jobs did not return any job submission string",
)
# The first element in the internal sequence should just be the job name
# The second one (batch_cmd_index) should be the actual batch submission command
batch_cmd_index = 1
# The prerequisite should be applied to all jobs, though we're only expecting one
for batch_cmd in batch_commands:
self.assertTrue(
isinstance(batch_cmd, collections.Sequence),
"case.submit_jobs did not return a sequence of sequences",
)
self.assertTrue(
len(batch_cmd) > batch_cmd_index,
"case.submit_jobs returned internal sequences with length <= {}"
.format(batch_cmd_index),
)
self.assertTrue(
isinstance(batch_cmd[1], six.string_types),
"case.submit_jobs returned internal sequences without the batch command string as the second parameter: {}"
.format(batch_cmd[1]),
)
batch_cmd_args = batch_cmd[1]
jobid_ident = "jobid"
dep_str_fmt = case.get_env("batch").get_value("depend_string",
subgroup=None)
self.assertTrue(
jobid_ident in dep_str_fmt,
"dependency string doesn't include the jobid identifier {}"
.format(jobid_ident),
)
dep_str = dep_str_fmt[:dep_str_fmt.index(jobid_ident)]
prereq_substr = None
while dep_str in batch_cmd_args:
dep_id_pos = batch_cmd_args.find(dep_str) + len(dep_str)
batch_cmd_args = batch_cmd_args[dep_id_pos:]
prereq_substr = batch_cmd_args[:len(prereq_name)]
if prereq_substr == prereq_name:
break
self.assertTrue(
prereq_name in prereq_substr,
"Dependencies added, but not the user specified one",
)
with Case(testdir, read_only=False) as case:
case.set_value("CHARGE_ACCOUNT", "fred")
# this should not fail with a locked file issue
self.run_cmd_assert_result("./case.build", from_dir=testdir)
self.run_cmd_assert_result("./case.st_archive --test-all", from_dir=testdir)
# Trying to set values outside of context manager should fail
case = Case(testdir, read_only=False)
with self.assertRaises(utils.CIMEError):
case.set_value("NTASKS_ATM", 42)
# Trying to read_xml with pending changes should fail
with self.assertRaises(utils.CIMEError):
with Case(testdir, read_only=False) as case:
case.set_value("CHARGE_ACCOUNT", "fouc")
case.read_xml()
cls._do_teardown.append(testdir)
def test_aa_no_flush_on_instantiate(self):
testdir = os.path.join(self.__class__._testroot, "testcreatenewcase")
with Case(testdir, read_only=False) as case:
for env_file in case._files:
self.assertFalse(
env_file.needsrewrite,
msg="Instantiating a case should not trigger a flush call",
)
with Case(testdir, read_only=False) as case:
class TestCreateNewcase(base.BaseTestCase):
@classmethod
def setUpClass(cls):
cls._testdirs = []
cls._do_teardown = []
cls._testroot = os.path.join(cls.TEST_ROOT, "TestCreateNewcase")
cls._root_dir = os.getcwd()
def tearDown(self):
cls = self.__class__
os.chdir(cls._root_dir)
def test_a_createnewcase(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testcreatenewcase")
if os.path.exists(testdir):
shutil.rmtree(testdir)
args = " --case %s --compset X --output-root %s --handle-preexisting-dirs=r" % (
testdir,
cls._testroot,
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._testdirs.append(testdir)
self.run_cmd_assert_result(
"./create_newcase %s" % (args), from_dir=self.SCRIPT_DIR
)
self.assertTrue(os.path.exists(testdir))
self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
with Case(testdir, read_only=False) as case:
ntasks = case.get_value("NTASKS_ATM")
case.set_value("NTASKS_ATM", ntasks + 1)
# this should fail with a locked file issue
self.run_cmd_assert_result("./case.build", from_dir=testdir, expected_stat=1)
self.run_cmd_assert_result("./case.setup --reset", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
with Case(testdir, read_only=False) as case:
case.set_value("CHARGE_ACCOUNT", "fred")
# this should not fail with a locked file issue
self.run_cmd_assert_result("./case.build", from_dir=testdir)
self.run_cmd_assert_result("./case.st_archive --test-all", from_dir=testdir)
# Trying to set values outside of context manager should fail
case = Case(testdir, read_only=False)
with self.assertRaises(utils.CIMEError):
case.set_value("NTASKS_ATM", 42)
def test_ka_createnewcase_extra_machines_dir(self):
# Test that we pick up changes in both config_machines.xml and
# cmake macros in a directory specified with the --extra-machines-dir
# argument to create_newcase.
cls = self.__class__
casename = "testcreatenewcase_extra_machines_dir"
# Setup: stage some xml files in a temporary directory
extra_machines_dir = os.path.join(cls._testroot,
"{}_machine_config".format(casename))
os.makedirs(os.path.join(extra_machines_dir, "cmake_macros"))
cls._do_teardown.append(extra_machines_dir)
newmachfile = os.path.join(
utils.get_cime_root(),
"CIME",
"data",
"config",
"xml_schemas",
"config_machines_template.xml",
)
utils.safe_copy(
newmachfile, os.path.join(extra_machines_dir,
"config_machines.xml"))
cmake_macro_text = """\
set(NETCDF_PATH /my/netcdf/path)
"""
cmake_macro_path = os.path.join(extra_machines_dir, "cmake_macros",
"mymachine.cmake")
with open(cmake_macro_path, "w") as cmake_macro:
cmake_macro.write(cmake_macro_text)
# Create the case
testdir = os.path.join(cls._testroot, casename)
if os.path.exists(testdir):
shutil.rmtree(testdir)
# In the following, note that 'mymachine' is the machine name defined in
# config_machines_template.xml
args = (" --case {testdir} --compset X --mach mymachine"
" --output-root {testroot} --non-local"
" --extra-machines-dir {extra_machines_dir}".format(
testdir=testdir,
testroot=cls._testroot,
extra_machines_dir=extra_machines_dir,
))
if utils.get_model() == "cesm":
args += " --run-unsupported"
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
self.run_cmd_assert_result("./create_newcase {}".format(args),
from_dir=self.SCRIPT_DIR)
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._do_teardown.append(testdir)
# Run case.setup
self.run_cmd_assert_result("./case.setup --non-local",
from_dir=testdir)
# Make sure Macros file contains expected text
with Case(testdir, non_local=True) as case:
with CmakeTmpBuildDir(macroloc=testdir) as cmaketmp:
macros_contents = cmaketmp.get_makefile_vars(case=case)
expected_re = re.compile("NETCDF_PATH.*/my/netcdf/path")
self.assertTrue(
expected_re.search(macros_contents),
msg="{} not found in:\n{}".format(expected_re.pattern,
macros_contents),
)
def test_h_primary_component(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testprimarycomponent")
if os.path.exists(testdir):
shutil.rmtree(testdir)
cls._testdirs.append(testdir)
args = (
" --case CreateNewcaseTest --script-root %s --compset X --output-root %s --handle-preexisting-dirs u"
% (testdir, cls._testroot)
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args += " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args += " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"%s/create_newcase %s" % (self.SCRIPT_DIR, args), from_dir=self.SCRIPT_DIR
)
self.assertTrue(os.path.exists(testdir))
self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
with Case(testdir, read_only=False) as case:
case._compsetname = case.get_value("COMPSET")
case.set_comp_classes(case.get_values("COMP_CLASSES"))
primary = case._find_primary_component()
self.assertEqual(
primary,
"drv",
msg="primary component test expected drv but got %s" % primary,
)
# now we are going to corrupt the case so that we can do more primary_component testing
case.set_valid_values("COMP_GLC", "%s,fred" % case.get_value("COMP_GLC"))
case.set_value("COMP_GLC", "fred")
primary = case._find_primary_component()
self.assertEqual(
primary,
"fred",
msg="primary component test expected fred but got %s" % primary,
)
case.set_valid_values("COMP_ICE", "%s,wilma" % case.get_value("COMP_ICE"))
case.set_value("COMP_ICE", "wilma")
primary = case._find_primary_component()
self.assertEqual(
primary,
"wilma",
msg="primary component test expected wilma but got %s" % primary,
)
case.set_valid_values(
"COMP_OCN", "%s,bambam,docn" % case.get_value("COMP_OCN")
)
case.set_value("COMP_OCN", "bambam")
primary = case._find_primary_component()
self.assertEqual(
primary,
"bambam",
msg="primary component test expected bambam but got %s" % primary,
)
case.set_valid_values("COMP_LND", "%s,barney" % case.get_value("COMP_LND"))
case.set_value("COMP_LND", "barney")
primary = case._find_primary_component()
# This is a "J" compset
self.assertEqual(
primary,
"allactive",
msg="primary component test expected allactive but got %s" % primary,
)
case.set_value("COMP_OCN", "docn")
case.set_valid_values("COMP_LND", "%s,barney" % case.get_value("COMP_LND"))
case.set_value("COMP_LND", "barney")
primary = case._find_primary_component()
self.assertEqual(
primary,
"barney",
msg="primary component test expected barney but got %s" % primary,
)
case.set_valid_values("COMP_ATM", "%s,wilma" % case.get_value("COMP_ATM"))
case.set_value("COMP_ATM", "wilma")
primary = case._find_primary_component()
self.assertEqual(
primary,
"wilma",
msg="primary component test expected wilma but got %s" % primary,
)
# this is a "E" compset
case._compsetname = case._compsetname.replace("XOCN", "DOCN%SOM")
primary = case._find_primary_component()
self.assertEqual(
primary,
"allactive",
msg="primary component test expected allactive but got %s" % primary,
)
# finally a "B" compset
case.set_value("COMP_OCN", "bambam")
primary = case._find_primary_component()
self.assertEqual(
primary,
"allactive",
msg="primary component test expected allactive but got %s" % primary,
)
utils.safe_copy(backup, active)
env_run = EnvRun(casedir, read_only=True)
self.assertEqual(env_run.get_value("RUN_TYPE"), "startup")
with Case(casedir, read_only=False) as case:
self.assertEqual(case.get_value("RUN_TYPE"), "startup")
case.set_value("RUN_TYPE", "branch")
# behind the back detection.
with self.assertRaises(utils.CIMEError):
with Case(casedir, read_only=False) as case:
case.set_value("RUN_TYPE", "startup")
time.sleep(0.2)
utils.safe_copy(backup, active)
with Case(casedir, read_only=False) as case:
case.set_value("RUN_TYPE", "branch")
# If there's no modications within CIME, the files should not be written
# and therefore no timestamp check
with Case(casedir) as case:
time.sleep(0.2)
utils.safe_copy(backup, active)
def test_configure(self):
testname = "SMS.f09_g16.X"
casedir = self._create_test([testname, "--no-build"],
test_id=self._baseline_name)
manual_config_dir = os.path.join(casedir, "manual_config")
os.mkdir(manual_config_dir)
