def test_user_concurrent_mods(self):
# Put this inside any test that's slow
if self.FAST_ONLY:
self.skipTest("Skipping slow test")
casedir = self._create_test(
[
"--walltime=0:30:00",
"TESTRUNUSERXMLCHANGE_Mmpi-serial.f19_g16.X"
],
test_id=self._baseline_name,
)
with utils.Timeout(3000):
while True:
with open(os.path.join(casedir, "CaseStatus"), "r") as fd:
self._wait_for_tests(self._baseline_name)
contents = fd.read()
if contents.count("model execution success") == 2:
break
time.sleep(5)
rundir = utils.run_cmd_no_fail("./xmlquery RUNDIR --value",
from_dir=casedir)
if utils.get_cime_default_driver() == "nuopc":
chk_file = "nuopc.runconfig"
else:
chk_file = "drv_in"
with open(os.path.join(rundir, chk_file), "r") as fd:
contents = fd.read()
self.assertTrue("stop_n = 6" in contents)
def get_all_checkable_files():
###############################################################################
cimeroot = get_cime_root()
all_git_files = run_cmd_no_fail("git ls-files",
from_dir=cimeroot,
verbose=False).splitlines()
if get_cime_default_driver() == "nuopc":
srcroot = get_src_root()
nuopc_git_files = []
try:
nuopc_git_files = run_cmd_no_fail(
"git ls-files",
from_dir=os.path.join(srcroot, "components", "cmeps"),
verbose=False,
).splitlines()
except:
logger.warning("No nuopc driver found in source")
all_git_files.extend([
os.path.join(srcroot, "components", "cmeps", _file)
for _file in nuopc_git_files
])
files_to_test = [
item for item in all_git_files
if ((item.endswith(".py") or is_python_executable(
os.path.join(cimeroot, item))) and not _should_pylint_skip(item))
]
return files_to_test
def test_n_createnewcase_bad_compset(self):
cls = self.__class__
model = utils.get_model()
testdir = os.path.join(cls._testroot, "testcreatenewcase_bad_compset")
if os.path.exists(testdir):
shutil.rmtree(testdir)
args = (
" --case %s --compset InvalidCompsetName --output-root %s --handle-preexisting-dirs=r "
% (testdir, cls._testroot)
)
if model == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"./create_newcase %s" % (args), from_dir=self.SCRIPT_DIR, expected_stat=1
)
self.assertFalse(os.path.exists(testdir))
def _create_test(self, extra_args, test_id=None, run_errors=False, env_changes=""):
"""
Convenience wrapper around create_test. Returns list of full paths to created cases. If multiple cases,
the order of the returned list is not guaranteed to match the order of the arguments.
"""
# All stub model not supported in nuopc driver
driver = utils.get_cime_default_driver()
if driver == "nuopc" and "cime_developer" in extra_args:
extra_args.append(
" ^SMS_Ln3.T42_T42.S ^PRE.f19_f19.ADESP_TEST ^PRE.f19_f19.ADESP ^DAE.ww3a.ADWAV"
)
test_id = (
"{}-{}".format(self._baseline_name, utils.get_timestamp())
if test_id is None
else test_id
)
extra_args.append("-t {}".format(test_id))
extra_args.append("--baseline-root {}".format(self._baseline_area))
if self.NO_BATCH:
extra_args.append("--no-batch")
if self.TEST_COMPILER and (
[extra_arg for extra_arg in extra_args if "--compiler" in extra_arg] == []
):
extra_args.append("--compiler={}".format(self.TEST_COMPILER))
if self.TEST_MPILIB and (
[extra_arg for extra_arg in extra_args if "--mpilib" in extra_arg] == []
):
extra_args.append("--mpilib={}".format(self.TEST_MPILIB))
if [extra_arg for extra_arg in extra_args if "--machine" in extra_arg] == []:
extra_args.append(f"--machine {self.MACHINE.get_machine_name()}")
extra_args.append("--test-root={0} --output-root={0}".format(self._testroot))
full_run = (
set(extra_args)
& set(["-n", "--namelist-only", "--no-setup", "--no-build", "--no-run"])
) == set()
if full_run and not self.NO_BATCH:
extra_args.append("--wait")
expected_stat = 0 if not run_errors else utils.TESTS_FAILED_ERR_CODE
output = self.run_cmd_assert_result(
"{} {}/create_test {}".format(
env_changes, self.SCRIPT_DIR, " ".join(extra_args)
),
expected_stat=expected_stat,
)
cases = []
for line in output.splitlines():
if "Case dir:" in line:
casedir = line.split()[-1]
self.assertTrue(
os.path.isdir(casedir), msg="Missing casedir {}".format(casedir)
)
cases.append(casedir)
self.assertTrue(len(cases) > 0, "create_test made no cases")
return cases[0] if len(cases) == 1 else cases
def simple_test(self, manual_timing=False):
if self.NO_FORTRAN_RUN:
self.skipTest("Skipping fortran test")
timing_flag = "" if manual_timing else "--save-timing"
driver = utils.get_cime_default_driver()
if driver == "mct":
walltime = "00:15:00"
else:
walltime = "00:30:00"
self._create_test(
[
"SMS_Ln9_P1.f19_g16_rx1.A", timing_flag,
"--walltime=" + walltime
],
test_id=self._baseline_name,
)
statuses = glob.glob("%s/*%s/TestStatus" %
(self._testroot, self._baseline_name))
self.assertEqual(
len(statuses),
1,
msg="Should have had exactly one match, found %s" % statuses,
)
casedir = os.path.dirname(statuses[0])
with Case(casedir, read_only=True) as case:
lids = utils.get_lids(case)
timing_dir = case.get_value("SAVE_TIMING_DIR")
casename = case.get_value("CASE")
self.assertEqual(len(lids), 1, msg="Expected one LID, found %s" % lids)
if manual_timing:
self.run_cmd_assert_result("cd %s && %s/save_provenance postrun" %
(casedir, self.TOOLS_DIR))
if utils.get_model() == "e3sm":
provenance_glob = os.path.join(
timing_dir,
"performance_archive",
getpass.getuser(),
casename,
lids[0] + "*",
)
provenance_dirs = glob.glob(provenance_glob)
self.assertEqual(
len(provenance_dirs),
1,
msg=
"wrong number of provenance dirs, expected 1, got {}, looked for {}"
.format(provenance_dirs, provenance_glob),
)
self.verify_perms("".join(provenance_dirs))
def test_f_createnewcase_with_user_compset(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testcreatenewcase_with_user_compset")
if os.path.exists(testdir):
shutil.rmtree(testdir)
cls._testdirs.append(testdir)
if utils.get_model() == "cesm":
if utils.get_cime_default_driver() == "nuopc":
pesfile = os.path.join(
utils.get_src_root(),
"components",
"cmeps",
"cime_config",
"config_pes.xml",
)
else:
pesfile = os.path.join(
utils.get_src_root(),
"components",
"cpl7",
"driver",
"cime_config",
"config_pes.xml",
)
else:
pesfile = os.path.join(
utils.get_src_root(), "driver-mct", "cime_config", "config_pes.xml"
)
args = (
"--case %s --compset 2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --pesfile %s --res f19_g16 --output-root %s --handle-preexisting-dirs=r"
% (testdir, pesfile, cls._testroot)
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args += " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"%s/create_newcase %s" % (self.SCRIPT_DIR, args), from_dir=self.SCRIPT_DIR
)
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
cls._do_teardown.append(testdir)
def test_b_user_mods(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testusermods")
if os.path.exists(testdir):
shutil.rmtree(testdir)
cls._testdirs.append(testdir)
user_mods_dir = os.path.join(
utils.get_python_libs_root(), "..", "tests", "user_mods_test1"
)
args = (
" --case %s --compset X --user-mods-dir %s --output-root %s --handle-preexisting-dirs=r"
% (testdir, user_mods_dir, cls._testroot)
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"%s/create_newcase %s " % (self.SCRIPT_DIR, args), from_dir=self.SCRIPT_DIR
)
self.assertTrue(
os.path.isfile(
os.path.join(testdir, "SourceMods", "src.drv", "somefile.F90")
),
msg="User_mods SourceMod missing",
)
with open(os.path.join(testdir, "user_nl_cpl"), "r") as fd:
contents = fd.read()
self.assertTrue(
"a different cpl test option" in contents,
msg="User_mods contents of user_nl_cpl missing",
)
self.assertTrue(
"a cpl namelist option" in contents,
msg="User_mods contents of user_nl_cpl missing",
)
cls._do_teardown.append(testdir)
def test_a_createnewcase(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testcreatenewcase")
if os.path.exists(testdir):
shutil.rmtree(testdir)
args = " --case %s --compset X --output-root %s --handle-preexisting-dirs=r" % (
testdir,
cls._testroot,
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args = args + " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args = args + " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._testdirs.append(testdir)
self.run_cmd_assert_result("./create_newcase %s" % (args),
from_dir=self.SCRIPT_DIR)
self.assertTrue(os.path.exists(testdir))
self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
with Case(testdir, read_only=False) as case:
ntasks = case.get_value("NTASKS_ATM")
case.set_value("NTASKS_ATM", ntasks + 1)
# this should fail with a locked file issue
self.run_cmd_assert_result("./case.build",
from_dir=testdir,
expected_stat=1)
self.run_cmd_assert_result("./case.setup --reset", from_dir=testdir)
self.run_cmd_assert_result("./case.build", from_dir=testdir)
with Case(testdir, read_only=False) as case:
case.set_value("CHARGE_ACCOUNT", "fred")
def test_run_restart(self):
if self.NO_FORTRAN_RUN:
self.skipTest("Skipping fortran test")
driver = utils.get_cime_default_driver()
if driver == "mct":
walltime = "00:15:00"
else:
walltime = "00:30:00"
casedir = self._create_test(
["--walltime " + walltime, "NODEFAIL_P1.f09_g16.X"],
test_id=self._baseline_name,
)
rundir = utils.run_cmd_no_fail("./xmlquery RUNDIR --value",
from_dir=casedir)
fail_sentinel = os.path.join(rundir, "FAIL_SENTINEL")
self.assertTrue(os.path.exists(fail_sentinel),
msg="Missing %s" % fail_sentinel)
self.assertEqual(open(fail_sentinel, "r").read().count("FAIL"), 3)
def get_all_checkable_files():
###############################################################################
cimeroot = get_cime_root()
all_git_files = run_cmd_no_fail("git ls-files",
from_dir=cimeroot,
verbose=False).splitlines()
if get_cime_default_driver() == "nuopc":
nuopc_git_files = run_cmd_no_fail("git ls-files",
from_dir=os.path.join(
cimeroot, "src", "drivers",
"nuopc"),
verbose=False).splitlines()
all_git_files.extend([
os.path.join("src", "drivers", "nuopc", _file)
for _file in nuopc_git_files
])
files_to_test = [
item for item in all_git_files
if ((item.endswith(".py") or is_python_executable(
os.path.join(cimeroot, item))) and not _should_pylint_skip(item))
]
return files_to_test
def test_full_system(self):
# Put this inside any test that's slow
if self.FAST_ONLY:
self.skipTest("Skipping slow test")
driver = utils.get_cime_default_driver()
if driver == "mct":
cases = self._create_test(["--walltime=0:15:00", "cime_developer"],
test_id=self._baseline_name)
else:
cases = self._create_test(["--walltime=0:30:00", "cime_developer"],
test_id=self._baseline_name)
self.run_cmd_assert_result(
"%s/cs.status.%s" % (self._testroot, self._baseline_name),
from_dir=self._testroot,
)
# Ensure that we can get test times
for case_dir in cases:
tstatus = os.path.join(case_dir, "TestStatus")
test_time = wait_for_tests.get_test_time(os.path.dirname(tstatus))
self.assertIs(type(test_time),
int,
msg="get time did not return int for %s" % tstatus)
self.assertTrue(test_time > 0,
msg="test time was zero for %s" % tstatus)
# Test that re-running works
skip_tests = None
if utils.get_cime_default_driver() == "nuopc":
skip_tests = [
"SMS_Ln3.T42_T42.S",
"PRE.f19_f19.ADESP_TEST",
"PRE.f19_f19.ADESP",
"DAE.ww3a.ADWAV",
]
tests = get_tests.get_test_suite(
"cime_developer",
machine=self._machine,
compiler=self._compiler,
skip_tests=skip_tests,
)
for test in tests:
casedir = self.get_casedir(test, cases)
# Subtle issue: The run phases of these tests will be in the PASS state until
# the submitted case.test script is run, which could take a while if the system is
# busy. This potentially leaves a window where the wait_for_tests command below will
# not wait for the re-submitted jobs to run because it sees the original PASS.
# The code below forces things back to PEND to avoid this race condition. Note
# that we must use the MEMLEAK phase, not the RUN phase, because RUN being in a non-PEND
# state is how system tests know they are being re-run and must reset certain
# case settings.
if self._hasbatch:
with test_status.TestStatus(test_dir=casedir) as ts:
ts.set_status(test_status.MEMLEAK_PHASE,
test_status.TEST_PEND_STATUS)
self.run_cmd_assert_result("./case.submit --skip-preview-namelist",
from_dir=casedir)
self._wait_for_tests(self._baseline_name)
def test_ka_createnewcase_extra_machines_dir(self):
# Test that we pick up changes in both config_machines.xml and
# config_compilers.xml in a directory specified with the --extra-machines-dir
# argument to create_newcase.
cls = self.__class__
casename = "testcreatenewcase_extra_machines_dir"
# Setup: stage some xml files in a temporary directory
extra_machines_dir = os.path.join(
cls._testroot, "{}_machine_config".format(casename)
)
os.makedirs(extra_machines_dir)
cls._do_teardown.append(extra_machines_dir)
newmachfile = os.path.join(
utils.get_cime_root(),
"config",
"xml_schemas",
"config_machines_template.xml",
)
utils.safe_copy(
newmachfile, os.path.join(extra_machines_dir, "config_machines.xml")
)
os.environ["CIME_NO_CMAKE_MACRO"] = "ON"
config_compilers_text = """\
<?xml version="1.0" encoding="UTF-8"?>
<config_compilers version="2.0">
<compiler MACH="mymachine">
<NETCDF_PATH>/my/netcdf/path</NETCDF_PATH>
</compiler>
</config_compilers>
"""
config_compilers_path = os.path.join(extra_machines_dir, "config_compilers.xml")
with open(config_compilers_path, "w") as config_compilers:
config_compilers.write(config_compilers_text)
# Create the case
testdir = os.path.join(cls._testroot, casename)
if os.path.exists(testdir):
shutil.rmtree(testdir)
# In the following, note that 'mymachine' is the machine name defined in
# config_machines_template.xml
args = (
" --case {testdir} --compset X --mach mymachine"
" --output-root {testroot} --non-local"
" --extra-machines-dir {extra_machines_dir}".format(
testdir=testdir,
testroot=cls._testroot,
extra_machines_dir=extra_machines_dir,
)
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
self.run_cmd_assert_result(
"./create_newcase {}".format(args), from_dir=self.SCRIPT_DIR
)
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._do_teardown.append(testdir)
# Run case.setup
self.run_cmd_assert_result("./case.setup", from_dir=testdir)
# Make sure Macros file contains expected text
if utils.get_model() != "e3sm":
macros_file_name = os.path.join(testdir, "Macros.make")
self.assertTrue(os.path.isfile(macros_file_name))
with open(macros_file_name) as macros_file:
macros_contents = macros_file.read()
expected_re = re.compile("NETCDF_PATH.*/my/netcdf/path")
self.assertTrue(expected_re.search(macros_contents))
del os.environ["CIME_NO_CMAKE_MACRO"]
def test_j_createnewcase_user_compset_vs_alias(self):
"""
Create a compset using the alias and another compset using the full compset name
and make sure they are the same by comparing the namelist files in CaseDocs.
Ignore the modelio files and clean the directory names out first.
"""
cls = self.__class__
testdir1 = os.path.join(cls._testroot, "testcreatenewcase_user_compset")
if os.path.exists(testdir1):
shutil.rmtree(testdir1)
cls._testdirs.append(testdir1)
args = " --case CreateNewcaseTest --script-root {} --compset 2000_DATM%NYF_SLND_SICE_DOCN%SOMAQP_SROF_SGLC_SWAV --output-root {} --handle-preexisting-dirs u".format(
testdir1, cls._testroot
)
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args += " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args += " --mpilib %s" % self.TEST_MPILIB
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"{}/create_newcase {}".format(self.SCRIPT_DIR, args),
from_dir=self.SCRIPT_DIR,
)
self.run_cmd_assert_result("./case.setup ", from_dir=testdir1)
self.run_cmd_assert_result("./preview_namelists ", from_dir=testdir1)
dir1 = os.path.join(testdir1, "CaseDocs")
dir2 = os.path.join(testdir1, "CleanCaseDocs")
os.mkdir(dir2)
for _file in os.listdir(dir1):
if "modelio" in _file:
continue
with open(os.path.join(dir1, _file), "r") as fi:
file_text = fi.read()
file_text = file_text.replace(os.path.basename(testdir1), "PATH")
file_text = re.sub(r"logfile =.*", "", file_text)
with open(os.path.join(dir2, _file), "w") as fo:
fo.write(file_text)
cleancasedocs1 = dir2
testdir2 = os.path.join(cls._testroot, "testcreatenewcase_alias_compset")
if os.path.exists(testdir2):
shutil.rmtree(testdir2)
cls._testdirs.append(testdir2)
args = " --case CreateNewcaseTest --script-root {} --compset ADSOMAQP --output-root {} --handle-preexisting-dirs u".format(
testdir2, cls._testroot
)
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args += " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args += " --mpilib %s" % self.TEST_MPILIB
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"{}/create_newcase {}".format(self.SCRIPT_DIR, args),
from_dir=self.SCRIPT_DIR,
)
self.run_cmd_assert_result("./case.setup ", from_dir=testdir2)
self.run_cmd_assert_result("./preview_namelists ", from_dir=testdir2)
dir1 = os.path.join(testdir2, "CaseDocs")
dir2 = os.path.join(testdir2, "CleanCaseDocs")
os.mkdir(dir2)
for _file in os.listdir(dir1):
if "modelio" in _file:
continue
with open(os.path.join(dir1, _file), "r") as fi:
file_text = fi.read()
file_text = file_text.replace(os.path.basename(testdir2), "PATH")
file_text = re.sub(r"logfile =.*", "", file_text)
with open(os.path.join(dir2, _file), "w") as fo:
fo.write(file_text)
cleancasedocs2 = dir2
dcmp = filecmp.dircmp(cleancasedocs1, cleancasedocs2)
self.assertTrue(
len(dcmp.diff_files) == 0, "CaseDocs differ {}".format(dcmp.diff_files)
)
cls._do_teardown.append(testdir1)
cls._do_teardown.append(testdir2)
def test_h_primary_component(self):
cls = self.__class__
testdir = os.path.join(cls._testroot, "testprimarycomponent")
if os.path.exists(testdir):
shutil.rmtree(testdir)
cls._testdirs.append(testdir)
args = (
" --case CreateNewcaseTest --script-root %s --compset X --output-root %s --handle-preexisting-dirs u"
% (testdir, cls._testroot)
)
if utils.get_model() == "cesm":
args += " --run-unsupported"
if self.TEST_COMPILER is not None:
args += " --compiler %s" % self.TEST_COMPILER
if self.TEST_MPILIB is not None:
args += " --mpilib %s" % self.TEST_MPILIB
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
args += f" --machine {self.MACHINE.get_machine_name()}"
self.run_cmd_assert_result(
"%s/create_newcase %s" % (self.SCRIPT_DIR, args), from_dir=self.SCRIPT_DIR
)
self.assertTrue(os.path.exists(testdir))
self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
with Case(testdir, read_only=False) as case:
case._compsetname = case.get_value("COMPSET")
case.set_comp_classes(case.get_values("COMP_CLASSES"))
primary = case._find_primary_component()
self.assertEqual(
primary,
"drv",
msg="primary component test expected drv but got %s" % primary,
)
# now we are going to corrupt the case so that we can do more primary_component testing
case.set_valid_values("COMP_GLC", "%s,fred" % case.get_value("COMP_GLC"))
case.set_value("COMP_GLC", "fred")
primary = case._find_primary_component()
self.assertEqual(
primary,
"fred",
msg="primary component test expected fred but got %s" % primary,
)
case.set_valid_values("COMP_ICE", "%s,wilma" % case.get_value("COMP_ICE"))
case.set_value("COMP_ICE", "wilma")
primary = case._find_primary_component()
self.assertEqual(
primary,
"wilma",
msg="primary component test expected wilma but got %s" % primary,
)
case.set_valid_values(
"COMP_OCN", "%s,bambam,docn" % case.get_value("COMP_OCN")
)
case.set_value("COMP_OCN", "bambam")
primary = case._find_primary_component()
self.assertEqual(
primary,
"bambam",
msg="primary component test expected bambam but got %s" % primary,
)
case.set_valid_values("COMP_LND", "%s,barney" % case.get_value("COMP_LND"))
case.set_value("COMP_LND", "barney")
primary = case._find_primary_component()
# This is a "J" compset
self.assertEqual(
primary,
"allactive",
msg="primary component test expected allactive but got %s" % primary,
)
case.set_value("COMP_OCN", "docn")
case.set_valid_values("COMP_LND", "%s,barney" % case.get_value("COMP_LND"))
case.set_value("COMP_LND", "barney")
primary = case._find_primary_component()
self.assertEqual(
primary,
"barney",
msg="primary component test expected barney but got %s" % primary,
)
case.set_valid_values("COMP_ATM", "%s,wilma" % case.get_value("COMP_ATM"))
case.set_value("COMP_ATM", "wilma")
primary = case._find_primary_component()
self.assertEqual(
primary,
"wilma",
msg="primary component test expected wilma but got %s" % primary,
)
# this is a "E" compset
case._compsetname = case._compsetname.replace("XOCN", "DOCN%SOM")
primary = case._find_primary_component()
self.assertEqual(
primary,
"allactive",
msg="primary component test expected allactive but got %s" % primary,
)
# finally a "B" compset
case.set_value("COMP_OCN", "bambam")
primary = case._find_primary_component()
self.assertEqual(
primary,
"allactive",
msg="primary component test expected allactive but got %s" % primary,
)
def parse_command_line(args, cimeroot, description):
###############################################################################
parser = argparse.ArgumentParser(
description=description, formatter_class=RawTextHelpFormatter
)
CIME.utils.setup_standard_logging_options(parser)
try:
cime_config = get_cime_config()
except Exception:
cime_config = None
parser.add_argument(
"--case",
"-case",
required=True,
metavar="CASENAME",
help="(required) Specify the case name. "
"\nIf this is simply a name (not a path), the case directory is created in the current working directory."
"\nThis can also be a relative or absolute path specifying where the case should be created;"
"\nwith this usage, the name of the case will be the last component of the path.",
)
parser.add_argument(
"--compset",
"-compset",
required=True,
help="(required) Specify a compset. "
"\nTo see list of current compsets, use the utility ./query_config --compsets in this directory.\n",
)
parser.add_argument(
"--res",
"-res",
required=True,
metavar="GRID",
help="(required) Specify a model grid resolution. "
"\nTo see list of current model resolutions, use the utility "
"\n./query_config --grids in this directory.",
)
parser.add_argument(
"--machine",
"-mach",
help="Specify a machine. "
"The default value is the match to NODENAME_REGEX in config_machines.xml. To see "
"\nthe list of current machines, invoke ./query_config --machines.",
)
parser.add_argument(
"--compiler",
"-compiler",
help="Specify a compiler. "
"\nTo see list of supported compilers for each machine, use the utility "
"\n./query_config --machines in this directory. "
"\nThe default value will be the first one listed.",
)
parser.add_argument(
"--multi-driver",
action="store_true",
help="Specify that --ninst should modify the number of driver/coupler instances. "
"\nThe default is to have one driver/coupler supporting multiple component instances.",
)
parser.add_argument(
"--ninst",
default=1,
type=int,
help="Specify number of model ensemble instances. "
"\nThe default is multiple components and one driver/coupler. "
"\nUse --multi-driver to run multiple driver/couplers in the ensemble.",
)
parser.add_argument(
"--mpilib",
"-mpilib",
help="Specify the MPI library. "
"To see list of supported mpilibs for each machine, invoke ./query_config --machines."
"\nThe default is the first listing in MPILIBS in config_machines.xml.\n",
)
parser.add_argument(
"--project",
"-project",
help="Specify a project id for the case (optional)."
"\nUsed for accounting and directory permissions when on a batch system."
"\nThe default is user or machine specified by PROJECT."
"\nAccounting (only) may be overridden by user or machine specified CHARGE_ACCOUNT.",
)
parser.add_argument(
"--pecount",
"-pecount",
default="M",
help="Specify a target size description for the number of cores. "
"\nThis is used to query the appropriate config_pes.xml file and find the "
"\noptimal PE-layout for your case - if it exists there. "
"\nAllowed options are ('S','M','L','X1','X2','[0-9]x[0-9]','[0-9]').\n",
)
# This option supports multiple values, hence the plural ("user-mods-dirs"). However,
# we support the singular ("user-mods-dir") for backwards compatibility (and because
# the singular may be more intuitive for someone who only wants to use a single
# directory).
parser.add_argument(
"--user-mods-dirs",
"--user-mods-dir",
nargs="*",
help="Full pathname to a directory containing any combination of user_nl_* files "
"\nand a shell_commands script (typically containing xmlchange commands). "
"\nThe directory can also contain an SourceMods/ directory with the same structure "
"\nas would be found in a case directory."
"\nIt can also contain a file named 'include_user_mods' which gives the path to"
"\none or more other directories that should be included."
"\nMultiple directories can be given to the --user-mods-dirs argument,"
"\nin which case changes from all of them are applied."
"\n(If there are conflicts, later directories take precedence.)"
"\n(Care is needed if multiple directories include the same directory via 'include_user_mods':"
"\nin this case, the included directory will be applied multiple times.)",
)
parser.add_argument(
"--pesfile",
help="Full pathname of an optional pes specification file. "
"\nThe file can follow either the config_pes.xml or the env_mach_pes.xml format.",
)
parser.add_argument(
"--gridfile",
help="Full pathname of config grid file to use. "
"\nThis should be a copy of config/config_grids.xml with the new user grid changes added to it. \n",
)
if cime_config and cime_config.has_option("main", "workflow"):
workflow_default = cime_config.get("main", "workflow")
else:
workflow_default = "default"
parser.add_argument(
"--workflow",
default=workflow_default,
help="A workflow from config_workflow.xml to apply to this case. ",
)
if cime_config:
model = get_model()
else:
model = None
srcroot_default = get_src_root()
parser.add_argument(
"--srcroot",
default=srcroot_default,
help="Alternative pathname for source root directory. "
f"The default is {srcroot_default}",
)
parser.add_argument(
"--output-root",
help="Alternative pathname for the directory where case output is written.",
)
# The following is a deprecated option
parser.add_argument(
"--script-root", dest="script_root", default=None, help=argparse.SUPPRESS
)
if model == "cesm":
parser.add_argument(
"--run-unsupported",
action="store_true",
help="Force the creation of a case that is not tested or supported by CESM developers.",
)
# hidden argument indicating called from create_test
# Indicates that create_newcase was called from create_test - do not use otherwise.
parser.add_argument("--test", "-test", action="store_true", help=argparse.SUPPRESS)
parser.add_argument(
"--walltime",
default=os.getenv("CIME_GLOBAL_WALLTIME"),
help="Set the wallclock limit for this case in the format (the usual format is HH:MM:SS). "
"\nYou may use env var CIME_GLOBAL_WALLTIME to set this. "
"\nIf CIME_GLOBAL_WALLTIME is not defined in the environment, then the walltime"
"\nwill be the maximum allowed time defined for the queue in config_batch.xml.",
)
parser.add_argument(
"-q",
"--queue",
default=None,
help="Force batch system to use the specified queue. ",
)
parser.add_argument(
"--handle-preexisting-dirs",
dest="answer",
choices=("a", "r", "u"),
default=None,
help="Do not query how to handle pre-existing bld/exe dirs. "
"\nValid options are (a)bort (r)eplace or (u)se existing. "
"\nThis can be useful if you need to run create_newcase non-iteractively.",
)
parser.add_argument(
"-i",
"--input-dir",
help="Use a non-default location for input files. This will change the xml value of DIN_LOC_ROOT.",
)
parser.add_argument(
"--driver",
default=get_cime_default_driver(),
choices=("mct", "nuopc", "moab"),
help=argparse.SUPPRESS,
)
parser.add_argument(
"-n",
"--non-local",
action="store_true",
help="Use when you've requested a machine that you aren't on. "
"Will reduce errors for missing directories etc.",
)
parser.add_argument(
"--extra-machines-dir",
help="Optional path to a directory containing one or more of:"
"\nconfig_machines.xml, config_compilers.xml, config_batch.xml."
"\nIf provided, the contents of these files will be appended to"
"\nthe standard machine files (and any files in ~/.cime).",
)
parser.add_argument("--case-group", help="Add this case to a case group")
parser.add_argument(
"--ngpus-per-node",
default=0,
type=int,
help="Specify number of GPUs used for simulation. ",
)
args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
if args.srcroot is not None:
expect(
os.path.isdir(args.srcroot),
"Input non-default directory srcroot {} does not exist ".format(
args.srcroot
),
)
args.srcroot = os.path.abspath(args.srcroot)
if args.gridfile is not None:
expect(
os.path.isfile(args.gridfile),
"Grid specification file {} does not exist ".format(args.gridfile),
)
if args.pesfile is not None:
expect(
os.path.isfile(args.pesfile),
"Pes specification file {} cannot be found ".format(args.pesfile),
)
run_unsupported = False
if model == "cesm":
run_unsupported = args.run_unsupported
expect(
CIME.utils.check_name(args.case, fullpath=True),
"Illegal case name argument provided",
)
if args.input_dir is not None:
args.input_dir = os.path.abspath(args.input_dir)
elif cime_config and cime_config.has_option("main", "input_dir"):
args.input_dir = os.path.abspath(cime_config.get("main", "input_dir"))
return (
args.case,
args.compset,
args.res,
args.machine,
args.compiler,
args.mpilib,
args.project,
args.pecount,
args.user_mods_dirs,
args.pesfile,
args.gridfile,
args.srcroot,
args.test,
args.multi_driver,
args.ninst,
args.walltime,
args.queue,
args.output_root,
args.script_root,
run_unsupported,
args.answer,
args.input_dir,
args.driver,
args.workflow,
args.non_local,
args.extra_machines_dir,
args.case_group,
args.ngpus_per_node,
)
def test_ka_createnewcase_extra_machines_dir(self):
# Test that we pick up changes in both config_machines.xml and
# cmake macros in a directory specified with the --extra-machines-dir
# argument to create_newcase.
cls = self.__class__
casename = "testcreatenewcase_extra_machines_dir"
# Setup: stage some xml files in a temporary directory
extra_machines_dir = os.path.join(cls._testroot,
"{}_machine_config".format(casename))
os.makedirs(os.path.join(extra_machines_dir, "cmake_macros"))
cls._do_teardown.append(extra_machines_dir)
newmachfile = os.path.join(
utils.get_cime_root(),
"CIME",
"data",
"config",
"xml_schemas",
"config_machines_template.xml",
)
utils.safe_copy(
newmachfile, os.path.join(extra_machines_dir,
"config_machines.xml"))
cmake_macro_text = """\
set(NETCDF_PATH /my/netcdf/path)
"""
cmake_macro_path = os.path.join(extra_machines_dir, "cmake_macros",
"mymachine.cmake")
with open(cmake_macro_path, "w") as cmake_macro:
cmake_macro.write(cmake_macro_text)
# Create the case
testdir = os.path.join(cls._testroot, casename)
if os.path.exists(testdir):
shutil.rmtree(testdir)
# In the following, note that 'mymachine' is the machine name defined in
# config_machines_template.xml
args = (" --case {testdir} --compset X --mach mymachine"
" --output-root {testroot} --non-local"
" --extra-machines-dir {extra_machines_dir}".format(
testdir=testdir,
testroot=cls._testroot,
extra_machines_dir=extra_machines_dir,
))
if utils.get_model() == "cesm":
args += " --run-unsupported"
if utils.get_cime_default_driver() == "nuopc":
args += " --res f19_g17 "
else:
args += " --res f19_g16 "
self.run_cmd_assert_result("./create_newcase {}".format(args),
from_dir=self.SCRIPT_DIR)
args += f" --machine {self.MACHINE.get_machine_name()}"
cls._do_teardown.append(testdir)
# Run case.setup
self.run_cmd_assert_result("./case.setup --non-local",
from_dir=testdir)
# Make sure Macros file contains expected text
with Case(testdir, non_local=True) as case:
with CmakeTmpBuildDir(macroloc=testdir) as cmaketmp:
macros_contents = cmaketmp.get_makefile_vars(case=case)
expected_re = re.compile("NETCDF_PATH.*/my/netcdf/path")
self.assertTrue(
expected_re.search(macros_contents),
msg="{} not found in:\n{}".format(expected_re.pattern,
macros_contents),
)
