def set_batch_system(self, batchobj, batch_system_type=None):
if batch_system_type is not None:
self.set_batch_system_type(batch_system_type)
if batchobj.batch_system_node is not None and batchobj.machine_node is not None:
for node in batchobj.get_children("", root=batchobj.machine_node):
name = self.name(node)
if name != 'directives':
oldnode = batchobj.get_optional_child(
name, root=batchobj.batch_system_node)
if oldnode is not None:
logger.debug("Replacing {}".format(self.name(oldnode)))
batchobj.remove_child(oldnode,
root=batchobj.batch_system_node)
if batchobj.batch_system_node is not None:
self.add_child(self.copy(batchobj.batch_system_node))
if batchobj.machine_node is not None:
self.add_child(self.copy(batchobj.machine_node))
if os.path.exists(
os.path.join(self._caseroot, "LockedFiles", "env_batch.xml")):
unlock_file(os.path.basename(batchobj.filename),
caseroot=self._caseroot)
self.set_value("BATCH_SYSTEM", batch_system_type)
if os.path.exists(os.path.join(self._caseroot, "LockedFiles")):
lock_file(os.path.basename(batchobj.filename),
caseroot=self._caseroot)
def post_build(case, logs, build_complete=False):
###############################################################################
logdir = case.get_value("LOGDIR")
#zip build logs to CASEROOT/logs
if logdir:
bldlogdir = os.path.join(logdir, "bld")
if not os.path.exists(bldlogdir):
os.makedirs(bldlogdir)
for log in logs:
logger.info("Copying build log {} to {}".format(log, bldlogdir))
with open(log, 'rb') as f_in:
with gzip.open("{}.gz".format(log), 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
if "sharedlibroot" not in log:
shutil.copy("{}.gz".format(log), os.path.join(bldlogdir, "{}.gz".format(os.path.basename(log))))
os.remove(log)
if build_complete:
# Set XML to indicate build complete
case.set_value("BUILD_COMPLETE", True)
case.set_value("BUILD_STATUS", 0)
if "SMP_VALUE" in os.environ:
case.set_value("SMP_BUILD", os.environ["SMP_VALUE"])
case.flush()
lock_file("env_build.xml")
# must ensure there's an lid
lid = os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S")
save_build_provenance(case, lid=lid)
def _init_locked_files(self, caseroot, expected):
"""
If the locked env_run.orig.xml does not exist, copy the current
env_run.xml file. If it does exist, restore values changed in a previous
run of the test.
"""
if is_locked("env_run.orig.xml"):
self.compare_env_run(expected=expected)
elif os.path.isfile(os.path.join(caseroot, "env_run.xml")):
lock_file("env_run.xml", caseroot=caseroot, newname="env_run.orig.xml")
def post_build(case, logs, build_complete=False, save_build_provenance=True):
###############################################################################
for log in logs:
gzip_existing_file(log)
if build_complete:
# must ensure there's an lid
lid = os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S")
if save_build_provenance:
save_build_provenance_sub(case, lid=lid)
# Set XML to indicate build complete
case.set_value("BUILD_COMPLETE", True)
case.set_value("BUILD_STATUS", 0)
if "SMP_VALUE" in os.environ:
case.set_value("SMP_BUILD", os.environ["SMP_VALUE"])
case.flush()
lock_file("env_build.xml")
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file, "CPL")
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname {} -testroot {}".format(test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate {}".format(self._baseline_gen_name)
basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test)
logger.debug("basegen_case is {}".format(basegen_case_fullpath))
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare {}".format(self._baseline_cmp_name)
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
tput_tolerance = self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False)
envtest.set_value("TEST_TPUT_TOLERANCE", 0.25 if tput_tolerance is None else tput_tolerance)
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are {} ".format(case_opts))
for opt in case_opts: # pylint: disable=not-an-iterable
logger.debug("case_opt is {}".format(opt))
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
logger.debug (" USE_ESMF_LIB set to TRUE")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to {}".format(opt))
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to {}".format(stop_option[opt]))
logger.debug (" STOP_N set to {}".format(opti))
elif opt.startswith('R'):
# R option is for testing in PTS_MODE or Single Column Model
# (SCM) mode
envtest.set_test_parameter("PTS_MODE", "TRUE")
# For PTS_MODE, compile with mpi-serial
envtest.set_test_parameter("MPILIB", "mpi-serial")
elif (opt.startswith('I') or # Marker to distinguish tests with same name - ignored
opt.startswith('M') or # handled in create_newcase
opt.startswith('P') or # handled in create_newcase
opt.startswith('N') or # handled in create_newcase
opt.startswith('C') or # handled in create_newcase
opt.startswith('V')): # handled in create_newcase
pass
elif opt.startswith('IOP'):
logger.warning("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '{}' ".format(opt))
envtest.write()
lock_file("env_run.xml", caseroot=test_dir, newname="env_run.orig.xml")
with Case(test_dir, read_only=False) as case:
if self._output_root is None:
self._output_root = case.get_value("CIME_OUTPUT_ROOT")
# if we are running a single test we don't need sharedlibroot
if len(self._tests) > 1 and self._cime_model != "e3sm":
case.set_value("SHAREDLIBROOT",
os.path.join(self._output_root,
"sharedlibroot.{}".format(self._test_id)))
envtest.set_initial_values(case)
case.set_value("TEST", True)
case.set_value("SAVE_TIMING", self._save_timing)
# Scale back build parallelism on systems with few cores
if self._model_build_cost > self._proc_pool:
case.set_value("GMAKE_J", self._proc_pool)
self._model_build_cost = self._proc_pool
return True, ""
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_CPL_FILE")
drv_comp = Component(drv_config_file, "CPL")
envtest.add_elements_by_group(files, {}, "env_test.xml")
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
if test in self._test_data and "options" in self._test_data[test] and \
"memleak_tolerance" in self._test_data[test]['options']:
envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance'])
test_argv = "-testname {} -testroot {}".format(test, self._test_root)
if self._baseline_gen_name:
test_argv += " -generate {}".format(self._baseline_gen_name)
basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test)
logger.debug("basegen_case is {}".format(basegen_case_fullpath))
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._baseline_cmp_name:
test_argv += " -compare {}".format(self._baseline_cmp_name)
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
tput_tolerance = self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False)
envtest.set_value("TEST_TPUT_TOLERANCE", 0.25 if tput_tolerance is None else tput_tolerance)
# Add the test instructions from config_test to env_test in the case
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are {} ".format(case_opts))
for opt in case_opts: # pylint: disable=not-an-iterable
logger.debug("case_opt is {}".format(opt))
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
logger.debug (" USE_ESMF_LIB set to TRUE")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to {}".format(opt))
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to {}".format(stop_option[opt]))
logger.debug (" STOP_N set to {}".format(opti))
elif opt.startswith('R'):
# R option is for testing in PTS_MODE or Single Column Model
# (SCM) mode
envtest.set_test_parameter("PTS_MODE", "TRUE")
# For PTS_MODE, compile with mpi-serial
envtest.set_test_parameter("MPILIB", "mpi-serial")
elif (opt.startswith('I') or # Marker to distinguish tests with same name - ignored
opt.startswith('M') or # handled in create_newcase
opt.startswith('P') or # handled in create_newcase
opt.startswith('N') or # handled in create_newcase
opt.startswith('C') or # handled in create_newcase
opt.startswith('V')): # handled in create_newcase
pass
elif opt.startswith('IOP'):
logger.warning("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '{}' ".format(opt))
envtest.write()
lock_file("env_run.xml", caseroot=test_dir, newname="env_run.orig.xml")
with Case(test_dir, read_only=False) as case:
if self._output_root is None:
self._output_root = case.get_value("CIME_OUTPUT_ROOT")
# if we are running a single test we don't need sharedlibroot
if len(self._tests) > 1 and self._cime_model != "e3sm":
case.set_value("SHAREDLIBROOT",
os.path.join(self._output_root,
"sharedlibroot.{}".format(self._test_id)))
envtest.set_initial_values(case)
case.set_value("TEST", True)
case.set_value("SAVE_TIMING", self._save_timing)
# Scale back build parallelism on systems with few cores
if self._model_build_cost > self._proc_pool:
case.set_value("GMAKE_J", self._proc_pool)
self._model_build_cost = self._proc_pool
if build_complete:
# must ensure there's an lid
lid = os.environ["LID"] if "LID" in os.environ else get_timestamp(
"%y%m%d-%H%M%S")
if save_build_provenance:
save_build_provenance_sub(case, lid=lid)
# Set XML to indicate build complete
case.set_value("BUILD_COMPLETE", True)
case.set_value("BUILD_STATUS", 0)
if "SMP_VALUE" in os.environ:
case.set_value("SMP_BUILD", os.environ["SMP_VALUE"])
case.flush()
lock_file("env_build.xml", caseroot=case.get_value("CASEROOT"))
###############################################################################
def case_build(caseroot,
case,
sharedlib_only=False,
model_only=False,
buildlist=None,
save_build_provenance=True):
###############################################################################
functor = lambda: _case_build_impl(caseroot, case, sharedlib_only,
model_only, buildlist,
save_build_provenance)
cb = "case.build"
if (sharedlib_only == True):
def create_clone(self,
newcase,
keepexe=False,
mach_dir=None,
project=None,
cime_output_root=None,
exeroot=None,
rundir=None,
user_mods_dir=None):
"""
Create a case clone
If exeroot or rundir are provided (not None), sets these directories
to the given paths; if not provided, uses default values for these
directories. It is an error to provide exeroot if keepexe is True.
"""
if cime_output_root is None:
cime_output_root = self.get_value("CIME_OUTPUT_ROOT")
newcaseroot = os.path.abspath(newcase)
expect(not os.path.isdir(newcaseroot),
"New caseroot directory {} already exists".format(newcaseroot))
newcasename = os.path.basename(newcaseroot)
expect(check_name(newcasename),
"New case name invalid {} ".format(newcasename))
newcase_cimeroot = os.path.abspath(get_cime_root())
# create clone from case to case
clone_cimeroot = self.get_value("CIMEROOT")
if newcase_cimeroot != clone_cimeroot:
logger.warning(" case CIMEROOT is {} ".format(newcase_cimeroot))
logger.warning(" clone CIMEROOT is {} ".format(clone_cimeroot))
logger.warning(
" It is NOT recommended to clone cases from different versions of CIME."
)
# *** create case object as deepcopy of clone object ***
srcroot = os.path.join(newcase_cimeroot, "..")
newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
newcase.set_value("CIMEROOT", newcase_cimeroot)
# if we are cloning to a different user modify the output directory
olduser = self.get_value("USER")
newuser = os.environ.get("USER")
if olduser != newuser:
cime_output_root = cime_output_root.replace(olduser, newuser)
newcase.set_value("USER", newuser)
newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root)
# try to make the new output directory and raise an exception
# on any error other than directory already exists.
if os.path.isdir(cime_output_root):
expect(
os.access(cime_output_root, os.W_OK),
"Directory {} is not writable "
"by this user. Use the --cime-output-root flag to provide a writable "
"scratch directory".format(cime_output_root))
else:
try:
os.makedirs(cime_output_root)
except:
if not os.path.isdir(cime_output_root):
raise
# determine if will use clone executable or not
if keepexe:
orig_exeroot = self.get_value("EXEROOT")
newcase.set_value("EXEROOT", orig_exeroot)
newcase.set_value("BUILD_COMPLETE", "TRUE")
orig_bld_complete = self.get_value("BUILD_COMPLETE")
if not orig_bld_complete:
logger.warning(
"\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone"
)
logger.warning(
"Avoid this message by building case one before you clone.\n")
else:
newcase.set_value("BUILD_COMPLETE", "FALSE")
# set machdir
if mach_dir is not None:
newcase.set_value("MACHDIR", mach_dir)
# set exeroot and rundir if requested
if exeroot is not None:
expect(
not keepexe, "create_case_clone: if keepexe is True, "
"then exeroot cannot be set")
newcase.set_value("EXEROOT", exeroot)
if rundir is not None:
newcase.set_value("RUNDIR", rundir)
# Set project id
# Note: we do not just copy this from the clone because it seems likely that
# users will want to change this sometimes, especially when cloning another
# user's case. However, note that, if a project is not given, the fallback will
# be to copy it from the clone, just like other xml variables are copied.
if project is None:
project = self.get_value("PROJECT", subgroup=self.get_primary_job())
if project is not None:
newcase.set_value("PROJECT", project)
# create caseroot
newcase.create_caseroot(clone=True)
newcase.flush(flushall=True)
# copy user_ files
cloneroot = self.get_case_root()
files = glob.glob(cloneroot + '/user_*')
for item in files:
safe_copy(item, newcaseroot)
# copy SourceMod and Buildconf files
# if symlinks exist, copy rather than follow links
for casesub in ("SourceMods", "Buildconf"):
shutil.copytree(os.path.join(cloneroot, casesub),
os.path.join(newcaseroot, casesub),
symlinks=True)
# lock env_case.xml in new case
lock_file("env_case.xml", newcaseroot)
# apply user_mods if appropriate
newcase_root = newcase.get_value("CASEROOT")
if user_mods_dir is not None:
if keepexe:
# If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
# env_build.xml and verify that it has not been modified
safe_copy(
os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
# Now apply contents of user_mods directory
apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)
# Determine if env_build.xml has changed
if keepexe:
success, comment = compare_files(
os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
if not success:
logger.warning(comment)
shutil.rmtree(newcase_root)
expect(
False,
"env_build.xml cannot be changed via usermods if keepexe is an option: \n "
"Failed to clone case, removed {}\n".format(newcase_root))
# if keep executable, then remove the new case SourceMods directory and link SourceMods to
# the clone directory
if keepexe:
shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
os.symlink(os.path.join(cloneroot, "SourceMods"),
os.path.join(newcase_root, "SourceMods"))
# Update README.case
fclone = open(cloneroot + "/README.case", "r")
fnewcase = open(newcaseroot + "/README.case", "a")
fnewcase.write("\n *** original clone README follows ****")
fnewcase.write("\n " + fclone.read())
clonename = self.get_value("CASE")
logger.info(" Successfully created new case {} from clone case {} ".format(
newcasename, clonename))
newcase.case_setup()
return newcase
def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False,
resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None,
batch_args=None):
if job is None:
job = case.get_primary_job()
# Check if CONTINUE_RUN value makes sense
if job != "case.test" and case.get_value("CONTINUE_RUN"):
rundir = case.get_value("RUNDIR")
caseroot = case.get_value("CASEROOT")
expect(os.path.isdir(rundir),
"CONTINUE_RUN is true but RUNDIR {} does not exist".format(rundir))
expect(len(glob.glob(os.path.join(rundir, "*.nc"))) > 0,
"CONTINUE_RUN is true but this case does not appear to have been run before (no .nc files in RUNDIR)")
expect(does_file_have_string(os.path.join(caseroot, "CaseStatus"), "case.run {}".format(CASE_SUCCESS)),
"CONTINUE_RUN is true but this case does not appear to have ever run successfully")
# if case.submit is called with the no_batch flag then we assume that this
# flag will stay in effect for the duration of the RESUBMITs
env_batch = case.get_env("batch")
if resubmit:
if env_batch.get_batch_system_type() == "none":
no_batch = True
# This is a resubmission, do not reinitialize test values
if job == "case.test":
case.set_value("IS_FIRST_RUN", False)
resub = case.get_value("RESUBMIT")
logger.info("Submitting job '{}', resubmit={:d}".format(job, resub))
case.set_value("RESUBMIT", resub-1)
if case.get_value("RESUBMIT_SETS_CONTINUE_RUN"):
case.set_value("CONTINUE_RUN", True)
else:
if job == "case.test":
case.set_value("IS_FIRST_RUN", True)
if no_batch:
batch_system = "none"
else:
batch_system = env_batch.get_batch_system_type()
case.set_value("BATCH_SYSTEM", batch_system)
env_batch_has_changed = False
try:
case.check_lockedfile(os.path.basename(env_batch.filename))
except SystemExit:
env_batch_has_changed = True
if env_batch.get_batch_system_type() != "none" and env_batch_has_changed:
# May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
logger.warning(\
"""
env_batch.xml appears to have changed, regenerating batch scripts
manual edits to these file will be lost!
""")
env_batch.make_all_batch_files(case)
unlock_file(os.path.basename(env_batch.filename))
lock_file(os.path.basename(env_batch.filename))
if job == case.get_primary_job():
case.check_case()
case.check_DA_settings()
if case.get_value("MACH") == "mira":
with open(".original_host", "w") as fd:
fd.write( socket.gethostname())
#Load Modules
case.load_env()
case.flush()
logger.warning("submit_jobs {}".format(job))
job_ids = case.submit_jobs(no_batch=no_batch, job=job, prereq=prereq,
skip_pnl=skip_pnl, resubmit_immediate=resubmit_immediate,
allow_fail=allow_fail, mail_user=mail_user,
mail_type=mail_type, batch_args=batch_args)
xml_jobids = []
for jobname, jobid in job_ids.items():
logger.info("Submitted job {} with id {}".format(jobname, jobid))
if jobid:
xml_jobids.append("{}:{}".format(jobname, jobid))
xml_jobid_text = ", ".join(xml_jobids)
if xml_jobid_text:
case.set_value("JOB_IDS", xml_jobid_text)
return xml_jobid_text
for log in logs:
gzip_existing_file(log)
if build_complete:
# must ensure there's an lid
lid = os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S")
if save_build_provenance:
save_build_provenance_sub(case, lid=lid)
# Set XML to indicate build complete
case.set_value("BUILD_COMPLETE", True)
case.set_value("BUILD_STATUS", 0)
if "SMP_VALUE" in os.environ:
case.set_value("SMP_BUILD", os.environ["SMP_VALUE"])
case.flush()
lock_file("env_build.xml")
###############################################################################
def case_build(caseroot, case, sharedlib_only=False, model_only=False, buildlist=None, save_build_provenance=True):
###############################################################################
functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist,
save_build_provenance)
return run_and_log_case_status(functor, "case.build", caseroot=caseroot)
###############################################################################
def clean(case, cleanlist=None, clean_all=False, clean_depends=None):
###############################################################################
functor = lambda: _clean_impl(case, cleanlist, clean_all, clean_depends)
return run_and_log_case_status(functor, "build.clean", caseroot=case.get_value("CASEROOT"))
def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
###############################################################################
os.chdir(caseroot)
# Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
din_loc_root = case.get_value("DIN_LOC_ROOT")
testcase = case.get_value("TESTCASE")
expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
"inputdata root is not a directory: {}".format(din_loc_root))
# Remove batch scripts
if reset or clean:
# clean batch script
batch_script = get_batch_script_for_job(case.get_primary_job())
if os.path.exists(batch_script):
os.remove(batch_script)
logger.info("Successfully cleaned batch script {}".format(batch_script))
if not test_mode:
# rebuild the models (even on restart)
case.set_value("BUILD_COMPLETE", False)
if not clean:
case.load_env()
models = case.get_values("COMP_CLASSES")
mach = case.get_value("MACH")
compiler = case.get_value("COMPILER")
debug = case.get_value("DEBUG")
mpilib = case.get_value("MPILIB")
sysos = case.get_value("OS")
expect(mach is not None, "xml variable MACH is not set")
# creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler
# and env_mach_specific.xml if they don't already exist.
if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"):
configure(Machines(machine=mach), caseroot, ["Makefile"], compiler, mpilib, debug, sysos)
# Set tasks to 1 if mpi-serial library
if mpilib == "mpi-serial":
for vid, value in case:
if vid.startswith("NTASKS") and value != 1:
case.set_value(vid, 1)
# Check ninst.
# In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
multi_driver = case.get_value("MULTI_DRIVER")
nthrds = 1
for comp in models:
ntasks = case.get_value("NTASKS_{}".format(comp))
nthrds = max(nthrds,case.get_value("NTHRDS_{}".format(comp)))
if comp == "CPL":
continue
ninst = case.get_value("NINST_{}".format(comp))
if multi_driver:
expect(case.get_value("NINST_LAYOUT_{}".format(comp)) == "concurrent",
"If multi_driver is TRUE, NINST_LAYOUT_{} must be concurrent".format(comp))
case.set_value("NTASKS_PER_INST_{}".format(comp), ntasks)
else:
if ninst > ntasks:
if ntasks == 1:
case.set_value("NTASKS_{}".format(comp), ninst)
ntasks = ninst
else:
expect(False, "NINST_{} value {:d} greater than NTASKS_{} {:d}".format(comp, ninst, comp, ntasks))
case.set_value("NTASKS_PER_INST_{}".format(comp), int(ntasks / ninst))
if nthrds > 1:
case.set_value("BUILD_THREADED",True)
if os.path.exists(get_batch_script_for_job(case.get_primary_job())):
logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
case.initialize_derived_attributes()
case.set_value("SMP_PRESENT", case.get_build_threaded())
else:
case.check_pelayouts_require_rebuild(models)
unlock_file("env_build.xml")
unlock_file("env_batch.xml")
case.flush()
case.check_lockedfiles()
case.initialize_derived_attributes()
cost_per_node = 16 if case.get_value("MACH") == "yellowstone" else case.get_value("MAX_MPITASKS_PER_NODE")
case.set_value("COST_PES", case.num_nodes * cost_per_node)
case.set_value("TOTALPES", case.total_tasks)
case.set_value("SMP_PRESENT", case.get_build_threaded())
# create batch files
env_batch = case.get_env("batch")
env_batch.make_all_batch_files(case)
if get_model() == "e3sm" and not case.get_value("TEST"):
input_batch_script = os.path.join(case.get_value("MACHDIR"), "template.case.run.sh")
env_batch.make_batch_script(input_batch_script, "case.run", case, outfile=get_batch_script_for_job("case.run.sh"))
# May need to select new batch settings if pelayout changed (e.g. problem is now too big for prev-selected queue)
env_batch.set_job_defaults([(case.get_primary_job(), {})], case)
case.schedule_rewrite(env_batch)
# Make a copy of env_mach_pes.xml in order to be able
# to check that it does not change once case.setup is invoked
case.flush()
logger.debug("at copy TOTALPES = {}".format(case.get_value("TOTALPES")))
lock_file("env_mach_pes.xml")
lock_file("env_batch.xml")
# Create user_nl files for the required number of instances
if not os.path.exists("user_nl_cpl"):
logger.info("Creating user_nl_xxx files for components and cpl")
# loop over models
for model in models:
comp = case.get_value("COMP_{}".format(model))
logger.debug("Building {} usernl files".format(model))
_build_usernl_files(case, model, comp)
if comp == "cism":
glcroot = case.get_value("COMP_ROOT_DIR_GLC")
run_cmd_no_fail("{}/cime_config/cism.template {}".format(glcroot, caseroot))
_build_usernl_files(case, "drv", "cpl")
# Create needed directories for case
case.create_dirs()
logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")
# Some tests need namelists created here (ERP) - so do this if we are in test mode
if test_mode or get_model() == "e3sm":
logger.info("Generating component namelists as part of setup")
case.create_namelists()
# Record env information
env_module = case.get_env("mach_specific")
env_module.make_env_mach_specific_file("sh", case)
env_module.make_env_mach_specific_file("csh", case)
env_module.save_all_env_info("software_environment.txt")
def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None,
cime_output_root=None, exeroot=None, rundir=None,
user_mods_dir=None):
"""
Create a case clone
If exeroot or rundir are provided (not None), sets these directories
to the given paths; if not provided, uses default values for these
directories. It is an error to provide exeroot if keepexe is True.
"""
if cime_output_root is None:
cime_output_root = self.get_value("CIME_OUTPUT_ROOT")
newcaseroot = os.path.abspath(newcase)
expect(not os.path.isdir(newcaseroot),
"New caseroot directory {} already exists".format(newcaseroot))
newcasename = os.path.basename(newcaseroot)
expect(check_name(newcasename),
"New case name invalid {} ".format(newcasename))
newcase_cimeroot = os.path.abspath(get_cime_root())
# create clone from case to case
clone_cimeroot = self.get_value("CIMEROOT")
if newcase_cimeroot != clone_cimeroot:
logger.warning(" case CIMEROOT is {} ".format(newcase_cimeroot))
logger.warning(" clone CIMEROOT is {} ".format(clone_cimeroot))
logger.warning(" It is NOT recommended to clone cases from different versions of CIME.")
# *** create case object as deepcopy of clone object ***
srcroot = os.path.join(newcase_cimeroot,"..")
newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
newcase.set_value("CIMEROOT", newcase_cimeroot)
# if we are cloning to a different user modify the output directory
olduser = self.get_value("USER")
newuser = os.environ.get("USER")
if olduser != newuser:
cime_output_root = cime_output_root.replace(olduser, newuser)
newcase.set_value("USER", newuser)
newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root)
# try to make the new output directory and raise an exception
# on any error other than directory already exists.
if os.path.isdir(cime_output_root):
expect(os.access(cime_output_root, os.W_OK), "Directory {} is not writable "
"by this user. Use the --cime-output-root flag to provide a writable "
"scratch directory".format(cime_output_root))
else:
try:
os.makedirs(cime_output_root)
except:
if not os.path.isdir(cime_output_root):
raise
# determine if will use clone executable or not
if keepexe:
orig_exeroot = self.get_value("EXEROOT")
newcase.set_value("EXEROOT", orig_exeroot)
newcase.set_value("BUILD_COMPLETE","TRUE")
orig_bld_complete = self.get_value("BUILD_COMPLETE")
if not orig_bld_complete:
logger.warning("\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone")
logger.warning("Avoid this message by building case one before you clone.\n")
else:
newcase.set_value("BUILD_COMPLETE","FALSE")
# set machdir
if mach_dir is not None:
newcase.set_value("MACHDIR", mach_dir)
# set exeroot and rundir if requested
if exeroot is not None:
expect(not keepexe, "create_case_clone: if keepexe is True, "
"then exeroot cannot be set")
newcase.set_value("EXEROOT", exeroot)
if rundir is not None:
newcase.set_value("RUNDIR", rundir)
# Set project id
# Note: we do not just copy this from the clone because it seems likely that
# users will want to change this sometimes, especially when cloning another
# user's case. However, note that, if a project is not given, the fallback will
# be to copy it from the clone, just like other xml variables are copied.
if project is None:
project = self.get_value("PROJECT", subgroup=self.get_primary_job())
if project is not None:
newcase.set_value("PROJECT", project)
# create caseroot
newcase.create_caseroot(clone=True)
newcase.flush(flushall=True)
# copy user_ files
cloneroot = self.get_case_root()
files = glob.glob(cloneroot + '/user_*')
for item in files:
safe_copy(item, newcaseroot)
# copy SourceMod and Buildconf files
# if symlinks exist, copy rather than follow links
for casesub in ("SourceMods", "Buildconf"):
shutil.copytree(os.path.join(cloneroot, casesub),
os.path.join(newcaseroot, casesub),
symlinks=True)
# lock env_case.xml in new case
lock_file("env_case.xml", newcaseroot)
# apply user_mods if appropriate
newcase_root = newcase.get_value("CASEROOT")
if user_mods_dir is not None:
if keepexe:
# If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
# env_build.xml and verify that it has not been modified
safe_copy(os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
# Now apply contents of user_mods directory
apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)
# Determine if env_build.xml has changed
if keepexe:
success, comment = compare_files(os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
if not success:
logger.warning(comment)
shutil.rmtree(newcase_root)
expect(False, "env_build.xml cannot be changed via usermods if keepexe is an option: \n "
"Failed to clone case, removed {}\n".format(newcase_root))
# if keep executable, then remove the new case SourceMods directory and link SourceMods to
# the clone directory
if keepexe:
shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
os.symlink(os.path.join(cloneroot, "SourceMods"),
os.path.join(newcase_root, "SourceMods"))
# Update README.case
fclone = open(cloneroot + "/README.case", "r")
fnewcase = open(newcaseroot + "/README.case", "a")
fnewcase.write("\n *** original clone README follows ****")
fnewcase.write("\n " + fclone.read())
clonename = self.get_value("CASE")
logger.info(" Successfully created new case {} from clone case {} ".format(newcasename, clonename))
newcase.case_setup()
return newcase
if not "cam.i" in newfile:
if os.path.lexists(newfile):
os.unlink(newfile)
os.symlink(reffile, newfile)
def per_run_case_updates(case, date, sdrestdir, user_mods_dir, rundir):
caseroot = case.get_value("CASEROOT")
basecasename = os.path.basename(caseroot)[:-6]
member = os.path.basename(caseroot)[-2:]
unlock_file("env_case.xml", caseroot=caseroot)
casename = basecasename + "." + date + "." + member
case.set_value("CASE", casename)
case.flush()
lock_file("env_case.xml", caseroot=caseroot)
case.set_value("CONTINUE_RUN", False)
case.set_value("RUN_REFDATE", date)
case.set_value("RUN_STARTDATE", date)
case.set_value("RUN_REFDIR", sdrestdir)
case.set_value("REST_OPTION", 'none')
case.set_value("PROJECT", "P93300007")
# dout_s_root = case.get_value("DOUT_S_ROOT")
# dout_s_root = os.path.join(os.path.dirname(dout_s_root),casename)
# if dout_s_root.startswith("/glade/scratch"):
# dout_s_root = dout_s_root.replace("/glade/scratch/","/glade/p/nsc/ncgd0042/")
# case.set_value("DOUT_S_ROOT",dout_s_root)
# restage user_nl files for each run
for usermod in glob.iglob(user_mods_dir + "/user*"):
safe_copy(usermod, caseroot)
if not external_workflow:
try:
case.check_lockedfile(os.path.basename(env_batch.filename),
caseroot=caseroot)
except:
env_batch_has_changed = True
if batch_system != "none" and env_batch_has_changed and not external_workflow:
# May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
logger.warning("""
env_batch.xml appears to have changed, regenerating batch scripts
manual edits to these file will be lost!
""")
env_batch.make_all_batch_files(case)
case.flush()
lock_file(os.path.basename(env_batch.filename), caseroot=caseroot)
if resubmit:
# This is a resubmission, do not reinitialize test values
if job == "case.test":
case.set_value("IS_FIRST_RUN", False)
resub = case.get_value("RESUBMIT")
logger.info("Submitting job '{}', resubmit={:d}".format(job, resub))
case.set_value("RESUBMIT", resub - 1)
if case.get_value("RESUBMIT_SETS_CONTINUE_RUN"):
case.set_value("CONTINUE_RUN", True)
else:
if job == "case.test":
case.set_value("IS_FIRST_RUN", True)
env_batch.make_all_batch_files(case)
if get_model() == "e3sm" and not case.get_value("TEST"):
input_batch_script = os.path.join(case.get_value("MACHDIR"),
"template.case.run.sh")
env_batch.make_batch_script(
input_batch_script,
"case.run",
case,
outfile=get_batch_script_for_job("case.run.sh"))
# Make a copy of env_mach_pes.xml in order to be able
# to check that it does not change once case.setup is invoked
case.flush()
logger.debug("at copy TOTALPES = {}".format(
case.get_value("TOTALPES")))
lock_file("env_mach_pes.xml")
lock_file("env_batch.xml")
# Create user_nl files for the required number of instances
if not os.path.exists("user_nl_cpl"):
logger.info("Creating user_nl_xxx files for components and cpl")
# loop over models
for model in models:
comp = case.get_value("COMP_{}".format(model))
logger.debug("Building {} usernl files".format(model))
_build_usernl_files(case, model, comp)
if comp == "cism":
glcroot = case.get_value("COMP_ROOT_DIR_GLC")
run_cmd_no_fail("{}/cime_config/cism.template {}".format(
glcroot, caseroot))
try:
case.check_lockedfile(os.path.basename(env_batch.filename))
except:
env_batch_has_changed = True
if batch_system != "none" and env_batch_has_changed and not external_workflow:
# May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
logger.warning(\
"""
env_batch.xml appears to have changed, regenerating batch scripts
manual edits to these file will be lost!
""")
env_batch.make_all_batch_files(case)
unlock_file(os.path.basename(env_batch.filename))
lock_file(os.path.basename(env_batch.filename))
if resubmit:
# This is a resubmission, do not reinitialize test values
if job == "case.test":
case.set_value("IS_FIRST_RUN", False)
resub = case.get_value("RESUBMIT")
logger.info("Submitting job '{}', resubmit={:d}".format(job, resub))
case.set_value("RESUBMIT", resub-1)
if case.get_value("RESUBMIT_SETS_CONTINUE_RUN"):
case.set_value("CONTINUE_RUN", True)
else:
if job == "case.test":
case.set_value("IS_FIRST_RUN", True)
def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False,
resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None,
batch_args=None):
if job is None:
job = case.get_primary_job()
# Check if CONTINUE_RUN value makes sense
if job != "case.test" and case.get_value("CONTINUE_RUN"):
rundir = case.get_value("RUNDIR")
expect(os.path.isdir(rundir),
"CONTINUE_RUN is true but RUNDIR {} does not exist".format(rundir))
# only checks for the first instance in a multidriver case
if case.get_value("MULTI_DRIVER"):
rpointer = "rpointer.drv_0001"
else:
rpointer = "rpointer.drv"
expect(os.path.exists(os.path.join(rundir,rpointer)),
"CONTINUE_RUN is true but this case does not appear to have restart files staged in {}".format(rundir))
# Finally we open the rpointer file and check that it's correct
casename = case.get_value("CASE")
with open(os.path.join(rundir,rpointer), "r") as fd:
ncfile = fd.readline().strip()
expect(ncfile.startswith(casename) and
os.path.exists(os.path.join(rundir,ncfile)),
"File {ncfile} not present or does not match case {casename}".
format(ncfile=os.path.join(rundir,ncfile),casename=casename))
# if case.submit is called with the no_batch flag then we assume that this
# flag will stay in effect for the duration of the RESUBMITs
env_batch = case.get_env("batch")
if resubmit and env_batch.get_batch_system_type() == "none":
no_batch = True
if no_batch:
batch_system = "none"
else:
batch_system = env_batch.get_batch_system_type()
case.set_value("BATCH_SYSTEM", batch_system)
env_batch_has_changed = False
try:
case.check_lockedfile(os.path.basename(env_batch.filename))
except:
env_batch_has_changed = True
if batch_system != "none" and env_batch_has_changed:
# May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
logger.warning(\
"""
env_batch.xml appears to have changed, regenerating batch scripts
manual edits to these file will be lost!
""")
env_batch.make_all_batch_files(case)
unlock_file(os.path.basename(env_batch.filename))
lock_file(os.path.basename(env_batch.filename))
if resubmit:
# This is a resubmission, do not reinitialize test values
if job == "case.test":
case.set_value("IS_FIRST_RUN", False)
resub = case.get_value("RESUBMIT")
logger.info("Submitting job '{}', resubmit={:d}".format(job, resub))
case.set_value("RESUBMIT", resub-1)
if case.get_value("RESUBMIT_SETS_CONTINUE_RUN"):
case.set_value("CONTINUE_RUN", True)
else:
if job == "case.test":
case.set_value("IS_FIRST_RUN", True)
if no_batch:
batch_system = "none"
else:
batch_system = env_batch.get_batch_system_type()
case.set_value("BATCH_SYSTEM", batch_system)
env_batch_has_changed = False
try:
case.check_lockedfile(os.path.basename(env_batch.filename))
except CIMEError:
env_batch_has_changed = True
if env_batch.get_batch_system_type() != "none" and env_batch_has_changed:
# May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc)
logger.warning(\
"""
env_batch.xml appears to have changed, regenerating batch scripts
manual edits to these file will be lost!
""")
env_batch.make_all_batch_files(case)
unlock_file(os.path.basename(env_batch.filename))
lock_file(os.path.basename(env_batch.filename))
if job == case.get_primary_job():
case.check_case()
case.check_DA_settings()
if case.get_value("MACH") == "mira":
with open(".original_host", "w") as fd:
fd.write( socket.gethostname())
#Load Modules
case.load_env()
case.flush()
logger.warning("submit_jobs {}".format(job))
job_ids = case.submit_jobs(no_batch=no_batch, job=job, prereq=prereq,
skip_pnl=skip_pnl, resubmit_immediate=resubmit_immediate,
allow_fail=allow_fail, mail_user=mail_user,
mail_type=mail_type, batch_args=batch_args)
xml_jobids = []
for jobname, jobid in job_ids.items():
logger.info("Submitted job {} with id {}".format(jobname, jobid))
if jobid:
xml_jobids.append("{}:{}".format(jobname, jobid))
xml_jobid_text = ", ".join(xml_jobids)
if xml_jobid_text:
case.set_value("JOB_IDS", xml_jobid_text)
return xml_jobid_text
