def pre_run_check(case):
###############################################################################
# Pre run initialization code..
caseroot = case.get_value("CASEROOT")
din_loc_root = case.get_value("DIN_LOC_ROOT")
batchsubmit = case.get_value("BATCHSUBMIT")
mpilib = case.get_value("MPILIB")
rundir = case.get_value("RUNDIR")
build_complete = case.get_value("BUILD_COMPLETE")
# check for locked files.
check_lockedfiles(case.get_value("CASEROOT"))
logger.debug("check_lockedfiles OK")
# check that build is done
expect(build_complete,
"BUILD_COMPLETE is not true\nPlease rebuild the model interactively")
logger.debug("build complete is %s " %build_complete)
# load the module environment...
case.load_env()
# set environment variables
# This is a requirement for yellowstone only
if mpilib == "mpi-serial" and "MP_MPILIB" in os.environ:
del os.environ["MP_MPILIB"]
else:
os.environ["MPILIB"] = mpilib
if batchsubmit is None or len(batchsubmit) == 0:
os.environ["LBQUERY"] = "FALSE"
os.environ["BATCHQUERY"] = "undefined"
elif batchsubmit == 'UNSET':
os.environ["LBQUERY"] = "FALSE"
os.environ["BATCHQUERY"] = "undefined"
else:
os.environ["LBQUERY"] = "TRUE"
# create the timing directories, optionally cleaning them if needed.
if not os.path.isdir(rundir):
os.mkdir(rundir)
if os.path.isdir(os.path.join(rundir, "timing")):
shutil.rmtree(os.path.join(rundir, "timing"))
os.makedirs(os.path.join(rundir, "timing", "checkpoints"))
# This needs to be done everytime the LID changes in order for log files to be set up correctly
create_namelists(case)
# document process
append_status("Run started ", caseroot=caseroot,
sfile="CaseStatus")
logger.info("-------------------------------------------------------------------------")
logger.info(" - To prestage required restarts, untar a restart.tar file into %s" %(rundir))
logger.info(" - Case input data directory (DIN_LOC_ROOT) is %s " %(din_loc_root))
logger.info(" - Checking for required input datasets in DIN_LOC_ROOT")
logger.info("-------------------------------------------------------------------------")
def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None, generate_name=None, baseline_root=None, logfile_name="TestStatus.log"):
expect(case.get_value("TEST"), "Only makes sense to run this for a test case")
caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID")
if not compare:
compare = case.get_value("COMPARE_BASELINE")
if not generate:
generate = case.get_value("GENERATE_BASELINE")
if not compare and not generate:
logging.debug("No namelists compares requested")
return True
# create namelists for case if they haven't been already
casedocs = os.path.join(caseroot, "CaseDocs")
if not os.path.exists(os.path.join(casedocs, "drv_in")):
create_namelists(case)
test_name = casebaseid if casebaseid is not None else case.get_value("CASE")
with TestStatus(test_dir=caseroot, test_name=test_name) as ts:
try:
# Inside this try are where we catch non-fatal errors, IE errors involving
# baseline operations which may not directly impact the functioning of the viability of this case
if compare and not compare_name:
compare_name = case.get_value("BASELINE_NAME_CMP")
expect(compare_name, "Was asked to do baseline compare but unable to determine baseline name")
logging.info("Comparing namelists with baselines '{}'".format(compare_name))
if generate and not generate_name:
generate_name = case.get_value("BASELINE_NAME_GEN")
expect(generate_name, "Was asked to do baseline generation but unable to determine baseline name")
logging.info("Generating namelists to baselines '{}'".format(generate_name))
success = True
output = ""
if compare:
success, output = _do_full_nl_comp(case, test_name, compare_name, baseline_root)
if not success and ts.get_status(RUN_PHASE) is not None:
run_warn = \
"""NOTE: It is not necessarily safe to compare namelists after RUN
phase has completed. Running a case can pollute namelists. The namelists
kept in the baselines are pre-RUN namelists."""
output += run_warn
logging.info(run_warn)
if generate:
_do_full_nl_gen(case, test_name, generate_name, baseline_root)
except:
ts.set_status(NAMELIST_PHASE, TEST_FAIL_STATUS)
success = False
warn = "Exception during namelist operations:\n{}\n{}".format(sys.exc_info()[1], traceback.format_exc())
output += warn
logging.warning(warn)
finally:
ts.set_status(NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS)
try:
append_status(output, logfile_name, caseroot=caseroot)
except IOError:
pass
return success
def check_case(case):
check_lockedfiles(case)
create_namelists(case) # Must be called before check_all_input_data
logger.info("Checking that inputdata is available as part of case submission")
check_all_input_data(case)
expect(case.get_value("BUILD_COMPLETE"), "Build complete is "
"not True please rebuild the model by calling case.build")
logger.info("Check case OK")
def check_case(case, caseroot):
check_lockedfiles(caseroot)
create_namelists(case) # Must be called before check_all_input_data
check_all_input_data(case)
# Now that we have baselines, do baseline operations
if case.get_value("TEST"):
case_cmpgen_namelists(case)
expect(case.get_value("BUILD_COMPLETE"), "Build complete is "
"not True please rebuild the model by calling case.build")
logger.info("Check case OK")
def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None, generate_name=None, baseline_root=None, logfile_name="TestStatus.log"):
expect(case.get_value("TEST"), "Only makes sense to run this for a test case")
caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID")
if not compare:
compare = case.get_value("COMPARE_BASELINE")
if not generate:
generate = case.get_value("GENERATE_BASELINE")
if not compare and not generate:
logging.info("Nothing to do")
return True
# create namelists for case if they haven't been already
casedocs = os.path.join(caseroot, "CaseDocs")
if not os.path.exists(os.path.join(casedocs, "drv_in")):
create_namelists(case)
test_name = casebaseid if casebaseid is not None else case.get_value("CASE")
with TestStatus(test_dir=caseroot, test_name=test_name) as ts:
try:
# Inside this try are where we catch non-fatal errors, IE errors involving
# baseline operations which may not directly impact the functioning of the viability of this case
if compare and not compare_name:
compare_name = case.get_value("BASELINE_NAME_CMP")
compare_name = get_current_branch() if compare_name is None else compare_name
expect(compare_name, "Was asked to do baseline compare but unable to determine baseline name")
logging.info("Comparing namelists with baselines '%s'" % compare_name)
if generate and not generate_name:
generate_name = case.get_value("BASELINE_NAME_GEN")
generate_name = get_current_branch() if generate_name is None else generate_name
expect(generate_name, "Was asked to do baseline generation but unable to determine baseline name")
logging.info("Generating namelists to baselines '%s'" % generate_name)
success = True
output = ""
if compare:
success, output = _do_full_nl_comp(case, test_name, compare_name, baseline_root)
if generate:
_do_full_nl_gen(case, test_name, generate_name, baseline_root)
except:
ts.set_status(NAMELIST_PHASE, TEST_FAIL_STATUS)
success = False
warn = "Exception during namelist operations:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc())
output += warn
logging.warning(warn)
finally:
ts.set_status(NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS)
append_status(output, caseroot=caseroot, sfile=logfile_name)
return success
def build_checks(case, build_threaded, comp_interface, use_esmf_lib,
debug, compiler, mpilib, complist, ninst_build, smp_value,
model_only):
###############################################################################
"""
check if a build needs to be done and warn if a clean is warrented first
returns the relative sharedpath directory for sharedlibraries
"""
ninst_value = case.get_value("NINST_VALUE")
smp_build = case.get_value("SMP_BUILD")
build_status = case.get_value("BUILD_STATUS")
smpstr = ""
inststr = ""
for model, _, nthrds, ninst, _ in complist:
if nthrds > 1:
build_threaded = True
if build_threaded:
smpstr += "%s1"%model[0]
else:
smpstr += "%s0"%model[0]
inststr += "%s%d"%(model[0],ninst)
if build_threaded:
os.environ["SMP"] = "TRUE"
else:
os.environ["SMP"] = "FALSE"
case.set_value("SMP_VALUE", smpstr)
os.environ["SMP_VALUE"] = smpstr
case.set_value("NINST_VALUE", inststr)
os.environ["NINST_VALUE"] = inststr
debugdir = "debug" if debug else "nodebug"
threaddir = "threads" if (os.environ["SMP"] == "TRUE" or build_threaded) else "nothreads"
sharedpath = os.path.join(compiler, mpilib, debugdir, threaddir)
logger.debug("compiler=%s mpilib=%s debugdir=%s threaddir=%s"%
(compiler,mpilib,debugdir,threaddir))
expect( ninst_build == ninst_value or ninst_build == "0",
"""
ERROR, NINST VALUES HAVE CHANGED
NINST_BUILD = %s
NINST_VALUE = %s
A manual clean of your obj directories is strongly recommended
You should execute the following:
./case.build --clean
Then rerun the build script interactively
---- OR ----
You can override this error message at your own risk by executing:
./xmlchange -file env_build.xml -id NINST_BUILD -val 0
Then rerun the build script interactively
""" % (ninst_build, ninst_value))
expect( smp_build == smpstr or smp_build == "0",
"""
ERROR, SMP VALUES HAVE CHANGED
SMP_BUILD = %s
SMP_VALUE = %s
smpstr = %s
A manual clean of your obj directories is strongly recommended
You should execute the following:
./case.build --clean
Then rerun the build script interactively
---- OR ----
You can override this error message at your own risk by executing:
./xmlchange -file env_build.xml -id SMP_BUILD -val 0
Then rerun the build script interactively
""" % (smp_build, smp_value, smpstr))
expect(build_status == 0,
"""
ERROR env_build HAS CHANGED
A manual clean of your obj directories is required
You should execute the following:
./case.build --clean-all
""")
expect(comp_interface != "ESMF" or use_esmf_lib,
"""
ERROR COMP_INTERFACE IS ESMF BUT USE_ESMF_LIB IS NOT TRUE
SET USE_ESMF_LIB to TRUE with:
./xmlchange -file env_build.xml -id USE_ESMF_LIB -value TRUE
""")
expect(mpilib != "mpi-serial" or not use_esmf_lib,
"""
ERROR MPILIB is mpi-serial and USE_ESMF_LIB IS TRUE
MPILIB can only be used with an ESMF library built with mpiuni on
Set USE_ESMF_LIB to FALSE with
./xmlchange -file env_build.xml -id USE_ESMF_LIB -val FALSE
---- OR ----
Make sure the ESMF_LIBDIR used was built with mipuni (or change it to one that was)
And comment out this if block in Tools/models_buildexe
""")
case.set_value("BUILD_COMPLETE", False)
case.flush()
if not model_only:
create_namelists(case)
return sharedpath
def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
###############################################################################
os.chdir(caseroot)
msg = "case.setup starting"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
cimeroot = get_cime_root(case)
# Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
din_loc_root = case.get_value("DIN_LOC_ROOT")
testcase = case.get_value("TESTCASE")
expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
"inputdata root is not a directory: \"$din_loc_root\" ")
# Check that userdefine settings are specified before expanding variable
for vid, value in case:
expect(not (type(value) is str and "USERDEFINED_required_build" in value),
"Parameter '%s' must be defined" % vid)
# Create batch script
if reset or clean:
# Clean batch script
backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S")
if not os.path.isdir(backup_dir):
os.makedirs(backup_dir)
# back up relevant files
for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]:
for filename in glob.glob(fileglob):
shutil.copy(filename, backup_dir)
if os.path.exists("case.run"):
os.remove("case.run")
# only do the following if are NOT in testmode
if not test_mode:
# rebuild the models (even on restart)
case.set_value("BUILD_COMPLETE", False)
# backup and then clean test script
if os.path.exists("case.test"):
shutil.copy("case.test", backup_dir)
os.remove("case.test")
logger.info("Successfully cleaned test script case.test")
if os.path.exists("case.testdriver"):
shutil.copy("case.testdriver", backup_dir)
os.remove("case.testdriver")
logger.info("Successfully cleaned test script case.testdriver")
logger.info("Successfully cleaned batch script case.run")
logger.info("Successfully cleaned batch script case.run")
logger.info("Some files have been saved to %s" % backup_dir)
msg = "case.setup clean complete"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
if not clean:
case.load_env()
models = case.get_values("COMP_CLASSES")
mach = case.get_value("MACH")
compiler = case.get_value("COMPILER")
debug = case.get_value("DEBUG")
mpilib = case.get_value("MPILIB")
sysos = case.get_value("OS")
expect(mach is not None, "xml variable MACH is not set")
# creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler
# and env_mach_specific.xml if they don't already exist.
if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"):
configure(Machines(machine=mach), caseroot, ["Makefile"], compiler, mpilib, debug, sysos)
# Set tasks to 1 if mpi-serial library
if mpilib == "mpi-serial":
for vid, value in case:
if vid.startswith("NTASKS_") and value != 1:
case.set_value(vid, 1)
# Check ninst.
# In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
for comp in models:
if comp == "DRV":
continue
ninst = case.get_value("NINST_%s" % comp)
ntasks = case.get_value("NTASKS_%s" % comp)
if ninst > ntasks:
if ntasks == 1:
case.set_value("NTASKS_%s" % comp, ninst)
else:
expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst, comp, ntasks))
if os.path.exists("case.run"):
logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
else:
_check_pelayouts_require_rebuild(case, models)
if os.path.exists("LockedFiles/env_build.xml"):
os.remove("LockedFiles/env_build.xml")
case.flush()
check_lockedfiles()
env_mach_pes = case.get_env("mach_pes")
pestot = env_mach_pes.get_total_tasks(models)
logger.debug("at update TOTALPES = %s"%pestot)
case.set_value("TOTALPES", pestot)
thread_count = env_mach_pes.get_max_thread_count(models)
if thread_count > 1:
case.set_value("BUILD_THREADED", True)
expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"),
"it is not possible to run with OpenMP if using the NAG Fortran compiler")
cost_pes = env_mach_pes.get_cost_pes(pestot, thread_count, machine=case.get_value("MACH"))
case.set_value("COST_PES", cost_pes)
# create batch file
logger.info("Creating batch script case.run")
# Use BatchFactory to get the appropriate instance of a BatchMaker,
# use it to create our batch scripts
env_batch = case.get_env("batch")
num_nodes = env_mach_pes.get_total_nodes(pestot, thread_count)
tasks_per_node = env_mach_pes.get_tasks_per_node(pestot, thread_count)
for job in env_batch.get_jobs():
input_batch_script = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job))
if job == "case.test" and testcase is not None and not test_mode:
logger.info("Writing %s script" % job)
testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase)
# Short term fix to be removed when csh tests are removed
if not os.path.exists(testscript):
env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count)
elif job != "case.test":
logger.info("Writing %s script from input template %s" % (job, input_batch_script))
env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count)
# Make a copy of env_mach_pes.xml in order to be able
# to check that it does not change once case.setup is invoked
logger.info("Locking file env_mach_pes.xml")
case.flush()
logger.debug("at copy TOTALPES = %s"%case.get_value("TOTALPES"))
shutil.copy("env_mach_pes.xml", "LockedFiles")
# Create user_nl files for the required number of instances
if not os.path.exists("user_nl_cpl"):
logger.info("Creating user_nl_xxx files for components and cpl")
# loop over models
for model in models:
comp = case.get_value("COMP_%s" % model)
logger.info("Building %s usernl files"%model)
_build_usernl_files(case, model, comp)
if comp == "cism":
run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot))
_build_usernl_files(case, "drv", "cpl")
# Create needed directories for case
create_dirs(case)
logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")
# Create test script if appropriate
# Short term fix to be removed when csh tests are removed
if os.path.exists("env_test.xml"):
if not os.path.exists("case.test"):
logger.info("Starting testcase.setup")
run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot)
logger.info("Finished testcase.setup")
# some tests need namelists created here (ERP)
if test_mode:
create_namelists(case)
msg = "case.setup complete"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
# Record env information
env_module = case.get_env("mach_specific")
env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
with open("software_environment.txt", "w") as f:
f.write(env_module.list_modules())
run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
env >> software_environment.txt")
def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
###############################################################################
pre_run_check(case, lid, skip_pnl=skip_pnl, da_cycle=da_cycle)
model = case.get_value("MODEL")
# Set OMP_NUM_THREADS
env_mach_pes = case.get_env("mach_pes")
comp_classes = case.get_values("COMP_CLASSES")
thread_count = env_mach_pes.get_max_thread_count(comp_classes)
os.environ["OMP_NUM_THREADS"] = str(thread_count)
# Run the model
logger.info("{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
cmd = case.get_mpirun_cmd(job="case.run")
cmd = case.get_resolved_value(cmd)
logger.info("run command is {} ".format(cmd))
rundir = case.get_value("RUNDIR")
loop = True
while loop:
loop = False
stat = run_cmd(cmd, from_dir=rundir)[0]
model_logfile = os.path.join(rundir, model + ".log." + lid)
# Determine if failure was due to a failed node, if so, try to restart
if stat != 0:
node_fail_re = case.get_value("NODE_FAIL_REGEX")
if node_fail_re:
node_fail_regex = re.compile(node_fail_re)
model_logfile = os.path.join(rundir, model + ".log." + lid)
if os.path.exists(model_logfile):
num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read()))
if num_fails > 0 and case.spare_nodes >= num_fails:
# We failed due to node failure!
logger.warning("Detected model run failed due to node failure, restarting")
# Archive the last consistent set of restart files and restore them
case_st_archive(case, no_resubmit=True)
restore_from_archive(case)
orig_cont = case.get_value("CONTINUE_RUN")
if not orig_cont:
case.set_value("CONTINUE_RUN", True)
create_namelists(case)
lid = new_lid()
loop = True
case.spare_nodes -= num_fails
if not loop:
# We failed and we're not restarting
expect(False, "RUN FAIL: Command '{}' failed\nSee log file for details: {}".format(cmd, model_logfile))
logger.info("{} MODEL EXECUTION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
post_run_check(case, lid)
return lid
def pre_run_check(case, lid, skip_pnl=False, da_cycle=0):
###############################################################################
# Pre run initialization code..
if da_cycle > 0:
create_namelists(case, component='cpl')
return
caseroot = case.get_value("CASEROOT")
din_loc_root = case.get_value("DIN_LOC_ROOT")
batchsubmit = case.get_value("BATCHSUBMIT")
mpilib = case.get_value("MPILIB")
rundir = case.get_value("RUNDIR")
build_complete = case.get_value("BUILD_COMPLETE")
if case.get_value("TESTCASE") == "PFS":
env_mach_pes = os.path.join(caseroot,"env_mach_pes.xml")
shutil.copy(env_mach_pes,"{}.{}".format(env_mach_pes, lid))
# check for locked files.
check_lockedfiles(case)
logger.debug("check_lockedfiles OK")
# check that build is done
expect(build_complete,
"BUILD_COMPLETE is not true\nPlease rebuild the model interactively")
logger.debug("build complete is {} ".format(build_complete))
# load the module environment...
case.load_env()
# set environment variables
# This is a requirement for yellowstone only
if mpilib == "mpi-serial" and "MP_MPILIB" in os.environ:
del os.environ["MP_MPILIB"]
else:
os.environ["MPILIB"] = mpilib
if batchsubmit is None or len(batchsubmit) == 0:
os.environ["LBQUERY"] = "FALSE"
os.environ["BATCHQUERY"] = "undefined"
elif batchsubmit == 'UNSET':
os.environ["LBQUERY"] = "FALSE"
os.environ["BATCHQUERY"] = "undefined"
else:
os.environ["LBQUERY"] = "TRUE"
# create the timing directories, optionally cleaning them if needed.
if not os.path.isdir(rundir):
os.mkdir(rundir)
if os.path.isdir(os.path.join(rundir, "timing")):
shutil.rmtree(os.path.join(rundir, "timing"))
os.makedirs(os.path.join(rundir, "timing", "checkpoints"))
# This needs to be done everytime the LID changes in order for log files to be set up correctly
# The following also needs to be called in case a user changes a user_nl_xxx file OR an env_run.xml
# variable while the job is in the queue
if not skip_pnl:
create_namelists(case)
logger.info("-------------------------------------------------------------------------")
logger.info(" - Prestage required restarts into {}".format(rundir))
logger.info(" - Case input data directory (DIN_LOC_ROOT) is {} ".format(din_loc_root))
logger.info(" - Checking for required input datasets in DIN_LOC_ROOT")
logger.info("-------------------------------------------------------------------------")
def _build_checks(case, build_threaded, comp_interface, use_esmf_lib,
debug, compiler, mpilib, complist, ninst_build, smp_value,
model_only, buildlist):
###############################################################################
"""
check if a build needs to be done and warn if a clean is warrented first
returns the relative sharedpath directory for sharedlibraries
"""
ninst_value = case.get_value("NINST_VALUE")
smp_build = case.get_value("SMP_BUILD")
build_status = case.get_value("BUILD_STATUS")
expect(comp_interface == "mct", "Only supporting mct comp_interface at this time")
smpstr = ""
inststr = ""
for model, _, nthrds, ninst, _ in complist:
if nthrds > 1:
build_threaded = True
if build_threaded:
smpstr += "{}1".format(model[0])
else:
smpstr += "{}0".format(model[0])
inststr += "{}{:d}".format((model[0]),ninst)
if build_threaded:
os.environ["SMP"] = "TRUE"
else:
os.environ["SMP"] = "FALSE"
case.set_value("SMP_VALUE", smpstr)
os.environ["SMP_VALUE"] = smpstr
case.set_value("NINST_VALUE", inststr)
os.environ["NINST_VALUE"] = inststr
debugdir = "debug" if debug else "nodebug"
threaddir = "threads" if (os.environ["SMP"] == "TRUE" or build_threaded) else "nothreads"
sharedpath = os.path.join(compiler, mpilib, debugdir, threaddir)
logger.debug("compiler={} mpilib={} debugdir={} threaddir={}"
.format(compiler,mpilib,debugdir,threaddir))
expect(ninst_build == ninst_value or ninst_build == "0",
"""
ERROR, NINST VALUES HAVE CHANGED
NINST_BUILD = {}
NINST_VALUE = {}
A manual clean of your obj directories is strongly recommended
You should execute the following:
./case.build --clean
Then rerun the build script interactively
---- OR ----
You can override this error message at your own risk by executing:
./xmlchange -file env_build.xml -id NINST_BUILD -val 0
Then rerun the build script interactively
""".format(ninst_build, ninst_value))
expect(smp_build == smpstr or smp_build == "0",
"""
ERROR, SMP VALUES HAVE CHANGED
SMP_BUILD = {}
SMP_VALUE = {}
smpstr = {}
A manual clean of your obj directories is strongly recommended
You should execute the following:
./case.build --clean
Then rerun the build script interactively
---- OR ----
You can override this error message at your own risk by executing:
./xmlchange -file env_build.xml -id SMP_BUILD -val 0
Then rerun the build script interactively
""".format(smp_build, smp_value, smpstr))
expect(build_status == 0,
"""
ERROR env_build HAS CHANGED
A manual clean of your obj directories is required
You should execute the following:
./case.build --clean-all
""")
expect(mpilib != "mpi-serial" or not use_esmf_lib,
"""
ERROR MPILIB is mpi-serial and USE_ESMF_LIB IS TRUE
MPILIB can only be used with an ESMF library built with mpiuni on
Set USE_ESMF_LIB to FALSE with
./xmlchange -file env_build.xml -id USE_ESMF_LIB -val FALSE
---- OR ----
Make sure the ESMF_LIBDIR used was built with mipuni (or change it to one that was)
And comment out this if block in Tools/models_buildexe
""")
case.set_value("BUILD_COMPLETE", False)
# User may have rm -rf their build directory
create_dirs(case)
case.flush()
if not model_only and not buildlist:
logger.info("Generating component namelists as part of build")
create_namelists(case)
return sharedpath
def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
###############################################################################
os.chdir(caseroot)
# Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
din_loc_root = case.get_value("DIN_LOC_ROOT")
testcase = case.get_value("TESTCASE")
expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
"inputdata root is not a directory: {}".format(din_loc_root))
# Remove batch scripts
if reset or clean:
case_run, case_test = get_batch_script_for_job("case.run"), get_batch_script_for_job("case.test")
if os.path.exists(case_run):
os.remove(case_run)
if not test_mode:
# rebuild the models (even on restart)
case.set_value("BUILD_COMPLETE", False)
# backup and then clean test script
if os.path.exists(case_test):
os.remove(case_test)
logger.info("Successfully cleaned test script {}".format(case_test))
logger.info("Successfully cleaned batch script case.run")
if not clean:
case.load_env()
models = case.get_values("COMP_CLASSES")
mach = case.get_value("MACH")
compiler = case.get_value("COMPILER")
debug = case.get_value("DEBUG")
mpilib = case.get_value("MPILIB")
sysos = case.get_value("OS")
expect(mach is not None, "xml variable MACH is not set")
# creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler
# and env_mach_specific.xml if they don't already exist.
if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"):
configure(Machines(machine=mach), caseroot, ["Makefile"], compiler, mpilib, debug, sysos)
# Set tasks to 1 if mpi-serial library
if mpilib == "mpi-serial":
for vid, value in case:
if vid.startswith("NTASKS") and value != 1:
case.set_value(vid, 1)
# Check ninst.
# In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
multi_driver = case.get_value("MULTI_DRIVER")
nthrds = 1
for comp in models:
ntasks = case.get_value("NTASKS_{}".format(comp))
nthrds = max(nthrds,case.get_value("NTHRDS_{}".format(comp)))
if comp == "CPL":
continue
ninst = case.get_value("NINST_{}".format(comp))
if multi_driver:
expect(case.get_value("NINST_LAYOUT_{}".format(comp)) == "concurrent",
"If multi_driver is TRUE, NINST_LAYOUT_{} must be concurrent".format(comp))
case.set_value("NTASKS_PER_INST_{}".format(comp), ntasks)
else:
if ninst > ntasks:
if ntasks == 1:
case.set_value("NTASKS_{}".format(comp), ninst)
ntasks = ninst
else:
expect(False, "NINST_{} value {:d} greater than NTASKS_{} {:d}".format(comp, ninst, comp, ntasks))
case.set_value("NTASKS_PER_INST_{}".format(comp), int(ntasks / ninst))
if nthrds > 1:
case.set_value("BUILD_THREADED",True)
if os.path.exists(get_batch_script_for_job("case.run")):
logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
case.initialize_derived_attributes()
case.set_value("SMP_PRESENT", case.get_build_threaded())
else:
check_pelayouts_require_rebuild(case, models)
unlock_file("env_build.xml")
unlock_file("env_batch.xml")
case.flush()
check_lockedfiles(case)
case.initialize_derived_attributes()
cost_per_node = 16 if case.get_value("MACH") == "yellowstone" else case.get_value("MAX_MPITASKS_PER_NODE")
case.set_value("COST_PES", case.num_nodes * cost_per_node)
case.set_value("TOTALPES", case.total_tasks)
case.set_value("SMP_PRESENT", case.get_build_threaded())
# create batch files
env_batch = case.get_env("batch")
env_batch.make_all_batch_files(case, test_mode=test_mode)
# May need to select new batch settings if pelayout changed (e.g. problem is now too big for prev-selected queue)
env_batch.set_job_defaults([(("case.test" if case.get_value("TEST") else "case.run"), {})], case)
case.schedule_rewrite(env_batch)
# Make a copy of env_mach_pes.xml in order to be able
# to check that it does not change once case.setup is invoked
case.flush()
logger.debug("at copy TOTALPES = {}".format(case.get_value("TOTALPES")))
lock_file("env_mach_pes.xml")
lock_file("env_batch.xml")
# Create user_nl files for the required number of instances
if not os.path.exists("user_nl_cpl"):
logger.info("Creating user_nl_xxx files for components and cpl")
# loop over models
for model in models:
comp = case.get_value("COMP_{}".format(model))
logger.debug("Building {} usernl files".format(model))
_build_usernl_files(case, model, comp)
if comp == "cism":
glcroot = case.get_value("COMP_ROOT_DIR_GLC")
run_cmd_no_fail("{}/cime_config/cism.template {}".format(glcroot, caseroot))
_build_usernl_files(case, "drv", "cpl")
# Create needed directories for case
create_dirs(case)
logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")
# Some tests need namelists created here (ERP) - so do this if we are in test mode
if test_mode or get_model() == "e3sm":
logger.info("Generating component namelists as part of setup")
create_namelists(case)
# Record env information
env_module = case.get_env("mach_specific")
env_module.make_env_mach_specific_file("sh", case)
env_module.make_env_mach_specific_file("csh", case)
env_module.save_all_env_info("software_environment.txt")
