def _do_external(script_name, caseroot, rundir, lid, prefix):
###############################################################################
expect(os.path.isfile(script_name), "External script {} not found".format(script_name))
filename = "{}.external.log.{}".format(prefix, lid)
outfile = os.path.join(rundir, filename)
append_status("Starting script {}".format(script_name), "CaseStatus")
run_sub_or_cmd(script_name, [caseroot], (os.path.basename(script_name).split('.',1))[0], [caseroot], logfile=outfile) # For sub, use case?
append_status("Completed script {}".format(script_name), "CaseStatus")
def _do_external(script_name, caseroot, rundir, lid, prefix):
###############################################################################
expect(os.path.isfile(script_name), "External script {} not found".format(script_name))
filename = "{}.external.log.{}".format(prefix, lid)
outfile = os.path.join(rundir, filename)
append_status("Starting script {}".format(script_name), "CaseStatus")
run_sub_or_cmd(script_name, [caseroot], (os.path.basename(script_name).split('.',1))[0], [caseroot], logfile=outfile)
append_status("Completed script {}".format(script_name), "CaseStatus")
def _do_data_assimilation(da_script, caseroot, cycle, lid, rundir):
###############################################################################
expect(os.path.isfile(da_script),
"Data Assimilation script {} not found".format(da_script))
filename = "da.log.{}".format(lid)
outfile = os.path.join(rundir, filename)
run_sub_or_cmd(da_script, [caseroot, cycle],
os.path.basename(da_script), [caseroot, cycle],
logfile=outfile) # For sub, use case?
def do_external(script_name, caseroot, rundir, lid, prefix):
###############################################################################
expect(os.path.isfile(script_name),
"External script {} not found".format(script_name))
filename = "{}.external.log.{}".format(prefix, lid)
outfile = os.path.join(rundir, filename)
run_sub_or_cmd(script_name, [caseroot],
os.path.basename(script_name), [caseroot],
logfile=outfile)
def create_namelists(self, component=None):
"""
Create component namelists
"""
self.flush()
create_dirs(self)
casebuild = self.get_value("CASEBUILD")
caseroot = self.get_value("CASEROOT")
rundir = self.get_value("RUNDIR")
docdir = os.path.join(caseroot, "CaseDocs")
# Load modules
self.load_env()
self.stage_refcase()
logger.info("Creating component namelists")
# Create namelists - must have cpl last in the list below
# Note - cpl must be last in the loop below so that in generating its namelist,
# it can use xml vars potentially set by other component's buildnml scripts
models = self.get_values("COMP_CLASSES")
models += [models.pop(0)]
for model in models:
model_str = model.lower()
logger.info(" {} {}".format(time.strftime("%Y-%m-%d %H:%M:%S"),model_str))
config_file = self.get_value("CONFIG_{}_FILE".format(model_str.upper()))
config_dir = os.path.dirname(config_file)
if model_str == "cpl":
compname = "drv"
else:
compname = self.get_value("COMP_{}".format(model_str.upper()))
if component is None or component == model_str:
# first look in the case SourceMods directory
cmd = os.path.join(caseroot, "SourceMods", "src."+compname, "buildnml")
if os.path.isfile(cmd):
logger.warning("\nWARNING: Using local buildnml file {}\n".format(cmd))
else:
# otherwise look in the component config_dir
cmd = os.path.join(config_dir, "buildnml")
expect(os.path.isfile(cmd), "Could not find buildnml file for component {}".format(compname))
run_sub_or_cmd(cmd, (caseroot), "buildnml", (self, caseroot, compname), case=self)
logger.info("Finished creating component namelists")
# Save namelists to docdir
if (not os.path.isdir(docdir)):
os.makedirs(docdir)
try:
with open(os.path.join(docdir, "README"), "w") as fd:
fd.write(" CESM Resolved Namelist Files\n For documentation only DO NOT MODIFY\n")
except (OSError, IOError) as e:
expect(False, "Failed to write {}/README: {}".format(docdir, e))
for cpglob in ["*_in_[0-9]*", "*modelio*", "*_in", "cesm.runconfig*",
"*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]:
for file_to_copy in glob.glob(os.path.join(rundir, cpglob)):
logger.debug("Copy file from '{}' to '{}'".format(file_to_copy, docdir))
safe_copy(file_to_copy, docdir)
# Copy over chemistry mechanism docs if they exist
if (os.path.isdir(os.path.join(casebuild, "camconf"))):
for file_to_copy in glob.glob(os.path.join(casebuild, "camconf", "*chem_mech*")):
safe_copy(file_to_copy, docdir)
def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist, comp_interface):
###############################################################################
shared_lib = os.path.join(exeroot, sharedpath, "lib")
shared_inc = os.path.join(exeroot, sharedpath, "include")
for shared_item in [shared_lib, shared_inc]:
if (not os.path.exists(shared_item)):
os.makedirs(shared_item)
mpilib = case.get_value("MPILIB")
libs = ["gptl", "mct", "pio", "csm_share"]
if mpilib == "mpi-serial":
libs.insert(0, mpilib)
logs = []
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
for lib in libs:
if buildlist is not None and lib not in buildlist:
continue
if lib == "csm_share":
# csm_share adds its own dir name
full_lib_path = os.path.join(sharedlibroot, sharedpath)
elif lib == "mpi-serial":
full_lib_path = os.path.join(sharedlibroot, sharedpath, "mct", lib)
else:
full_lib_path = os.path.join(sharedlibroot, sharedpath, lib)
# pio build creates its own directory
if (lib != "pio" and not os.path.exists(full_lib_path)):
os.makedirs(full_lib_path)
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(lib, lid))
my_file = os.path.join(cimeroot, "src", "build_scripts", "buildlib.{}".format(lib))
logger.info("Building {} with output to file {}".format(lib,file_build))
run_sub_or_cmd(my_file, [full_lib_path, os.path.join(exeroot, sharedpath), caseroot], 'buildlib',
[full_lib_path, os.path.join(exeroot, sharedpath), caseroot], logfile=file_build)
analyze_build_log(lib, file_build, compiler)
logs.append(file_build)
if lib == "pio":
bldlog = open(file_build, "r")
for line in bldlog:
if re.search("Current setting for", line):
logger.warning(line)
# clm not a shared lib for E3SM
if get_model() != "e3sm" and (buildlist is None or "lnd" in buildlist):
comp_lnd = case.get_value("COMP_LND")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
if comp_lnd == "clm" and "clm4_0" not in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf"
bldroot = os.path.join(sharedlibroot, sharedpath, comp_interface, esmfdir, "clm","obj" )
libroot = os.path.join(exeroot, sharedpath, comp_interface, esmfdir, "lib")
incroot = os.path.join(exeroot, sharedpath, comp_interface, esmfdir, "include")
file_build = os.path.join(exeroot, "lnd.bldlog.{}".format( lid))
config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE"))
for ndir in [bldroot, libroot, incroot]:
if (not os.path.isdir(ndir)):
os.makedirs(ndir)
smp = "SMP" in os.environ and os.environ["SMP"] == "TRUE"
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
thread_bad_results = []
_build_model_thread(config_lnd_dir, "lnd", comp_lnd, caseroot, libroot, bldroot, incroot,
file_build, thread_bad_results, smp, compiler)
logs.append(file_build)
expect(not thread_bad_results, "\n".join(thread_bad_results))
return logs
if lib == "csm_share":
# csm_share adds its own dir name
full_lib_path = os.path.join(sharedlibroot, sharedpath)
elif lib == "mpi-serial":
full_lib_path = os.path.join(sharedlibroot, sharedpath, comp_interface, lib)
else:
full_lib_path = os.path.join(sharedlibroot, sharedpath, lib)
# pio build creates its own directory
if (lib != "pio" and not os.path.exists(full_lib_path)):
os.makedirs(full_lib_path)
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(lib, lid))
my_file = os.path.join(cimeroot, "src", "build_scripts", "buildlib.{}".format(lib))
logger.info("Building {} with output to file {}".format(lib,file_build))
run_sub_or_cmd(my_file, [full_lib_path, os.path.join(exeroot, sharedpath), caseroot], 'buildlib',
[full_lib_path, os.path.join(exeroot, sharedpath), caseroot], logfile=file_build)
analyze_build_log(lib, file_build, compiler)
logs.append(file_build)
if lib == "pio":
bldlog = open(file_build, "r")
for line in bldlog:
if re.search("Current setting for", line):
logger.warning(line)
# clm not a shared lib for E3SM
if get_model() != "e3sm" and (buildlist is None or "lnd" in buildlist):
comp_lnd = case.get_value("COMP_LND")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
if comp_lnd == "clm" and "clm4_0" not in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
def _do_data_assimilation(da_script, caseroot, cycle, lid, rundir):
###############################################################################
expect(os.path.isfile(da_script), "Data Assimilation script {} not found".format(da_script))
filename = "da.log.{}".format(lid)
outfile = os.path.join(rundir, filename)
run_sub_or_cmd(da_script, [caseroot, cycle], os.path.basename(da_script), [caseroot, cycle], logfile=outfile)
def do_external(script_name, caseroot, rundir, lid, prefix):
###############################################################################
expect(os.path.isfile(script_name), "External script {} not found".format(script_name))
filename = "{}.external.log.{}".format(prefix, lid)
outfile = os.path.join(rundir, filename)
run_sub_or_cmd(script_name, [caseroot], os.path.basename(script_name), [caseroot], logfile=outfile)