def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning("WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname))
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {} {} {} {} "
.format(compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot),
from_dir=bldroot, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build))
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning("WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname))
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {} {} {} {} "
.format(compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot),
from_dir=bldroot, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build))
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(
config_dir,
compclass,
compname,
caseroot,
libroot,
bldroot,
incroot,
file_build,
thread_bad_results,
smp,
compiler,
):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning(
"WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd),
"Could not find buildlib for {}".format(compname))
compile_cmd = "COMP_CLASS={compclass} COMP_NAME={compname} {cmd} {caseroot} {libroot} {bldroot} ".format(
compclass=compclass,
compname=compname,
cmd=cmd,
caseroot=caseroot,
libroot=libroot,
bldroot=bldroot,
)
if get_model() != "ufs":
compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd)
if is_python_executable(cmd):
logging_options = get_logging_options()
if logging_options != "":
compile_cmd = compile_cmd + logging_options
with open(file_build, "w") as fd:
stat = run_cmd(compile_cmd,
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
if stat != 0:
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compname, file_build))
analyze_build_log(compclass, file_build, compiler)
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot,
incroot, file_build, thread_bad_results, smp,
compiler):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {}/buildlib {} {} {} ".format(
compclass, stringify_bool(smp), config_dir, caseroot, libroot,
bldroot),
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compclass, file_build))
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compclass, (t2 - t1)))
def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler):
###############################################################################
with open(file_build, "w") as fd:
stat = run_cmd("MODEL=%s SMP=%s %s/buildlib %s %s %s " %
(compclass, stringify_bool(smp), config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=True, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append("ERROR: %s.buildlib failed, see %s" % (compclass, file_build))
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot,
bldroot, incroot, file_build, thread_bad_results, smp,
compiler, case):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning(
"WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd),
"Could not find buildlib for {}".format(compname))
# Add to this list as components are converted to python/cmake
if compname in ["cam"] and get_model() == "e3sm":
try:
stat = 0
run_sub_or_cmd(cmd, [caseroot, libroot, bldroot],
"buildlib", [bldroot, libroot, case],
logfile=file_build)
except Exception:
stat = 1
else:
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {} {} {} {} ".format(
compclass, stringify_bool(smp), cmd, caseroot, libroot,
bldroot),
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if stat != 0:
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compname, file_build))
analyze_build_log(compclass, file_build, compiler)
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot,
incroot, file_build, thread_bad_results, smp,
compiler):
###############################################################################
with open(file_build, "w") as fd:
stat = run_cmd("MODEL=%s SMP=%s %s/buildlib %s %s %s " %
(compclass, stringify_bool(smp), config_dir, caseroot,
libroot, bldroot),
from_dir=bldroot,
verbose=True,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append("ERROR: %s.buildlib failed, see %s" %
(compclass, file_build))
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
def _build_model(build_threaded, exeroot, incroot, complist, lid, caseroot,
cimeroot, compiler, buildlist, comp_interface, case):
###############################################################################
logs = []
thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
if buildlist is not None and model.lower() not in buildlist:
continue
# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
logger.debug("Skip aquap ocn build here")
continue
# coupler handled seperately
if model == "cpl":
continue
# special case for clm
# clm 4_5 and newer is a shared (as in sharedlibs, shared by all tests) library
# (but not in E3SM) and should be built in build_libraries
if get_model() != "e3sm" and comp == "clm":
continue
else:
logger.info(" - Building {} Library ".format(model))
smp = nthrds > 1 or build_threaded
bldroot = os.path.join(exeroot, model, "obj")
libroot = os.path.join(exeroot, "lib")
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(model, lid))
logger.debug("bldroot is {}".format(bldroot))
logger.debug("libroot is {}".format(libroot))
# make sure bldroot and libroot exist
for build_dir in [bldroot, libroot]:
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# build the component library
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
t = threading.Thread(target=_build_model_thread,
args=(config_dir, model, comp, caseroot, libroot,
bldroot, incroot, file_build,
thread_bad_results, smp, compiler, case))
t.start()
logs.append(file_build)
# Wait for threads to finish
while (threading.active_count() > 1):
time.sleep(1)
expect(not thread_bad_results, "\n".join(thread_bad_results))
#
# Now build the executable
#
if not buildlist:
cime_model = get_model()
file_build = os.path.join(exeroot,
"{}.bldlog.{}".format(cime_model, lid))
config_dir = os.path.join(cimeroot, "src", "drivers", comp_interface,
"cime_config")
if not os.path.isdir(config_dir):
config_dir = os.path.join(cimeroot, "..", "src", "model", "NEMS",
"cime", "cime_config")
expect(os.path.exists(config_dir),
"Config directory not found {}".format(config_dir))
if "cpl" in complist:
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
os.makedirs(bldroot)
logger.info("Building {} with output to {} ".format(
cime_model, file_build))
with open(file_build, "w") as fd:
stat = run_cmd("{}/buildexe {} {} {} ".format(
config_dir, caseroot, libroot, bldroot),
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0,
"BUILD FAIL: buildexe failed, cat {}".format(file_build))
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
safe_copy("{}/{}.exe".format(exeroot, cime_model),
"{}/{}.exe.{}".format(exeroot, cime_model, lid))
logs.append(file_build)
return logs
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(lib, lid))
my_file = os.path.join(cimeroot, "src", "build_scripts",
"buildlib.{}".format(lib))
logger.info("Building {} with output to file {}".format(
lib, file_build))
run_sub_or_cmd(
my_file,
[full_lib_path,
os.path.join(exeroot, sharedpath), caseroot],
'buildlib',
[full_lib_path,
os.path.join(exeroot, sharedpath), case],
logfile=file_build)
analyze_build_log(lib, file_build, compiler)
logs.append(file_build)
if lib == "pio":
bldlog = open(file_build, "r")
for line in bldlog:
if re.search("Current setting for", line):
logger.warning(line)
# clm not a shared lib for E3SM
if get_model() != "e3sm" and (buildlist is None or "lnd" in buildlist):
comp_lnd = case.get_value("COMP_LND")
if comp_lnd == "clm":
logging.info(" - Building clm library ")
esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf"
bldroot = os.path.join(sharedlibroot, sharedpath, comp_interface,
esmfdir, "clm", "obj")
def build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler):
###############################################################################
shared_lib = os.path.join(exeroot, sharedpath, "lib")
shared_inc = os.path.join(exeroot, sharedpath, "include")
for shared_item in [shared_lib, shared_inc]:
if (not os.path.exists(shared_item)):
os.makedirs(shared_item)
libs = ["mct", "gptl", "pio", "csm_share"]
logs = []
sharedlibroot = case.get_value("SHAREDLIBROOT")
for lib in libs:
if lib == "csm_share":
# csm_share adds its own dir name
full_lib_path = os.path.join(sharedlibroot, sharedpath)
else:
full_lib_path = os.path.join(sharedlibroot, sharedpath, lib)
# pio build creates its own directory
if (lib != "pio" and not os.path.exists(full_lib_path)):
os.makedirs(full_lib_path)
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (lib, lid))
my_file = os.path.join(cimeroot, "cime_config", "buildlib.%s" % lib)
if lib == "pio":
my_file = "PYTHONPATH=%s:%s:$PYTHONPATH %s"%(os.path.join(cimeroot,"scripts","Tools"),
os.path.join(cimeroot,"utils","python"), my_file)
with open(file_build, "w") as fd:
stat = run_cmd("%s %s %s %s" %
(my_file, full_lib_path, os.path.join(exeroot,sharedpath), caseroot),
from_dir=exeroot,
verbose=True, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(lib, file_build, compiler)
expect(stat == 0, "ERROR: buildlib.%s failed, cat %s" % (lib, file_build))
logs.append(file_build)
if lib == "pio":
bldlog = open(file_build, "r")
for line in bldlog:
if re.search("Current setting for", line):
logger.warn(line)
comp_lnd = case.get_value("COMP_LND")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
if comp_lnd == "clm" and not "clm4_0" in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf"
bldroot = os.path.join(sharedlibroot, sharedpath, case.get_value("COMP_INTERFACE"), esmfdir, "clm","obj" )
libroot = os.path.join(exeroot, sharedpath, case.get_value("COMP_INTERFACE"), esmfdir, "lib")
incroot = os.path.join(exeroot, sharedpath, case.get_value("COMP_INTERFACE"), esmfdir, "include")
file_build = os.path.join(exeroot, "lnd.bldlog.%s" % lid)
config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE"))
for ndir in [bldroot, libroot, incroot]:
if (not os.path.isdir(ndir)):
os.makedirs(ndir)
smp = "SMP" in os.environ and os.environ["SMP"] == "TRUE"
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
thread_bad_results = []
_build_model_thread(config_lnd_dir, "lnd", caseroot, libroot, bldroot, incroot,
file_build, thread_bad_results, smp, compiler)
logs.append(file_build)
expect(not thread_bad_results, "\n".join(thread_bad_results))
return logs
def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler):
###############################################################################
logs = []
thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
logger.debug("Skip aquap ocn build here")
continue
# coupler handled seperately
if model == "cpl":
continue
# special case for clm
# clm 4_0 is not a shared library and must be built here
# clm 4_5 and newer is a shared library and should be built in build_libraries
if comp == "clm":
if "clm4_0" in clm_config_opts:
logger.info(" - Building clm4_0 Library ")
else:
continue
smp = nthrds > 1 or build_threaded
bldroot = os.path.join(exeroot, model, "obj")
libroot = os.path.join(exeroot, "lib")
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (model, lid))
logger.debug("bldroot is %s" % bldroot)
logger.debug("libroot is %s" % libroot)
# make sure bldroot and libroot exist
for build_dir in [bldroot, libroot]:
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# build the component library
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
t = threading.Thread(target=_build_model_thread,
args=(config_dir, model, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler))
t.start()
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
logs.append(file_build)
# Wait for threads to finish
while(threading.active_count() > 1):
time.sleep(1)
# aquap has a dependancy on atm so we build it after the threaded loop
for model, comp, nthrds, _, config_dir in complist:
smp = nthrds > 1 or build_threaded
if comp == "aquap":
logger.debug("Now build aquap ocn component")
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
_build_model_thread(config_dir, comp, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler)
logs.append(file_build)
expect(not thread_bad_results, "\n".join(thread_bad_results))
#
# Now build the executable
#
cime_model = get_model()
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (cime_model, lid))
config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
stat = run_cmd("%s/buildexe %s %s %s" %
(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=True, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("%s exe"%cime_model, file_build, compiler)
expect(stat == 0, "ERROR: buildexe failed, cat %s" % file_build)
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("%s/%s.exe" % (exeroot, cime_model), "%s/%s.exe.%s" % (exeroot, cime_model, lid))
logs.append(file_build)
return logs
def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist, comp_interface):
###############################################################################
shared_lib = os.path.join(exeroot, sharedpath, "lib")
shared_inc = os.path.join(exeroot, sharedpath, "include")
for shared_item in [shared_lib, shared_inc]:
if (not os.path.exists(shared_item)):
os.makedirs(shared_item)
mpilib = case.get_value("MPILIB")
libs = ["gptl", "mct", "pio", "csm_share"]
if mpilib == "mpi-serial":
libs.insert(0, mpilib)
logs = []
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
for lib in libs:
if buildlist is not None and lib not in buildlist:
continue
if lib == "csm_share":
# csm_share adds its own dir name
full_lib_path = os.path.join(sharedlibroot, sharedpath)
elif lib == "mpi-serial":
full_lib_path = os.path.join(sharedlibroot, sharedpath, "mct", lib)
else:
full_lib_path = os.path.join(sharedlibroot, sharedpath, lib)
# pio build creates its own directory
if (lib != "pio" and not os.path.exists(full_lib_path)):
os.makedirs(full_lib_path)
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(lib, lid))
my_file = os.path.join(cimeroot, "src", "build_scripts", "buildlib.{}".format(lib))
logger.info("Building {} with output to file {}".format(lib,file_build))
run_sub_or_cmd(my_file, [full_lib_path, os.path.join(exeroot, sharedpath), caseroot], 'buildlib',
[full_lib_path, os.path.join(exeroot, sharedpath), caseroot], logfile=file_build)
analyze_build_log(lib, file_build, compiler)
logs.append(file_build)
if lib == "pio":
bldlog = open(file_build, "r")
for line in bldlog:
if re.search("Current setting for", line):
logger.warning(line)
# clm not a shared lib for E3SM
if get_model() != "e3sm" and (buildlist is None or "lnd" in buildlist):
comp_lnd = case.get_value("COMP_LND")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
if comp_lnd == "clm" and "clm4_0" not in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf"
bldroot = os.path.join(sharedlibroot, sharedpath, comp_interface, esmfdir, "clm","obj" )
libroot = os.path.join(exeroot, sharedpath, comp_interface, esmfdir, "lib")
incroot = os.path.join(exeroot, sharedpath, comp_interface, esmfdir, "include")
file_build = os.path.join(exeroot, "lnd.bldlog.{}".format( lid))
config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE"))
for ndir in [bldroot, libroot, incroot]:
if (not os.path.isdir(ndir)):
os.makedirs(ndir)
smp = "SMP" in os.environ and os.environ["SMP"] == "TRUE"
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
thread_bad_results = []
_build_model_thread(config_lnd_dir, "lnd", comp_lnd, caseroot, libroot, bldroot, incroot,
file_build, thread_bad_results, smp, compiler)
logs.append(file_build)
expect(not thread_bad_results, "\n".join(thread_bad_results))
return logs
def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler, buildlist, comp_interface):
###############################################################################
logs = []
thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
if buildlist is not None and model.lower() not in buildlist:
continue
# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
logger.debug("Skip aquap ocn build here")
continue
# coupler handled seperately
if model == "cpl":
continue
# special case for clm
# clm 4_0 is not a shared (as in sharedlibs, shared by all tests) library and must be built here
# clm 4_5 and newer is a shared library (but not in E3SM) and should be built in build_libraries
if get_model() != "e3sm":
if comp == "clm":
if "clm4_0" in clm_config_opts:
logger.info(" - Building clm4_0 Library ")
else:
continue
else:
logger.info(" - Building {} Library ".format(model))
smp = nthrds > 1 or build_threaded
bldroot = os.path.join(exeroot, model, "obj")
libroot = os.path.join(exeroot, "lib")
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(model, lid))
logger.debug("bldroot is {}".format(bldroot))
logger.debug("libroot is {}".format(libroot))
# make sure bldroot and libroot exist
for build_dir in [bldroot, libroot]:
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# build the component library
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
t = threading.Thread(target=_build_model_thread,
args=(config_dir, model, comp, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler))
t.start()
logs.append(file_build)
# Wait for threads to finish
while(threading.active_count() > 1):
time.sleep(1)
expect(not thread_bad_results, "\n".join(thread_bad_results))
#
# Now build the executable
#
if not buildlist:
cime_model = get_model()
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))
config_dir = os.path.join(cimeroot, "src", "drivers", comp_interface, "cime_config")
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
os.makedirs(bldroot)
logger.info("Building {} with output to {} ".format(cime_model, file_build))
with open(file_build, "w") as fd:
stat = run_cmd("{}/buildexe {} {} {} "
.format(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
safe_copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))
logs.append(file_build)
return logs
def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler):
###############################################################################
logs = []
thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
logger.debug("Skip aquap ocn build here")
continue
# coupler handled seperately
if model == "cpl":
continue
# special case for clm
# clm 4_0 is not a shared library and must be built here
# clm 4_5 and newer is a shared library and should be built in build_libraries
if comp == "clm":
if "clm4_0" in clm_config_opts:
logger.info(" - Building clm4_0 Library ")
else:
continue
smp = nthrds > 1 or build_threaded
bldroot = os.path.join(exeroot, model, "obj")
libroot = os.path.join(exeroot, "lib")
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (model, lid))
logger.debug("bldroot is %s" % bldroot)
logger.debug("libroot is %s" % libroot)
# make sure bldroot and libroot exist
for build_dir in [bldroot, libroot]:
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# build the component library
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
t = threading.Thread(target=_build_model_thread,
args=(config_dir, model, caseroot, libroot,
bldroot, incroot, file_build,
thread_bad_results, smp, compiler))
t.start()
for mod_file in glob.glob(
os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
logs.append(file_build)
# Wait for threads to finish
while (threading.active_count() > 1):
time.sleep(1)
# aquap has a dependancy on atm so we build it after the threaded loop
for model, comp, nthrds, _, config_dir in complist:
smp = nthrds > 1 or build_threaded
if comp == "aquap":
logger.debug("Now build aquap ocn component")
# thread_bad_results captures error output from thread (expected to be empty)
# logs is a list of log files to be compressed and added to the case logs/bld directory
_build_model_thread(config_dir, comp, caseroot, libroot, bldroot,
incroot, file_build, thread_bad_results, smp,
compiler)
logs.append(file_build)
expect(not thread_bad_results, "\n".join(thread_bad_results))
#
# Now build the executable
#
cime_model = get_model()
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (cime_model, lid))
config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
stat = run_cmd("%s/buildexe %s %s %s" %
(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot,
verbose=True,
arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("%s exe" % cime_model, file_build, compiler)
expect(stat == 0, "ERROR: buildexe failed, cat %s" % file_build)
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("%s/%s.exe" % (exeroot, cime_model),
"%s/%s.exe.%s" % (exeroot, cime_model, lid))
logs.append(file_build)
return logs
