def _clean_impl(case, cleanlist, clean_all, clean_depends):
###############################################################################
exeroot = os.path.abspath(case.get_value("EXEROOT"))
if clean_all:
# If cleanlist is empty just remove the bld directory
expect(exeroot is not None,"No EXEROOT defined in case")
if os.path.isdir(exeroot):
logging.info("cleaning directory {}".format(exeroot))
shutil.rmtree(exeroot)
# if clean_all is True also remove the sharedlibpath
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case")
if sharedlibroot != exeroot and os.path.isdir(sharedlibroot):
logging.warning("cleaning directory {}".format(sharedlibroot))
shutil.rmtree(sharedlibroot)
else:
expect((cleanlist is not None and len(cleanlist) > 0) or
(clean_depends is not None and len(clean_depends)),"Empty cleanlist not expected")
debug = case.get_value("DEBUG")
use_esmf_lib = case.get_value("USE_ESMF_LIB")
build_threaded = case.get_build_threaded()
gmake = case.get_value("GMAKE")
caseroot = os.path.abspath(case.get_value("CASEROOT"))
casetools = case.get_value("CASETOOLS")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib)
os.environ["BUILD_THREADED"] = stringify_bool(build_threaded)
os.environ["CASEROOT"] = caseroot
os.environ["COMP_INTERFACE"] = case.get_value("COMP_INTERFACE")
os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION"))
os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else ""
cmd = gmake + " -f " + os.path.join(casetools, "Makefile")
if cleanlist is not None:
for item in cleanlist:
tcmd = cmd + " clean" + item
logger.info("calling {} ".format(tcmd))
run_cmd_no_fail(tcmd)
else:
for item in clean_depends:
tcmd = cmd + " clean_depends" + item
logger.info("calling {} ".format(tcmd))
run_cmd_no_fail(tcmd)
# unlink Locked files directory
unlock_file("env_build.xml")
# reset following values in xml files
case.set_value("SMP_BUILD",str(0))
case.set_value("NINST_BUILD",str(0))
case.set_value("BUILD_STATUS",str(0))
case.set_value("BUILD_COMPLETE","FALSE")
case.flush()
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning("WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname))
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {} {} {} {} "
.format(compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot),
from_dir=bldroot, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build))
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build,
thread_bad_results, smp, compiler):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning("WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname))
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {} {} {} {} "
.format(compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot),
from_dir=bldroot, arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build))
for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(
config_dir,
compclass,
compname,
caseroot,
libroot,
bldroot,
incroot,
file_build,
thread_bad_results,
smp,
compiler,
):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning(
"WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd),
"Could not find buildlib for {}".format(compname))
compile_cmd = "COMP_CLASS={compclass} COMP_NAME={compname} {cmd} {caseroot} {libroot} {bldroot} ".format(
compclass=compclass,
compname=compname,
cmd=cmd,
caseroot=caseroot,
libroot=libroot,
bldroot=bldroot,
)
if get_model() != "ufs":
compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd)
if is_python_executable(cmd):
logging_options = get_logging_options()
if logging_options != "":
compile_cmd = compile_cmd + logging_options
with open(file_build, "w") as fd:
stat = run_cmd(compile_cmd,
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
if stat != 0:
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compname, file_build))
analyze_build_log(compclass, file_build, compiler)
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot,
incroot, file_build, thread_bad_results, smp,
compiler):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {}/buildlib {} {} {} ".format(
compclass, stringify_bool(smp), config_dir, caseroot, libroot,
bldroot),
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if (stat != 0):
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compclass, file_build))
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
shutil.copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compclass, (t2 - t1)))
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot,
bldroot, incroot, file_build, thread_bad_results, smp,
compiler, case):
###############################################################################
logger.info("Building {} with output to {}".format(compclass, file_build))
t1 = time.time()
cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
if os.path.isfile(cmd):
logger.warning(
"WARNING: using local buildlib script for {}".format(compname))
else:
cmd = os.path.join(config_dir, "buildlib")
expect(os.path.isfile(cmd),
"Could not find buildlib for {}".format(compname))
# Add to this list as components are converted to python/cmake
if compname in ["cam"] and get_model() == "e3sm":
try:
stat = 0
run_sub_or_cmd(cmd, [caseroot, libroot, bldroot],
"buildlib", [bldroot, libroot, case],
logfile=file_build)
except Exception:
stat = 1
else:
with open(file_build, "w") as fd:
stat = run_cmd("MODEL={} SMP={} {} {} {} {} ".format(
compclass, stringify_bool(smp), cmd, caseroot, libroot,
bldroot),
from_dir=bldroot,
arg_stdout=fd,
arg_stderr=subprocess.STDOUT)[0]
analyze_build_log(compclass, file_build, compiler)
if stat != 0:
thread_bad_results.append(
"BUILD FAIL: {}.buildlib failed, cat {}".format(
compname, file_build))
analyze_build_log(compclass, file_build, compiler)
for mod_file in glob.glob(os.path.join(bldroot,
"*_[Cc][Oo][Mm][Pp]_*.mod")):
safe_copy(mod_file, incroot)
t2 = time.time()
logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _main():
output, build_dir, build_optimized, clean,\
cmake_args, compiler, enable_genf90, machine, machines_dir,\
make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
use_openmp, xml_test_list, verbose \
= parse_command_line(sys.argv)
#=================================================
# Find directory and file paths.
#=================================================
suite_specs = []
# TODO: this violates cime policy of direct access to xml
# should be moved to CIME/XML
if xml_test_list is not None:
test_xml_tree = ElementTree()
test_xml_tree.parse(xml_test_list)
known_paths = {
"here": os.path.abspath(os.path.dirname(xml_test_list)),
}
suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
if test_spec_dir is not None:
suite_specs.append(
TestSuiteSpec("__command_line_test__", ["__command_line_test__"],
[os.path.abspath(test_spec_dir)]))
if machines_dir is not None:
machines_file = os.path.join(machines_dir, "config_machines.xml")
machobj = Machines(infile=machines_file, machine=machine)
else:
machobj = Machines(machine=machine)
# Create build directory if necessary.
build_dir = os.path.abspath(build_dir)
if not os.path.isdir(build_dir):
os.mkdir(build_dir)
# Switch to the build directory.
os.chdir(build_dir)
if clean:
pwd_contents = os.listdir(os.getcwd())
# Clear CMake cache.
for file_ in pwd_contents:
if file_ in ("Macros.cmake", "env_mach_specific.xml") \
or file_.startswith('Depends') or file_.startswith(".env_mach_specific"):
os.remove(file_)
#=================================================
# Functions to perform various stages of build.
#=================================================
if not use_mpi:
mpilib = "mpi-serial"
elif mpilib is None:
mpilib = machobj.get_default_MPIlib()
logger.info("Using mpilib: {}".format(mpilib))
if compiler is None:
compiler = machobj.get_default_compiler()
logger.info("Compiler is {}".format(compiler))
compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)
pfunit_path = find_pfunit(compilerobj,
mpilib=mpilib,
use_openmp=use_openmp)
debug = not build_optimized
os_ = machobj.get_value("OS")
# Create the environment, and the Macros.cmake file
#
#
configure(machobj,
build_dir, ["CMake"],
compiler,
mpilib,
debug,
os_,
unit_testing=True)
machspecific = EnvMachSpecific(build_dir, unit_testing=True)
fake_case = FakeCase(compiler, mpilib, debug)
machspecific.load_env(fake_case)
os.environ["OS"] = os_
os.environ["COMPILER"] = compiler
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["MPILIB"] = mpilib
if use_openmp:
os.environ["compile_threaded"] = "true"
else:
os.environ["compile_threaded"] = "false"
os.environ["UNIT_TEST_HOST"] = socket.gethostname()
if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
# The CMake Netcdf find utility that we use (from pio2) seems to key off
# of the environment variable NETCDF, but not NETCDF_PATH
logger.info("Setting NETCDF environment variable: {}".format(
os.environ["NETCDF_PATH"]))
os.environ["NETCDF"] = os.environ["NETCDF_PATH"]
if not use_mpi:
mpirun_command = ""
elif mpirun_command is None:
mpi_attribs = {
"compiler": compiler,
"mpilib": mpilib,
"threaded": use_openmp,
"unit_testing": True
}
# We can get away with specifying case=None since we're using exe_only=True
mpirun_command, _ = machspecific.get_mpirun(None,
mpi_attribs,
None,
exe_only=True)
mpirun_command = machspecific.get_resolved_value(mpirun_command)
logger.info("mpirun command is '{}'".format(mpirun_command))
#=================================================
# Run tests.
#=================================================
for spec in suite_specs:
os.chdir(build_dir)
if os.path.isdir(spec.name):
if clean:
rmtree(spec.name)
if not os.path.isdir(spec.name):
os.mkdir(spec.name)
for label, directory in spec:
os.chdir(os.path.join(build_dir, spec.name))
if not os.path.isdir(label):
os.mkdir(label)
os.chdir(label)
name = spec.name + "/" + label
if not os.path.islink("Macros.cmake"):
os.symlink(os.path.join(build_dir, "Macros.cmake"),
"Macros.cmake")
use_mpiserial = not use_mpi
cmake_stage(name,
directory,
build_optimized,
use_mpiserial,
mpirun_command,
output,
pfunit_path,
verbose=verbose,
enable_genf90=enable_genf90,
cmake_args=cmake_args)
make_stage(name, output, make_j, clean=clean, verbose=verbose)
for spec in suite_specs:
os.chdir(os.path.join(build_dir, spec.name))
for label, directory in spec:
name = spec.name + "/" + label
output.print_header("Running CTest tests for " + name + ".")
ctest_command = ["ctest", "--output-on-failure"]
if verbose:
ctest_command.append("-VV")
if ctest_args is not None:
ctest_command.extend(ctest_args.split(" "))
run_cmd_no_fail(" ".join(ctest_command),
from_dir=label,
arg_stdout=None,
arg_stderr=subprocess.STDOUT)
stringify_bool(uses_kokkos(case)))
for var in _CMD_ARGS_FOR_BUILD:
cmake_args += xml_to_make_variable(case, var, cmake=True)
# Disable compiler checks
cmake_args += " -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 -DCMAKE_Fortran_COMPILER_WORKS=1"
return cmake_args
def xml_to_make_variable(case, varname, cmake=False):
varvalue = case.get_value(varname)
if varvalue is None:
return ""
if type(varvalue) == type(True):
varvalue = stringify_bool(varvalue)
return "{}{}=\"{}\" ".format("-D" if cmake else "", varname, varvalue)
###############################################################################
def uses_kokkos(case):
###############################################################################
cam_target = case.get_value("CAM_TARGET")
return get_model() == "e3sm" and cam_target in ("preqx_kokkos", "theta-l")
###############################################################################
def _build_model(build_threaded, exeroot, incroot, complist, lid, caseroot,
cimeroot, compiler, buildlist, comp_interface, case):
###############################################################################
logs = []
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case")
if sharedlibroot != exeroot and os.path.isdir(sharedlibroot):
logging.warning("cleaning directory {}".format(sharedlibroot))
shutil.rmtree(sharedlibroot)
else:
expect((cleanlist is not None and len(cleanlist) > 0) or
(clean_depends is not None and len(clean_depends)),"Empty cleanlist not expected")
debug = case.get_value("DEBUG")
use_esmf_lib = case.get_value("USE_ESMF_LIB")
build_threaded = case.get_build_threaded()
gmake = case.get_value("GMAKE")
caseroot = os.path.abspath(case.get_value("CASEROOT"))
casetools = case.get_value("CASETOOLS")
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib)
os.environ["BUILD_THREADED"] = stringify_bool(build_threaded)
os.environ["CASEROOT"] = caseroot
os.environ["COMP_INTERFACE"] = case.get_value("COMP_INTERFACE")
os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION"))
cmd = gmake + " -f " + os.path.join(casetools, "Makefile")
if cleanlist is not None:
for item in cleanlist:
tcmd = cmd + " clean" + item
logger.info("calling {} ".format(tcmd))
run_cmd_no_fail(tcmd)
else:
for item in clean_depends:
tcmd = cmd + " clean_depends" + item
def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist,
save_build_provenance):
###############################################################################
t1 = time.time()
expect(not (sharedlib_only and model_only),
"Contradiction: both sharedlib_only and model_only")
logger.info("Building case in directory {}".format(caseroot))
logger.info("sharedlib_only is {}".format(sharedlib_only))
logger.info("model_only is {}".format(model_only))
expect(os.path.isdir(caseroot), "'{}' is not a valid directory".format(caseroot))
os.chdir(caseroot)
expect(os.path.exists(get_batch_script_for_job(case.get_primary_job())),
"ERROR: must invoke case.setup script before calling build script ")
cimeroot = case.get_value("CIMEROOT")
comp_classes = case.get_values("COMP_CLASSES")
case.check_lockedfiles(skip="env_batch")
# Retrieve relevant case data
# This environment variable gets set for cesm Make and
# needs to be unset before building again.
if "MODEL" in os.environ:
del os.environ["MODEL"]
build_threaded = case.get_build_threaded()
casetools = case.get_value("CASETOOLS")
exeroot = os.path.abspath(case.get_value("EXEROOT"))
incroot = os.path.abspath(case.get_value("INCROOT"))
libroot = os.path.abspath(case.get_value("LIBROOT"))
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
multi_driver = case.get_value("MULTI_DRIVER")
complist = []
ninst = 1
for comp_class in comp_classes:
if comp_class == "CPL":
config_dir = None
if multi_driver:
ninst = case.get_value("NINST_MAX")
else:
config_dir = os.path.dirname(case.get_value("CONFIG_{}_FILE".format(comp_class)))
if multi_driver:
ninst = 1
else:
ninst = case.get_value("NINST_{}".format(comp_class))
comp = case.get_value("COMP_{}".format(comp_class))
thrds = case.get_value("NTHRDS_{}".format(comp_class))
expect(ninst is not None,"Failed to get ninst for comp_class {}".format(comp_class))
complist.append((comp_class.lower(), comp, thrds, ninst, config_dir ))
os.environ["COMP_{}".format(comp_class)] = comp
ocn_submodel = case.get_value("OCN_SUBMODEL")
profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE")
compiler = case.get_value("COMPILER")
comp_interface = case.get_value("COMP_INTERFACE")
mpilib = case.get_value("MPILIB")
use_esmf_lib = case.get_value("USE_ESMF_LIB")
debug = case.get_value("DEBUG")
ninst_build = case.get_value("NINST_BUILD")
smp_value = case.get_value("SMP_VALUE")
clm_use_petsc = case.get_value("CLM_USE_PETSC")
cism_use_trilinos = case.get_value("CISM_USE_TRILINOS")
mali_use_albany = case.get_value("MALI_USE_ALBANY")
use_moab = case.get_value("USE_MOAB")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
cam_config_opts = case.get_value("CAM_CONFIG_OPTS")
pio_config_opts = case.get_value("PIO_CONFIG_OPTS")
ninst_value = case.get_value("NINST_VALUE")
mach = case.get_value("MACH")
os_ = case.get_value("OS")
# Load some params into env
os.environ["CIMEROOT"] = cimeroot
os.environ["CASETOOLS"] = casetools
os.environ["EXEROOT"] = exeroot
os.environ["INCROOT"] = incroot
os.environ["LIBROOT"] = libroot
os.environ["SHAREDLIBROOT"] = sharedlibroot
os.environ["CASEROOT"] = caseroot
os.environ["COMPILER"] = compiler
os.environ["COMP_INTERFACE"] = comp_interface
os.environ["NINST_VALUE"] = str(ninst_value)
os.environ["BUILD_THREADED"] = stringify_bool(build_threaded)
os.environ["MACH"] = mach
os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib)
os.environ["MPILIB"] = mpilib
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["OS"] = os_
os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else ""
os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else ""
os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else ""
os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else ""
os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable)
os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc)
os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos)
os.environ["MALI_USE_ALBANY"] = stringify_bool(mali_use_albany)
os.environ["USE_MOAB"] = stringify_bool(use_moab)
if get_model() == "e3sm" and mach == "titan" and compiler == "pgiacc":
case.set_value("CAM_TARGET", "preqx_acc")
# This is a timestamp for the build , not the same as the testid,
# and this case may not be a test anyway. For a production
# experiment there may be many builds of the same case.
lid = get_timestamp("%y%m%d-%H%M%S")
os.environ["LID"] = lid
# Set the overall USE_PETSC variable to TRUE if any of the
# *_USE_PETSC variables are TRUE.
# For now, there is just the one CLM_USE_PETSC variable, but in
# the future there may be others -- so USE_PETSC will be true if
# ANY of those are true.
use_petsc = clm_use_petsc
case.set_value("USE_PETSC", use_petsc)
os.environ["USE_PETSC"] = stringify_bool(use_petsc)
# Set the overall USE_TRILINOS variable to TRUE if any of the
# *_USE_TRILINOS variables are TRUE.
# For now, there is just the one CISM_USE_TRILINOS variable, but in
# the future there may be others -- so USE_TRILINOS will be true if
# ANY of those are true.
use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos
case.set_value("USE_TRILINOS", use_trilinos)
os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos)
# Set the overall USE_ALBANY variable to TRUE if any of the
# *_USE_ALBANY variables are TRUE.
# For now, there is just the one MALI_USE_ALBANY variable, but in
# the future there may be others -- so USE_ALBANY will be true if
# ANY of those are true.
use_albany = stringify_bool(mali_use_albany)
case.set_value("USE_ALBANY", use_albany)
os.environ["USE_ALBANY"] = use_albany
# Load modules
case.load_env()
sharedpath = _build_checks(case, build_threaded, comp_interface,
use_esmf_lib, debug, compiler, mpilib,
complist, ninst_build, smp_value, model_only, buildlist)
t2 = time.time()
logs = []
if not model_only:
logs = _build_libraries(case, exeroot, sharedpath, caseroot,
cimeroot, libroot, lid, compiler, buildlist, comp_interface)
if not sharedlib_only:
os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators
logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler, buildlist, comp_interface))
if not buildlist:
# in case component build scripts updated the xml files, update the case object
case.read_xml()
# Note, doing buildlists will never result in the system thinking the build is complete
post_build(case, logs, build_complete=not (buildlist or sharedlib_only),
save_build_provenance=save_build_provenance)
t3 = time.time()
if not sharedlib_only:
logger.info("Time spent not building: {:f} sec".format(t2 - t1))
logger.info("Time spent building: {:f} sec".format(t3 - t2))
logger.info("MODEL BUILD HAS FINISHED SUCCESSFULLY")
return True
def _main():
output, build_dir, build_optimized, clean,\
cmake_args, compiler, enable_genf90, machine, machines_dir,\
make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
use_openmp, xml_test_list, verbose \
= parse_command_line(sys.argv)
#=================================================
# Find directory and file paths.
#=================================================
suite_specs = []
# TODO: this violates cime policy of direct access to xml
# should be moved to CIME/XML
if xml_test_list is not None:
test_xml_tree = ElementTree()
test_xml_tree.parse(xml_test_list)
known_paths = {
"here": os.path.abspath(os.path.dirname(xml_test_list)),
}
suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
if test_spec_dir is not None:
suite_specs.append(
TestSuiteSpec("__command_line_test__",
["__command_line_test__"],
[os.path.abspath(test_spec_dir)])
)
if machines_dir is not None:
machines_file = os.path.join(machines_dir, "config_machines.xml")
machobj = Machines(infile=machines_file, machine=machine)
else:
machobj = Machines(machine=machine)
# Create build directory if necessary.
build_dir = os.path.abspath(build_dir)
if not os.path.isdir(build_dir):
os.mkdir(build_dir)
# Switch to the build directory.
os.chdir(build_dir)
#=================================================
# Functions to perform various stages of build.
#=================================================
if not use_mpi:
mpilib = "mpi-serial"
elif mpilib is None:
mpilib = machobj.get_default_MPIlib()
logger.info("Using mpilib: {}".format(mpilib))
if compiler is None:
compiler = machobj.get_default_compiler()
logger.info("Compiler is {}".format(compiler))
compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)
pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp)
debug = not build_optimized
os_ = machobj.get_value("OS")
# Create the environment, and the Macros.cmake file
#
#
configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_,
unit_testing=True)
machspecific = EnvMachSpecific(build_dir, unit_testing=True)
fake_case = FakeCase(compiler, mpilib, debug)
machspecific.load_env(fake_case)
os.environ["OS"] = os_
os.environ["COMPILER"] = compiler
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["MPILIB"] = mpilib
if use_openmp:
os.environ["compile_threaded"] = "true"
else:
os.environ["compile_threaded"] = "false"
os.environ["UNIT_TEST_HOST"] = socket.gethostname()
if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
# The CMake Netcdf find utility that we use (from pio2) seems to key off
# of the environment variable NETCDF, but not NETCDF_PATH
logger.info("Setting NETCDF environment variable: {}".format(os.environ["NETCDF_PATH"]))
os.environ["NETCDF"] = os.environ["NETCDF_PATH"]
if not use_mpi:
mpirun_command = ""
elif mpirun_command is None:
mpi_attribs = {
"compiler" : compiler,
"mpilib" : mpilib,
"threaded" : use_openmp,
"unit_testing" : True
}
# We can get away with specifying case=None since we're using exe_only=True
mpirun_command, _ = machspecific.get_mpirun(case=None, attribs=mpi_attribs, exe_only=True)
mpirun_command = machspecific.get_resolved_value(mpirun_command)
logger.info("mpirun command is '{}'".format(mpirun_command))
#=================================================
# Run tests.
#=================================================
for spec in suite_specs:
os.chdir(build_dir)
if os.path.isdir(spec.name):
if clean:
rmtree(spec.name)
if not os.path.isdir(spec.name):
os.mkdir(spec.name)
for label, directory in spec:
os.chdir(os.path.join(build_dir,spec.name))
if not os.path.isdir(label):
os.mkdir(label)
os.chdir(label)
name = spec.name+"/"+label
if not os.path.islink("Macros.cmake"):
os.symlink(os.path.join(build_dir,"Macros.cmake"), "Macros.cmake")
use_mpiserial = not use_mpi
cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose,
enable_genf90=enable_genf90, cmake_args=cmake_args)
make_stage(name, output, make_j, clean=clean, verbose=verbose)
for spec in suite_specs:
os.chdir(os.path.join(build_dir,spec.name))
for label, directory in spec:
name = spec.name+"/"+label
output.print_header("Running CTest tests for "+name+".")
ctest_command = ["ctest", "--output-on-failure"]
if verbose:
ctest_command.append("-VV")
if ctest_args is not None:
ctest_command.extend(ctest_args.split(" "))
run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)
def configure(
machobj,
output_dir,
macros_format,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing=False,
noenv=False,
threaded=False,
extra_machines_dir=None,
):
"""Add Macros, Depends, and env_mach_specific files to a directory.
Arguments:
machobj - Machines argument for this machine.
output_dir - Directory in which to place output.
macros_format - Container containing the string 'Makefile' to produce
Makefile Macros output, and/or 'CMake' for CMake output.
compiler - String containing the compiler vendor to configure for.
mpilib - String containing the MPI implementation to configure for.
debug - Boolean specifying whether debugging options are enabled.
unit_testing - Boolean specifying whether we're running unit tests (as
opposed to a system run)
extra_machines_dir - String giving path to an additional directory that will be
searched for a config_compilers.xml file.
"""
# Macros generation.
suffixes = {"Makefile": "make", "CMake": "cmake"}
new_cmake_macros_dir = Files(
comp_interface=comp_interface).get_value("CMAKE_MACROS_DIR")
macro_maker = None
for form in macros_format:
if (new_cmake_macros_dir is not None
and os.path.exists(new_cmake_macros_dir)
and not "CIME_NO_CMAKE_MACRO" in os.environ):
if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")):
safe_copy(os.path.join(new_cmake_macros_dir, "Macros.cmake"),
output_dir)
if not os.path.exists(os.path.join(output_dir, "cmake_macros")):
shutil.copytree(new_cmake_macros_dir,
os.path.join(output_dir, "cmake_macros"))
# Grab macros from extra machine dir if it was provided
if extra_machines_dir:
extra_cmake_macros = glob.glob(
"{}/cmake_macros/*.cmake".format(extra_machines_dir))
for extra_cmake_macro in extra_cmake_macros:
safe_copy(extra_cmake_macro, new_cmake_macros_dir)
if form == "Makefile":
# Use the cmake macros to generate the make macros
cmake_args = " -DOS={} -DMACH={} -DCOMPILER={} -DDEBUG={} -DMPILIB={} -Dcompile_threaded={} -DCASEROOT={}".format(
sysos,
machobj.get_machine_name(),
compiler,
stringify_bool(debug),
mpilib,
stringify_bool(threaded),
output_dir,
)
with CmakeTmpBuildDir(macroloc=output_dir) as cmaketmp:
output = cmaketmp.get_makefile_vars(cmake_args=cmake_args)
with open(os.path.join(output_dir, "Macros.make"), "w") as fd:
fd.write(output)
else:
logger.warning("Using deprecated CIME makefile generators")
if macro_maker is None:
macro_maker = Compilers(
machobj,
compiler=compiler,
mpilib=mpilib,
extra_machines_dir=extra_machines_dir,
)
out_file_name = os.path.join(output_dir,
"Macros." + suffixes[form])
macro_maker.write_macros_file(macros_file=out_file_name,
output_format=suffixes[form])
copy_depends_files(machobj.get_machine_name(), machobj.machines_dir,
output_dir, compiler)
generate_env_mach_specific(
output_dir,
machobj,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing,
threaded,
noenv=noenv,
)
