def _has_unit_test_support(self):
if self.TEST_COMPILER is None:
default_compiler = self.MACHINE.get_default_compiler()
compiler = Compilers(self.MACHINE, compiler=default_compiler)
else:
compiler = Compilers(self.MACHINE, compiler=self.TEST_COMPILER)
attrs = {"MPILIB": "mpi-serial", "compile_threaded": "FALSE"}
pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH",
attributes=attrs)
if pfunit_path is None:
return False
else:
return True
def configure(machobj,
output_dir,
macros_format,
compiler,
mpilib,
debug,
sysos,
unit_testing=False):
"""Add Macros, Depends, and env_mach_specific files to a directory.
Arguments:
machobj - Machines argument for this machine.
output_dir - Directory in which to place output.
macros_format - Container containing the string 'Makefile' to produce
Makefile Macros output, and/or 'CMake' for CMake output.
compiler - String containing the compiler vendor to configure for.
mpilib - String containing the MPI implementation to configure for.
debug - Boolean specifying whether debugging options are enabled.
unit_testing - Boolean specifying whether we're running unit tests (as
opposed to a system run)
"""
# Macros generation.
suffixes = {'Makefile': 'make', 'CMake': 'cmake'}
macro_maker = Compilers(machobj, compiler=compiler, mpilib=mpilib)
for form in macros_format:
out_file_name = os.path.join(output_dir, "Macros." + suffixes[form])
macro_maker.write_macros_file(macros_file=out_file_name,
output_format=suffixes[form])
_copy_depends_files(machobj.get_machine_name(), machobj.machines_dir,
output_dir, compiler)
_generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
sysos, unit_testing)
def test_script_is_callable(self):
"""The test script can be called on valid output without dying."""
# This is really more a smoke test of this script than anything else.
maker = Compilers(test_utils.MockMachines("mymachine", "SomeOS"),
version=2.0)
test_xml = test_utils._wrap_config_compilers_xml(
"<compiler><SUPPORTS_CXX>FALSE</SUPPORTS_CXX></compiler>")
test_utils.get_macros(maker, test_xml, "Makefile")
def test_script_rejects_bad_build_system(self):
"""The macro writer rejects a bad build system string."""
maker = Compilers(test_utils.MockMachines("mymachine", "SomeOS"),
version=2.0)
bad_string = "argle-bargle."
with self.assertRaisesRegex(
utils.CIMEError,
"Unrecognized build system provided to write_macros: " +
bad_string,
):
test_utils.get_macros(maker, "This string is irrelevant.",
bad_string)
def setUp(self):
self.test_os = "SomeOS"
self.test_machine = "mymachine"
self.test_compiler = (self.MACHINE.get_default_compiler()
if self.TEST_COMPILER is None else
self.TEST_COMPILER)
self.test_mpilib = (self.MACHINE.get_default_MPIlib(
attributes={"compiler": self.test_compiler})
if self.TEST_MPILIB is None else self.TEST_MPILIB)
self._maker = Compilers(test_utils.MockMachines(
self.test_machine, self.test_os),
version=2.0)
super().setUp()
def setUp(self):
super().setUp()
if "CIME_NO_CMAKE_MACRO" not in os.environ:
self.skipTest("Skipping test of old macro system")
self.test_os = "SomeOS"
self.test_machine = "mymachine"
self.test_compiler = (self.MACHINE.get_default_compiler()
if self.TEST_COMPILER is None else
self.TEST_COMPILER)
self.test_mpilib = (self.MACHINE.get_default_MPIlib(
attributes={"compiler": self.test_compiler})
if self.TEST_MPILIB is None else self.TEST_MPILIB)
self._maker = Compilers(test_utils.MockMachines(
self.test_machine, self.test_os),
version=2.0)
def _main():
output, build_dir, build_optimized, clean,\
cmake_args, compiler, enable_genf90, machine, machines_dir,\
make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
use_openmp, xml_test_list, verbose \
= parse_command_line(sys.argv)
#=================================================
# Find directory and file paths.
#=================================================
suite_specs = []
# TODO: this violates cime policy of direct access to xml
# should be moved to CIME/XML
if xml_test_list is not None:
test_xml_tree = ElementTree()
test_xml_tree.parse(xml_test_list)
known_paths = {
"here": os.path.abspath(os.path.dirname(xml_test_list)),
}
suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
if test_spec_dir is not None:
suite_specs.append(
TestSuiteSpec("__command_line_test__", ["__command_line_test__"],
[os.path.abspath(test_spec_dir)]))
if machines_dir is not None:
machines_file = os.path.join(machines_dir, "config_machines.xml")
machobj = Machines(infile=machines_file, machine=machine)
else:
machobj = Machines(machine=machine)
# Create build directory if necessary.
build_dir = os.path.abspath(build_dir)
if not os.path.isdir(build_dir):
os.mkdir(build_dir)
# Switch to the build directory.
os.chdir(build_dir)
if clean:
pwd_contents = os.listdir(os.getcwd())
# Clear CMake cache.
for file_ in pwd_contents:
if file_ in ("Macros.cmake", "env_mach_specific.xml") \
or file_.startswith('Depends') or file_.startswith(".env_mach_specific"):
os.remove(file_)
#=================================================
# Functions to perform various stages of build.
#=================================================
if not use_mpi:
mpilib = "mpi-serial"
elif mpilib is None:
mpilib = machobj.get_default_MPIlib()
logger.info("Using mpilib: {}".format(mpilib))
if compiler is None:
compiler = machobj.get_default_compiler()
logger.info("Compiler is {}".format(compiler))
compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)
pfunit_path = find_pfunit(compilerobj,
mpilib=mpilib,
use_openmp=use_openmp)
debug = not build_optimized
os_ = machobj.get_value("OS")
# Create the environment, and the Macros.cmake file
#
#
configure(machobj,
build_dir, ["CMake"],
compiler,
mpilib,
debug,
os_,
unit_testing=True)
machspecific = EnvMachSpecific(build_dir, unit_testing=True)
fake_case = FakeCase(compiler, mpilib, debug)
machspecific.load_env(fake_case)
os.environ["OS"] = os_
os.environ["COMPILER"] = compiler
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["MPILIB"] = mpilib
if use_openmp:
os.environ["compile_threaded"] = "true"
else:
os.environ["compile_threaded"] = "false"
os.environ["UNIT_TEST_HOST"] = socket.gethostname()
if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
# The CMake Netcdf find utility that we use (from pio2) seems to key off
# of the environment variable NETCDF, but not NETCDF_PATH
logger.info("Setting NETCDF environment variable: {}".format(
os.environ["NETCDF_PATH"]))
os.environ["NETCDF"] = os.environ["NETCDF_PATH"]
if not use_mpi:
mpirun_command = ""
elif mpirun_command is None:
mpi_attribs = {
"compiler": compiler,
"mpilib": mpilib,
"threaded": use_openmp,
"unit_testing": True
}
# We can get away with specifying case=None since we're using exe_only=True
mpirun_command, _ = machspecific.get_mpirun(None,
mpi_attribs,
None,
exe_only=True)
mpirun_command = machspecific.get_resolved_value(mpirun_command)
logger.info("mpirun command is '{}'".format(mpirun_command))
#=================================================
# Run tests.
#=================================================
for spec in suite_specs:
os.chdir(build_dir)
if os.path.isdir(spec.name):
if clean:
rmtree(spec.name)
if not os.path.isdir(spec.name):
os.mkdir(spec.name)
for label, directory in spec:
os.chdir(os.path.join(build_dir, spec.name))
if not os.path.isdir(label):
os.mkdir(label)
os.chdir(label)
name = spec.name + "/" + label
if not os.path.islink("Macros.cmake"):
os.symlink(os.path.join(build_dir, "Macros.cmake"),
"Macros.cmake")
use_mpiserial = not use_mpi
cmake_stage(name,
directory,
build_optimized,
use_mpiserial,
mpirun_command,
output,
pfunit_path,
verbose=verbose,
enable_genf90=enable_genf90,
cmake_args=cmake_args)
make_stage(name, output, make_j, clean=clean, verbose=verbose)
for spec in suite_specs:
os.chdir(os.path.join(build_dir, spec.name))
for label, directory in spec:
name = spec.name + "/" + label
output.print_header("Running CTest tests for " + name + ".")
ctest_command = ["ctest", "--output-on-failure"]
if verbose:
ctest_command.append("-VV")
if ctest_args is not None:
ctest_command.extend(ctest_args.split(" "))
run_cmd_no_fail(" ".join(ctest_command),
from_dir=label,
arg_stdout=None,
arg_stderr=subprocess.STDOUT)
msg = "case.setup clean complete"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")
if not clean:
models = case.get_values("COMP_CLASSES")
mach, compiler, debug, mpilib = \
case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB")
expect(mach is not None, "xml variable MACH is not set")
# Create Macros file only if it does not exist
if not os.path.exists("Macros"):
logger.debug("Creating Macros file for %s" % mach)
compilers = Compilers(compiler=compiler,
machine=mach,
os_=case.get_value("OS"),
mpilib=mpilib)
compilers.write_macros_file()
else:
logger.debug("Macros script already created ...skipping")
# Set tasks to 1 if mpi-serial library
if mpilib == "mpi-serial":
for vid, value in case:
if vid.startswith("NTASKS_") and value != 1:
case.set_value(vid, 1)
# Check ninst.
# In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
# Save ninst in a dict to use later in apply_user_mods
def test_script_rejects_bad_xml(self):
"""The macro writer rejects input that's not valid XML."""
maker = Compilers(test_utils.MockMachines("mymachine", "SomeOS"),
version=2.0)
with self.assertRaises(ParseError):
test_utils.get_macros(maker, "This is not valid XML.", "Makefile")
def configure(machobj,
output_dir,
macros_format,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing=False,
noenv=False,
threaded=False,
extra_machines_dir=None):
"""Add Macros, Depends, and env_mach_specific files to a directory.
Arguments:
machobj - Machines argument for this machine.
output_dir - Directory in which to place output.
macros_format - Container containing the string 'Makefile' to produce
Makefile Macros output, and/or 'CMake' for CMake output.
compiler - String containing the compiler vendor to configure for.
mpilib - String containing the MPI implementation to configure for.
debug - Boolean specifying whether debugging options are enabled.
unit_testing - Boolean specifying whether we're running unit tests (as
opposed to a system run)
extra_machines_dir - String giving path to an additional directory that will be
searched for a config_compilers.xml file.
"""
# Macros generation.
suffixes = {'Makefile': 'make', 'CMake': 'cmake'}
new_cmake_macros_dir = Files(
comp_interface=comp_interface).get_value("CMAKE_MACROS_DIR")
macro_maker = None
for form in macros_format:
if form == "CMake" and new_cmake_macros_dir is not None and os.path.exists(
new_cmake_macros_dir
) and not "CIME_NO_CMAKE_MACRO" in os.environ:
if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")):
safe_copy(os.path.join(new_cmake_macros_dir, "Macros.cmake"),
output_dir)
if not os.path.exists(os.path.join(output_dir, "cmake_macros")):
shutil.copytree(new_cmake_macros_dir,
os.path.join(output_dir, "cmake_macros"))
else:
logger.warning("Using deprecated CIME makefile generators")
if macro_maker is None:
macro_maker = Compilers(machobj,
compiler=compiler,
mpilib=mpilib,
extra_machines_dir=extra_machines_dir)
out_file_name = os.path.join(output_dir,
"Macros." + suffixes[form])
macro_maker.write_macros_file(macros_file=out_file_name,
output_format=suffixes[form])
copy_depends_files(machobj.get_machine_name(), machobj.machines_dir,
output_dir, compiler)
generate_env_mach_specific(output_dir,
machobj,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing,
threaded,
noenv=noenv)
def configure(
machobj,
output_dir,
macros_format,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing=False,
noenv=False,
threaded=False,
extra_machines_dir=None,
):
"""Add Macros, Depends, and env_mach_specific files to a directory.
Arguments:
machobj - Machines argument for this machine.
output_dir - Directory in which to place output.
macros_format - Container containing the string 'Makefile' to produce
Makefile Macros output, and/or 'CMake' for CMake output.
compiler - String containing the compiler vendor to configure for.
mpilib - String containing the MPI implementation to configure for.
debug - Boolean specifying whether debugging options are enabled.
unit_testing - Boolean specifying whether we're running unit tests (as
opposed to a system run)
extra_machines_dir - String giving path to an additional directory that will be
searched for a config_compilers.xml file.
"""
# Macros generation.
suffixes = {"Makefile": "make", "CMake": "cmake"}
new_cmake_macros_dir = Files(
comp_interface=comp_interface).get_value("CMAKE_MACROS_DIR")
macro_maker = None
for form in macros_format:
if (new_cmake_macros_dir is not None
and os.path.exists(new_cmake_macros_dir)
and not "CIME_NO_CMAKE_MACRO" in os.environ):
if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")):
safe_copy(os.path.join(new_cmake_macros_dir, "Macros.cmake"),
output_dir)
if not os.path.exists(os.path.join(output_dir, "cmake_macros")):
shutil.copytree(new_cmake_macros_dir,
os.path.join(output_dir, "cmake_macros"))
# Grab macros from extra machine dir if it was provided
if extra_machines_dir:
extra_cmake_macros = glob.glob(
"{}/cmake_macros/*.cmake".format(extra_machines_dir))
for extra_cmake_macro in extra_cmake_macros:
safe_copy(extra_cmake_macro, new_cmake_macros_dir)
if form == "Makefile":
# Use the cmake macros to generate the make macros
cmake_args = " -DOS={} -DMACH={} -DCOMPILER={} -DDEBUG={} -DMPILIB={} -Dcompile_threaded={} -DCASEROOT={}".format(
sysos,
machobj.get_machine_name(),
compiler,
stringify_bool(debug),
mpilib,
stringify_bool(threaded),
output_dir,
)
with CmakeTmpBuildDir(macroloc=output_dir) as cmaketmp:
output = cmaketmp.get_makefile_vars(cmake_args=cmake_args)
with open(os.path.join(output_dir, "Macros.make"), "w") as fd:
fd.write(output)
else:
logger.warning("Using deprecated CIME makefile generators")
if macro_maker is None:
macro_maker = Compilers(
machobj,
compiler=compiler,
mpilib=mpilib,
extra_machines_dir=extra_machines_dir,
)
out_file_name = os.path.join(output_dir,
"Macros." + suffixes[form])
macro_maker.write_macros_file(macros_file=out_file_name,
output_format=suffixes[form])
copy_depends_files(machobj.get_machine_name(), machobj.machines_dir,
output_dir, compiler)
generate_env_mach_specific(
output_dir,
machobj,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing,
threaded,
noenv=noenv,
)
