def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
sysos, unit_testing):
"""
env_mach_specific generation.
"""
ems_path = os.path.join(output_dir, "env_mach_specific.xml")
if os.path.exists(ems_path):
logger.warning("{} already exists, delete to replace".format(ems_path))
return
ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing)
ems_file.populate(machobj)
ems_file.write()
fake_case = FakeCase(compiler, mpilib, debug)
ems_file.load_env(fake_case)
for shell in ('sh', 'csh'):
ems_file.make_env_mach_specific_file(shell, fake_case)
shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
with open(shell_path, 'a') as shell_file:
if shell == 'sh':
shell_file.write("\nexport COMPILER={}\n".format(compiler))
shell_file.write("export MPILIB={}\n".format(mpilib))
shell_file.write("export DEBUG={}\n".format(
repr(debug).upper()))
shell_file.write("export OS={}\n".format(sysos))
else:
shell_file.write("\nsetenv COMPILER {}\n".format(compiler))
shell_file.write("setenv MPILIB {}\n".format(mpilib))
shell_file.write("setenv DEBUG {}\n".format(
repr(debug).upper()))
shell_file.write("setenv OS {}\n".format(sysos))
def generate_env_mach_specific(
output_dir,
machobj,
compiler,
mpilib,
debug,
comp_interface,
sysos,
unit_testing,
threaded,
noenv=False,
):
"""
env_mach_specific generation.
"""
ems_path = os.path.join(output_dir, "env_mach_specific.xml")
if os.path.exists(ems_path):
logger.warning("{} already exists, delete to replace".format(ems_path))
return
ems_file = EnvMachSpecific(output_dir,
unit_testing=unit_testing,
standalone_configure=True)
ems_file.populate(
machobj,
attributes={
"mpilib": mpilib,
"compiler": compiler,
"threaded": threaded
},
)
ems_file.write()
if noenv:
return
fake_case = FakeCase(compiler, mpilib, debug, comp_interface)
ems_file.load_env(fake_case)
for shell in ("sh", "csh"):
ems_file.make_env_mach_specific_file(shell,
fake_case,
output_dir=output_dir)
shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
with open(shell_path, "a") as shell_file:
if shell == "sh":
shell_file.write("\nexport COMPILER={}\n".format(compiler))
shell_file.write("export MPILIB={}\n".format(mpilib))
shell_file.write("export DEBUG={}\n".format(
repr(debug).upper()))
shell_file.write("export OS={}\n".format(sysos))
else:
shell_file.write("\nsetenv COMPILER {}\n".format(compiler))
shell_file.write("setenv MPILIB {}\n".format(mpilib))
shell_file.write("setenv DEBUG {}\n".format(
repr(debug).upper()))
shell_file.write("setenv OS {}\n".format(sysos))
def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
sysos, unit_testing):
"""
env_mach_specific generation.
"""
ems_path = os.path.join(output_dir, "env_mach_specific.xml")
if os.path.exists(ems_path):
logger.warning("{} already exists, delete to replace".format(ems_path))
return
ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing)
ems_file.populate(machobj)
ems_file.write()
ems_file.load_env(compiler, debug, mpilib)
for shell in ('sh', 'csh'):
ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell)
shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
with open(shell_path, 'a') as shell_file:
if shell == 'sh':
shell_file.write("\nexport COMPILER={}\n".format(compiler))
shell_file.write("export MPILIB={}\n".format(mpilib))
shell_file.write("export DEBUG={}\n".format(repr(debug).upper()))
shell_file.write("export OS={}\n".format(sysos))
else:
shell_file.write("\nsetenv COMPILER {}\n".format(compiler))
shell_file.write("setenv MPILIB {}\n".format(mpilib))
shell_file.write("setenv DEBUG {}\n".format(repr(debug).upper()))
shell_file.write("setenv OS {}\n".format(sysos))
def load_env_for_case(self):
mach_specific = EnvMachSpecific(caseroot=self._caseroot)
module_nodes = mach_specific.get_node("modules")
env_nodes = mach_specific.get_node("environment_variables")
if (module_nodes is not None):
modules_to_load = self._compute_module_actions(module_nodes)
self.load_modules(modules_to_load)
if (env_nodes is not None):
envs_to_set = self._compute_env_actions(env_nodes)
self.load_envs(envs_to_set)
def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
sysos, unit_testing):
"""
env_mach_specific generation.
"""
ems_path = os.path.join(output_dir, "env_mach_specific.xml")
if os.path.exists(ems_path):
logger.warn("%s already exists, delete to replace" % ems_path)
return
ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing)
ems_file.populate(machobj)
ems_file.write()
for shell in ('sh', 'csh'):
ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell)
shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
with open(shell_path, 'a') as shell_file:
if shell == 'sh':
shell_file.write("\nexport COMPILER=%s\n" % compiler)
shell_file.write("export MPILIB=%s\n" % mpilib)
shell_file.write("export DEBUG=%s\n" % repr(debug).upper())
shell_file.write("export OS=%s\n" % sysos)
else:
shell_file.write("\nsetenv COMPILER %s\n" % compiler)
shell_file.write("setenv MPILIB %s\n" % mpilib)
shell_file.write("setenv DEBUG %s\n" % repr(debug).upper())
shell_file.write("setenv OS %s\n" % sysos)
def read_xml(self):
if (len(self._env_files_that_need_rewrite) > 0):
files = ""
for env_file in self._env_files_that_need_rewrite:
files += " " + env_file.filename
expect(
False,
"Object(s) %s seem to have newer data than the corresponding case file"
% files)
self._env_entryid_files = []
self._env_entryid_files.append(EnvCase(self._caseroot,
components=None))
components = self._env_entryid_files[0].get_values("COMP_CLASSES")
self._env_entryid_files.append(
EnvRun(self._caseroot, components=components))
self._env_entryid_files.append(
EnvBuild(self._caseroot, components=components))
self._env_entryid_files.append(
EnvMachPes(self._caseroot, components=components))
if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")):
self._env_entryid_files.append(
EnvTest(self._caseroot, components=components))
self._env_generic_files = []
self._env_generic_files.append(EnvBatch(self._caseroot))
self._env_generic_files.append(EnvMachSpecific(self._caseroot))
self._env_generic_files.append(EnvArchive(self._caseroot))
self._files = self._env_entryid_files + self._env_generic_files
def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos):
"""
env_mach_specific generation.
"""
ems_path = os.path.join(output_dir, "env_mach_specific.xml")
if os.path.exists(ems_path):
logger.warn("%s already exists, delete to replace"%ems_path)
return
ems_file = EnvMachSpecific(output_dir)
ems_file.populate(machobj)
ems_file.write()
for shell in ('sh', 'csh'):
ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell)
shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
with open(shell_path, 'a') as shell_file:
if shell == 'sh':
shell_file.write("\nexport COMPILER=%s\n" % compiler)
shell_file.write("export MPILIB=%s\n" % mpilib)
shell_file.write("export DEBUG=%s\n" % repr(debug).upper())
shell_file.write("export OS=%s\n" % sysos)
else:
shell_file.write("\nsetenv COMPILER %s\n" % compiler)
shell_file.write("setenv MPILIB %s\n" % mpilib)
shell_file.write("setenv DEBUG %s\n" % repr(debug).upper())
shell_file.write("setenv OS %s\n" % sysos)
def _main():
output, build_dir, build_optimized, clean,\
cmake_args, compiler, enable_genf90, machine, machines_dir,\
make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
use_openmp, xml_test_list, verbose \
= parse_command_line(sys.argv)
#=================================================
# Find directory and file paths.
#=================================================
suite_specs = []
# TODO: this violates cime policy of direct access to xml
# should be moved to CIME/XML
if xml_test_list is not None:
test_xml_tree = ElementTree()
test_xml_tree.parse(xml_test_list)
known_paths = {
"here": os.path.abspath(os.path.dirname(xml_test_list)),
}
suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
if test_spec_dir is not None:
suite_specs.append(
TestSuiteSpec("__command_line_test__", ["__command_line_test__"],
[os.path.abspath(test_spec_dir)]))
if machines_dir is not None:
machines_file = os.path.join(machines_dir, "config_machines.xml")
machobj = Machines(infile=machines_file, machine=machine)
else:
machobj = Machines(machine=machine)
# Create build directory if necessary.
build_dir = os.path.abspath(build_dir)
if not os.path.isdir(build_dir):
os.mkdir(build_dir)
# Switch to the build directory.
os.chdir(build_dir)
if clean:
pwd_contents = os.listdir(os.getcwd())
# Clear CMake cache.
for file_ in pwd_contents:
if file_ in ("Macros.cmake", "env_mach_specific.xml") \
or file_.startswith('Depends') or file_.startswith(".env_mach_specific"):
os.remove(file_)
#=================================================
# Functions to perform various stages of build.
#=================================================
if not use_mpi:
mpilib = "mpi-serial"
elif mpilib is None:
mpilib = machobj.get_default_MPIlib()
logger.info("Using mpilib: {}".format(mpilib))
if compiler is None:
compiler = machobj.get_default_compiler()
logger.info("Compiler is {}".format(compiler))
compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)
pfunit_path = find_pfunit(compilerobj,
mpilib=mpilib,
use_openmp=use_openmp)
debug = not build_optimized
os_ = machobj.get_value("OS")
# Create the environment, and the Macros.cmake file
#
#
configure(machobj,
build_dir, ["CMake"],
compiler,
mpilib,
debug,
os_,
unit_testing=True)
machspecific = EnvMachSpecific(build_dir, unit_testing=True)
fake_case = FakeCase(compiler, mpilib, debug)
machspecific.load_env(fake_case)
os.environ["OS"] = os_
os.environ["COMPILER"] = compiler
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["MPILIB"] = mpilib
if use_openmp:
os.environ["compile_threaded"] = "true"
else:
os.environ["compile_threaded"] = "false"
os.environ["UNIT_TEST_HOST"] = socket.gethostname()
if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
# The CMake Netcdf find utility that we use (from pio2) seems to key off
# of the environment variable NETCDF, but not NETCDF_PATH
logger.info("Setting NETCDF environment variable: {}".format(
os.environ["NETCDF_PATH"]))
os.environ["NETCDF"] = os.environ["NETCDF_PATH"]
if not use_mpi:
mpirun_command = ""
elif mpirun_command is None:
mpi_attribs = {
"compiler": compiler,
"mpilib": mpilib,
"threaded": use_openmp,
"unit_testing": True
}
# We can get away with specifying case=None since we're using exe_only=True
mpirun_command, _ = machspecific.get_mpirun(None,
mpi_attribs,
None,
exe_only=True)
mpirun_command = machspecific.get_resolved_value(mpirun_command)
logger.info("mpirun command is '{}'".format(mpirun_command))
#=================================================
# Run tests.
#=================================================
for spec in suite_specs:
os.chdir(build_dir)
if os.path.isdir(spec.name):
if clean:
rmtree(spec.name)
if not os.path.isdir(spec.name):
os.mkdir(spec.name)
for label, directory in spec:
os.chdir(os.path.join(build_dir, spec.name))
if not os.path.isdir(label):
os.mkdir(label)
os.chdir(label)
name = spec.name + "/" + label
if not os.path.islink("Macros.cmake"):
os.symlink(os.path.join(build_dir, "Macros.cmake"),
"Macros.cmake")
use_mpiserial = not use_mpi
cmake_stage(name,
directory,
build_optimized,
use_mpiserial,
mpirun_command,
output,
pfunit_path,
verbose=verbose,
enable_genf90=enable_genf90,
cmake_args=cmake_args)
make_stage(name, output, make_j, clean=clean, verbose=verbose)
for spec in suite_specs:
os.chdir(os.path.join(build_dir, spec.name))
for label, directory in spec:
name = spec.name + "/" + label
output.print_header("Running CTest tests for " + name + ".")
ctest_command = ["ctest", "--output-on-failure"]
if verbose:
ctest_command.append("-VV")
if ctest_args is not None:
ctest_command.extend(ctest_args.split(" "))
run_cmd_no_fail(" ".join(ctest_command),
from_dir=label,
arg_stdout=None,
arg_stderr=subprocess.STDOUT)
def _main():
output, build_dir, build_optimized, clean,\
cmake_args, compiler, enable_genf90, machine, machines_dir,\
make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
use_openmp, xml_test_list, verbose \
= parse_command_line(sys.argv)
#=================================================
# Find directory and file paths.
#=================================================
suite_specs = []
# TODO: this violates cime policy of direct access to xml
# should be moved to CIME/XML
if xml_test_list is not None:
test_xml_tree = ElementTree()
test_xml_tree.parse(xml_test_list)
known_paths = {
"here": os.path.abspath(os.path.dirname(xml_test_list)),
}
suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
if test_spec_dir is not None:
suite_specs.append(
TestSuiteSpec("__command_line_test__",
["__command_line_test__"],
[os.path.abspath(test_spec_dir)])
)
if machines_dir is not None:
machines_file = os.path.join(machines_dir, "config_machines.xml")
machobj = Machines(infile=machines_file, machine=machine)
else:
machobj = Machines(machine=machine)
# Create build directory if necessary.
build_dir = os.path.abspath(build_dir)
if not os.path.isdir(build_dir):
os.mkdir(build_dir)
# Switch to the build directory.
os.chdir(build_dir)
#=================================================
# Functions to perform various stages of build.
#=================================================
if not use_mpi:
mpilib = "mpi-serial"
elif mpilib is None:
mpilib = machobj.get_default_MPIlib()
logger.info("Using mpilib: {}".format(mpilib))
if compiler is None:
compiler = machobj.get_default_compiler()
logger.info("Compiler is {}".format(compiler))
compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)
pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp)
debug = not build_optimized
os_ = machobj.get_value("OS")
# Create the environment, and the Macros.cmake file
#
#
configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_,
unit_testing=True)
machspecific = EnvMachSpecific(build_dir, unit_testing=True)
fake_case = FakeCase(compiler, mpilib, debug)
machspecific.load_env(fake_case)
os.environ["OS"] = os_
os.environ["COMPILER"] = compiler
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["MPILIB"] = mpilib
if use_openmp:
os.environ["compile_threaded"] = "true"
else:
os.environ["compile_threaded"] = "false"
os.environ["UNIT_TEST_HOST"] = socket.gethostname()
if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
# The CMake Netcdf find utility that we use (from pio2) seems to key off
# of the environment variable NETCDF, but not NETCDF_PATH
logger.info("Setting NETCDF environment variable: {}".format(os.environ["NETCDF_PATH"]))
os.environ["NETCDF"] = os.environ["NETCDF_PATH"]
if not use_mpi:
mpirun_command = ""
elif mpirun_command is None:
mpi_attribs = {
"compiler" : compiler,
"mpilib" : mpilib,
"threaded" : use_openmp,
"unit_testing" : True
}
# We can get away with specifying case=None since we're using exe_only=True
mpirun_command, _ = machspecific.get_mpirun(case=None, attribs=mpi_attribs, exe_only=True)
mpirun_command = machspecific.get_resolved_value(mpirun_command)
logger.info("mpirun command is '{}'".format(mpirun_command))
#=================================================
# Run tests.
#=================================================
for spec in suite_specs:
os.chdir(build_dir)
if os.path.isdir(spec.name):
if clean:
rmtree(spec.name)
if not os.path.isdir(spec.name):
os.mkdir(spec.name)
for label, directory in spec:
os.chdir(os.path.join(build_dir,spec.name))
if not os.path.isdir(label):
os.mkdir(label)
os.chdir(label)
name = spec.name+"/"+label
if not os.path.islink("Macros.cmake"):
os.symlink(os.path.join(build_dir,"Macros.cmake"), "Macros.cmake")
use_mpiserial = not use_mpi
cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose,
enable_genf90=enable_genf90, cmake_args=cmake_args)
make_stage(name, output, make_j, clean=clean, verbose=verbose)
for spec in suite_specs:
os.chdir(os.path.join(build_dir,spec.name))
for label, directory in spec:
name = spec.name+"/"+label
output.print_header("Running CTest tests for "+name+".")
ctest_command = ["ctest", "--output-on-failure"]
if verbose:
ctest_command.append("-VV")
if ctest_args is not None:
ctest_command.extend(ctest_args.split(" "))
run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)
