def validate_executors(self):
slurm_executors = deep_get(self.target_config, "executors", "slurm")
lsf_executors = deep_get(self.target_config, "executors", "lsf")
cobalt_executors = deep_get(self.target_config, "executors", "cobalt")
pbs_executors = deep_get(self.target_config, "executors", "pbs")
if slurm_executors:
logger.debug("Checking slurm executors")
self._validate_slurm_executors(slurm_executors)
if lsf_executors:
logger.debug("Checking lsf executors")
self._validate_lsf_executors(lsf_executors)
if cobalt_executors:
logger.debug("Checking cobalt executors")
self._validate_cobalt_executors(cobalt_executors)
if pbs_executors:
logger.debug("Checking pbs executors")
self._validate_pbs_executors(pbs_executors)
if (self.target_config.get("moduletool") != "N/A"
and self.target_config.get("moduletool") !=
system.system["moduletool"]):
raise BuildTestError(
f"Cannot find modules_tool: {self.target_config('moduletool')} from configuration, please confirm if you have environment-modules or lmod and specify the appropriate tool."
)
def load(self):
"""Load the a Cobalt executor configuration from buildtest settings."""
self.launcher = self._settings.get("launcher") or deep_get(
self._buildtestsettings.target_config, "executors", "defaults", "launcher"
)
self.launcher_opts = self._settings.get("options")
self.queue = self._settings.get("queue")
self.account = self._settings.get("account") or deep_get(
self._buildtestsettings.target_config, "executors", "defaults", "account"
)
self.max_pend_time = self._settings.get("max_pend_time") or deep_get(
self._buildtestsettings, "executors", "defaults", "max_pend_time"
)
def _run_cmd(self):
"""This method builds the run command which refers to how to run the
generated binary after compilation.
"""
# order of precedence on how to generate run line when executing binary.
# 1. Check in 'config' section within compiler
# 2. Check in 'default' section within compiler group
# 3. Check in 'default' section within 'all' section
# 4. Last resort run binary standalone
run_line = (deep_get(
self.compiler_section, "config", self.compiler, "run") or deep_get(
self.compiler_section, "default", self.compiler_group, "run")
or deep_get(self.compiler_section, "default", "all", "run")
or "./$_EXEC")
return run_line
def __init__(self, settings_file=None, debug=False):
"""
:param settings_file: Specify an alternate settings file to use when finding compilers
:param settings_file: str, optional
:param compilers: compiler section from buildtest configuration.
:type compilers: dict
"""
self.configuration = site_config.target_config
# if settings_file is provided, let's load settings into BuildtestConfiguration
# and set self.configuration to loaded configuration
if settings_file:
bc = BuildtestConfiguration(settings_file)
self.configuration = bc.target_config
self.debug = debug
if not deep_get(self.configuration, "compilers", "compiler"):
raise BuildTestError("compiler section not defined")
self.compilers = self.configuration["compilers"]["compiler"]
self.names = []
self.compiler_name_to_group = {}
for name in self.compilers:
if isinstance(self.compilers[name], dict):
self.names += self.compilers[name].keys()
for compiler in self.compilers[name].keys():
self.compiler_name_to_group[compiler] = name
def load(self):
"""Load the a slurm executor configuration from buildtest settings."""
self.launcher = self._settings.get("launcher") or deep_get(
self._buildtestsettings.target_config, "executors", "defaults", "launcher"
)
self.launcher_opts = self._settings.get("options")
self.cluster = self._settings.get("cluster")
self.partition = self._settings.get("partition")
self.qos = self._settings.get("qos")
self.account = self._settings.get("account") or deep_get(
self._buildtestsettings.target_config, "executors", "defaults", "account"
)
self.max_pend_time = self._settings.get("max_pend_time") or deep_get(
self._buildtestsettings.target_config,
"executors",
"defaults",
"max_pend_time",
)
def get_executors_by_type(self, executor_type):
"""Return list of executor names by given type of executor.
:param executor_type: type of executor (local, slurm, lsf, cobalt)
:type executor_type: string
"""
names = deep_get(self.target_config, "executors", executor_type)
if not names:
raise BuildTestError(
"Cannot fetch executors by type: %s in system: %s ",
executor_type,
self.name,
)
return list(names.keys())
def __init__(self, debug=False, settings_file=None):
"""
:param compilers: compiler section from buildtest configuration.
:type compilers: dict
"""
self.configuration = load_settings(settings_file)
self.debug = debug
if not deep_get(self.configuration, "compilers", "compiler"):
raise BuildTestError("compiler section not defined")
self.compilers = self.configuration["compilers"]["compiler"]
self.names = []
self.compiler_name_to_group = {}
for name in self.compilers:
if isinstance(self.compilers[name], dict):
self.names += self.compilers[name].keys()
for compiler in self.compilers[name].keys():
self.compiler_name_to_group[compiler] = name
def setup(self):
"""The setup method is responsible for process compiler section, getting modules
pre_build, post_build, pre_run, post_run section and generate compilation
and run command. This method invokes other methods and set values in class
variables. This method is called by self.generate_script method.
"""
self._resolve_source()
self.lang = self._detect_lang(self.sourcefile)
# set executable name and assign to self.executable
self.executable = "%s.exe" % os.path.basename(self.sourcefile)
self.exec_variable = f"_EXEC={self.executable}"
self._process_compiler_config()
# get environment variables
self.envvars = (
deep_get(self.compiler_section, "config", self.compiler, "env")
or deep_get(self.compiler_section, "default", self.compiler_group, "env")
or deep_get(self.compiler_section, "default", "all", "env")
)
# get environment variables
self.vars = (
deep_get(self.compiler_section, "config", self.compiler, "vars")
or deep_get(self.compiler_section, "default", self.compiler_group, "vars")
or deep_get(self.compiler_section, "default", "all", "vars")
)
# get status
self.status = (
deep_get(self.compiler_section, "config", self.compiler, "status")
or deep_get(self.compiler_section, "default", self.compiler_group, "status")
or deep_get(self.compiler_section, "default", "all", "status")
)
# compiler set in compilers 'config' section, we try to get module lines using self._get_modules
self.modules = self._get_modules(
deep_get(self.compiler_section, "config", self.compiler, "module")
)
if not self.modules:
self.modules = self._get_modules(self.bc_compiler.get("module"))
self.pre_build = (
deep_get(self.compiler_section, "config", self.compiler, "pre_build")
or deep_get(
self.compiler_section, "default", self.compiler_group, "pre_build"
)
or deep_get(self.compiler_section, "default", "all", "pre_build")
)
self.post_build = (
deep_get(self.compiler_section, "config", self.compiler, "post_build")
or deep_get(
self.compiler_section, "default", self.compiler_group, "post_build"
)
or deep_get(self.compiler_section, "default", "all", "post_build")
)
self.pre_run = (
deep_get(self.compiler_section, "config", self.compiler, "pre_run")
or deep_get(
self.compiler_section, "default", self.compiler_group, "pre_run"
)
or deep_get(self.compiler_section, "default", "all", "pre_run")
)
self.post_run = (
deep_get(self.compiler_section, "config", self.compiler, "post_run")
or deep_get(
self.compiler_section, "default", self.compiler_group, "post_run"
)
or deep_get(self.compiler_section, "default", "all", "post_run")
)
self.compile_cmd = self._compile_cmd()
self.run_cmd = self._run_cmd()
def _process_compiler_config(self):
"""This method is responsible for setting cc, fc, cxx class variables based
on compiler selection. The order of precedence is ``config``, ``default``,
then buildtest setting. Compiler settings in 'config' takes highest precedence,
this overrides any configuration in 'default'. Finally we resort to compiler
configuration in buildtest setting if none defined. This method is responsible
for setting cc, fc, cxx, cflags, cxxflags, fflags, ldflags, and cppflags.
"""
bc = BuildtestCompilers()
self.compiler_group = bc.compiler_name_to_group[self.compiler]
self.logger.debug(
f"[{self.name}]: compiler: {self.compiler} belongs to compiler group: {self.compiler_group}"
)
# compiler from buildtest settings
self.bc_compiler = self.settings["compilers"]["compiler"][self.compiler_group][
self.compiler
]
self.logger.debug(self.bc_compiler)
# set compiler values based on 'default' property in buildspec. This can override
# compiler setting defined in configuration file. If default is not set we load from buildtest settings for appropriate compiler.
# set compiler variables to ones defined in buildtest configuration
self.cc = self.bc_compiler["cc"]
self.cxx = self.bc_compiler["cxx"]
self.fc = self.bc_compiler["fc"]
self.logger.debug(
f"[{self.name}]: Compiler setting for {self.compiler} from configuration file"
)
self.logger.debug(
f"[{self.name}]: {self.compiler}: {json.dumps(self.bc_compiler, indent=2)}"
)
# if default compiler setting provided in buildspec let's assign it.
if deep_get(self.compiler_section, "default", self.compiler_group):
self.cc = (
self.compiler_section["default"][self.compiler_group].get("cc")
or self.cc
)
self.fc = (
self.compiler_section["default"][self.compiler_group].get("fc")
or self.fc
)
self.cxx = (
self.compiler_section["default"][self.compiler_group].get("cxx")
or self.fc
)
self.cflags = self.compiler_section["default"][self.compiler_group].get(
"cflags"
)
self.cxxflags = self.compiler_section["default"][self.compiler_group].get(
"cxxflags"
)
self.fflags = self.compiler_section["default"][self.compiler_group].get(
"fflags"
)
self.ldflags = self.compiler_section["default"][self.compiler_group].get(
"ldflags"
)
self.cppflags = self.compiler_section["default"][self.compiler_group].get(
"cppflags"
)
# if compiler instance defined in config section read from buildspec. This overrides default section if specified
if deep_get(self.compiler_section, "config", self.compiler):
self.logger.debug(
f"[{self.name}]: Detected compiler: {self.compiler} in 'config' scope overriding default compiler group setting for: {self.compiler_group}"
)
self.cc = (
self.compiler_section["config"][self.compiler].get("cc") or self.cc
)
self.fc = (
self.compiler_section["config"][self.compiler].get("fc") or self.fc
)
self.cxx = (
self.compiler_section["config"][self.compiler].get("cxx") or self.cxx
)
self.cflags = (
self.compiler_section["config"][self.compiler].get("cflags")
or self.cflags
)
self.cxxflags = (
self.compiler_section["config"][self.compiler].get("cxxflags")
or self.cxxflags
)
self.fflags = (
self.compiler_section["config"][self.compiler].get("fflags")
or self.fflags
)
self.cppflags = (
self.compiler_section["config"][self.compiler].get("cppflags")
or self.cppflags
)
self.ldflags = (
self.compiler_section["config"][self.compiler].get("ldflags")
or self.ldflags
)
self.logger.debug(
f"cc: {self.cc}, cxx: {self.cxx} fc: {self.fc} cppflags: {self.cppflags} cflags: {self.cflags} fflags: {self.fflags} ldflags: {self.ldflags}"
)
# this condition is a safety check before compiling code to ensure if all C, C++, Fortran compiler not set we raise error
if not self.cc and not self.cxx and not self.fc:
raise BuildTestError(
"Unable to set C, C++, and Fortran compiler wrapper, please specify 'cc', 'cxx','fc' in your compiler settings in buildtest configuration or specify in buildspec file. "
)
def generate_script(self):
"""This method is responsible for generating test script for compiler schema.
The method ``generate_script`` is implemented in each subclass because
implementation on test generation differs across schema types.
This method will add the lines into list which comprise content
of test. The method will return a list containing lines of test script.
"""
self.setup()
# every test starts with shebang line
lines = [self.shebang]
batch_dict = {}
cray_dict = {}
# get sbatch, bsub, cobalt, batch property and store in batch dictionary.
# The order of lookup is in order of precedence
for batch in ["sbatch", "bsub", "cobalt", "batch"]:
batch_dict[batch] = (
deep_get(self.compiler_section, "config", self.compiler, batch)
or deep_get(
self.compiler_section, "default", self.compiler_group, batch
)
or deep_get(self.compiler_section, "default", "all", batch)
)
batch_directives_lines = self._get_scheduler_directives(
bsub=batch_dict["bsub"],
sbatch=batch_dict["sbatch"],
cobalt=batch_dict["cobalt"],
batch=batch_dict["batch"],
)
if batch_directives_lines:
lines += batch_directives_lines
# get cray burst buffer (BB) and datawarp (DW) fields in order of precedence.
for name in ["BB", "DW"]:
cray_dict[name] = (
deep_get(self.compiler_section, "config", self.compiler, name)
or deep_get(self.compiler_section, "default", self.compiler_group, name)
or deep_get(self.compiler_section, "default", "all", name)
)
burst_buffer_lines = self._get_burst_buffer(cray_dict["BB"])
if burst_buffer_lines:
lines += burst_buffer_lines
data_warp_lines = self._get_data_warp(cray_dict["DW"])
if data_warp_lines:
lines += data_warp_lines
lines += [
f"source {os.path.join(executor_root, self.executor, 'before_script.sh')}"
]
lines += [self.exec_variable]
lines += self.get_environment(self.envvars)
# get variables
lines += self.get_variables(self.vars)
# if 'module' defined in Buildspec add modules to test
if self.modules:
lines += self.modules
if self.pre_build:
lines.append(self.pre_build)
lines.append(self.compile_cmd)
if self.post_build:
lines.append(self.post_build)
if self.pre_run:
lines.append(self.pre_run)
# add run command
lines.append(self.run_cmd)
if self.post_run:
lines.append(self.post_run)
lines += [
f"source {os.path.join(executor_root, self.executor, 'after_script.sh')}"
]
return lines
