def func_config_system(args=None, settings_file=None):
"""This method implements command ``buildtest config systems`` which displays
system details from configuration file in table format.
"""
# settings_file = settings_file or resolve_settings_file()
# print(settings_file)
bc = check_settings(settings_path=settings_file, executor_check=False)
table = {
"system": [],
"description": [],
"hostnames": [],
"moduletool": []
}
for name in bc.config["system"].keys():
table["system"].append(name)
table["description"].append(
bc.config["system"][name].get("description"))
table["moduletool"].append(bc.config["system"][name]["moduletool"])
table["hostnames"].append(bc.config["system"][name]["hostnames"])
if os.getenv("BUILDTEST_COLOR") == "True":
print(
tabulate(
table,
headers=[
colored(field, "blue", attrs=["bold"])
for field in table.keys()
],
tablefmt="grid",
))
return
print(tabulate(table, headers=table.keys(), tablefmt="grid"))
def func_config_validate(args=None):
"""This method implements ``buildtest config validate`` which attempts to
validate buildtest settings with schema. If it not validate an exception
an exception of type SystemError is raised. We invoke ``check_settings``
method which will validate the configuration, if it fails we except an exception
of type ValidationError which we catch and print message.
"""
settings_file = resolve_settings_file()
try:
check_settings(settings_path=settings_file, executor_check=True)
except (ValidationError, SystemExit) as err:
print(err)
raise sys.exit(f"{settings_file} is not valid")
print(f"{settings_file} is valid")
def test_cori():
if os.getenv("NERSC_HOST") != "cori":
pytest.skip("Test runs only on Cori")
here = os.path.dirname(os.path.abspath(__file__))
cori_configuration = os.path.join(here, "settings", "cori.config.yml")
settings = check_settings(cori_configuration, retrieve_settings=True)
buildspec_files = os.path.join(here, "examples", "cori_buildspecs",
"hostname.yml")
class args:
buildspec = [buildspec_files]
debug = False
stage = None
testdir = None
exclude = None
tags = None
executor = None
filter_tags = None
rebuild = None
# test job submission on Cori
func_build_subcmd(args, settings)
def test_check_settings():
settings = check_settings(
settings_path=DEFAULT_SETTINGS_FILE,
executor_check=False,
)
assert settings
assert isinstance(settings, BuildtestConfiguration)
def test_cori_configuration(tmp_path):
if os.getenv("NERSC_HOST") != "cori":
pytest.skip("Test runs only on Cori")
here = os.path.dirname(os.path.abspath(__file__))
cori_configuration = os.path.join(here, "settings", "cori.config.yml")
settings = check_settings(cori_configuration, retrieve_settings=True)
assert isinstance(settings, dict)
be = BuildExecutor(settings)
assert be.list_executors() == [
"local.bash",
"local.sh",
"local.csh",
"local.python",
"slurm.haswell_debug",
]
def __init__(
self,
rebuild=False,
filterfields=None,
formatfields=None,
roots=None,
settings_file=None,
):
"""The initializer method for BuildspecCache class is responsible for
loading and finding buildspecs into buildspec cache. This method is called
when using ``buildtest buildspec find`` command.
:param rebuild: rebuild the buildspec cache by validating all buildspecs. The --rebuild is passed to this argument
:type rebuild: bool, required
:param filterfields: The --filter option contains list of key value pairs for filtering buildspecs
:type filterfields: str, required
:param formatfields: The --format option contains list of key value pairs for formating buildspecs
:type formatfields: str, required
:param roots: List of directories to search for buildspecs. This argument contains value of --roots
:type roots: list, required
"""
self.settings = settings_file or DEFAULT_SETTINGS_FILE
self.configuration = check_settings(self.settings, executor_check=True)
self.filter = filterfields
self.format = formatfields
# if --root is not specified we set to empty list instead of None
self.roots = roots or []
# list of buildspec directories to search for .yml files
self.paths = []
self.rebuild = rebuild
self.cache = {}
self.load_paths()
self.build()
self.check_filter_fields()
self.check_format_fields()
self.find_buildspecs()
def test_check_settings():
settings = check_settings(executor_check=False, retrieve_settings=True)
assert isinstance(settings, dict)
assert "executors" in settings.keys()
assert "moduletool" in settings.keys()
assert "load_default_buildspecs" in settings.keys()
def main():
"""Entry point to buildtest."""
if not os.getenv("BUILDTEST_COLOR"):
os.environ["BUILDTEST_COLOR"] = "True"
# create a temporary file to store logfile and we don't delete file by setting 'delete=False'
# by default tempfile will delete file upon exit.
tf = tempfile.NamedTemporaryFile(prefix="buildtest_",
delete=False,
suffix=".log")
dest_logfile = tf.name
logger = init_logfile(dest_logfile)
logger.info("Starting buildtest log")
create_dir(BUILDTEST_USER_HOME)
create_dir(var_root)
# Create a build test system, and check requirements
# BuildTestSystem()
system.check()
parser = BuildTestParser()
args = parser.parse_options()
if args.debug:
streamlog(args.debug)
if args.subcommands == "build":
# settings_file = resolve_settings_file(args.config)
# check_settings(args.config)
cmd = BuildTest(
config_file=args.config,
buildspecs=args.buildspec,
exclude_buildspecs=args.exclude,
executors=args.executor,
tags=args.tags,
filter_tags=args.filter_tags,
rebuild=args.rebuild,
stage=args.stage,
testdir=args.testdir,
)
cmd.build()
logdir = buildtest_configuration.target_config.get("logdir")
if not logdir:
print(f"Writing Logfile to: {dest_logfile}")
sys.exit(0)
logdir = resolve_path(logdir, exist=False)
if logdir:
create_dir(logdir)
fname = os.path.basename(dest_logfile)
logpath = os.path.join(logdir, fname)
shutil.copy2(dest_logfile, logpath)
print(f"Writing Logfile to: {logpath}")
else:
print(f"Writing Logfile to: {dest_logfile}")
# store copy of logfile at $BUILDTEST_ROOT/buildtest.log. A convenient location for user to
# find logfile for last build, this will be overwritten for every subsequent build.
shutil.copy2(
dest_logfile,
os.path.join(os.getenv("BUILDTEST_ROOT"), "buildtest.log"))
return
settings_file = resolve_settings_file()
logger.info(f"Processing buildtest configuration file: {settings_file}")
check_settings(settings_file)
# implementation for 'buildtest buildspec find'
if args.subcommands == "buildspec":
buildspec_find(args=args, settings_file=settings_file)
return
if args.subcommands and args.func:
args.func(args)
def main():
"""Entry point to buildtest."""
# create a temporary file to store logfile and we don't delete file by setting 'delete=False'
# by default tempfile will delete file upon exit.
tf = tempfile.NamedTemporaryFile(prefix="buildtest_",
delete=False,
suffix=".log")
dest_logfile = tf.name
logger = init_logfile(dest_logfile)
logger.info("Starting buildtest log")
create_dir(BUILDTEST_USER_HOME)
create_dir(var_root)
# Create a build test system, and check requirements
# BuildTestSystem()
system.check()
parser = BuildTestParser()
args = parser.parse_options()
if args.debug:
streamlog(args.debug)
# invoking load_settings will attempt to initialize buildtest settings and
# load the schema
settings_file = resolve_settings_file()
logger.info(f"Processing buildtest configuration file: {settings_file}")
buildtest_configuration = check_settings(settings_file,
retrieve_settings=True)
if args.subcommands == "build":
func_build_subcmd(args, buildtest_configuration)
else:
if args.subcommands and args.func:
args.func(args)
return
logdir = buildtest_configuration.get("logdir")
if not logdir:
print(f"Writing Logfile to: {dest_logfile}")
sys.exit(0)
logdir = resolve_path(logdir, exist=False)
if logdir:
create_dir(logdir)
fname = os.path.basename(dest_logfile)
logpath = os.path.join(logdir, fname)
shutil.copy2(dest_logfile, logpath)
print(f"Writing Logfile to: {logpath}")
else:
print(f"Writing Logfile to: {dest_logfile}")
# store copy of logfile at $BUILDTEST_ROOT/buildtest.log. A convenient location for user to
# find logfile for last build, this will be overwritten for every subsequent build.
shutil.copy2(dest_logfile,
os.path.join(os.getenv("BUILDTEST_ROOT"), "buildtest.log"))