def get_test_configs(pav_cfg,
host: str,
test_files: List[Union[str, Path]] = None,
tests: List[str] = None,
modes: List[str] = None,
overrides: List[str] = None,
outfile: TextIO = StringIO(),
conditions: Dict[str, Dict[str, List[str]]] = None):
"""Translate a general set of pavilion test configs into the final,
resolved configurations.
:param pav_cfg: The pavilion config
:param host: The host config to target these tests with
:param modes: The mode configs to use.
:param test_files: Files containing a newline separated list of tests.
:param tests: The tests to run.
:param overrides: Overrides to apply to the configurations.
name) of lists of tuples test configs and their variable managers.
:param conditions: A dict containing the only_if and not_if conditions.
:param outfile: Where to print user error messages.
"""
overrides = overrides if overrides else []
tests = tests if tests else []
modes = modes if modes else []
test_files = test_files if test_files else []
conditions = conditions if conditions else {}
resolver = test_config.TestConfigResolver(pav_cfg)
tests = list(tests)
for file in test_files:
try:
with pathlib.PosixPath(file).open() as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
raise commands.CommandError(
"Could not read test file {}: {}".format(file, err.args[0]))
try:
resolved_cfgs = resolver.load(
tests,
host,
modes,
overrides,
conditions=conditions,
output_file=outfile,
)
except TestConfigError as err:
raise commands.CommandError(err.args[0])
return resolved_cfgs
def _get_test_configs(self, pav_cfg, host, test_files, tests, modes,
overrides):
"""Translate a general set of pavilion test configs into the final,
resolved configurations. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:param pav_cfg: The pavilion config
:param str host: The host config to target these tests with
:param list(str) modes: The mode configs to use.
:param list(Path) test_files: Files containing a newline separated
list of tests.
:param list(str) tests: The tests to run.
:param list(str) overrides: Overrides to apply to the configurations.
name) of lists of tuples
test configs and their variable managers.
"""
self.logger.debug("Finding Configs")
resolver = test_config.TestConfigResolver(pav_cfg)
tests = list(tests)
for file in test_files:
try:
with pathlib.PosixPath(file).open() as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
try:
resolved_cfgs = resolver.load(
tests,
host,
modes,
overrides,
output_file=self.outfile,
)
except TestConfigError as err:
raise commands.CommandError(err.args[0])
tests_by_scheduler = defaultdict(lambda: [])
for cfg, var_man in resolved_cfgs:
tests_by_scheduler[cfg['scheduler']].append((cfg, var_man))
return tests_by_scheduler
def _configs_to_tests(pav_cfg, configs_by_sched, mb_tracker=None,
build_only=False, rebuild=False):
"""Convert the dictionary of test configs by scheduler into actual
tests.
:param pav_cfg: The Pavilion config
:param dict[str,list] configs_by_sched: A dictionary of lists of test
configs.
:param Union[MultiBuildTracker,None] mb_tracker: The build tracker.
:param bool build_only: Whether to only build these tests.
:param bool rebuild: After figuring out what build to use, rebuild it.
:return:
"""
tests_by_sched = {}
for sched_name in configs_by_sched.keys():
tests_by_sched[sched_name] = []
try:
for i in range(len(configs_by_sched[sched_name])):
cfg, var_man = configs_by_sched[sched_name][i]
tests_by_sched[sched_name].append(TestRun(
pav_cfg=pav_cfg,
config=cfg,
var_man=var_man,
build_tracker=mb_tracker,
build_only=build_only,
rebuild=rebuild,
))
except (TestRunError, TestConfigError) as err:
raise commands.CommandError(err)
return tests_by_sched
def get_statuses(pav_cfg, args, errfile):
"""Get the statuses of the listed tests or series.
:param pav_cfg: The pavilion config.
:param argparse namespace args: The tests via the command line args.
:param errfile: stream to output errors as needed.
:returns: List of dictionary objects with the test id, name, state,
time that the most recent status was set, and the associated
note.
"""
if not args.tests:
# Get the last series ran by this user.
series_id = series.TestSeries.load_user_series_id(pav_cfg)
if series_id is not None:
args.tests.append(series_id)
else:
raise commands.CommandError(
"No tests specified and no last series was found.")
test_list = []
for test_id in args.tests:
if test_id.startswith('s'):
try:
test_list.extend(
series.TestSeries.from_id(pav_cfg, int(test_id[1:])).tests)
except series.TestSeriesError as err:
utils.fprint("Suite {} could not be found.\n{}".format(
test_id[1:], err),
file=errfile,
color=utils.RED)
continue
else:
test_list.append(test_id)
test_list = list(map(int, test_list))
test_statuses = []
test_obj_list = []
for test_id in test_list:
try:
test = PavTest.load(pav_cfg, test_id)
test_obj_list.append(test)
except (PavTestError, PavTestNotFoundError) as err:
test_statuses.append({
'test_id': test_id,
'name': "",
'state': STATES.UNKNOWN,
'time': "",
'note': "Test not found.",
})
statuses = status_from_test_obj(pav_cfg, test_obj_list)
if statuses is not None:
test_statuses = test_statuses + statuses
return test_statuses
def _configs_to_tests(pav_cfg,
configs_by_sched,
mb_tracker=None,
build_only=False,
rebuild=False,
outfile=None):
"""Convert the dictionary of test configs by scheduler into actual
tests.
:param pav_cfg: The Pavilion config
:param dict[str,list] configs_by_sched: A dictionary of lists of test
configs.
:param Union[MultiBuildTracker,None] mb_tracker: The build tracker.
:param bool build_only: Whether to only build these tests.
:param bool rebuild: After figuring out what build to use, rebuild it.
:return:
"""
tests_by_sched = {}
progress = 0
tot_tests = sum([len(tests) for tests in configs_by_sched.values()])
for sched_name in configs_by_sched.keys():
tests_by_sched[sched_name] = []
try:
for i in range(len(configs_by_sched[sched_name])):
cfg, var_man = configs_by_sched[sched_name][i]
tests_by_sched[sched_name].append(
TestRun(
pav_cfg=pav_cfg,
config=cfg,
var_man=var_man,
build_tracker=mb_tracker,
build_only=build_only,
rebuild=rebuild,
))
progress += 1.0 / tot_tests
if outfile is not None:
fprint("Creating Test Runs: {:.0%}".format(progress),
file=outfile,
end='\r')
except (TestRunError, TestConfigError) as err:
raise commands.CommandError(err)
if outfile is not None:
fprint('', file=outfile)
return tests_by_sched
def run(self, pav_cfg, args):
"""Wait for the requested tests to complete."""
# get start time
start_time = time.time()
tests = []
# if args.tests is empty, then retrieve the last series
if not args.tests:
series_id = series_util.load_user_series_id(pav_cfg)
if series_id is not None:
series_obj = series.TestSeries.from_id(pav_cfg, series_id)
# if this is a series made from a series file, add the
# whole series id to the list of tests
if Path(series_obj.path / 'series.pgid').exists():
tests.append(series_id)
else:
tests.extend(
status_utils.get_tests(pav_cfg, args.tests,
self.errfile))
else:
raise commands.CommandError(
"No tests specified and no last series found")
else:
tests_cli = copy.deepcopy(args.tests)
for test_id in tests_cli:
if test_id.startswith('s'):
series_obj = series.TestSeries.from_id(pav_cfg, test_id)
if Path(series_obj.path / 'series.pgid').exists():
tests.append(test_id)
args.tests.remove(test_id)
tests.extend(
status_utils.get_tests(pav_cfg, args.tests, self.errfile))
# determine timeout time, if there is one
end_time = None
if args.timeout is not None:
end_time = start_time + float(args.timeout)
self.wait(pav_cfg, tests, end_time, args.out_mode)
return 0
def get_tests(pav_cfg, args, errfile):
"""
Gets the tests depending on arguments.
:param pav_cfg: The pavilion config
:param argparse namespace args: The tests via command line args.
:param errfile: stream to output errors as needed
:return: List of test objects
"""
if not args.tests:
# Get the last series ran by this user
series_id = series.TestSeries.load_user_series_id(pav_cfg)
if series_id is not None:
args.tests.append(series_id)
else:
raise commands.CommandError(
"No tests specified and no last series was found."
)
test_list = []
for test_id in args.tests:
# Series
if test_id.startswith('s'):
try:
test_list.extend(series.TestSeries.from_id(pav_cfg,
test_id).tests)
except series.TestSeriesError as err:
output.fprint(
"Suite {} could not be found.\n{}"
.format(test_id, err),
file=errfile,
color=output.RED
)
continue
# Test
else:
test_list.append(test_id)
test_list = list(map(int, test_list))
return test_list
def configs_to_tests(pav_cfg,
proto_tests: List[test_config.ProtoTest],
mb_tracker: Union[MultiBuildTracker, None] = None,
build_only: bool = False,
rebuild: bool = False,
outfile: TextIO = None) -> List[TestRun]:
"""Convert configs/var_man tuples into actual
tests.
:param pav_cfg: The Pavilion config
:param proto_tests: A list of test configs.
:param mb_tracker: The build tracker.
:param build_only: Whether to only build these tests.
:param rebuild: After figuring out what build to use, rebuild it.
:param outfile: Output file for printing messages
"""
test_list = []
progress = 0
tot_tests = len(proto_tests)
for ptest in proto_tests:
try:
test_list.append(
TestRun(pav_cfg=pav_cfg,
config=ptest.config,
var_man=ptest.var_man,
build_tracker=mb_tracker,
build_only=build_only,
rebuild=rebuild))
progress += 1.0 / tot_tests
if outfile is not None:
output.fprint("Creating Test Runs: {:.0%}".format(progress),
file=outfile,
end='\r')
except (TestRunError, TestConfigError) as err:
raise commands.CommandError(err.args[0])
if outfile is not None:
output.fprint('', file=outfile)
return test_list
def get_tests(pav_cfg, tests: List['str'], errfile: TextIO) -> List[int]:
"""Convert a list of test id's and series id's into a list of test id's.
:param pav_cfg: The pavilion config
:param tests: A list of tests or test series names.
:param errfile: stream to output errors as needed
:return: List of test objects
"""
tests = [str(test) for test in tests.copy()]
if not tests:
# Get the last series ran by this user
series_id = series_util.load_user_series_id(pav_cfg)
if series_id is not None:
tests.append(series_id)
else:
raise commands.CommandError(
"No tests specified and no last series was found.")
test_list = []
for test_id in tests:
# Series start with 's', like 'snake'.
if test_id.startswith('s'):
try:
test_list.extend(
series.TestSeries.from_id(pav_cfg, test_id).tests)
except series_util.TestSeriesError as err:
output.fprint("Suite {} could not be found.\n{}".format(
test_id, err),
file=errfile,
color=output.RED)
continue
# Test
else:
test_list.append(test_id)
return list(map(int, test_list))
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides,
sys_vars):
"""Translate a general set of pavilion test configs into the final,
resolved configurations. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:param pav_cfg: The pavilion config
:param str host: The host config to target these tests with
:param list(str) modes: The mode configs to use.
:param list(Path) test_files: Files containing a newline separated
list of tests.
:param list(str) tests: The tests to run.
:param list(str) overrides: Overrides to apply to the configurations.
:param system_variables.SysVarDict sys_vars: The system variables dict.
:returns: A dictionary (by scheduler type name) of lists of test
configs.
"""
self.logger.debug("Finding Configs")
# Use the sys_host if a host isn't specified.
if host is None:
host = sys_vars.get('sys_name')
tests = list(tests)
for file in test_files:
try:
with file.open() as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
try:
raw_tests = test_config.load_test_configs(pav_cfg, host, modes,
tests)
except test_config.TestConfigError as err:
self.logger.error(str(err))
raise commands.CommandError(str(err))
raw_tests_by_sched = defaultdict(lambda: [])
tests_by_scheduler = defaultdict(lambda: [])
# Apply config overrides.
for test_cfg in raw_tests:
# Apply the overrides to each of the config values.
try:
test_config.apply_overrides(test_cfg, overrides)
except test_config.TestConfigError as err:
msg = 'Error applying overrides to test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Resolve all configuration permutations.
try:
p_cfg, permutes = test_config.resolve_permutations(
test_cfg, pav_cfg.pav_vars, sys_vars)
for p_var_man in permutes:
sched = p_cfg['scheduler'].resolve(p_var_man)
raw_tests_by_sched[sched].append((p_cfg, p_var_man))
except test_config.TestConfigError as err:
msg = 'Error resolving permutations for test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Get the schedulers for the tests, and the scheduler variables.
# The scheduler variables are based on all of the
for sched_name in raw_tests_by_sched.keys():
try:
sched = schedulers.get_scheduler_plugin(sched_name)
except KeyError:
msg = "Could not find scheduler '{}'.".format(sched_name)
self.logger.error(msg)
raise commands.CommandError(msg)
nondeferred_cfg_sctns = schedulers.list_scheduler_plugins()
# Builds must have the values of all their variables now.
nondeferred_cfg_sctns.append('build')
# Set the scheduler variables for each test.
for test_cfg, test_var_man in raw_tests_by_sched[sched_name]:
test_var_man.add_var_set('sched', sched.get_vars(test_cfg))
# Resolve all variables for the test.
try:
resolved_config = test_config.resolve_all_vars(
test_cfg,
test_var_man,
no_deferred_allowed=nondeferred_cfg_sctns)
except (ResolveError, KeyError) as err:
msg = "Error resolving variables in config at '{}': {}"\
.format(test_cfg['suite_path'].resolve(test_var_man),
err)
self.logger.error(msg)
raise commands.CommandError(msg)
tests_by_scheduler[sched.name].append(resolved_config)
return tests_by_scheduler
def run(self, pav_cfg, args):
"""Run this command."""
cutoff_date = datetime.today() - timedelta(days=30)
if args.older_than:
args.older_than = args.older_than.split()
if len(args.older_than) == 2:
if not args.older_than[0].isdigit():
raise commands.CommandError(
"Invalid `--older-than` value."
)
if args.older_than[1] in ['minute', 'minutes']:
cutoff_date = datetime.today() - timedelta(
minutes=int(args.older_than[0]))
elif args.older_than[1] in ['hour', 'hours']:
cutoff_date = datetime.today() - timedelta(
hours=int(args.older_than[0]))
elif args.older_than[1] in ['day', 'days']:
cutoff_date = datetime.today() - timedelta(
days=int(args.older_than[0]))
elif args.older_than[1] in ['week', 'weeks']:
cutoff_date = datetime.today() - timedelta(
weeks=int(args.older_than[0]))
elif args.older_than[1] in ['month', 'months']:
cutoff_date = datetime.today() - timedelta(
days=30*int(args.older_than[0]))
elif len(args.older_than) == 3:
date = ' '.join(args.older_than)
try:
cutoff_date = datetime.strptime(date, '%b %d %Y')
except (TypeError, ValueError):
output.fprint("{} is not a valid date."
.format(args.older_than),
file=self.errfile, color=output.RED)
return errno.EINVAL
else:
output.fprint(
"Invalid `--older-than` value.", file=self.errfile,
color=output.RED
)
return errno.EINVAL
elif args.all:
cutoff_date = datetime.today()
tests_dir = pav_cfg.working_dir / 'test_runs' # type: Path
series_dir = pav_cfg.working_dir / 'series' # type: Path
build_dir = pav_cfg.working_dir / 'builds' # type: Path
removed_tests = 0
removed_series = 0
removed_builds = 0
used_builds = set()
# Clean Tests
output.fprint("Removing Tests...", file=self.outfile,
color=output.GREEN)
for test_path in tests_dir.iterdir():
test = test_path.name
try:
int(test)
except ValueError:
# Skip files that aren't numeric
continue
# Skip non-directories.
if not test_path.is_dir():
continue
try:
test_time = datetime.fromtimestamp(test_path.lstat().st_mtime)
except FileNotFoundError:
# The file no longer exists. This is a race condition.
continue
build_origin_symlink = test_path/'build_origin'
# 'None' will probably end up in used_builds, but that's ok.
build_origin = None
if (build_origin_symlink.exists() and
build_origin_symlink.is_symlink() and
build_origin_symlink.resolve().exists()):
build_origin = build_origin_symlink.resolve()
if test_time > cutoff_date:
used_builds.add(build_origin)
continue
state = None
try:
test_obj = TestRun.load(pav_cfg, int(test))
state = test_obj.status.current().state
except (TestRunError, TestRunNotFoundError):
# It's ok if this happens, we'll still remove by date.
# It is possible the test isn't completely written (a race
# condition).
pass
except PermissionError as err:
err = str(err).split("'")
output.fprint("Permission Error: {} cannot be removed"
.format(err[1]), file=self.errfile, color=31)
continue
if state in (STATES.RUNNING, STATES.SCHEDULED):
used_builds.add(build_origin)
continue
try:
shutil.rmtree(test_path.as_posix())
if args.verbose:
output.fprint("Removed test {}".format(test_path),
file=self.outfile)
removed_tests += 1
except OSError as err:
output.fprint(
"Could not remove test {}: {}"
.format(test_path, err),
color=output.YELLOW, file=self.errfile)
# Clean Series
output.fprint("Removing Series...", file=self.outfile,
color=output.GREEN)
for series in series_dir.iterdir():
for test in series.iterdir():
if (test.is_symlink() and
test.exists() and
test.resolve().exists()):
# This test is still present, so keep the series.
break
else:
# This series has no remaining tests, we can delete it.
try:
shutil.rmtree(series.as_posix())
removed_series += 1
except OSError as err:
output.fprint(
"Could not remove series {}: {}"
.format(series, err),
color=output.YELLOW, file=self.errfile
)
# Clean Builds
output.fprint("Removing Builds...", file=self.outfile,
color=output.GREEN)
for build in build_dir.iterdir():
if build in used_builds:
continue
try:
shutil.rmtree(build.as_posix())
if args.verbose:
output.fprint("Removed build", build, file=self.outfile)
except OSError as err:
output.fprint(
"Could not remove build {}: {}"
.format(build, err),
color=output.YELLOW, file=self.errfile)
output.fprint("Removed {tests} tests, {series} series, and {builds} "
"builds."
.format(tests=removed_tests, series=removed_series,
builds=removed_builds),
color=output.GREEN, file=self.outfile)
return 0
def get_tests(self, pav_config, args):
"""Translate a general set of pavilion test configs into the final,
resolved configuration objects. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:returns: A dictionary (by scheduler type name) of lists of test
objects
"""
self.logger.DEBUG("Finding Configs")
# Use the sys_host if a host isn't specified.
if args.host is None:
host = pav_config.sys_vars.get('sys_host')
else:
host = args.host
tests = args.tests
for file in args.files:
try:
with open(file) as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
raw_tests = config_utils.get_tests(pav_config, host, args.mode, tests)
raw_tests_by_sched = defaultdict(lambda: [])
tests_by_scheduler = defaultdict(lambda: [])
# Apply config overrides.
for test_cfg in raw_tests:
# Apply the overrides to each of the config values.
try:
config_utils.apply_overrides(test_cfg, args.overrides)
except config_utils.TestConfigError as err:
msg = 'Error applying overrides to test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Resolve all configuration permutations.
try:
for p_cfg, p_var_man in config_utils.resolve_permutations(
test_cfg, pav_config.pav_vars, pav_config.sys_vars):
sched = p_cfg['scheduler']
raw_tests_by_sched[sched].append((p_cfg, p_var_man))
except config_utils.TestConfigError as err:
msg = 'Error resolving permutations for test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Get the schedulers for the tests, and the scheduler variables.
# The scheduler variables are based on all of the
for sched_name in raw_tests_by_sched.keys():
try:
sched = schedulers.get_scheduler_plugin(sched_name)
except KeyError:
msg = "Could not find scheduler '{}'.".format(sched_name)
self.logger.error(msg)
raise commands.CommandError(msg)
nondeferred_cfg_sctns = schedulers.list_scheduler_plugins()
# Builds must have the values of all their variables now.
nondeferred_cfg_sctns.append('build')
# Set the echeduler variables for each test.
for test_cfg, test_var_man in raw_tests_by_sched[sched_name]:
test_var_man.add_var_set('sched', sched)
# Resolve all variables for the test.
try:
resolved_config = config_utils.resolve_all_vars(
test_cfg,
test_var_man,
no_deferred_allowed=nondeferred_cfg_sctns)
except (ResolveError, KeyError) as err:
msg = 'Error resolving variables in config: {}'.format(err)
self.logger.error(msg)
raise commands.CommandError(msg)
test = PavTest(pav_config, resolved_config)
tests_by_scheduler[sched.name].append(test)
return tests_by_scheduler
