def run_suite(self, job, test_suite):
summary = set()
test_suite.tests, _ = nrunner.check_runnables_runner_requirements(
test_suite.tests)
job.result.tests_total = test_suite.variants.get_number_of_tests(
test_suite.tests)
listen = test_suite.config.get('nrunner.status_server_listen')
self._start_status_server(listen)
# pylint: disable=W0201
self.runtime_tasks = self._get_all_runtime_tasks(test_suite)
if test_suite.config.get('nrunner.shuffle'):
random.shuffle(self.runtime_tasks)
test_ids = [
rt.task.identifier for rt in self.runtime_tasks
if rt.task.category == 'test'
]
tsm = TaskStateMachine(self.runtime_tasks, self.status_repo)
spawner_name = test_suite.config.get('nrunner.spawner')
spawner = SpawnerDispatcher(test_suite.config)[spawner_name].obj
max_running = min(test_suite.config.get('nrunner.max_parallel_tasks'),
len(self.runtime_tasks))
timeout = test_suite.config.get('task.timeout.running')
workers = [
Worker(state_machine=tsm,
spawner=spawner,
max_running=max_running,
task_timeout=timeout).run() for _ in range(max_running)
]
asyncio.ensure_future(self._update_status(job))
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(
asyncio.wait_for(asyncio.gather(*workers), job.timeout
or None))
except (KeyboardInterrupt, asyncio.TimeoutError, TestFailFast) as ex:
LOG_JOB.info(str(ex))
job.interrupted_reason = str(ex)
summary.add("INTERRUPTED")
# Wait until all messages may have been processed by the
# status_updater. This should be replaced by a mechanism
# that only waits if there are missing status messages to
# be processed, and, only for a given amount of time.
# Tests with non received status will always show as SKIP
# because of result reconciliation.
loop.run_until_complete(asyncio.sleep(0.05))
job.result.end_tests()
self.status_server.close()
# Update the overall summary with found test statuses, which will
# determine the Avocado command line exit status
summary.update([
status.upper()
for status in self.status_repo.get_result_set_for_tasks(test_ids)
])
return summary
def run_tests(self):
"""
The actual test execution phase
"""
self._log_job_debug_info()
jobdata.record(self, sys.argv)
if self.size == 0:
msg = ('Unable to resolve any reference or "resolver.references"'
"is empty.")
LOG_UI.error(msg)
if not self.test_suites:
self.exitcode |= exit_codes.AVOCADO_JOB_FAIL
return self.exitcode
summary = set()
for suite in self.test_suites:
summary |= suite.run(self)
# If it's all good so far, set job status to 'PASS'
if self.status == 'RUNNING':
self.status = 'PASS'
LOG_JOB.info('Test results available in %s', self.logdir)
if 'INTERRUPTED' in summary:
self.exitcode |= exit_codes.AVOCADO_JOB_INTERRUPTED
if 'FAIL' in summary or 'ERROR' in summary:
self.exitcode |= exit_codes.AVOCADO_TESTS_FAIL
return self.exitcode
def run(self, args):
"""
Take care of command line overwriting, parameter preparation,
setup and cleanup chains, and paths/utilities for all host controls.
"""
log.info("Manual setup chain started.")
# set English environment (command output might be localized, need to be safe)
os.environ['LANG'] = 'en_US.UTF-8'
cmd_parser.params_from_cmd(args)
run_params = param.prepare_params(
base_file="guest-base.cfg",
ovrwrt_dict={"vms": " ".join(args.selected_vms)},
ovrwrt_file=param.vms_ovrwrt_file,
ovrwrt_str=args.param_str)
# prepare a setup step or a chain of such
run_params["count"] = 0
setup_chain = run_params["setup"].split()
for setup_step in setup_chain:
setup_func = getattr(intertest, setup_step)
setup_func(args, run_params, "0m%s" % run_params["count"])
run_params["count"] += 1
log.info("Manual setup chain finished.")
def _log_avocado_version():
version_log = version.VERSION
git_version = get_avocado_git_version()
if git_version is not None:
version_log += git_version
LOG_JOB.info('Avocado version: %s', version_log)
LOG_JOB.info('')
def sigtstp_handler(signum, frame): # pylint: disable=W0613
""" SIGSTOP all test processes on SIGTSTP """
if not proc: # Ignore ctrl+z when proc not yet started
return
with sigtstp:
msg = "ctrl+z pressed, %%s test (%s)" % proc.pid
app_log_msg = '\n%s' % msg
if self.sigstopped:
APP_LOG.info(app_log_msg, "resumming")
TEST_LOG.info(msg, "resumming")
process.kill_process_tree(proc.pid, signal.SIGCONT, False)
self.sigstopped = False
else:
APP_LOG.info(app_log_msg, "stopping")
TEST_LOG.info(msg, "stopping")
process.kill_process_tree(proc.pid, signal.SIGSTOP, False)
self.sigstopped = True
def run(self, config):
"""
Take care of command line overwriting, parameter preparation,
setup and cleanup chains, and paths/utilities for all host controls.
"""
log.info("Manual setup chain started.")
# set English environment (command output might be localized, need to be safe)
os.environ['LANG'] = 'en_US.UTF-8'
cmd_parser.params_from_cmd(config)
intertest.load_addons_tools()
run_params = config["vms_params"]
# prepare a setup step or a chain of such
setup_chain = run_params.objects("setup")
for i, setup_step in enumerate(setup_chain):
run_params["count"] = i
setup_func = getattr(intertest, setup_step)
setup_func(config, "0m%s" % i)
log.info("Manual setup chain finished.")
def permubuntu(config, tag=""):
"""
Perform all extra setup needed for the ubuntu permanent vms.
:param config: command line arguments and run configuration
:type config: {str, str}
:param str tag: extra name identifier for the test to be run
"""
l, r = config["graph"].l, config["graph"].r
selected_vms = sorted(config["vm_strs"].keys())
LOG_UI.info("Starting permanent vm setup for %s (%s)",
", ".join(selected_vms), os.path.basename(r.job.logdir))
for test_object in l.parse_objects(config["param_dict"],
config["vm_strs"]):
if test_object.key != "vms":
continue
vm = test_object
# parse individual net only for the current vm
net = l.parse_object_from_objects([vm],
param_dict=config["param_dict"])
logging.info("Performing extra setup for the permanent %s", vm.suffix)
# consider this as a special kind of state converting test which concerns
# permanent objects (i.e. instead of transition from customize to on
# root, it is a transition from supposedly "permanentized" vm to the root)
logging.info("Booting %s for the first permanent on state", vm.suffix)
setup_dict = config["param_dict"].copy()
setup_dict.update({"set_state_vms": "ready"})
setup_str = param.re_str("all..internal..manage.start")
test_node = l.parse_node_from_object(net,
setup_dict,
setup_str,
prefix=tag)
to_run = r.run_test_node(test_node)
asyncio.get_event_loop().run_until_complete(
asyncio.wait_for(to_run, r.job.timeout or None))
LOG_UI.info("Finished permanent vm setup")
def run(self, config):
"""
Take care of command line overwriting, parameter preparation,
setup and cleanup chains, and paths/utilities for all host controls.
"""
log.info("Manual setup chain started.")
# set English environment (command output might be localized, need to be safe)
os.environ['LANG'] = 'en_US.UTF-8'
config["run.test_runner"] = "traverser"
config["params"] = config["i2n.manu.params"]
try:
cmd_parser.params_from_cmd(config)
except param.EmptyCartesianProduct as error:
LOG_UI.error(error)
return 1
intertest.load_addons_tools()
run_params = config["vms_params"]
# prepare a setup step or a chain of such
setup_chain = run_params.objects("setup")
retcode = 0
for i, setup_step in enumerate(setup_chain):
run_params["count"] = i
setup_func = getattr(intertest, setup_step)
try:
# TODO: drop the consideration of None in the future if the
# functions from the intertest module do not return this value.
if setup_func(config, "0m%s" % i) not in [None, 0]:
# return 1 if at least one of the steps fails
retcode = 1
except Exception as error:
LOG_UI.error(error)
retcode = 1
log.info("Manual setup chain finished.")
return retcode
def _log_avocado_config(self):
LOG_JOB.info('Avocado config:')
LOG_JOB.info('')
for line in pprint.pformat(self.config).splitlines():
LOG_JOB.info(line)
LOG_JOB.info('')
def _log_avocado_datadir(self):
LOG_JOB.info('Avocado Data Directories:')
LOG_JOB.info('')
LOG_JOB.info('base %s', self.config.get('datadir.paths.base_dir'))
LOG_JOB.info('tests %s', data_dir.get_test_dir())
LOG_JOB.info('data %s', self.config.get('datadir.paths.data_dir'))
LOG_JOB.info('logs %s', self.logdir)
LOG_JOB.info('')
def params_from_cmd(config):
"""
Take care of command line overwriting, parameter preparation,
setup and cleanup chains, and paths/utilities for all host controls.
:param config: command line arguments
:type config: {str, str}
:raises: :py:class:`ValueError` if a command line selected vm is not available
from the configuration and thus supported or internal tests are
restricted from the command line
.. todo:: Any dynamically created config keys here are usually entire data
structures like dictionaries and lists and only used internally during
the run which makes them unfit for displaying to the user and putting
in a namespace scope like the officially registered plugin settings.
Let's wait to see if the multi-suite support in avocado would establish
some standards for doing this first. Until then, the user won't directly
interact with these keys anyway.
"""
suite_path = settings.as_dict().get('i2n.common.suite_path', ".")
sys.path.insert(1, os.path.join(suite_path, "utils"))
# validate typed vm names and possible vm specific restrictions
available_vms = param.all_objects("vms")
available_restrictions = param.all_restrictions()
# defaults usage vs command line overriding
use_tests_default = True
with_nontrivial_restrictions = False
use_vms_default = {vm_name: True for vm_name in available_vms}
with_selected_vms = list(available_vms)
# the run string includes only pure parameters
param_dict = {}
# the tests string includes the test restrictions while the vm strings include the ones for the vm variants
tests_str, vm_strs = "", collections.OrderedDict([(vm, "")
for vm in available_vms])
# main tokenizing loop
for cmd_param in config["params"]:
re_param = re.match(r"(\w+)=(.*)", cmd_param)
if re_param is None:
ui.error(
"Found malformed parameter on the command line '%s' - "
"must be of the form <key>=<val>", cmd_param)
sys.exit(1)
(key, value) = re_param.group(1, 2)
if key == "only" or key == "no":
# detect if this is the primary restriction to escape defaults
for variant in re.split(r",|\.|\.\.", value):
if variant in available_restrictions:
use_tests_default = False
# else this is an auxiliary restriction
else:
with_nontrivial_restrictions = True
# main test restriction part
tests_str += "%s %s\n" % (key, value)
elif key.startswith("only_") or key.startswith("no_"):
for vm_name in available_vms:
if re.match("(only|no)_%s" % vm_name, key):
# escape defaults for this vm and use the command line
use_vms_default[vm_name] = False
# main vm restriction part
vm_strs[vm_name] += "%s %s\n" % (key.replace(
"_%s" % vm_name, ""), value)
# NOTE: comma in a parameter sense implies the same as space in config file
elif key == "vms":
# NOTE: no restrictions of the required vms are allowed during tests since
# these are specified by each test (allowed only for manual setup steps)
with_selected_vms[:] = value.split(",")
for vm_name in with_selected_vms:
if vm_name not in available_vms:
raise ValueError(
"The vm '%s' is not among the supported vms: "
"%s" % (vm_name, ", ".join(available_vms)))
else:
# NOTE: comma on the command line is space in a config file
value = value.replace(",", " ")
param_dict[key] = value
config["param_dict"] = param_dict
log.debug("Parsed param dict '%s'", param_dict)
# get minimal configurations and parse defaults if no command line arguments
config["vms_params"], config["vm_strs"] = full_vm_params_and_strs(
param_dict, vm_strs, use_vms_default)
config["vms_params"]["vms"] = " ".join(with_selected_vms)
config["available_vms"] = vm_strs.copy()
for vm_name in available_vms:
# the keys of vm strings must be equivalent to the selected vms
if vm_name not in with_selected_vms:
del config["vm_strs"][vm_name]
config["tests_params"], config["tests_str"] = full_tests_params_and_str(
param_dict, tests_str, use_tests_default)
config["available_restrictions"] = available_restrictions
# control against invoking only runnable tests and empty Cartesian products
control_config = param.Reparsable()
control_config.parse_next_batch(base_file="sets.cfg",
ovrwrt_file=param.tests_ovrwrt_file(),
ovrwrt_str=config["tests_str"],
ovrwrt_dict=config["param_dict"])
control_parser = control_config.get_parser()
if with_nontrivial_restrictions:
log.info("%s tests with nontrivial restriction %s",
len(list(control_parser.get_dicts())), config["tests_str"])
# prefix for all tests of the current run making it possible to perform multiple runs in one command
config["prefix"] = ""
# log into files for each major level the way it was done for autotest
config["run.store_logging_stream"] = [":10", ":20", ":30", ":40"]
# set default off and on state backends
from .states import lvm, qcow2, lxc, btrfs, ramfile, pool, vmnet
ss.BACKENDS = {
"lvm": lvm.LVMBackend,
"qcow2": qcow2.QCOW2Backend,
"lxc": lxc.LXCBackend,
"btrfs": btrfs.BtrfsBackend,
"pool": pool.QCOW2PoolBackend,
"qcow2vt": qcow2.QCOW2VTBackend,
"ramfile": ramfile.RamfileBackend,
"vmnet": vmnet.VMNetBackend
}
# attach environment processing hooks
env_process_hooks()
def _run_test(job, result, test_factory, queue):
"""
Run a test instance.
This code is the first thing that runs inside a new process, known here
as the test process. It communicates to the test runner by using
:param:`queue`. It's important that this early state is given to the
test runner in a reliable way.
:param test_factory: Test factory (test class and parameters).
:type test_factory: tuple of :class:`avocado.core.test.Test` and dict.
:param queue: Multiprocess queue.
:type queue: :class:`multiprocessing.Queue` instance.
"""
sys.stdout = output.LoggingFile(["[stdout] "], loggers=[TEST_LOG])
sys.stderr = output.LoggingFile(["[stderr] "], loggers=[TEST_LOG])
def sigterm_handler(signum, frame): # pylint: disable=W0613
""" Produce traceback on SIGTERM """
raise RuntimeError("Test interrupted by SIGTERM")
signal.signal(signal.SIGTERM, sigterm_handler)
# At this point, the original `sys.stdin` has already been
# closed and replaced with `os.devnull` by
# `multiprocessing.Process()` (not directly from Avocado
# code). Still, tests trying to use file descriptor 0 would
# be able to read from the tty, and would hang. Let's replace
# STDIN fd (0), with the same fd previously set by
# `multiprocessing.Process()`
os.dup2(sys.stdin.fileno(), 0)
instance = loader.load_test(test_factory)
if instance.runner_queue is None:
instance.set_runner_queue(queue)
early_state = instance.get_state()
early_state['early_status'] = True
try:
queue.put(early_state)
except queueFullException:
instance.error(stacktrace.str_unpickable_object(early_state))
result.start_test(early_state)
job.result_events_dispatcher.map_method('start_test',
result,
early_state)
if job.config.get('run.log_test_data_directories'):
data_sources = getattr(instance, "DATA_SOURCES", [])
if data_sources:
locations = []
for source in data_sources:
locations.append(instance.get_data("", source=source,
must_exist=False))
TEST_LOG.info('Test data directories: ')
for source, location in zip(data_sources, locations):
if location is not None:
TEST_LOG.info(' %s: %s', source, location)
TEST_LOG.info('')
try:
instance.run_avocado()
finally:
try:
state = instance.get_state()
queue.put(state)
except queueFullException:
instance.error(stacktrace.str_unpickable_object(state))
def _log_tmp_dir(self):
LOG_JOB.info("Temporary dir: %s", self.tmpdir)
LOG_JOB.info("")
def end_test(self, result, state):
if self.runner == 'nrunner':
LOG_JOB.info('%s: %s', self._get_name(state),
state.get("status", "ERROR"))
LOG_JOB.info('More information in %s', state.get('task_path', ''))
def _log_variants(variants):
lines = variants.to_str(summary=1, variants=1, use_utf8=False)
for line in lines.splitlines():
LOG_JOB.info(line)
def _log_job_id(self):
LOG_JOB.info('Job ID: %s', self.unique_id)
if self.replay_sourcejob is not None:
LOG_JOB.info('Replay of Job ID: %s', self.replay_sourcejob)
LOG_JOB.info('')
def run(self, args):
LOG_JOB.info(self.description)
def end_test(self, result, state):
LOG_JOB.info("%s: %s", self._get_name(state),
state.get("status", "ERROR"))
LOG_JOB.info("More information in %s", state.get("task_path", ""))
def start_test(self, result, state):
LOG_JOB.info("%s: STARTED", self._get_name(state))
def _log_cmdline():
cmdline = " ".join(sys.argv)
LOG_JOB.info("Command line: %s", cmdline)
LOG_JOB.info('')
def params_from_cmd(config):
"""
Take care of command line overwriting, parameter preparation,
setup and cleanup chains, and paths/utilities for all host controls.
:param config: command line arguments
:type config: {str, str}
:raises: :py:class:`ValueError` if a command line selected vm is not available
from the configuration and thus supported or internal tests are
restricted from the command line
"""
sys.path.insert(1, os.path.join(param.suite_path, "utils"))
# validate typed vm names and possible vm specific restrictions
available_vms = param.all_vms()
available_restrictions = param.all_restrictions()
# defaults usage vs command line overriding
use_tests_default = True
with_nontrivial_restrictions = False
use_vms_default = {vm_name: True for vm_name in available_vms}
with_selected_vms = list(available_vms)
# the run string includes only pure parameters
param_dict = {}
# the tests string includes the test restrictions while the vm strings include the ones for the vm variants
tests_str, vm_strs = "", {vm: "" for vm in available_vms}
# main tokenizing loop
for cmd_param in config["params"]:
re_param = re.match(r"(\w+)=(.*)", cmd_param)
if re_param is None:
log.error("Skipping malformed parameter on the command line '%s' - "
"must be of the form <key>=<val>", cmd_param)
continue
(key, value) = re_param.group(1, 2)
if key == "only" or key == "no":
# detect if this is the primary restriction to escape defaults
if value in available_restrictions:
use_tests_default = False
# else this is an auxiliary restriction
else:
with_nontrivial_restrictions = True
# main test restriction part
tests_str += "%s %s\n" % (key, value)
elif key.startswith("only_") or key.startswith("no_"):
for vm_name in available_vms:
if re.match("(only|no)_%s" % vm_name, key):
# escape defaults for this vm and use the command line
use_vms_default[vm_name] = False
# main vm restriction part
vm_strs[vm_name] += "%s %s\n" % (key.replace("_%s" % vm_name, ""), value)
# NOTE: comma in a parameter sense implies the same as space in config file
elif key == "vms":
# NOTE: no restrictions of the required vms are allowed during tests since
# these are specified by each test (allowed only for manual setup steps)
with_selected_vms[:] = value.split(",")
for vm_name in with_selected_vms:
if vm_name not in available_vms:
raise ValueError("The vm '%s' is not among the supported vms: "
"%s" % (vm_name, ", ".join(available_vms)))
else:
# NOTE: comma on the command line is space in a config file
value = value.replace(",", " ")
param_dict[key] = value
config["param_dict"] = param_dict
log.debug("Parsed param dict '%s'", param_dict)
# get minimal configurations and parse defaults if no command line arguments
config["vms_params"], config["vm_strs"] = full_vm_params_and_strs(param_dict, vm_strs,
use_vms_default)
config["vms_params"]["vms"] = " ".join(with_selected_vms)
config["available_vms"] = vm_strs.copy()
for vm_name in available_vms:
# the keys of vm strings must be equivalent to the selected vms
if vm_name not in with_selected_vms:
del config["vm_strs"][vm_name]
config["tests_params"], config["tests_str"] = full_tests_params_and_str(param_dict, tests_str,
use_tests_default)
config["available_restrictions"] = available_restrictions
# control against invoking only runnable tests and empty Cartesian products
control_config = param.Reparsable()
control_config.parse_next_batch(base_file="sets.cfg",
ovrwrt_file=param.tests_ovrwrt_file(),
ovrwrt_str=config["tests_str"],
ovrwrt_dict=config["param_dict"])
control_parser = control_config.get_parser()
if with_nontrivial_restrictions:
log.info("%s tests with nontrivial restriction %s",
len(list(control_parser.get_dicts())), config["tests_str"])
# prefix for all tests of the current run making it possible to perform multiple runs in one command
config["prefix"] = ""
# log into files for each major level the way it was done for autotest
config["run.store_logging_stream"] = [":10", ":20", ":30", ":40"]
# attach environment processing hooks
env_process_hooks()
def _log_tmp_dir(self):
LOG_JOB.info('Temporary dir: %s', self.tmpdir)
LOG_JOB.info('')
def run_suite(self, job, test_suite):
summary = set()
if not test_suite.enabled:
job.interrupted_reason = f"Suite {test_suite.name} is disabled."
return summary
test_suite.tests, missing_requirements = check_runnables_runner_requirements(
test_suite.tests)
self._update_avocado_configuration_used_on_runnables(
test_suite.tests, test_suite.config)
self._abort_if_missing_runners(missing_requirements)
job.result.tests_total = test_suite.variants.get_number_of_tests(
test_suite.tests)
self._create_status_server(test_suite, job)
graph = RuntimeTaskGraph(
test_suite.get_test_variants(),
test_suite.name,
self.status_server.uri,
job.unique_id,
)
# pylint: disable=W0201
self.runtime_tasks = graph.get_tasks_in_topological_order()
# Start the status server
asyncio.ensure_future(self.status_server.serve_forever())
if test_suite.config.get("nrunner.shuffle"):
random.shuffle(self.runtime_tasks)
test_ids = [
rt.task.identifier for rt in self.runtime_tasks
if rt.task.category == "test"
]
tsm = TaskStateMachine(self.runtime_tasks, self.status_repo)
spawner_name = test_suite.config.get("nrunner.spawner")
spawner = SpawnerDispatcher(test_suite.config, job)[spawner_name].obj
max_running = min(test_suite.config.get("nrunner.max_parallel_tasks"),
len(self.runtime_tasks))
timeout = test_suite.config.get("task.timeout.running")
failfast = test_suite.config.get("run.failfast")
workers = [
Worker(
state_machine=tsm,
spawner=spawner,
max_running=max_running,
task_timeout=timeout,
failfast=failfast,
).run() for _ in range(max_running)
]
asyncio.ensure_future(self._update_status(job))
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(
asyncio.wait_for(asyncio.gather(*workers), job.timeout
or None))
except (KeyboardInterrupt, asyncio.TimeoutError, TestFailFast) as ex:
LOG_JOB.info(str(ex))
job.interrupted_reason = str(ex)
summary.add("INTERRUPTED")
# Wait until all messages may have been processed by the
# status_updater. This should be replaced by a mechanism
# that only waits if there are missing status messages to
# be processed, and, only for a given amount of time.
# Tests with non received status will always show as SKIP
# because of result reconciliation.
loop.run_until_complete(asyncio.sleep(0.05))
job.result.end_tests()
self.status_server.close()
if self.status_server_dir is not None:
self.status_server_dir.cleanup()
# Update the overall summary with found test statuses, which will
# determine the Avocado command line exit status
summary.update([
status.upper()
for status in self.status_repo.get_result_set_for_tasks(test_ids)
])
return summary
def start_test(self, result, state):
if self.runner == 'nrunner':
LOG_JOB.info('%s: STARTED', self._get_name(state))
def _log_avocado_datadir(self):
LOG_JOB.info("Avocado Data Directories:")
LOG_JOB.info("")
LOG_JOB.info("base %s", self.config.get("datadir.paths.base_dir"))
LOG_JOB.info("tests %s", data_dir.get_test_dir())
LOG_JOB.info("data %s", self.config.get("datadir.paths.data_dir"))
LOG_JOB.info("logs %s", self.logdir)
LOG_JOB.info("")