def test_requeue(self):
"""Requeue test.
Same as previous example except that all tests are requeued
once.
"""
results = {}
def collect(job):
if job.uid not in results:
results[job.uid] = True
return True
else:
return False
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(Scheduler.simple_provider(NopJob),
tokens=2, collect=collect)
s.run(dag)
assert s.max_active_jobs == 2
assert results['1']
assert results['2']
def test_keyboard_interrupt(self):
"""Ensure that jobs can be interrupted."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return NopJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
# fake log_state that will raise a KeyboardInterrupt
def fake_log_state():
raise KeyboardInterrupt
s.log_state = fake_log_state
with pytest.raises(KeyboardInterrupt):
s.run(dag)
for k, v in results.items():
assert v.interrupted
def test_minimal_run2(self):
"""Test with two interdependent jobs."""
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2', predecessors=['1'])
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2)
s.run(dag)
assert s.max_active_jobs == 1
def test_minimal_run(self):
"""Test with only two independent jobs."""
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2)
s.run(dag)
assert s.max_active_jobs == 2
def test_ordering(self):
"""Test that jobs are ordered correctly."""
results = []
def collect(job):
results.append(job.uid)
dag = DAG()
dag.add_vertex("3")
dag.add_vertex("0")
dag.add_vertex("1")
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=1, collect=collect)
s.run(dag)
assert tuple(results) == ("0", "1", "3")
def test_collect_feedback_scheme(self):
"""Collect feedback construction.
Scheme in which if a job predecessor "fails" then job is skipped
In order to do that get_job and collect should have access to
common data. Note that scheduler ensure that these functions
are called sequentially.
"""
class SchedulerContext(object):
def __init__(self):
# Save in results tuples with first element being a bool
# indicating success or failure and the second the job itself
self.results = {}
def get_job(self, uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
# If any of the predecessor failed skip the job
for k in predecessors:
if not self.results[k][0]:
result.should_skip = True
return result
def collect(self, job):
if job.should_skip:
# Skipped jobs are considered failed
self.results[job.uid] = [False, job]
else:
# Job '2' is always failing
if job.uid == "2":
self.results[job.uid] = [False, job]
else:
self.results[job.uid] = [True, job]
dag = DAG()
dag.add_vertex("1")
dag.add_vertex("2")
dag.add_vertex("3", predecessors=["1", "2"])
dag.add_vertex("4", predecessors=["3"])
c = SchedulerContext()
s = Scheduler(c.get_job, tokens=2, collect=c.collect)
s.run(dag)
assert (
not c.results["2"][1].should_skip and not c.results["2"][0]
), 'job "2" is run and should be marked as failed'
assert c.results["3"][1].should_skip, 'job "3" should be skipped'
assert c.results["4"][1].should_skip, 'job "4" should be skipped'
def __init__(self, actions):
"""Object initializer.
:param actions: DAG of actions to perform.
:type actions: DAG
"""
self.actions = actions
self.new_fingerprints = {}
self.job_status = {}
self.set_scheduling_params()
self.scheduler = Scheduler(job_provider=self.get_job,
collect=self.collect,
queues=self.queues,
tokens=self.tokens,
job_timeout=self.job_timeout)
self.scheduler.run(self.actions)
def test_timeout(self):
"""Ensure that jobs are interrupted correctly on timeout."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return SleepJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
s.run(dag)
for k, v in results.items():
assert v.interrupted
def __init__(self, actions: DAG):
"""Object initializer.
:param actions: DAG of actions to perform.
"""
self.actions = actions
self.prev_fingerprints: Dict[str, Optional[Fingerprint]] = {}
self.new_fingerprints: Dict[str, Optional[Fingerprint]] = {}
self.job_status: Dict[str, ReturnValue] = {}
self.set_scheduling_params()
self.failure_source: Dict[str, Set[str]] = {}
self.scheduler = Scheduler(
job_provider=self.get_job,
collect=self.collect, # type: ignore
queues=self.queues,
tokens=self.tokens,
job_timeout=self.job_timeout,
)
self.scheduler.run(self.actions)
def test_skip(self):
"""Simple example in which all the tests are skipped."""
results = {}
def get_job(uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
result.should_skip = True
return result
def collect(job):
results[job.uid] = job.timing_info
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect)
s.run(dag)
# Check start_time end_time to be sure tests have not been run
for k, v in results.items():
assert v.start_time is None
assert v.stop_time is None
def run(self, action_list):
sch = Scheduler(self.get_job, self.collect)
sch.run(action_list)
def run_standard_mainloop(self, dag: DAG) -> None:
"""Run the main loop to execute test fragments in threads."""
assert self.main.args is not None
from e3.job import Job
from e3.testsuite.fragment import FragmentData, ThreadTestFragment
def job_factory(
uid: str,
data: Any,
predecessors: FrozenSet[str],
notify_end: Callable[[str], None],
) -> ThreadTestFragment:
"""Turn a DAG item into a ThreadTestFragment instance."""
assert isinstance(data, FragmentData)
# When passing return values from predecessors, remove current test
# name from the keys to ease referencing by user (the short
# fragment name can then be used by user without knowing the full
# node id).
key_prefix = data.driver.test_name + "."
key_prefix_len = len(key_prefix)
def filter_key(k: str) -> str:
if k.startswith(key_prefix):
return k[key_prefix_len:]
else:
return k
return ThreadTestFragment(
uid,
data.driver,
data.callback,
{filter_key(k): self.return_values[k]
for k in predecessors},
notify_end,
self.running_status,
)
def collect_result(job: Job) -> bool:
"""Collect test results from the given fragment."""
assert isinstance(job, ThreadTestFragment)
self.return_values[job.uid] = job.return_value
self.collect_result(job)
# In the e3.job.scheduler API, collect returning "True" means
# "requeue the job". We never want to do that.
return False
# Create a scheduler to run all fragments for the testsuite main loop
scheduler = Scheduler(
job_provider=job_factory,
tokens=self.main.args.jobs,
collect=collect_result,
)
# Run the tests. Note that when the testsuite aborts because of too
# many consecutive test failures, we still want to produce a report and
# exit through regular ways, to catch KeyboardInterrupt exceptions,
# which e3's scheduler uses to abort the execution loop, but only in
# such cases. In other words, let the exception propagates if it's the
# user that interrupted the testsuite.
try:
scheduler.run(dag)
except KeyboardInterrupt:
if not self.aborted_too_many_failures: # interactive-only
raise
def testsuite_main(self, args=None):
"""Main for the main testsuite script.
:param args: command line arguments. If None use sys.argv
:type args: list[str] | None
"""
self.main = Main(platform_args=self.CROSS_SUPPORT)
# Add common options
parser = self.main.argument_parser
parser.add_argument("-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir")
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"--max-consecutive-failures",
default=0,
help="If there are more than N consecutive failures, the testsuite"
" is aborted. If set to 0 (default) then the testsuite will never"
" be stopped")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)")
parser.add_argument("--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space")
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously")
parser.add_argument(
"--show-error-output",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.")
parser.add_argument('sublist',
metavar='tests',
nargs='*',
default=[],
help='test')
# Add user defined options
self.add_options()
# parse options
self.main.parse_args(args)
self.env = BaseEnv.from_env()
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths
# Keep the working dir as short as possible, to avoid the risk
# of having a path that's too long (a problem often seen on
# Windows, or when using WRS tools that have their own max path
# limitations).
# Note that we do make sure that working_dir is an absolute
# path, as we are likely to be changing directories when
# running each test. A relative path would no longer work
# under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, 'new')
self.old_output_dir = os.path.join(d, 'old')
if not os.path.isdir(self.main.args.temp_dir):
logging.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
'', 'tmp', os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results
self.setup_result_dir()
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.tear_up()
# Retrieve the list of test
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(job_provider=self.job_factory,
collect=self.collect_result,
tokens=self.main.args.jobs)
actions = DAG()
for test in self.test_list:
self.parse_test(actions, test)
with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd:
fd.write(actions.as_dot())
self.scheduler.run(actions)
self.dump_testsuite_result()
# Clean everything
self.tear_down()
return 0
def testsuite_main(self, args: Optional[List[str]] = None) -> int:
"""Main for the main testsuite script.
:param args: Command line arguments. If None, use `sys.argv`.
:return: The testsuite status code (0 for success, a positive for
failure).
"""
self.main = Main(platform_args=True)
# Add common options
parser = self.main.argument_parser
parser.add_argument(
"-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir",
)
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"-d",
"--dev-temp",
nargs="?",
default=None,
const="tmp",
help="Unlike --temp-dir, use this very directory to store"
" testsuite temporaries (i.e. no random subdirectory). Also"
" automatically disable temp dir cleanup, to be developer"
" friendly. If no directory is provided, use the local"
" \"tmp\" directory")
parser.add_argument(
"--max-consecutive-failures",
"-M",
metavar="N",
type=int,
default=self.default_max_consecutive_failures,
help="Number of test failures (FAIL or ERROR) that trigger the"
" abortion of the testuite. If zero, this behavior is disabled. In"
" some cases, aborting the testsuite when there are just too many"
" failures saves time and costs: the software to test/environment"
" is too broken, there is no point to continue running the"
" testsuite.")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)",
)
parser.add_argument(
"--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space",
)
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously",
)
parser.add_argument(
"--show-error-output",
"-E",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.",
)
parser.add_argument(
"--show-time-info",
action="store_true",
help="Display time information for test results, if available")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.",
)
parser.add_argument(
"--xunit-output",
dest="xunit_output",
metavar="FILE",
help="Output testsuite report to the given file in the standard"
" XUnit XML format. This is useful to display results in"
" continuous build systems such as Jenkins.",
)
parser.add_argument(
"--gaia-output",
action="store_true",
help="Output a GAIA-compatible testsuite report next to the YAML"
" report.")
parser.add_argument(
"--truncate-logs",
"-T",
metavar="N",
type=int,
default=200,
help="When outputs (for instance subprocess outputs) exceed 2*N"
" lines, only include the first and last N lines in logs. This is"
" necessary when storage for testsuite results have size limits,"
" and the useful information is generally either at the beginning"
" or the end of such outputs. If 0, never truncate logs.")
parser.add_argument(
"--failure-exit-code",
metavar="N",
type=int,
default=self.default_failure_exit_code,
help="Exit code the testsuite must use when at least one test"
" result shows a failure/error. By default, this is"
f" {self.default_failure_exit_code}. This option is useful when"
" running a testsuite in a continuous integration setup, as this"
" can make the testing process stop when there is a regression.")
parser.add_argument("sublist",
metavar="tests",
nargs="*",
default=[],
help="test")
# Add user defined options
self.add_options(parser)
# Parse options
self.main.parse_args(args)
assert self.main.args is not None
# If there is a chance for the logging to end up in a non-tty stream,
# disable colors. If not, be user-friendly and automatically show error
# outputs.
if (self.main.args.log_file or not isatty(sys.stdout)
or not isatty(sys.stderr)):
enable_colors = False
else: # interactive-only
enable_colors = True
self.main.args.show_error_output = True
self.colors = ColorConfig(enable_colors)
self.Fore = self.colors.Fore
self.Style = self.colors.Style
self.env = BaseEnv.from_env()
self.env.enable_colors = enable_colors
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths Keep the working dir as
# short as possible, to avoid the risk of having a path that's too long
# (a problem often seen on Windows, or when using WRS tools that have
# their own max path limitations).
#
# Note that we do make sure that working_dir is an absolute path, as we
# are likely to be changing directories when running each test. A
# relative path would no longer work under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, "new")
self.old_output_dir = os.path.join(d, "old")
if self.main.args.dev_temp:
# Use a temporary directory for developers: make sure it is an
# empty directory and disable cleanup to ease post-mortem
# investigation.
self.working_dir = os.path.abspath(self.main.args.dev_temp)
rm(self.working_dir, recursive=True)
mkdir(self.working_dir)
self.main.args.enable_cleanup = False
else:
# If the temp dir is supposed to be randomized, we need to create a
# subdirectory, so check that the parent directory exists first.
if not os.path.isdir(self.main.args.temp_dir):
logger.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
"", "tmp", os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results and create
# an index for it.
self.setup_result_dir()
self.report_index = ReportIndex(self.output_dir)
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.set_up()
# Retrieve the list of test
self.has_error = False
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(
job_provider=self.job_factory,
tokens=self.main.args.jobs,
# correct_result expects specifically TestFragment instances (a Job
# subclass), while Scheduler only guarantees Job instances.
# Test drivers are supposed to register only TestFragment
# instances, so the following cast should be fine.
collect=cast(Any, self.collect_result),
)
actions = DAG()
for parsed_test in self.test_list:
if not self.add_test(actions, parsed_test):
self.has_error = True
actions.check()
with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
fd.write(actions.as_dot())
# Run the tests. Note that when the testsuite aborts because of too
# many consecutive test failures, we still want to produce a report and
# exit through regular ways, to catch KeyboardInterrupt exceptions,
# which e3's scheduler uses to abort the execution loop, but only in
# such cases. In other words, let the exception propagates if it's the
# user that interrupted the testsuite.
try:
self.scheduler.run(actions)
except KeyboardInterrupt:
if not self.aborted_too_many_failures: # interactive-only
raise
self.report_index.write()
self.dump_testsuite_result()
if self.main.args.xunit_output:
dump_xunit_report(self, self.main.args.xunit_output)
if self.main.args.gaia_output:
dump_gaia_report(self, self.output_dir)
# Clean everything
self.tear_down()
# Return the appropriate status code: 1 when there is a framework
# issue, the failure status code from the --failure-exit-code=N option
# when there is a least one testcase failure, or 0.
statuses = {
s
for s, count in self.report_index.status_counters.items() if count
}
if self.has_error:
return 1
elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
return self.main.args.failure_exit_code
else:
return 0