def test_keyboard_interrupt(self):
"""Ensure that jobs can be interrupted."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return NopJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
# fake log_state that will raise a KeyboardInterrupt
def fake_log_state():
raise KeyboardInterrupt
s.log_state = fake_log_state
with pytest.raises(KeyboardInterrupt):
s.run(dag)
for k, v in results.items():
assert v.interrupted
def test_requeue(self):
"""Requeue test.
Same as previous example except that all tests are requeued
once.
"""
results = {}
def collect(job):
if job.uid not in results:
results[job.uid] = True
return True
else:
return False
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(Scheduler.simple_provider(NopJob),
tokens=2, collect=collect)
s.run(dag)
assert s.max_active_jobs == 2
assert results['1']
assert results['2']
def test_minimal_run2(self):
"""Test with two interdependent jobs."""
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2', predecessors=['1'])
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2)
s.run(dag)
assert s.max_active_jobs == 1
def test_minimal_run(self):
"""Test with only two independent jobs."""
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2)
s.run(dag)
assert s.max_active_jobs == 2
def test_ordering(self):
"""Test that jobs are ordered correctly."""
results = []
def collect(job):
results.append(job.uid)
dag = DAG()
dag.add_vertex("3")
dag.add_vertex("0")
dag.add_vertex("1")
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=1, collect=collect)
s.run(dag)
assert tuple(results) == ("0", "1", "3")
def test_collect_feedback_scheme(self):
"""Collect feedback construction.
Scheme in which if a job predecessor "fails" then job is skipped
In order to do that get_job and collect should have access to
common data. Note that scheduler ensure that these functions
are called sequentially.
"""
class SchedulerContext(object):
def __init__(self):
# Save in results tuples with first element being a bool
# indicating success or failure and the second the job itself
self.results = {}
def get_job(self, uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
# If any of the predecessor failed skip the job
for k in predecessors:
if not self.results[k][0]:
result.should_skip = True
return result
def collect(self, job):
if job.should_skip:
# Skipped jobs are considered failed
self.results[job.uid] = [False, job]
else:
# Job '2' is always failing
if job.uid == "2":
self.results[job.uid] = [False, job]
else:
self.results[job.uid] = [True, job]
dag = DAG()
dag.add_vertex("1")
dag.add_vertex("2")
dag.add_vertex("3", predecessors=["1", "2"])
dag.add_vertex("4", predecessors=["3"])
c = SchedulerContext()
s = Scheduler(c.get_job, tokens=2, collect=c.collect)
s.run(dag)
assert (
not c.results["2"][1].should_skip and not c.results["2"][0]
), 'job "2" is run and should be marked as failed'
assert c.results["3"][1].should_skip, 'job "3" should be skipped'
assert c.results["4"][1].should_skip, 'job "4" should be skipped'
def __init__(self, actions):
"""Object initializer.
:param actions: DAG of actions to perform.
:type actions: DAG
"""
self.actions = actions
self.new_fingerprints = {}
self.job_status = {}
self.set_scheduling_params()
self.scheduler = Scheduler(job_provider=self.get_job,
collect=self.collect,
queues=self.queues,
tokens=self.tokens,
job_timeout=self.job_timeout)
self.scheduler.run(self.actions)
def test_timeout(self):
"""Ensure that jobs are interrupted correctly on timeout."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return SleepJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
s.run(dag)
for k, v in results.items():
assert v.interrupted
def __init__(self, actions: DAG):
"""Object initializer.
:param actions: DAG of actions to perform.
"""
self.actions = actions
self.prev_fingerprints: Dict[str, Optional[Fingerprint]] = {}
self.new_fingerprints: Dict[str, Optional[Fingerprint]] = {}
self.job_status: Dict[str, ReturnValue] = {}
self.set_scheduling_params()
self.failure_source: Dict[str, Set[str]] = {}
self.scheduler = Scheduler(
job_provider=self.get_job,
collect=self.collect, # type: ignore
queues=self.queues,
tokens=self.tokens,
job_timeout=self.job_timeout,
)
self.scheduler.run(self.actions)
def test_skip(self):
"""Simple example in which all the tests are skipped."""
results = {}
def get_job(uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
result.should_skip = True
return result
def collect(job):
results[job.uid] = job.timing_info
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect)
s.run(dag)
# Check start_time end_time to be sure tests have not been run
for k, v in results.items():
assert v.start_time is None
assert v.stop_time is None
def run(self, action_list):
sch = Scheduler(self.get_job, self.collect)
sch.run(action_list)
class Walk:
"""An abstract class scheduling and executing a DAG of actions.
:ivar actions: DAG of actions to perform.
:vartype actions: DAG
:ivar prev_fingerprints: A dict of e3.fingerprint.Fingerprint objects,
indexed by the corresponding job ID. This dictionary contains
the former fingerprints a given job or None if there was no
such fingerprint.
(with the job corresponding to a given entry in the DAG of
actions).
:vartype prev_fingerprints: Dict[str, Optional[Fingerprint]]
:ivar new_fingerprints: A dict of e3.fingerprint.Fingerprint objects,
indexed by the corresponding job ID. This dictionary contains
the fingerprints we compute each time we create a new job
(with the job corresponding to a given entry in the DAG of
actions).
:vartype new_fingerprints: Dict[str, Optional[Fingerprint]]
:ivar job_status: A dictionary of job status (ReturnValue), indexed by
job unique IDs.
:vartype job_status: Dict[str, ReturnValue]
:ivar scheduler: The scheduler used to schedule and execute all
the actions.
:vartype scheduler: e3.job.scheduler.Scheduler
"""
def __init__(self, actions: DAG):
"""Object initializer.
:param actions: DAG of actions to perform.
"""
self.actions = actions
self.prev_fingerprints: Dict[str, Optional[Fingerprint]] = {}
self.new_fingerprints: Dict[str, Optional[Fingerprint]] = {}
self.job_status: Dict[str, ReturnValue] = {}
self.queues: Dict[str, int] = {}
self.tokens = 1
self.job_timeout = DEFAULT_JOB_MAX_DURATION
self.set_scheduling_params()
self.failure_source: Dict[str, Set[str]] = {}
self.scheduler = Scheduler(
job_provider=self.get_job,
collect=self.collect, # type: ignore
queues=self.queues,
tokens=self.tokens,
job_timeout=self.job_timeout,
)
self.scheduler.run(self.actions)
def set_scheduling_params(self) -> None:
"""Set the parameters used when creating the scheduler.
This method is expected to set the following attributes:
- self.queues: Same as parameter of the same name
in e3.job.scheduler.Scheduler.__init__.
- self.tokens: Likewise.
- self.job_timeout: Likewise.
This method provides a default setup where the scheduler has
1 token and 1 queue, and where the job's maximum duration is
DEFAULT_JOB_MAX_DURATION. Child classes requiring different
scheduling parameters should override this method.
"""
pass
def compute_fingerprint(
self,
uid: str,
data: Any,
is_prediction: bool = False) -> Optional[Fingerprint]:
"""Compute the given action's Fingerprint.
This method is expected to return a Fingerprint corresponding
to the state of the system should the given action be executed
and succesful. It can also return None if keeping track of
the result of past actions is not necessary.
This implementation always returns None. Child classes requiring
a different behavior may override this method.
:param uid: A unique Job ID.
:param data: Data associated to the job.
:param is_prediction: If True this is an attempt to compute the
fingerprint before launching the job. In that case if the
function returns None then the job will always be launched.
When False, this is the computation done after running the
job (that will be the final fingerprint saved for future
comparison)
"""
return None
def save_fingerprint(self, uid: str,
fingerprint: Optional[Fingerprint]) -> None:
"""Save the given fingerprint.
For systems that require fingerprint persistence, this method
is expected to save the fingerprint somewhere -- typically
inside a file. Passing None as the fingerprint causes
the fingerprint to be deleted instead.
This implementation does nothing. Child classes taking advantage
of fingerprint support should override this method to save
the fingerprint at the location of their choice and using
the format that suits them.
:param uid: A unique Job ID.
:param fingerprint: The fingerprint corresponding to the given
job, or None, if the fingerprint should be deleted instead
of saved.
"""
pass
def load_previous_fingerprint(self, uid: str) -> Optional[Fingerprint]:
"""Get the fingerprint from the given action's previous execution.
This method is expected to have the following behavior:
- If the given action has already previously been executed
and its fingerprint saved (see method "save_fingerprint"
above), then load and return it;
- Otherwise, return None.
This implementation always returns None, providing a behavior
where re-executing a given action always results in the
corresponding job being executed. Child classes requiring
a different behavior may override this method.
:param uid: A unique Job ID.
"""
return None
def should_execute_action(
self,
uid: str,
previous_fingerprint: Optional[Fingerprint],
new_fingerprint: Optional[Fingerprint],
) -> bool:
"""Return True if the given action should be performed.
The purpose of this function is to provide a way to determine,
based on the previous fingerprint, and the new one, whether
the user of this class wants us to launch the action or not.
The default implementation implements the following strategy:
- when fingerprints are not in use, always execution
the given action;
- when fingerprints are in use, execute the given action
if the fingerprint has changed.
However, child classes may want to override this method
to implement alternative strategies.
:param uid: A unique Job ID.
:param previous_fingerprint: The fingerprint from the previous
from the action's previous run. None if the action has not
been previously executed.
:param new_fingerprint: The fingerprint from the previous
from the action's previous run. None if the action has not
been previously executed.
"""
if previous_fingerprint is None or new_fingerprint is None:
return True
for pred_uid in self.actions.get_predecessors(uid):
if self.new_fingerprints[str(pred_uid)] is None:
# One of the predecessors has no fingerprint, so
# this node's new_fingerprint cannot tell us whether
# this dependency has changed or not. We therefore
# need to run this action.
return True
return previous_fingerprint != new_fingerprint
def create_skipped_job(
self,
uid: str,
data: Any,
predecessors: FrozenSet[str],
reason: Optional[str],
notify_end: Callable[[str], None],
status: ReturnValue = ReturnValue.failure,
) -> Job:
"""Return a failed job.
This method always returns an EmptyJob. Deriving classes may
override this method if they need something more specific.
:param uid: A unique Job ID.
:param data: Data associated to the job to create.
:param predecessors: A list of predecessor jobs, or None.
:param reason: If not None, the reason for creating a failed job.
:notify_end: Same as the notify_end parameter in Job.__init__.
"""
return EmptyJob(uid, data, notify_end, status=status)
@abc.abstractmethod
def create_job(
self,
uid: str,
data: Any,
predecessors: FrozenSet[str],
notify_end: Callable[[str], None],
) -> ProcessJob:
"""Create a ProcessJob.
:param uid: A unique Job ID.
:param data: Data associated to the job to create.
:param predecessors: A list of predecessor jobs, or None.
:notify_end: Same as the notify_end parameter in Job.__init__.
"""
pass # all: no cover
@abc.abstractmethod
def request_requeue(self, job: ProcessJob) -> bool:
"""Requeue the given job.
Return True if the job has been requeued, False otherwise.
:param job: The job to requeue.
"""
pass # all: no cover
def get_job(
self,
uid: str,
data: Any,
predecessors: FrozenSet[str],
notify_end: Callable[[str], None],
) -> Job:
"""Return a Job.
Same as self.create_job except that this function first checks
whether any of the predecessors might have failed, in which case
the failed job (creating using the create_skipped_job method)
is returned.
"""
# Get the latest fingerprint
self.prev_fingerprints[uid] = self.load_previous_fingerprint(uid)
# And reset the fingerprint on disk
self.save_fingerprint(uid, None)
self.new_fingerprints[uid] = self.compute_fingerprint(
uid, data, is_prediction=True)
# Check our predecessors. If any of them failed, then return
# a failed job.
failed_predecessors = [
k for k in predecessors if self.job_status[k] not in (
ReturnValue.success,
ReturnValue.skip,
ReturnValue.force_skip,
ReturnValue.unchanged,
)
]
if failed_predecessors:
force_fail = "Event failed because of prerequisite failure:\n"
force_fail += "\n".join(
[" " + str(self.actions[k]) for k in failed_predecessors])
# Compute the set of job that originate that failure
self.failure_source[uid] = set(
chain.from_iterable([
self.failure_source.get(k, [k])
for k in failed_predecessors
]))
force_fail += "\n\nOrigin(s) of failure:\n"
force_fail += "\n".join([
" " + str(self.actions[k]) for k in self.failure_source[uid]
])
return self.create_skipped_job(
uid,
data,
predecessors,
force_fail,
notify_end,
status=ReturnValue.force_fail,
)
if self.should_execute_action(uid, self.prev_fingerprints[uid],
self.new_fingerprints[uid]):
return self.create_job(uid, data, predecessors, notify_end)
else:
return self.create_skipped_job(uid,
data,
predecessors,
"skipped",
notify_end,
status=ReturnValue.skip)
def collect(self, job: ProcessJob) -> bool:
"""Collect all the results from the given job.
:param job: The job whose results we need to collect.
:return: True if the job is requeued, False otherwise
"""
# Only save the fingerprint if the job went as expected (either
# success or skipped). Since we already removed the previous
# fingerprint when we created the job, not saving the fingerprint
# ensures that we try that action again next time (as opposed
# to skipping it).
if job.status in (
ReturnValue.success,
ReturnValue.force_skip,
ReturnValue.skip,
ReturnValue.unchanged,
):
self.new_fingerprints[job.uid] = self.compute_fingerprint(
job.uid, job.data)
self.save_fingerprint(job.uid, self.new_fingerprints[job.uid])
self.job_status[job.uid] = ReturnValue(job.status)
if job.should_skip:
if job.status not in (ReturnValue.force_fail,
ReturnValue.force_skip):
logging.info(
"[%-10s %-9s %4ds] %s",
job.queue_name,
self.job_status[job.uid].name,
0,
job.data,
)
return False
logging.info(
"[%-10s %-9s %4ds] %s",
job.queue_name,
job.status.name,
int(job.timing_info.duration),
job.data,
)
requeued = False
if self.job_status[job.uid] == ReturnValue.notready:
requeued = self.request_requeue(job)
return requeued
def run_standard_mainloop(self, dag: DAG) -> None:
"""Run the main loop to execute test fragments in threads."""
assert self.main.args is not None
from e3.job import Job
from e3.testsuite.fragment import FragmentData, ThreadTestFragment
def job_factory(
uid: str,
data: Any,
predecessors: FrozenSet[str],
notify_end: Callable[[str], None],
) -> ThreadTestFragment:
"""Turn a DAG item into a ThreadTestFragment instance."""
assert isinstance(data, FragmentData)
# When passing return values from predecessors, remove current test
# name from the keys to ease referencing by user (the short
# fragment name can then be used by user without knowing the full
# node id).
key_prefix = data.driver.test_name + "."
key_prefix_len = len(key_prefix)
def filter_key(k: str) -> str:
if k.startswith(key_prefix):
return k[key_prefix_len:]
else:
return k
return ThreadTestFragment(
uid,
data.driver,
data.callback,
{filter_key(k): self.return_values[k]
for k in predecessors},
notify_end,
self.running_status,
)
def collect_result(job: Job) -> bool:
"""Collect test results from the given fragment."""
assert isinstance(job, ThreadTestFragment)
self.return_values[job.uid] = job.return_value
self.collect_result(job)
# In the e3.job.scheduler API, collect returning "True" means
# "requeue the job". We never want to do that.
return False
# Create a scheduler to run all fragments for the testsuite main loop
scheduler = Scheduler(
job_provider=job_factory,
tokens=self.main.args.jobs,
collect=collect_result,
)
# Run the tests. Note that when the testsuite aborts because of too
# many consecutive test failures, we still want to produce a report and
# exit through regular ways, to catch KeyboardInterrupt exceptions,
# which e3's scheduler uses to abort the execution loop, but only in
# such cases. In other words, let the exception propagates if it's the
# user that interrupted the testsuite.
try:
scheduler.run(dag)
except KeyboardInterrupt:
if not self.aborted_too_many_failures: # interactive-only
raise
class Walk(object):
"""An abstract class scheduling and executing a DAG of actions.
:ivar actions: DAG of actions to perform.
:vartype actions: DAG
:ivar new_fingerprints: A dict of e3.fingerprint.Fingerprint objects,
indexed by the corresponding job ID. This dictionary contains
the fingerprints we compute each time we create a new job
(with the job corresponding to a given entry in the DAG of
actions).
Note that there are situations where the user might not be able
to compute the fingerprint without running the job and waiting
for it to be complete (see method "can_predict_new_fingerprint").
In that situation, the fingerprint is only inserted after the job
completes succesfully.
:vartype new_fingerprints: dict
:ivar job_status: A dictionary of job status (ReturnValue), indexed by
job unique IDs.
:vartype job_status: dict
:ivar scheduler: The scheduler used to schedule and execute all
the actions.
:vartype scheduler: e3.job.scheduler.Scheduler
"""
def __init__(self, actions):
"""Object initializer.
:param actions: DAG of actions to perform.
:type actions: DAG
"""
self.actions = actions
self.new_fingerprints = {}
self.job_status = {}
self.set_scheduling_params()
self.scheduler = Scheduler(job_provider=self.get_job,
collect=self.collect,
queues=self.queues,
tokens=self.tokens,
job_timeout=self.job_timeout)
self.scheduler.run(self.actions)
def set_scheduling_params(self):
"""Set the parameters used when creating the scheduler.
This method is expected to set the following attributes:
- self.queues: Same as parameter of the same name
in e3.job.scheduler.Scheduler.__init__.
- self.tokens: Likewise.
- self.job_timeout: Likewise.
This method provides a default setup where the scheduler has
1 token and 1 queue, and where the job's maximum duration is
DEFAULT_JOB_MAX_DURATION. Child classes requiring different
scheduling parameters should override this method.
"""
self.queues = None
self.tokens = 1
self.job_timeout = DEFAULT_JOB_MAX_DURATION
def can_predict_new_fingerprint(self, uid, data):
"""Return True if a job's fingerprint is computable before running it.
This function should return True for all the jobs where
the fingerprint can be calculated before running the job,
False otherwise.
Ideally, we should be able to compute the fingerprint of every
job before we actually run the job, as this then allows us to
use that fingerprint to decide whether or not the job should be
run or not. Unfortunately, there can be situations where this is
not possible; consider for instance the case where the job is
to fetch some data, and the fingerprint needs to include that
contents.
For jobs were the fingerprint cannot be predicted, what will
happen is that the scheduler will execute that job, and then
compute the fingerprint at the end, instead of computing it
before.
By default, this method assumes that all jobs have predictable
fingerprints, and so always returns True. Child classes should
override this method is they have jobs for which the fingerprint
cannot be computed ahead of running the job.
:param uid: A unique Job ID.
:type uid: str
:param data: Data associated to the job.
:type data: T
:rtype: bool
"""
return True
def compute_new_fingerprint(self, uid, data):
"""Compute the given action's Fingerprint.
This method is expected to return a Fingerprint corresponding
to the state of the system should the given action be executed
and succesful. It can also return None if keeping track of
the result of past actions is not necessary.
This implementation always returns None. Child classes requiring
a different behavior may override this method.
:param uid: A unique Job ID.
:type uid: str
:param data: Data associated to the job.
:type data: T
:rtype: e3.fingerprint.Fingerprint | None
"""
return None
def save_fingerprint(self, uid, fingerprint):
"""Save the given fingerprint.
For systems that require fingerprint persistence, this method
is expected to save the fingerprint somewhere -- typically
inside a file. Passing None as the fingerprint causes
the fingerprint to be deleted instead.
This implementation does nothing. Child classes taking advantage
of fingerprint support should override this method to save
the fingerprint at the location of their choice and using
the format that suits them.
:param uid: A unique Job ID.
:type uid: str
:param fingerprint: The fingerprint corresponding to the given
job, or None, if the fingerprint should be deleted instead
of saved.
:type fingerprint: e3.fingerprint.Fingerprint | None
:rtype: None
"""
pass
def load_previous_fingerprint(self, uid):
"""Get the fingerprint from the given action's previous execution.
This method is expected to have the following behavior:
- If the given action has already previously been executed
and its fingerprint saved (see method "save_fingerprint"
above), then load and return it;
- Otherwise, return None.
This implementation always returns None, providing a behavior
where re-executing a given action always results in the
corresponding job being executed. Child classes requiring
a different behavior may override this method.
:param uid: A unique Job ID.
:type uid: str
:rtype: e3.fingerprint.Fingerprint | None
"""
return None
def should_execute_action(self, uid, previous_fingerprint,
new_fingerprint):
"""Return True if the given action should be performed.
The purpose of this function is to provide a way to determine,
based on the previous fingerprint, and the new one, whether
the user of this class wants us to launch the action or not.
The default implementation implements the following strategy:
- when fingerprints are not in use, always execution
the given action;
- when fingerprints are in use, execute the given action
if the fingerprint has changed.
However, child classes may want to override this method
to implement alternative strategies.
:param uid: A unique Job ID.
:type uid: str
:param previous_fingerprint: The fingerprint from the previous
from the action's previous run. None if the action has not
been previously executed.
:type previous_fingerprint: e3.fingerprint.Fingerprint | None
:param new_fingerprint: The fingerprint from the previous
from the action's previous run. None if the action has not
been previously executed.
:type new_fingerprint: e3.fingerprint.Fingerprint | None
:rtype: bool
"""
if previous_fingerprint is None or new_fingerprint is None:
return True
for pred_uid in self.actions.get_predecessors(uid):
if self.new_fingerprints[pred_uid] is None:
# One of the predecessors has no fingerprint, so
# this node's new_fingerprint cannot tell us whether
# this dependency has changed or not. We therefore
# need to run this action.
return True
return previous_fingerprint != new_fingerprint
def create_failed_job(self, uid, data, predecessors, reason, notify_end):
"""Return a failed job.
This method always returns an EmptyJob. Deriving classes may
override this method if they need something more specific.
:param uid: A unique Job ID.
:type uid: str
:param data: Data associated to the job to create.
:type data: T
:param predecessors: A list of predecessor jobs, or None.
:type predecessors: list[e3.job.Job] | None
:param reason: If not None, the reason for creating a failed job.
:type reason: str | None
:notify_end: Same as the notify_end parameter in Job.__init__.
:type notify_end: str -> None
:rtype: Job
"""
return EmptyJob(uid, data, notify_end, status=ReturnValue.failure)
@abc.abstractmethod
def create_job(self, uid, data, predecessors, notify_end):
"""Create a ProcessJob.
:param uid: A unique Job ID.
:type uid: str
:param data: Data associated to the job to create.
:type data: T
:param predecessors: A list of predecessor jobs, or None.
:type predecessors: list[e3.job.Job] | None
:notify_end: Same as the notify_end parameter in Job.__init__.
:type notify_end: str -> None
:rtype: ProcessJob
"""
pass # all: no cover
@abc.abstractmethod
def request_requeue(self, job):
"""Requeue the given job.
Return True if the job has been requeued, False otherwise.
:param job: The job to requeue.
:type job: ProcessJob
:rtype: bool
"""
pass # all: no cover
def get_job(self, uid, data, predecessors, notify_end):
"""Return a Job.
Same as self.create_job except that this function first checks
whether any of the predecessors might have failed, in which case
the failed job (creating using the create_failed_job method)
is returned.
:rtype: Job
"""
prev_fingerprint = self.load_previous_fingerprint(uid)
self.save_fingerprint(uid, None)
can_predict_new_fingerprint = \
self.can_predict_new_fingerprint(uid, data)
if can_predict_new_fingerprint:
self.new_fingerprints[uid] = \
self.compute_new_fingerprint(uid, data)
else:
# Set the new fingerprint's value to None for now.
# The actual fingerprint will be provided when we collect
# the job's results if the job is succesful.
self.new_fingerprints[uid] = None
# Check our predecessors. If any of them failed, then return
# a failed job.
failed_predecessors = [
k for k in predecessors
if self.job_status[k] not in (ReturnValue.success,
ReturnValue.skip,
ReturnValue.force_skip)
]
if failed_predecessors:
force_fail = "Event failed because of prerequisite failure:\n"
force_fail += "\n".join(
[" " + str(self.actions[k]) for k in failed_predecessors])
return self.create_failed_job(uid, data, predecessors, force_fail,
notify_end)
if not can_predict_new_fingerprint or \
self.should_execute_action(uid, prev_fingerprint,
self.new_fingerprints[uid]):
return self.create_job(uid, data, predecessors, notify_end)
else:
return EmptyJob(uid, data, notify_end, status=ReturnValue.skip)
def collect(self, job):
"""Collect all the results from the given job.
:param job: The job whose results we need to collect.
:type job: ProcessJob
"""
# Only save the fingerprint if the job went as expected (either
# success or skipped). Since we already removed the previous
# fingerprint when we created the job, not saving the fingerprint
# ensures that we try that action again next time (as opposed
# to skipping it).
if job.status in (ReturnValue.success, ReturnValue.force_skip,
ReturnValue.skip):
if not self.can_predict_new_fingerprint(job.uid, job.data):
self.new_fingerprints[job.uid] = \
self.compute_new_fingerprint(job.uid, job.data)
self.save_fingerprint(job.uid, self.new_fingerprints[job.uid])
self.job_status[job.uid] = ReturnValue(job.status)
if job.should_skip:
if job.status not in (ReturnValue.force_fail,
ReturnValue.force_skip):
logging.info("[queue=%-10s status=%-10s time=%5ds] %s",
job.queue_name, self.job_status[job.uid].name, 0,
job.data)
return False
logging.info("[queue=%-10s status=%-10s time=%5ds] %s", job.queue_name,
job.status.name, int(job.timing_info.duration), job.data)
requeued = False
if self.job_status[job.uid] == ReturnValue.notready:
requeued = self.request_requeue(job)
return requeued
class TestsuiteCore(object):
"""Testsuite Core driver.
This class is the base of Testsuite class and should not be instanciated.
It's not recommended to override any of the functions declared in it.
See documentation of Testsuite class for overridable methods and
variables.
"""
def __init__(self, root_dir):
"""Testsuite constructor.
:param root_dir: root dir of the testsuite. Usually the directory in
which testsuite.py and runtest.py are located
:type root_dir: str | unicode
"""
self.root_dir = os.path.abspath(root_dir)
self.test_dir = os.path.join(self.root_dir, self.TEST_SUBDIR)
self.consecutive_failures = 0
self.return_values = {}
self.results = {}
self.test_counter = 0
self.test_status_counters = {s: 0 for s in TestStatus}
def test_result_filename(self, test_name):
"""Return the name of the file in which the result are stored.
:param test_case_file: path to a test case scenario relative to the
test directory
:type test_case_file: str | unicode
:param variant: the test variant
:type variant: str
:return: the test name. Note that test names should not contain path
separators
:rtype: str | unicode
"""
return os.path.join(self.output_dir, test_name + '.yaml')
def job_factory(self, uid, data, predecessors, notify_end):
"""Run internal function.
See e3.job.scheduler
"""
# we assume that data[0] is the test instance and data[1] the method
# to call
# When passing return values from predecessors, remove current test
# name from the keys to ease referencing by user (the short fragment
# name can then be used by user without knowing the full node id).
key_prefix = data[0].test_name + '.'
key_prefix_len = len(key_prefix)
def filter_key(k):
if k.startswith(key_prefix):
return k[key_prefix_len:]
else:
return k
return TestFragment(
uid, data[0], data[1],
{filter_key(k): self.return_values[k]
for k in predecessors}, notify_end)
def testsuite_main(self, args=None):
"""Main for the main testsuite script.
:param args: command line arguments. If None use sys.argv
:type args: list[str] | None
"""
self.main = Main(platform_args=self.CROSS_SUPPORT)
# Add common options
parser = self.main.argument_parser
parser.add_argument("-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir")
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"--max-consecutive-failures",
default=0,
help="If there are more than N consecutive failures, the testsuite"
" is aborted. If set to 0 (default) then the testsuite will never"
" be stopped")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)")
parser.add_argument("--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space")
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously")
parser.add_argument(
"--show-error-output",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.")
parser.add_argument('sublist',
metavar='tests',
nargs='*',
default=[],
help='test')
# Add user defined options
self.add_options()
# parse options
self.main.parse_args(args)
self.env = BaseEnv.from_env()
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths
# Keep the working dir as short as possible, to avoid the risk
# of having a path that's too long (a problem often seen on
# Windows, or when using WRS tools that have their own max path
# limitations).
# Note that we do make sure that working_dir is an absolute
# path, as we are likely to be changing directories when
# running each test. A relative path would no longer work
# under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, 'new')
self.old_output_dir = os.path.join(d, 'old')
if not os.path.isdir(self.main.args.temp_dir):
logging.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
'', 'tmp', os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results
self.setup_result_dir()
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.tear_up()
# Retrieve the list of test
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(job_provider=self.job_factory,
collect=self.collect_result,
tokens=self.main.args.jobs)
actions = DAG()
for test in self.test_list:
self.parse_test(actions, test)
with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd:
fd.write(actions.as_dot())
self.scheduler.run(actions)
self.dump_testsuite_result()
# Clean everything
self.tear_down()
return 0
def parse_test(self, actions, test_case_file):
"""Register a test.
:param actions: the dag of actions for the testsuite
:type actions: e3.collection.dag.DAG
:param test_case_file: filename containing the testcase
:type test_case_file: str
"""
# Load testcase file
test_env = load_with_config(
os.path.join(self.test_dir, test_case_file),
Env().to_dict())
# Ensure that the test_env act like a dictionary
if not isinstance(test_env, collections.Mapping):
test_env = {
'test_name': self.test_name(test_case_file),
'test_yaml_wrong_content': test_env
}
logger.error("abort test because of invalid test.yaml")
return
# Add to the test environment the directory in which the test.yaml is
# stored
test_env['test_dir'] = os.path.join(self.env.test_dir,
os.path.dirname(test_case_file))
test_env['test_case_file'] = test_case_file
test_env['test_name'] = self.test_name(test_case_file)
test_env['working_dir'] = os.path.join(self.env.working_dir,
test_env['test_name'])
if 'driver' in test_env:
driver = test_env['driver']
else:
driver = self.default_driver
logger.debug('set driver to %s' % driver)
if driver not in self.DRIVERS or \
not issubclass(self.DRIVERS[driver], TestDriver):
logger.error('cannot find driver for %s' % test_case_file)
return
try:
instance = self.DRIVERS[driver](self.env, test_env)
instance.add_test(actions)
except Exception as e:
error_msg = str(e)
error_msg += "Traceback:\n"
error_msg += "\n".join(traceback.format_tb(sys.exc_traceback))
logger.error(error_msg)
return
def dump_testsuite_result(self):
"""To be implemented."""
pass
def collect_result(self, job):
"""Run internal function.
:param job: a job that is finished
:type job: TestFragment
"""
self.return_values[job.uid] = job.return_value
while job.test_instance.result_queue:
result = job.test_instance.result_queue.pop()
logging.info('%-12s %s' % (str(result.status), result.test_name))
assert result.test_name not in self.results, \
'cannot push twice results for %s' % result.test_name
with open(self.test_result_filename(result.test_name), 'wb') as fd:
yaml.dump(result, fd)
self.results[result.test_name] = result.status
self.test_counter += 1
self.test_status_counters[result.status] += 1
return False
def setup_result_dir(self):
"""Create the output directory in which the results are stored."""
if os.path.isdir(self.old_output_dir):
rm(self.old_output_dir, True)
if os.path.isdir(self.output_dir):
mv(self.output_dir, self.old_output_dir)
mkdir(self.output_dir)
if self.main.args.dump_environ:
with open(os.path.join(self.output_dir, 'environ.sh'), 'w') as f:
for var_name in sorted(os.environ):
f.write('export %s=%s\n' %
(var_name, quote_arg(os.environ[var_name])))
def testsuite_main(self, args=None):
"""Main for the main testsuite script.
:param args: command line arguments. If None use sys.argv
:type args: list[str] | None
"""
self.main = Main(platform_args=self.CROSS_SUPPORT)
# Add common options
parser = self.main.argument_parser
parser.add_argument("-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir")
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"--max-consecutive-failures",
default=0,
help="If there are more than N consecutive failures, the testsuite"
" is aborted. If set to 0 (default) then the testsuite will never"
" be stopped")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)")
parser.add_argument("--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space")
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously")
parser.add_argument(
"--show-error-output",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.")
parser.add_argument('sublist',
metavar='tests',
nargs='*',
default=[],
help='test')
# Add user defined options
self.add_options()
# parse options
self.main.parse_args(args)
self.env = BaseEnv.from_env()
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths
# Keep the working dir as short as possible, to avoid the risk
# of having a path that's too long (a problem often seen on
# Windows, or when using WRS tools that have their own max path
# limitations).
# Note that we do make sure that working_dir is an absolute
# path, as we are likely to be changing directories when
# running each test. A relative path would no longer work
# under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, 'new')
self.old_output_dir = os.path.join(d, 'old')
if not os.path.isdir(self.main.args.temp_dir):
logging.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
'', 'tmp', os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results
self.setup_result_dir()
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.tear_up()
# Retrieve the list of test
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(job_provider=self.job_factory,
collect=self.collect_result,
tokens=self.main.args.jobs)
actions = DAG()
for test in self.test_list:
self.parse_test(actions, test)
with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd:
fd.write(actions.as_dot())
self.scheduler.run(actions)
self.dump_testsuite_result()
# Clean everything
self.tear_down()
return 0
class TestsuiteCore:
"""Testsuite Core driver.
This class is the base of Testsuite class and should not be instanciated.
It's not recommended to override any of the functions declared in it.
See documentation of Testsuite class for overridable methods and
variables.
"""
def __init__(self,
root_dir: Optional[str] = None,
testsuite_name: str = "Untitled testsute") -> None:
"""Testsuite constructor.
:param root_dir: Root directory for the testsuite. If left to None, use
the directory containing the Python module that created self's
class.
:param testsuite_name: Name for this testsuite. It can be used to
provide a title in some report formats.
"""
if root_dir is None:
root_dir = os.path.dirname(inspect.getfile(type(self)))
self.root_dir = os.path.abspath(root_dir)
self.test_dir = os.path.join(self.root_dir, self.tests_subdir)
logger.debug("Test directory: %s", self.test_dir)
self.consecutive_failures = 0
self.return_values: Dict[str, Any] = {}
self.result_tracebacks: Dict[str, List[str]] = {}
self.testsuite_name = testsuite_name
self.aborted_too_many_failures = False
"""
Whether the testsuite aborted because of too many consecutive test
failures (see the --max-consecutive-failures command-line option).
"""
# Mypy does not support decorators on properties, so keep the actual
# implementations for deprecated properties in methods.
@deprecated(2)
def _test_counter(self) -> int:
return len(self.report_index.entries)
@deprecated(2)
def _test_status_counters(self) -> Dict[TestStatus, int]:
return self.report_index.status_counters
@deprecated(2)
def _results(self) -> Dict[str, TestStatus]:
return {
e.test_name: e.status
for e in self.report_index.entries.values()
}
@property
def test_counter(self) -> int:
"""Return the number of test results in the report.
Warning: this method is obsolete and will be removed in the future.
"""
return self._test_counter()
@property
def test_status_counters(self) -> Dict[TestStatus, int]:
"""Return test result counts per test status.
Warning: this method is obsolete and will be removed in the future.
"""
return self._test_status_counters()
@property
def results(self) -> Dict[str, TestStatus]:
"""Return a mapping from test names to results.
Warning: this method is obsolete and will be removed in the future.
"""
return self._results()
def test_result_filename(self, test_name: str) -> str:
"""Return the name of the file in which the result are stored.
:param test_name: Name of the test for this result file.
"""
return os.path.join(self.output_dir, test_name + ".yaml")
def job_factory(self, uid: str, data: Any, predecessors: FrozenSet[str],
notify_end: Callable[[str], None]) -> TestFragment:
"""Run internal function.
See e3.job.scheduler
"""
# We assume that data[0] is the test instance and data[1] the method
# to call.
# When passing return values from predecessors, remove current test
# name from the keys to ease referencing by user (the short fragment
# name can then be used by user without knowing the full node id).
key_prefix = data[0].test_name + "."
key_prefix_len = len(key_prefix)
def filter_key(k: str) -> str:
if k.startswith(key_prefix):
return k[key_prefix_len:]
else:
return k
return TestFragment(
uid,
data[0],
data[1],
{filter_key(k): self.return_values[k]
for k in predecessors},
notify_end,
)
def testsuite_main(self, args: Optional[List[str]] = None) -> int:
"""Main for the main testsuite script.
:param args: Command line arguments. If None, use `sys.argv`.
:return: The testsuite status code (0 for success, a positive for
failure).
"""
self.main = Main(platform_args=True)
# Add common options
parser = self.main.argument_parser
parser.add_argument(
"-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir",
)
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"-d",
"--dev-temp",
nargs="?",
default=None,
const="tmp",
help="Unlike --temp-dir, use this very directory to store"
" testsuite temporaries (i.e. no random subdirectory). Also"
" automatically disable temp dir cleanup, to be developer"
" friendly. If no directory is provided, use the local"
" \"tmp\" directory")
parser.add_argument(
"--max-consecutive-failures",
"-M",
metavar="N",
type=int,
default=self.default_max_consecutive_failures,
help="Number of test failures (FAIL or ERROR) that trigger the"
" abortion of the testuite. If zero, this behavior is disabled. In"
" some cases, aborting the testsuite when there are just too many"
" failures saves time and costs: the software to test/environment"
" is too broken, there is no point to continue running the"
" testsuite.")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)",
)
parser.add_argument(
"--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space",
)
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously",
)
parser.add_argument(
"--show-error-output",
"-E",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.",
)
parser.add_argument(
"--show-time-info",
action="store_true",
help="Display time information for test results, if available")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.",
)
parser.add_argument(
"--xunit-output",
dest="xunit_output",
metavar="FILE",
help="Output testsuite report to the given file in the standard"
" XUnit XML format. This is useful to display results in"
" continuous build systems such as Jenkins.",
)
parser.add_argument(
"--gaia-output",
action="store_true",
help="Output a GAIA-compatible testsuite report next to the YAML"
" report.")
parser.add_argument(
"--truncate-logs",
"-T",
metavar="N",
type=int,
default=200,
help="When outputs (for instance subprocess outputs) exceed 2*N"
" lines, only include the first and last N lines in logs. This is"
" necessary when storage for testsuite results have size limits,"
" and the useful information is generally either at the beginning"
" or the end of such outputs. If 0, never truncate logs.")
parser.add_argument(
"--failure-exit-code",
metavar="N",
type=int,
default=self.default_failure_exit_code,
help="Exit code the testsuite must use when at least one test"
" result shows a failure/error. By default, this is"
f" {self.default_failure_exit_code}. This option is useful when"
" running a testsuite in a continuous integration setup, as this"
" can make the testing process stop when there is a regression.")
parser.add_argument("sublist",
metavar="tests",
nargs="*",
default=[],
help="test")
# Add user defined options
self.add_options(parser)
# Parse options
self.main.parse_args(args)
assert self.main.args is not None
# If there is a chance for the logging to end up in a non-tty stream,
# disable colors. If not, be user-friendly and automatically show error
# outputs.
if (self.main.args.log_file or not isatty(sys.stdout)
or not isatty(sys.stderr)):
enable_colors = False
else: # interactive-only
enable_colors = True
self.main.args.show_error_output = True
self.colors = ColorConfig(enable_colors)
self.Fore = self.colors.Fore
self.Style = self.colors.Style
self.env = BaseEnv.from_env()
self.env.enable_colors = enable_colors
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths Keep the working dir as
# short as possible, to avoid the risk of having a path that's too long
# (a problem often seen on Windows, or when using WRS tools that have
# their own max path limitations).
#
# Note that we do make sure that working_dir is an absolute path, as we
# are likely to be changing directories when running each test. A
# relative path would no longer work under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, "new")
self.old_output_dir = os.path.join(d, "old")
if self.main.args.dev_temp:
# Use a temporary directory for developers: make sure it is an
# empty directory and disable cleanup to ease post-mortem
# investigation.
self.working_dir = os.path.abspath(self.main.args.dev_temp)
rm(self.working_dir, recursive=True)
mkdir(self.working_dir)
self.main.args.enable_cleanup = False
else:
# If the temp dir is supposed to be randomized, we need to create a
# subdirectory, so check that the parent directory exists first.
if not os.path.isdir(self.main.args.temp_dir):
logger.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
"", "tmp", os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results and create
# an index for it.
self.setup_result_dir()
self.report_index = ReportIndex(self.output_dir)
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.set_up()
# Retrieve the list of test
self.has_error = False
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(
job_provider=self.job_factory,
tokens=self.main.args.jobs,
# correct_result expects specifically TestFragment instances (a Job
# subclass), while Scheduler only guarantees Job instances.
# Test drivers are supposed to register only TestFragment
# instances, so the following cast should be fine.
collect=cast(Any, self.collect_result),
)
actions = DAG()
for parsed_test in self.test_list:
if not self.add_test(actions, parsed_test):
self.has_error = True
actions.check()
with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
fd.write(actions.as_dot())
# Run the tests. Note that when the testsuite aborts because of too
# many consecutive test failures, we still want to produce a report and
# exit through regular ways, to catch KeyboardInterrupt exceptions,
# which e3's scheduler uses to abort the execution loop, but only in
# such cases. In other words, let the exception propagates if it's the
# user that interrupted the testsuite.
try:
self.scheduler.run(actions)
except KeyboardInterrupt:
if not self.aborted_too_many_failures: # interactive-only
raise
self.report_index.write()
self.dump_testsuite_result()
if self.main.args.xunit_output:
dump_xunit_report(self, self.main.args.xunit_output)
if self.main.args.gaia_output:
dump_gaia_report(self, self.output_dir)
# Clean everything
self.tear_down()
# Return the appropriate status code: 1 when there is a framework
# issue, the failure status code from the --failure-exit-code=N option
# when there is a least one testcase failure, or 0.
statuses = {
s
for s, count in self.report_index.status_counters.items() if count
}
if self.has_error:
return 1
elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
return self.main.args.failure_exit_code
else:
return 0
def get_test_list(self, sublist: List[str]) -> List[ParsedTest]:
"""Retrieve the list of tests.
:param sublist: A list of tests scenarios or patterns.
"""
# Use a mapping: test name -> ParsedTest when building the result, as
# several patterns in "sublist" may yield the same testcase.
testcases: Dict[str, ParsedTest] = {}
test_finders = self.test_finders
def add_testcase(test: ParsedTest) -> None:
testcases[test.test_name] = test
def helper(spec: str) -> None:
pattern: Optional[Pattern[str]] = None
# If the given pattern is a directory, do not go through the whole
# tests subdirectory.
if os.path.isdir(spec):
root = spec
else:
root = self.test_dir
try:
pattern = re.compile(spec)
except re.error as exc:
logger.debug(
"Test pattern is not a valid regexp, try to match it"
" as-is: {}".format(exc))
pattern = re.compile(re.escape(spec))
# For each directory in the requested subdir, ask our test finders
# to probe for a testcase. Register matches.
for dirpath, dirnames, filenames in os.walk(root,
followlinks=True):
# If the directory name does not match the given pattern, skip
# it.
if pattern is not None and not pattern.search(dirpath):
continue
# The first test finder that has a match "wins". When handling
# test data, we want to deal only with absolute paths, so get
# the absolute name now.
dirpath = os.path.abspath(dirpath)
for tf in test_finders:
try:
test_or_list = tf.probe(self, dirpath, dirnames,
filenames)
except ProbingError as exc:
self.has_error = True
logger.error(str(exc))
break
if isinstance(test_or_list, list):
for t in test_or_list:
add_testcase(t)
break
elif test_or_list is not None:
add_testcase(test_or_list)
break
# If specific tests are requested, only look for them. Otherwise, just
# look in the tests subdirectory.
if sublist:
for s in sublist:
helper(s)
else:
helper(self.test_dir)
result = list(testcases.values())
logger.info("Found {} tests".format(len(result)))
logger.debug("tests:\n " + "\n ".join(t.test_dir for t in result))
return result
def add_test(self, actions: DAG, parsed_test: ParsedTest) -> bool:
"""Register a test to run.
:param actions: The dag of actions for the testsuite.
:param parsed_test: Test to instantiate.
:return: Whether the test was successfully registered.
"""
test_name = parsed_test.test_name
# Complete the test environment
test_env = dict(parsed_test.test_env)
test_env["test_dir"] = parsed_test.test_dir
test_env["test_name"] = test_name
assert isinstance(self.env.working_dir, str)
test_env["working_dir"] = os.path.join(self.env.working_dir, test_name)
# Fetch the test driver to use
driver = parsed_test.driver_cls
if not driver:
if self.default_driver:
driver = self.test_driver_map[self.default_driver]
else:
logger.error("missing driver for test '{}'".format(test_name))
return False
# Finally run the driver instantiation
try:
instance = driver(self.env, test_env)
instance.Fore = self.Fore
instance.Style = self.Style
instance.add_test(actions)
except Exception as e:
error_msg = str(e)
error_msg += "\nTraceback:\n"
error_msg += "\n".join(traceback.format_tb(sys.exc_info()[2]))
logger.error(error_msg)
return False
return True
def dump_testsuite_result(self) -> None:
"""Log a summary of test results.
Subclasses are free to override this to do whatever is suitable for
them.
"""
lines = ['Summary:']
# Display test count for each status, but only for status that have
# at least one test. Sort them by status value, to get consistent
# order.
def sort_key(couple: Tuple[TestStatus, int]) -> Any:
status, _ = couple
return status.value
stats = sorted(
((status, count)
for status, count in self.report_index.status_counters.items()
if count),
key=sort_key)
for status, count in stats:
lines.append(' {}{: <12}{} {}'.format(status.color(self.colors),
status.name,
self.Style.RESET_ALL,
count))
if not stats:
lines.append(' <no test result>')
logger.info('\n'.join(lines))
# Dump the comment file
with open(os.path.join(self.output_dir, "comment"), "w") as f:
self.write_comment_file(f)
def collect_result(self, job: TestFragment) -> bool:
"""Run internal function.
:param job: A job that is finished.
"""
assert self.main.args
# Keep track of the number of consecutive failures seen so far if it
# reaches the maximum number allowed, we must abort the testsuite.
max_consecutive_failures = self.main.args.max_consecutive_failures
consecutive_failures = 0
self.return_values[job.uid] = job.return_value
while job.test_instance.result_queue:
result, tb = job.test_instance.result_queue.pop()
# The test results that reach this point are special: there were
# serialized/deserialized through YAML, so the Log layer
# disappeared.
assert result.status is not None
# Log the test result. If error output is requested and the test
# failed unexpectedly, show the detailed logs.
log_line = summary_line(result, self.colors,
self.main.args.show_time_info)
if (self.main.args.show_error_output and result.status
not in (TestStatus.PASS, TestStatus.XFAIL,
TestStatus.XPASS, TestStatus.SKIP)):
def format_log(log: Log) -> str:
return "\n" + str(log) + self.Style.RESET_ALL
if result.diff:
log_line += format_log(result.diff)
else:
log_line += format_log(result.log)
logger.info(log_line)
def indented_tb(tb: List[str]) -> str:
return "".join(" {}".format(line) for line in tb)
assert result.test_name not in self.report_index.entries, (
"cannot push twice results for {}"
"\nFirst push happened at:"
"\n{}"
"\nThis one happened at:"
"\n{}".format(
result.test_name,
indented_tb(self.result_tracebacks[result.test_name]),
indented_tb(tb),
))
with open(self.test_result_filename(result.test_name), "w") as fd:
yaml.dump(result, fd)
self.report_index.add_result(result)
self.result_tracebacks[result.test_name] = tb
# Update the number of consecutive failures, aborting the testsuite
# if appropriate
if result.status in (TestStatus.ERROR, TestStatus.FAIL):
consecutive_failures += 1
if (max_consecutive_failures > 0
and consecutive_failures >= max_consecutive_failures):
self.aborted_too_many_failures = True
logger.error(
"Too many consecutive failures, aborting the testsuite"
)
raise KeyboardInterrupt
else:
consecutive_failures = 0
return False
def setup_result_dir(self) -> None:
"""Create the output directory in which the results are stored."""
assert self.main.args
if os.path.isdir(self.old_output_dir):
rm(self.old_output_dir, True)
if os.path.isdir(self.output_dir):
mv(self.output_dir, self.old_output_dir)
mkdir(self.output_dir)
if self.main.args.dump_environ:
with open(os.path.join(self.output_dir, "environ.sh"), "w") as f:
for var_name in sorted(os.environ):
f.write("export {}={}\n".format(
var_name, quote_arg(os.environ[var_name])))
# Unlike the previous methods, the following ones are supposed to be
# overriden.
@property
def tests_subdir(self) -> str:
"""
Return the subdirectory in which tests are looked for.
The returned directory name is considered relative to the root
testsuite directory (self.root_dir).
"""
raise NotImplementedError
@property
def test_driver_map(self) -> Dict[str, Type[TestDriver]]:
"""Return a map from test driver names to TestDriver subclasses.
Test finders will be able to use this map to fetch the test drivers
referenced in testcases.
"""
raise NotImplementedError
@property
def default_driver(self) -> Optional[str]:
"""Return the name of the default driver for testcases.
When tests do not query a specific driver, the one associated to this
name is used instead. If this property returns None, all tests are
required to query a driver.
"""
raise NotImplementedError
def test_name(self, test_dir: str) -> str:
"""Compute the test name given a testcase spec.
This function can be overridden. By default it uses the name of the
test directory. Note that the test name should be a valid filename (not
dir seprators, or special characters such as ``:``, ...).
"""
raise NotImplementedError
@property
def test_finders(self) -> List[TestFinder]:
"""Return test finders to probe tests directories."""
raise NotImplementedError
def add_options(self, parser: argparse.ArgumentParser) -> None:
"""Add testsuite specific switches.
Subclasses can override this method to add their own testsuite
command-line options.
:param parser: Parser for command-line arguments. See
<https://docs.python.org/3/library/argparse.html> for usage.
"""
raise NotImplementedError
def set_up(self) -> None:
"""Execute operations before running the testsuite.
Before running this, command-line arguments were parsed. After this
returns, the testsuite will look for testcases.
By default, this does nothing. Overriding this method allows testsuites
to prepare the execution of the testsuite depending on their needs. For
instance:
* process testsuite-specific options;
* initialize environment variables;
* adjust self.env (object forwarded to test drivers).
"""
raise NotImplementedError
def tear_down(self) -> None:
"""Execute operation when finalizing the testsuite.
By default, this cleans the working (temporary) directory in which the
tests were run.
"""
raise NotImplementedError
def write_comment_file(self, comment_file: IO[str]) -> None:
"""Write the comment file's content.
:param comment_file: File descriptor for the comment file. Overriding
methods should only call its "write" method (or print to it).
"""
raise NotImplementedError
@property
def default_max_consecutive_failures(self) -> int:
"""Return the default maximum number of consecutive failures.
In some cases, aborting the testsuite when there are just too many
failures saves time and costs: the software to test/environment is too
broken, there is no point to continue running the testsuite.
This property must return the number of test failures (FAIL or ERROR)
that trigger the abortion of the testuite. If zero, this behavior is
disabled.
"""
raise NotImplementedError
@property
def default_failure_exit_code(self) -> int:
"""Return the default exit code when at least one test fails."""
raise NotImplementedError
def testsuite_main(self, args: Optional[List[str]] = None) -> int:
"""Main for the main testsuite script.
:param args: Command line arguments. If None, use `sys.argv`.
:return: The testsuite status code (0 for success, a positive for
failure).
"""
self.main = Main(platform_args=True)
# Add common options
parser = self.main.argument_parser
parser.add_argument(
"-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir",
)
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"-d",
"--dev-temp",
nargs="?",
default=None,
const="tmp",
help="Unlike --temp-dir, use this very directory to store"
" testsuite temporaries (i.e. no random subdirectory). Also"
" automatically disable temp dir cleanup, to be developer"
" friendly. If no directory is provided, use the local"
" \"tmp\" directory")
parser.add_argument(
"--max-consecutive-failures",
"-M",
metavar="N",
type=int,
default=self.default_max_consecutive_failures,
help="Number of test failures (FAIL or ERROR) that trigger the"
" abortion of the testuite. If zero, this behavior is disabled. In"
" some cases, aborting the testsuite when there are just too many"
" failures saves time and costs: the software to test/environment"
" is too broken, there is no point to continue running the"
" testsuite.")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)",
)
parser.add_argument(
"--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space",
)
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously",
)
parser.add_argument(
"--show-error-output",
"-E",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.",
)
parser.add_argument(
"--show-time-info",
action="store_true",
help="Display time information for test results, if available")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.",
)
parser.add_argument(
"--xunit-output",
dest="xunit_output",
metavar="FILE",
help="Output testsuite report to the given file in the standard"
" XUnit XML format. This is useful to display results in"
" continuous build systems such as Jenkins.",
)
parser.add_argument(
"--gaia-output",
action="store_true",
help="Output a GAIA-compatible testsuite report next to the YAML"
" report.")
parser.add_argument(
"--truncate-logs",
"-T",
metavar="N",
type=int,
default=200,
help="When outputs (for instance subprocess outputs) exceed 2*N"
" lines, only include the first and last N lines in logs. This is"
" necessary when storage for testsuite results have size limits,"
" and the useful information is generally either at the beginning"
" or the end of such outputs. If 0, never truncate logs.")
parser.add_argument(
"--failure-exit-code",
metavar="N",
type=int,
default=self.default_failure_exit_code,
help="Exit code the testsuite must use when at least one test"
" result shows a failure/error. By default, this is"
f" {self.default_failure_exit_code}. This option is useful when"
" running a testsuite in a continuous integration setup, as this"
" can make the testing process stop when there is a regression.")
parser.add_argument("sublist",
metavar="tests",
nargs="*",
default=[],
help="test")
# Add user defined options
self.add_options(parser)
# Parse options
self.main.parse_args(args)
assert self.main.args is not None
# If there is a chance for the logging to end up in a non-tty stream,
# disable colors. If not, be user-friendly and automatically show error
# outputs.
if (self.main.args.log_file or not isatty(sys.stdout)
or not isatty(sys.stderr)):
enable_colors = False
else: # interactive-only
enable_colors = True
self.main.args.show_error_output = True
self.colors = ColorConfig(enable_colors)
self.Fore = self.colors.Fore
self.Style = self.colors.Style
self.env = BaseEnv.from_env()
self.env.enable_colors = enable_colors
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths Keep the working dir as
# short as possible, to avoid the risk of having a path that's too long
# (a problem often seen on Windows, or when using WRS tools that have
# their own max path limitations).
#
# Note that we do make sure that working_dir is an absolute path, as we
# are likely to be changing directories when running each test. A
# relative path would no longer work under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, "new")
self.old_output_dir = os.path.join(d, "old")
if self.main.args.dev_temp:
# Use a temporary directory for developers: make sure it is an
# empty directory and disable cleanup to ease post-mortem
# investigation.
self.working_dir = os.path.abspath(self.main.args.dev_temp)
rm(self.working_dir, recursive=True)
mkdir(self.working_dir)
self.main.args.enable_cleanup = False
else:
# If the temp dir is supposed to be randomized, we need to create a
# subdirectory, so check that the parent directory exists first.
if not os.path.isdir(self.main.args.temp_dir):
logger.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
"", "tmp", os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results and create
# an index for it.
self.setup_result_dir()
self.report_index = ReportIndex(self.output_dir)
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.set_up()
# Retrieve the list of test
self.has_error = False
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(
job_provider=self.job_factory,
tokens=self.main.args.jobs,
# correct_result expects specifically TestFragment instances (a Job
# subclass), while Scheduler only guarantees Job instances.
# Test drivers are supposed to register only TestFragment
# instances, so the following cast should be fine.
collect=cast(Any, self.collect_result),
)
actions = DAG()
for parsed_test in self.test_list:
if not self.add_test(actions, parsed_test):
self.has_error = True
actions.check()
with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
fd.write(actions.as_dot())
# Run the tests. Note that when the testsuite aborts because of too
# many consecutive test failures, we still want to produce a report and
# exit through regular ways, to catch KeyboardInterrupt exceptions,
# which e3's scheduler uses to abort the execution loop, but only in
# such cases. In other words, let the exception propagates if it's the
# user that interrupted the testsuite.
try:
self.scheduler.run(actions)
except KeyboardInterrupt:
if not self.aborted_too_many_failures: # interactive-only
raise
self.report_index.write()
self.dump_testsuite_result()
if self.main.args.xunit_output:
dump_xunit_report(self, self.main.args.xunit_output)
if self.main.args.gaia_output:
dump_gaia_report(self, self.output_dir)
# Clean everything
self.tear_down()
# Return the appropriate status code: 1 when there is a framework
# issue, the failure status code from the --failure-exit-code=N option
# when there is a least one testcase failure, or 0.
statuses = {
s
for s, count in self.report_index.status_counters.items() if count
}
if self.has_error:
return 1
elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
return self.main.args.failure_exit_code
else:
return 0