def test_keyboard_interrupt(self): """Ensure that jobs can be interrupted.""" results = {} pytest.importorskip('psutil') def get_job(uid, data, predecessors, notify_end): return NopJob(uid, data, notify_end) def collect(job): results[job.uid] = job dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2) # fake log_state that will raise a KeyboardInterrupt def fake_log_state(): raise KeyboardInterrupt s.log_state = fake_log_state with pytest.raises(KeyboardInterrupt): s.run(dag) for k, v in results.items(): assert v.interrupted
def test_requeue(self): """Requeue test. Same as previous example except that all tests are requeued once. """ results = {} def collect(job): if job.uid not in results: results[job.uid] = True return True else: return False # This time test with two interdependent jobs dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2, collect=collect) s.run(dag) assert s.max_active_jobs == 2 assert results['1'] assert results['2']
def test_minimal_run2(self): """Test with two interdependent jobs.""" dag = DAG() dag.add_vertex('1') dag.add_vertex('2', predecessors=['1']) s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2) s.run(dag) assert s.max_active_jobs == 1
def test_minimal_run(self): """Test with only two independent jobs.""" dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2) s.run(dag) assert s.max_active_jobs == 2
def test_ordering(self): """Test that jobs are ordered correctly.""" results = [] def collect(job): results.append(job.uid) dag = DAG() dag.add_vertex("3") dag.add_vertex("0") dag.add_vertex("1") s = Scheduler(Scheduler.simple_provider(NopJob), tokens=1, collect=collect) s.run(dag) assert tuple(results) == ("0", "1", "3")
def test_collect_feedback_scheme(self): """Collect feedback construction. Scheme in which if a job predecessor "fails" then job is skipped In order to do that get_job and collect should have access to common data. Note that scheduler ensure that these functions are called sequentially. """ class SchedulerContext(object): def __init__(self): # Save in results tuples with first element being a bool # indicating success or failure and the second the job itself self.results = {} def get_job(self, uid, data, predecessors, notify_end): result = NopJob(uid, data, notify_end) # If any of the predecessor failed skip the job for k in predecessors: if not self.results[k][0]: result.should_skip = True return result def collect(self, job): if job.should_skip: # Skipped jobs are considered failed self.results[job.uid] = [False, job] else: # Job '2' is always failing if job.uid == "2": self.results[job.uid] = [False, job] else: self.results[job.uid] = [True, job] dag = DAG() dag.add_vertex("1") dag.add_vertex("2") dag.add_vertex("3", predecessors=["1", "2"]) dag.add_vertex("4", predecessors=["3"]) c = SchedulerContext() s = Scheduler(c.get_job, tokens=2, collect=c.collect) s.run(dag) assert ( not c.results["2"][1].should_skip and not c.results["2"][0] ), 'job "2" is run and should be marked as failed' assert c.results["3"][1].should_skip, 'job "3" should be skipped' assert c.results["4"][1].should_skip, 'job "4" should be skipped'
def __init__(self, actions): """Object initializer. :param actions: DAG of actions to perform. :type actions: DAG """ self.actions = actions self.new_fingerprints = {} self.job_status = {} self.set_scheduling_params() self.scheduler = Scheduler(job_provider=self.get_job, collect=self.collect, queues=self.queues, tokens=self.tokens, job_timeout=self.job_timeout) self.scheduler.run(self.actions)
def test_timeout(self): """Ensure that jobs are interrupted correctly on timeout.""" results = {} pytest.importorskip('psutil') def get_job(uid, data, predecessors, notify_end): return SleepJob(uid, data, notify_end) def collect(job): results[job.uid] = job dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2) s.run(dag) for k, v in results.items(): assert v.interrupted
def __init__(self, actions: DAG): """Object initializer. :param actions: DAG of actions to perform. """ self.actions = actions self.prev_fingerprints: Dict[str, Optional[Fingerprint]] = {} self.new_fingerprints: Dict[str, Optional[Fingerprint]] = {} self.job_status: Dict[str, ReturnValue] = {} self.set_scheduling_params() self.failure_source: Dict[str, Set[str]] = {} self.scheduler = Scheduler( job_provider=self.get_job, collect=self.collect, # type: ignore queues=self.queues, tokens=self.tokens, job_timeout=self.job_timeout, ) self.scheduler.run(self.actions)
def test_skip(self): """Simple example in which all the tests are skipped.""" results = {} def get_job(uid, data, predecessors, notify_end): result = NopJob(uid, data, notify_end) result.should_skip = True return result def collect(job): results[job.uid] = job.timing_info # This time test with two interdependent jobs dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(get_job, tokens=2, collect=collect) s.run(dag) # Check start_time end_time to be sure tests have not been run for k, v in results.items(): assert v.start_time is None assert v.stop_time is None
def run(self, action_list): sch = Scheduler(self.get_job, self.collect) sch.run(action_list)
class Walk: """An abstract class scheduling and executing a DAG of actions. :ivar actions: DAG of actions to perform. :vartype actions: DAG :ivar prev_fingerprints: A dict of e3.fingerprint.Fingerprint objects, indexed by the corresponding job ID. This dictionary contains the former fingerprints a given job or None if there was no such fingerprint. (with the job corresponding to a given entry in the DAG of actions). :vartype prev_fingerprints: Dict[str, Optional[Fingerprint]] :ivar new_fingerprints: A dict of e3.fingerprint.Fingerprint objects, indexed by the corresponding job ID. This dictionary contains the fingerprints we compute each time we create a new job (with the job corresponding to a given entry in the DAG of actions). :vartype new_fingerprints: Dict[str, Optional[Fingerprint]] :ivar job_status: A dictionary of job status (ReturnValue), indexed by job unique IDs. :vartype job_status: Dict[str, ReturnValue] :ivar scheduler: The scheduler used to schedule and execute all the actions. :vartype scheduler: e3.job.scheduler.Scheduler """ def __init__(self, actions: DAG): """Object initializer. :param actions: DAG of actions to perform. """ self.actions = actions self.prev_fingerprints: Dict[str, Optional[Fingerprint]] = {} self.new_fingerprints: Dict[str, Optional[Fingerprint]] = {} self.job_status: Dict[str, ReturnValue] = {} self.queues: Dict[str, int] = {} self.tokens = 1 self.job_timeout = DEFAULT_JOB_MAX_DURATION self.set_scheduling_params() self.failure_source: Dict[str, Set[str]] = {} self.scheduler = Scheduler( job_provider=self.get_job, collect=self.collect, # type: ignore queues=self.queues, tokens=self.tokens, job_timeout=self.job_timeout, ) self.scheduler.run(self.actions) def set_scheduling_params(self) -> None: """Set the parameters used when creating the scheduler. This method is expected to set the following attributes: - self.queues: Same as parameter of the same name in e3.job.scheduler.Scheduler.__init__. - self.tokens: Likewise. - self.job_timeout: Likewise. This method provides a default setup where the scheduler has 1 token and 1 queue, and where the job's maximum duration is DEFAULT_JOB_MAX_DURATION. Child classes requiring different scheduling parameters should override this method. """ pass def compute_fingerprint( self, uid: str, data: Any, is_prediction: bool = False) -> Optional[Fingerprint]: """Compute the given action's Fingerprint. This method is expected to return a Fingerprint corresponding to the state of the system should the given action be executed and succesful. It can also return None if keeping track of the result of past actions is not necessary. This implementation always returns None. Child classes requiring a different behavior may override this method. :param uid: A unique Job ID. :param data: Data associated to the job. :param is_prediction: If True this is an attempt to compute the fingerprint before launching the job. In that case if the function returns None then the job will always be launched. When False, this is the computation done after running the job (that will be the final fingerprint saved for future comparison) """ return None def save_fingerprint(self, uid: str, fingerprint: Optional[Fingerprint]) -> None: """Save the given fingerprint. For systems that require fingerprint persistence, this method is expected to save the fingerprint somewhere -- typically inside a file. Passing None as the fingerprint causes the fingerprint to be deleted instead. This implementation does nothing. Child classes taking advantage of fingerprint support should override this method to save the fingerprint at the location of their choice and using the format that suits them. :param uid: A unique Job ID. :param fingerprint: The fingerprint corresponding to the given job, or None, if the fingerprint should be deleted instead of saved. """ pass def load_previous_fingerprint(self, uid: str) -> Optional[Fingerprint]: """Get the fingerprint from the given action's previous execution. This method is expected to have the following behavior: - If the given action has already previously been executed and its fingerprint saved (see method "save_fingerprint" above), then load and return it; - Otherwise, return None. This implementation always returns None, providing a behavior where re-executing a given action always results in the corresponding job being executed. Child classes requiring a different behavior may override this method. :param uid: A unique Job ID. """ return None def should_execute_action( self, uid: str, previous_fingerprint: Optional[Fingerprint], new_fingerprint: Optional[Fingerprint], ) -> bool: """Return True if the given action should be performed. The purpose of this function is to provide a way to determine, based on the previous fingerprint, and the new one, whether the user of this class wants us to launch the action or not. The default implementation implements the following strategy: - when fingerprints are not in use, always execution the given action; - when fingerprints are in use, execute the given action if the fingerprint has changed. However, child classes may want to override this method to implement alternative strategies. :param uid: A unique Job ID. :param previous_fingerprint: The fingerprint from the previous from the action's previous run. None if the action has not been previously executed. :param new_fingerprint: The fingerprint from the previous from the action's previous run. None if the action has not been previously executed. """ if previous_fingerprint is None or new_fingerprint is None: return True for pred_uid in self.actions.get_predecessors(uid): if self.new_fingerprints[str(pred_uid)] is None: # One of the predecessors has no fingerprint, so # this node's new_fingerprint cannot tell us whether # this dependency has changed or not. We therefore # need to run this action. return True return previous_fingerprint != new_fingerprint def create_skipped_job( self, uid: str, data: Any, predecessors: FrozenSet[str], reason: Optional[str], notify_end: Callable[[str], None], status: ReturnValue = ReturnValue.failure, ) -> Job: """Return a failed job. This method always returns an EmptyJob. Deriving classes may override this method if they need something more specific. :param uid: A unique Job ID. :param data: Data associated to the job to create. :param predecessors: A list of predecessor jobs, or None. :param reason: If not None, the reason for creating a failed job. :notify_end: Same as the notify_end parameter in Job.__init__. """ return EmptyJob(uid, data, notify_end, status=status) @abc.abstractmethod def create_job( self, uid: str, data: Any, predecessors: FrozenSet[str], notify_end: Callable[[str], None], ) -> ProcessJob: """Create a ProcessJob. :param uid: A unique Job ID. :param data: Data associated to the job to create. :param predecessors: A list of predecessor jobs, or None. :notify_end: Same as the notify_end parameter in Job.__init__. """ pass # all: no cover @abc.abstractmethod def request_requeue(self, job: ProcessJob) -> bool: """Requeue the given job. Return True if the job has been requeued, False otherwise. :param job: The job to requeue. """ pass # all: no cover def get_job( self, uid: str, data: Any, predecessors: FrozenSet[str], notify_end: Callable[[str], None], ) -> Job: """Return a Job. Same as self.create_job except that this function first checks whether any of the predecessors might have failed, in which case the failed job (creating using the create_skipped_job method) is returned. """ # Get the latest fingerprint self.prev_fingerprints[uid] = self.load_previous_fingerprint(uid) # And reset the fingerprint on disk self.save_fingerprint(uid, None) self.new_fingerprints[uid] = self.compute_fingerprint( uid, data, is_prediction=True) # Check our predecessors. If any of them failed, then return # a failed job. failed_predecessors = [ k for k in predecessors if self.job_status[k] not in ( ReturnValue.success, ReturnValue.skip, ReturnValue.force_skip, ReturnValue.unchanged, ) ] if failed_predecessors: force_fail = "Event failed because of prerequisite failure:\n" force_fail += "\n".join( [" " + str(self.actions[k]) for k in failed_predecessors]) # Compute the set of job that originate that failure self.failure_source[uid] = set( chain.from_iterable([ self.failure_source.get(k, [k]) for k in failed_predecessors ])) force_fail += "\n\nOrigin(s) of failure:\n" force_fail += "\n".join([ " " + str(self.actions[k]) for k in self.failure_source[uid] ]) return self.create_skipped_job( uid, data, predecessors, force_fail, notify_end, status=ReturnValue.force_fail, ) if self.should_execute_action(uid, self.prev_fingerprints[uid], self.new_fingerprints[uid]): return self.create_job(uid, data, predecessors, notify_end) else: return self.create_skipped_job(uid, data, predecessors, "skipped", notify_end, status=ReturnValue.skip) def collect(self, job: ProcessJob) -> bool: """Collect all the results from the given job. :param job: The job whose results we need to collect. :return: True if the job is requeued, False otherwise """ # Only save the fingerprint if the job went as expected (either # success or skipped). Since we already removed the previous # fingerprint when we created the job, not saving the fingerprint # ensures that we try that action again next time (as opposed # to skipping it). if job.status in ( ReturnValue.success, ReturnValue.force_skip, ReturnValue.skip, ReturnValue.unchanged, ): self.new_fingerprints[job.uid] = self.compute_fingerprint( job.uid, job.data) self.save_fingerprint(job.uid, self.new_fingerprints[job.uid]) self.job_status[job.uid] = ReturnValue(job.status) if job.should_skip: if job.status not in (ReturnValue.force_fail, ReturnValue.force_skip): logging.info( "[%-10s %-9s %4ds] %s", job.queue_name, self.job_status[job.uid].name, 0, job.data, ) return False logging.info( "[%-10s %-9s %4ds] %s", job.queue_name, job.status.name, int(job.timing_info.duration), job.data, ) requeued = False if self.job_status[job.uid] == ReturnValue.notready: requeued = self.request_requeue(job) return requeued
def run_standard_mainloop(self, dag: DAG) -> None: """Run the main loop to execute test fragments in threads.""" assert self.main.args is not None from e3.job import Job from e3.testsuite.fragment import FragmentData, ThreadTestFragment def job_factory( uid: str, data: Any, predecessors: FrozenSet[str], notify_end: Callable[[str], None], ) -> ThreadTestFragment: """Turn a DAG item into a ThreadTestFragment instance.""" assert isinstance(data, FragmentData) # When passing return values from predecessors, remove current test # name from the keys to ease referencing by user (the short # fragment name can then be used by user without knowing the full # node id). key_prefix = data.driver.test_name + "." key_prefix_len = len(key_prefix) def filter_key(k: str) -> str: if k.startswith(key_prefix): return k[key_prefix_len:] else: return k return ThreadTestFragment( uid, data.driver, data.callback, {filter_key(k): self.return_values[k] for k in predecessors}, notify_end, self.running_status, ) def collect_result(job: Job) -> bool: """Collect test results from the given fragment.""" assert isinstance(job, ThreadTestFragment) self.return_values[job.uid] = job.return_value self.collect_result(job) # In the e3.job.scheduler API, collect returning "True" means # "requeue the job". We never want to do that. return False # Create a scheduler to run all fragments for the testsuite main loop scheduler = Scheduler( job_provider=job_factory, tokens=self.main.args.jobs, collect=collect_result, ) # Run the tests. Note that when the testsuite aborts because of too # many consecutive test failures, we still want to produce a report and # exit through regular ways, to catch KeyboardInterrupt exceptions, # which e3's scheduler uses to abort the execution loop, but only in # such cases. In other words, let the exception propagates if it's the # user that interrupted the testsuite. try: scheduler.run(dag) except KeyboardInterrupt: if not self.aborted_too_many_failures: # interactive-only raise
class Walk(object): """An abstract class scheduling and executing a DAG of actions. :ivar actions: DAG of actions to perform. :vartype actions: DAG :ivar new_fingerprints: A dict of e3.fingerprint.Fingerprint objects, indexed by the corresponding job ID. This dictionary contains the fingerprints we compute each time we create a new job (with the job corresponding to a given entry in the DAG of actions). Note that there are situations where the user might not be able to compute the fingerprint without running the job and waiting for it to be complete (see method "can_predict_new_fingerprint"). In that situation, the fingerprint is only inserted after the job completes succesfully. :vartype new_fingerprints: dict :ivar job_status: A dictionary of job status (ReturnValue), indexed by job unique IDs. :vartype job_status: dict :ivar scheduler: The scheduler used to schedule and execute all the actions. :vartype scheduler: e3.job.scheduler.Scheduler """ def __init__(self, actions): """Object initializer. :param actions: DAG of actions to perform. :type actions: DAG """ self.actions = actions self.new_fingerprints = {} self.job_status = {} self.set_scheduling_params() self.scheduler = Scheduler(job_provider=self.get_job, collect=self.collect, queues=self.queues, tokens=self.tokens, job_timeout=self.job_timeout) self.scheduler.run(self.actions) def set_scheduling_params(self): """Set the parameters used when creating the scheduler. This method is expected to set the following attributes: - self.queues: Same as parameter of the same name in e3.job.scheduler.Scheduler.__init__. - self.tokens: Likewise. - self.job_timeout: Likewise. This method provides a default setup where the scheduler has 1 token and 1 queue, and where the job's maximum duration is DEFAULT_JOB_MAX_DURATION. Child classes requiring different scheduling parameters should override this method. """ self.queues = None self.tokens = 1 self.job_timeout = DEFAULT_JOB_MAX_DURATION def can_predict_new_fingerprint(self, uid, data): """Return True if a job's fingerprint is computable before running it. This function should return True for all the jobs where the fingerprint can be calculated before running the job, False otherwise. Ideally, we should be able to compute the fingerprint of every job before we actually run the job, as this then allows us to use that fingerprint to decide whether or not the job should be run or not. Unfortunately, there can be situations where this is not possible; consider for instance the case where the job is to fetch some data, and the fingerprint needs to include that contents. For jobs were the fingerprint cannot be predicted, what will happen is that the scheduler will execute that job, and then compute the fingerprint at the end, instead of computing it before. By default, this method assumes that all jobs have predictable fingerprints, and so always returns True. Child classes should override this method is they have jobs for which the fingerprint cannot be computed ahead of running the job. :param uid: A unique Job ID. :type uid: str :param data: Data associated to the job. :type data: T :rtype: bool """ return True def compute_new_fingerprint(self, uid, data): """Compute the given action's Fingerprint. This method is expected to return a Fingerprint corresponding to the state of the system should the given action be executed and succesful. It can also return None if keeping track of the result of past actions is not necessary. This implementation always returns None. Child classes requiring a different behavior may override this method. :param uid: A unique Job ID. :type uid: str :param data: Data associated to the job. :type data: T :rtype: e3.fingerprint.Fingerprint | None """ return None def save_fingerprint(self, uid, fingerprint): """Save the given fingerprint. For systems that require fingerprint persistence, this method is expected to save the fingerprint somewhere -- typically inside a file. Passing None as the fingerprint causes the fingerprint to be deleted instead. This implementation does nothing. Child classes taking advantage of fingerprint support should override this method to save the fingerprint at the location of their choice and using the format that suits them. :param uid: A unique Job ID. :type uid: str :param fingerprint: The fingerprint corresponding to the given job, or None, if the fingerprint should be deleted instead of saved. :type fingerprint: e3.fingerprint.Fingerprint | None :rtype: None """ pass def load_previous_fingerprint(self, uid): """Get the fingerprint from the given action's previous execution. This method is expected to have the following behavior: - If the given action has already previously been executed and its fingerprint saved (see method "save_fingerprint" above), then load and return it; - Otherwise, return None. This implementation always returns None, providing a behavior where re-executing a given action always results in the corresponding job being executed. Child classes requiring a different behavior may override this method. :param uid: A unique Job ID. :type uid: str :rtype: e3.fingerprint.Fingerprint | None """ return None def should_execute_action(self, uid, previous_fingerprint, new_fingerprint): """Return True if the given action should be performed. The purpose of this function is to provide a way to determine, based on the previous fingerprint, and the new one, whether the user of this class wants us to launch the action or not. The default implementation implements the following strategy: - when fingerprints are not in use, always execution the given action; - when fingerprints are in use, execute the given action if the fingerprint has changed. However, child classes may want to override this method to implement alternative strategies. :param uid: A unique Job ID. :type uid: str :param previous_fingerprint: The fingerprint from the previous from the action's previous run. None if the action has not been previously executed. :type previous_fingerprint: e3.fingerprint.Fingerprint | None :param new_fingerprint: The fingerprint from the previous from the action's previous run. None if the action has not been previously executed. :type new_fingerprint: e3.fingerprint.Fingerprint | None :rtype: bool """ if previous_fingerprint is None or new_fingerprint is None: return True for pred_uid in self.actions.get_predecessors(uid): if self.new_fingerprints[pred_uid] is None: # One of the predecessors has no fingerprint, so # this node's new_fingerprint cannot tell us whether # this dependency has changed or not. We therefore # need to run this action. return True return previous_fingerprint != new_fingerprint def create_failed_job(self, uid, data, predecessors, reason, notify_end): """Return a failed job. This method always returns an EmptyJob. Deriving classes may override this method if they need something more specific. :param uid: A unique Job ID. :type uid: str :param data: Data associated to the job to create. :type data: T :param predecessors: A list of predecessor jobs, or None. :type predecessors: list[e3.job.Job] | None :param reason: If not None, the reason for creating a failed job. :type reason: str | None :notify_end: Same as the notify_end parameter in Job.__init__. :type notify_end: str -> None :rtype: Job """ return EmptyJob(uid, data, notify_end, status=ReturnValue.failure) @abc.abstractmethod def create_job(self, uid, data, predecessors, notify_end): """Create a ProcessJob. :param uid: A unique Job ID. :type uid: str :param data: Data associated to the job to create. :type data: T :param predecessors: A list of predecessor jobs, or None. :type predecessors: list[e3.job.Job] | None :notify_end: Same as the notify_end parameter in Job.__init__. :type notify_end: str -> None :rtype: ProcessJob """ pass # all: no cover @abc.abstractmethod def request_requeue(self, job): """Requeue the given job. Return True if the job has been requeued, False otherwise. :param job: The job to requeue. :type job: ProcessJob :rtype: bool """ pass # all: no cover def get_job(self, uid, data, predecessors, notify_end): """Return a Job. Same as self.create_job except that this function first checks whether any of the predecessors might have failed, in which case the failed job (creating using the create_failed_job method) is returned. :rtype: Job """ prev_fingerprint = self.load_previous_fingerprint(uid) self.save_fingerprint(uid, None) can_predict_new_fingerprint = \ self.can_predict_new_fingerprint(uid, data) if can_predict_new_fingerprint: self.new_fingerprints[uid] = \ self.compute_new_fingerprint(uid, data) else: # Set the new fingerprint's value to None for now. # The actual fingerprint will be provided when we collect # the job's results if the job is succesful. self.new_fingerprints[uid] = None # Check our predecessors. If any of them failed, then return # a failed job. failed_predecessors = [ k for k in predecessors if self.job_status[k] not in (ReturnValue.success, ReturnValue.skip, ReturnValue.force_skip) ] if failed_predecessors: force_fail = "Event failed because of prerequisite failure:\n" force_fail += "\n".join( [" " + str(self.actions[k]) for k in failed_predecessors]) return self.create_failed_job(uid, data, predecessors, force_fail, notify_end) if not can_predict_new_fingerprint or \ self.should_execute_action(uid, prev_fingerprint, self.new_fingerprints[uid]): return self.create_job(uid, data, predecessors, notify_end) else: return EmptyJob(uid, data, notify_end, status=ReturnValue.skip) def collect(self, job): """Collect all the results from the given job. :param job: The job whose results we need to collect. :type job: ProcessJob """ # Only save the fingerprint if the job went as expected (either # success or skipped). Since we already removed the previous # fingerprint when we created the job, not saving the fingerprint # ensures that we try that action again next time (as opposed # to skipping it). if job.status in (ReturnValue.success, ReturnValue.force_skip, ReturnValue.skip): if not self.can_predict_new_fingerprint(job.uid, job.data): self.new_fingerprints[job.uid] = \ self.compute_new_fingerprint(job.uid, job.data) self.save_fingerprint(job.uid, self.new_fingerprints[job.uid]) self.job_status[job.uid] = ReturnValue(job.status) if job.should_skip: if job.status not in (ReturnValue.force_fail, ReturnValue.force_skip): logging.info("[queue=%-10s status=%-10s time=%5ds] %s", job.queue_name, self.job_status[job.uid].name, 0, job.data) return False logging.info("[queue=%-10s status=%-10s time=%5ds] %s", job.queue_name, job.status.name, int(job.timing_info.duration), job.data) requeued = False if self.job_status[job.uid] == ReturnValue.notready: requeued = self.request_requeue(job) return requeued
class TestsuiteCore(object): """Testsuite Core driver. This class is the base of Testsuite class and should not be instanciated. It's not recommended to override any of the functions declared in it. See documentation of Testsuite class for overridable methods and variables. """ def __init__(self, root_dir): """Testsuite constructor. :param root_dir: root dir of the testsuite. Usually the directory in which testsuite.py and runtest.py are located :type root_dir: str | unicode """ self.root_dir = os.path.abspath(root_dir) self.test_dir = os.path.join(self.root_dir, self.TEST_SUBDIR) self.consecutive_failures = 0 self.return_values = {} self.results = {} self.test_counter = 0 self.test_status_counters = {s: 0 for s in TestStatus} def test_result_filename(self, test_name): """Return the name of the file in which the result are stored. :param test_case_file: path to a test case scenario relative to the test directory :type test_case_file: str | unicode :param variant: the test variant :type variant: str :return: the test name. Note that test names should not contain path separators :rtype: str | unicode """ return os.path.join(self.output_dir, test_name + '.yaml') def job_factory(self, uid, data, predecessors, notify_end): """Run internal function. See e3.job.scheduler """ # we assume that data[0] is the test instance and data[1] the method # to call # When passing return values from predecessors, remove current test # name from the keys to ease referencing by user (the short fragment # name can then be used by user without knowing the full node id). key_prefix = data[0].test_name + '.' key_prefix_len = len(key_prefix) def filter_key(k): if k.startswith(key_prefix): return k[key_prefix_len:] else: return k return TestFragment( uid, data[0], data[1], {filter_key(k): self.return_values[k] for k in predecessors}, notify_end) def testsuite_main(self, args=None): """Main for the main testsuite script. :param args: command line arguments. If None use sys.argv :type args: list[str] | None """ self.main = Main(platform_args=self.CROSS_SUPPORT) # Add common options parser = self.main.argument_parser parser.add_argument("-o", "--output-dir", metavar="DIR", default="./out", help="select output dir") parser.add_argument("-t", "--temp-dir", metavar="DIR", default=Env().tmp_dir) parser.add_argument( "--max-consecutive-failures", default=0, help="If there are more than N consecutive failures, the testsuite" " is aborted. If set to 0 (default) then the testsuite will never" " be stopped") parser.add_argument( "--keep-old-output-dir", default=False, action="store_true", help="This is default with this testsuite framework. The option" " is kept only to keep backward compatibility of invocation with" " former framework (gnatpython.testdriver)") parser.add_argument("--disable-cleanup", dest="enable_cleanup", action="store_false", default=True, help="disable cleanup of working space") parser.add_argument( "-j", "--jobs", dest="jobs", type=int, metavar="N", default=Env().build.cpu.cores, help="Specify the number of jobs to run simultaneously") parser.add_argument( "--show-error-output", action="store_true", help="When testcases fail, display their output. This is for" " convenience for interactive use.") parser.add_argument( "--dump-environ", dest="dump_environ", action="store_true", default=False, help="Dump all environment variables in a file named environ.sh," " located in the output directory (see --output-dir). This" " file can then be sourced from a Bourne shell to recreate" " the environement that existed when this testsuite was run" " to produce a given testsuite report.") parser.add_argument('sublist', metavar='tests', nargs='*', default=[], help='test') # Add user defined options self.add_options() # parse options self.main.parse_args(args) self.env = BaseEnv.from_env() self.env.root_dir = self.root_dir self.env.test_dir = self.test_dir # At this stage compute commonly used paths # Keep the working dir as short as possible, to avoid the risk # of having a path that's too long (a problem often seen on # Windows, or when using WRS tools that have their own max path # limitations). # Note that we do make sure that working_dir is an absolute # path, as we are likely to be changing directories when # running each test. A relative path would no longer work # under those circumstances. d = os.path.abspath(self.main.args.output_dir) self.output_dir = os.path.join(d, 'new') self.old_output_dir = os.path.join(d, 'old') if not os.path.isdir(self.main.args.temp_dir): logging.critical("temp dir '%s' does not exist", self.main.args.temp_dir) return 1 self.working_dir = tempfile.mkdtemp( '', 'tmp', os.path.abspath(self.main.args.temp_dir)) # Create the new output directory that will hold the results self.setup_result_dir() # Store in global env: target information and common paths self.env.output_dir = self.output_dir self.env.working_dir = self.working_dir self.env.options = self.main.args # User specific startup self.tear_up() # Retrieve the list of test self.test_list = self.get_test_list(self.main.args.sublist) # Launch the mainloop self.total_test = len(self.test_list) self.run_test = 0 self.scheduler = Scheduler(job_provider=self.job_factory, collect=self.collect_result, tokens=self.main.args.jobs) actions = DAG() for test in self.test_list: self.parse_test(actions, test) with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd: fd.write(actions.as_dot()) self.scheduler.run(actions) self.dump_testsuite_result() # Clean everything self.tear_down() return 0 def parse_test(self, actions, test_case_file): """Register a test. :param actions: the dag of actions for the testsuite :type actions: e3.collection.dag.DAG :param test_case_file: filename containing the testcase :type test_case_file: str """ # Load testcase file test_env = load_with_config( os.path.join(self.test_dir, test_case_file), Env().to_dict()) # Ensure that the test_env act like a dictionary if not isinstance(test_env, collections.Mapping): test_env = { 'test_name': self.test_name(test_case_file), 'test_yaml_wrong_content': test_env } logger.error("abort test because of invalid test.yaml") return # Add to the test environment the directory in which the test.yaml is # stored test_env['test_dir'] = os.path.join(self.env.test_dir, os.path.dirname(test_case_file)) test_env['test_case_file'] = test_case_file test_env['test_name'] = self.test_name(test_case_file) test_env['working_dir'] = os.path.join(self.env.working_dir, test_env['test_name']) if 'driver' in test_env: driver = test_env['driver'] else: driver = self.default_driver logger.debug('set driver to %s' % driver) if driver not in self.DRIVERS or \ not issubclass(self.DRIVERS[driver], TestDriver): logger.error('cannot find driver for %s' % test_case_file) return try: instance = self.DRIVERS[driver](self.env, test_env) instance.add_test(actions) except Exception as e: error_msg = str(e) error_msg += "Traceback:\n" error_msg += "\n".join(traceback.format_tb(sys.exc_traceback)) logger.error(error_msg) return def dump_testsuite_result(self): """To be implemented.""" pass def collect_result(self, job): """Run internal function. :param job: a job that is finished :type job: TestFragment """ self.return_values[job.uid] = job.return_value while job.test_instance.result_queue: result = job.test_instance.result_queue.pop() logging.info('%-12s %s' % (str(result.status), result.test_name)) assert result.test_name not in self.results, \ 'cannot push twice results for %s' % result.test_name with open(self.test_result_filename(result.test_name), 'wb') as fd: yaml.dump(result, fd) self.results[result.test_name] = result.status self.test_counter += 1 self.test_status_counters[result.status] += 1 return False def setup_result_dir(self): """Create the output directory in which the results are stored.""" if os.path.isdir(self.old_output_dir): rm(self.old_output_dir, True) if os.path.isdir(self.output_dir): mv(self.output_dir, self.old_output_dir) mkdir(self.output_dir) if self.main.args.dump_environ: with open(os.path.join(self.output_dir, 'environ.sh'), 'w') as f: for var_name in sorted(os.environ): f.write('export %s=%s\n' % (var_name, quote_arg(os.environ[var_name])))
def testsuite_main(self, args=None): """Main for the main testsuite script. :param args: command line arguments. If None use sys.argv :type args: list[str] | None """ self.main = Main(platform_args=self.CROSS_SUPPORT) # Add common options parser = self.main.argument_parser parser.add_argument("-o", "--output-dir", metavar="DIR", default="./out", help="select output dir") parser.add_argument("-t", "--temp-dir", metavar="DIR", default=Env().tmp_dir) parser.add_argument( "--max-consecutive-failures", default=0, help="If there are more than N consecutive failures, the testsuite" " is aborted. If set to 0 (default) then the testsuite will never" " be stopped") parser.add_argument( "--keep-old-output-dir", default=False, action="store_true", help="This is default with this testsuite framework. The option" " is kept only to keep backward compatibility of invocation with" " former framework (gnatpython.testdriver)") parser.add_argument("--disable-cleanup", dest="enable_cleanup", action="store_false", default=True, help="disable cleanup of working space") parser.add_argument( "-j", "--jobs", dest="jobs", type=int, metavar="N", default=Env().build.cpu.cores, help="Specify the number of jobs to run simultaneously") parser.add_argument( "--show-error-output", action="store_true", help="When testcases fail, display their output. This is for" " convenience for interactive use.") parser.add_argument( "--dump-environ", dest="dump_environ", action="store_true", default=False, help="Dump all environment variables in a file named environ.sh," " located in the output directory (see --output-dir). This" " file can then be sourced from a Bourne shell to recreate" " the environement that existed when this testsuite was run" " to produce a given testsuite report.") parser.add_argument('sublist', metavar='tests', nargs='*', default=[], help='test') # Add user defined options self.add_options() # parse options self.main.parse_args(args) self.env = BaseEnv.from_env() self.env.root_dir = self.root_dir self.env.test_dir = self.test_dir # At this stage compute commonly used paths # Keep the working dir as short as possible, to avoid the risk # of having a path that's too long (a problem often seen on # Windows, or when using WRS tools that have their own max path # limitations). # Note that we do make sure that working_dir is an absolute # path, as we are likely to be changing directories when # running each test. A relative path would no longer work # under those circumstances. d = os.path.abspath(self.main.args.output_dir) self.output_dir = os.path.join(d, 'new') self.old_output_dir = os.path.join(d, 'old') if not os.path.isdir(self.main.args.temp_dir): logging.critical("temp dir '%s' does not exist", self.main.args.temp_dir) return 1 self.working_dir = tempfile.mkdtemp( '', 'tmp', os.path.abspath(self.main.args.temp_dir)) # Create the new output directory that will hold the results self.setup_result_dir() # Store in global env: target information and common paths self.env.output_dir = self.output_dir self.env.working_dir = self.working_dir self.env.options = self.main.args # User specific startup self.tear_up() # Retrieve the list of test self.test_list = self.get_test_list(self.main.args.sublist) # Launch the mainloop self.total_test = len(self.test_list) self.run_test = 0 self.scheduler = Scheduler(job_provider=self.job_factory, collect=self.collect_result, tokens=self.main.args.jobs) actions = DAG() for test in self.test_list: self.parse_test(actions, test) with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd: fd.write(actions.as_dot()) self.scheduler.run(actions) self.dump_testsuite_result() # Clean everything self.tear_down() return 0
class TestsuiteCore: """Testsuite Core driver. This class is the base of Testsuite class and should not be instanciated. It's not recommended to override any of the functions declared in it. See documentation of Testsuite class for overridable methods and variables. """ def __init__(self, root_dir: Optional[str] = None, testsuite_name: str = "Untitled testsute") -> None: """Testsuite constructor. :param root_dir: Root directory for the testsuite. If left to None, use the directory containing the Python module that created self's class. :param testsuite_name: Name for this testsuite. It can be used to provide a title in some report formats. """ if root_dir is None: root_dir = os.path.dirname(inspect.getfile(type(self))) self.root_dir = os.path.abspath(root_dir) self.test_dir = os.path.join(self.root_dir, self.tests_subdir) logger.debug("Test directory: %s", self.test_dir) self.consecutive_failures = 0 self.return_values: Dict[str, Any] = {} self.result_tracebacks: Dict[str, List[str]] = {} self.testsuite_name = testsuite_name self.aborted_too_many_failures = False """ Whether the testsuite aborted because of too many consecutive test failures (see the --max-consecutive-failures command-line option). """ # Mypy does not support decorators on properties, so keep the actual # implementations for deprecated properties in methods. @deprecated(2) def _test_counter(self) -> int: return len(self.report_index.entries) @deprecated(2) def _test_status_counters(self) -> Dict[TestStatus, int]: return self.report_index.status_counters @deprecated(2) def _results(self) -> Dict[str, TestStatus]: return { e.test_name: e.status for e in self.report_index.entries.values() } @property def test_counter(self) -> int: """Return the number of test results in the report. Warning: this method is obsolete and will be removed in the future. """ return self._test_counter() @property def test_status_counters(self) -> Dict[TestStatus, int]: """Return test result counts per test status. Warning: this method is obsolete and will be removed in the future. """ return self._test_status_counters() @property def results(self) -> Dict[str, TestStatus]: """Return a mapping from test names to results. Warning: this method is obsolete and will be removed in the future. """ return self._results() def test_result_filename(self, test_name: str) -> str: """Return the name of the file in which the result are stored. :param test_name: Name of the test for this result file. """ return os.path.join(self.output_dir, test_name + ".yaml") def job_factory(self, uid: str, data: Any, predecessors: FrozenSet[str], notify_end: Callable[[str], None]) -> TestFragment: """Run internal function. See e3.job.scheduler """ # We assume that data[0] is the test instance and data[1] the method # to call. # When passing return values from predecessors, remove current test # name from the keys to ease referencing by user (the short fragment # name can then be used by user without knowing the full node id). key_prefix = data[0].test_name + "." key_prefix_len = len(key_prefix) def filter_key(k: str) -> str: if k.startswith(key_prefix): return k[key_prefix_len:] else: return k return TestFragment( uid, data[0], data[1], {filter_key(k): self.return_values[k] for k in predecessors}, notify_end, ) def testsuite_main(self, args: Optional[List[str]] = None) -> int: """Main for the main testsuite script. :param args: Command line arguments. If None, use `sys.argv`. :return: The testsuite status code (0 for success, a positive for failure). """ self.main = Main(platform_args=True) # Add common options parser = self.main.argument_parser parser.add_argument( "-o", "--output-dir", metavar="DIR", default="./out", help="select output dir", ) parser.add_argument("-t", "--temp-dir", metavar="DIR", default=Env().tmp_dir) parser.add_argument( "-d", "--dev-temp", nargs="?", default=None, const="tmp", help="Unlike --temp-dir, use this very directory to store" " testsuite temporaries (i.e. no random subdirectory). Also" " automatically disable temp dir cleanup, to be developer" " friendly. If no directory is provided, use the local" " \"tmp\" directory") parser.add_argument( "--max-consecutive-failures", "-M", metavar="N", type=int, default=self.default_max_consecutive_failures, help="Number of test failures (FAIL or ERROR) that trigger the" " abortion of the testuite. If zero, this behavior is disabled. In" " some cases, aborting the testsuite when there are just too many" " failures saves time and costs: the software to test/environment" " is too broken, there is no point to continue running the" " testsuite.") parser.add_argument( "--keep-old-output-dir", default=False, action="store_true", help="This is default with this testsuite framework. The option" " is kept only to keep backward compatibility of invocation with" " former framework (gnatpython.testdriver)", ) parser.add_argument( "--disable-cleanup", dest="enable_cleanup", action="store_false", default=True, help="disable cleanup of working space", ) parser.add_argument( "-j", "--jobs", dest="jobs", type=int, metavar="N", default=Env().build.cpu.cores, help="Specify the number of jobs to run simultaneously", ) parser.add_argument( "--show-error-output", "-E", action="store_true", help="When testcases fail, display their output. This is for" " convenience for interactive use.", ) parser.add_argument( "--show-time-info", action="store_true", help="Display time information for test results, if available") parser.add_argument( "--dump-environ", dest="dump_environ", action="store_true", default=False, help="Dump all environment variables in a file named environ.sh," " located in the output directory (see --output-dir). This" " file can then be sourced from a Bourne shell to recreate" " the environement that existed when this testsuite was run" " to produce a given testsuite report.", ) parser.add_argument( "--xunit-output", dest="xunit_output", metavar="FILE", help="Output testsuite report to the given file in the standard" " XUnit XML format. This is useful to display results in" " continuous build systems such as Jenkins.", ) parser.add_argument( "--gaia-output", action="store_true", help="Output a GAIA-compatible testsuite report next to the YAML" " report.") parser.add_argument( "--truncate-logs", "-T", metavar="N", type=int, default=200, help="When outputs (for instance subprocess outputs) exceed 2*N" " lines, only include the first and last N lines in logs. This is" " necessary when storage for testsuite results have size limits," " and the useful information is generally either at the beginning" " or the end of such outputs. If 0, never truncate logs.") parser.add_argument( "--failure-exit-code", metavar="N", type=int, default=self.default_failure_exit_code, help="Exit code the testsuite must use when at least one test" " result shows a failure/error. By default, this is" f" {self.default_failure_exit_code}. This option is useful when" " running a testsuite in a continuous integration setup, as this" " can make the testing process stop when there is a regression.") parser.add_argument("sublist", metavar="tests", nargs="*", default=[], help="test") # Add user defined options self.add_options(parser) # Parse options self.main.parse_args(args) assert self.main.args is not None # If there is a chance for the logging to end up in a non-tty stream, # disable colors. If not, be user-friendly and automatically show error # outputs. if (self.main.args.log_file or not isatty(sys.stdout) or not isatty(sys.stderr)): enable_colors = False else: # interactive-only enable_colors = True self.main.args.show_error_output = True self.colors = ColorConfig(enable_colors) self.Fore = self.colors.Fore self.Style = self.colors.Style self.env = BaseEnv.from_env() self.env.enable_colors = enable_colors self.env.root_dir = self.root_dir self.env.test_dir = self.test_dir # At this stage compute commonly used paths Keep the working dir as # short as possible, to avoid the risk of having a path that's too long # (a problem often seen on Windows, or when using WRS tools that have # their own max path limitations). # # Note that we do make sure that working_dir is an absolute path, as we # are likely to be changing directories when running each test. A # relative path would no longer work under those circumstances. d = os.path.abspath(self.main.args.output_dir) self.output_dir = os.path.join(d, "new") self.old_output_dir = os.path.join(d, "old") if self.main.args.dev_temp: # Use a temporary directory for developers: make sure it is an # empty directory and disable cleanup to ease post-mortem # investigation. self.working_dir = os.path.abspath(self.main.args.dev_temp) rm(self.working_dir, recursive=True) mkdir(self.working_dir) self.main.args.enable_cleanup = False else: # If the temp dir is supposed to be randomized, we need to create a # subdirectory, so check that the parent directory exists first. if not os.path.isdir(self.main.args.temp_dir): logger.critical("temp dir '%s' does not exist", self.main.args.temp_dir) return 1 self.working_dir = tempfile.mkdtemp( "", "tmp", os.path.abspath(self.main.args.temp_dir)) # Create the new output directory that will hold the results and create # an index for it. self.setup_result_dir() self.report_index = ReportIndex(self.output_dir) # Store in global env: target information and common paths self.env.output_dir = self.output_dir self.env.working_dir = self.working_dir self.env.options = self.main.args # User specific startup self.set_up() # Retrieve the list of test self.has_error = False self.test_list = self.get_test_list(self.main.args.sublist) # Launch the mainloop self.total_test = len(self.test_list) self.run_test = 0 self.scheduler = Scheduler( job_provider=self.job_factory, tokens=self.main.args.jobs, # correct_result expects specifically TestFragment instances (a Job # subclass), while Scheduler only guarantees Job instances. # Test drivers are supposed to register only TestFragment # instances, so the following cast should be fine. collect=cast(Any, self.collect_result), ) actions = DAG() for parsed_test in self.test_list: if not self.add_test(actions, parsed_test): self.has_error = True actions.check() with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd: fd.write(actions.as_dot()) # Run the tests. Note that when the testsuite aborts because of too # many consecutive test failures, we still want to produce a report and # exit through regular ways, to catch KeyboardInterrupt exceptions, # which e3's scheduler uses to abort the execution loop, but only in # such cases. In other words, let the exception propagates if it's the # user that interrupted the testsuite. try: self.scheduler.run(actions) except KeyboardInterrupt: if not self.aborted_too_many_failures: # interactive-only raise self.report_index.write() self.dump_testsuite_result() if self.main.args.xunit_output: dump_xunit_report(self, self.main.args.xunit_output) if self.main.args.gaia_output: dump_gaia_report(self, self.output_dir) # Clean everything self.tear_down() # Return the appropriate status code: 1 when there is a framework # issue, the failure status code from the --failure-exit-code=N option # when there is a least one testcase failure, or 0. statuses = { s for s, count in self.report_index.status_counters.items() if count } if self.has_error: return 1 elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses: return self.main.args.failure_exit_code else: return 0 def get_test_list(self, sublist: List[str]) -> List[ParsedTest]: """Retrieve the list of tests. :param sublist: A list of tests scenarios or patterns. """ # Use a mapping: test name -> ParsedTest when building the result, as # several patterns in "sublist" may yield the same testcase. testcases: Dict[str, ParsedTest] = {} test_finders = self.test_finders def add_testcase(test: ParsedTest) -> None: testcases[test.test_name] = test def helper(spec: str) -> None: pattern: Optional[Pattern[str]] = None # If the given pattern is a directory, do not go through the whole # tests subdirectory. if os.path.isdir(spec): root = spec else: root = self.test_dir try: pattern = re.compile(spec) except re.error as exc: logger.debug( "Test pattern is not a valid regexp, try to match it" " as-is: {}".format(exc)) pattern = re.compile(re.escape(spec)) # For each directory in the requested subdir, ask our test finders # to probe for a testcase. Register matches. for dirpath, dirnames, filenames in os.walk(root, followlinks=True): # If the directory name does not match the given pattern, skip # it. if pattern is not None and not pattern.search(dirpath): continue # The first test finder that has a match "wins". When handling # test data, we want to deal only with absolute paths, so get # the absolute name now. dirpath = os.path.abspath(dirpath) for tf in test_finders: try: test_or_list = tf.probe(self, dirpath, dirnames, filenames) except ProbingError as exc: self.has_error = True logger.error(str(exc)) break if isinstance(test_or_list, list): for t in test_or_list: add_testcase(t) break elif test_or_list is not None: add_testcase(test_or_list) break # If specific tests are requested, only look for them. Otherwise, just # look in the tests subdirectory. if sublist: for s in sublist: helper(s) else: helper(self.test_dir) result = list(testcases.values()) logger.info("Found {} tests".format(len(result))) logger.debug("tests:\n " + "\n ".join(t.test_dir for t in result)) return result def add_test(self, actions: DAG, parsed_test: ParsedTest) -> bool: """Register a test to run. :param actions: The dag of actions for the testsuite. :param parsed_test: Test to instantiate. :return: Whether the test was successfully registered. """ test_name = parsed_test.test_name # Complete the test environment test_env = dict(parsed_test.test_env) test_env["test_dir"] = parsed_test.test_dir test_env["test_name"] = test_name assert isinstance(self.env.working_dir, str) test_env["working_dir"] = os.path.join(self.env.working_dir, test_name) # Fetch the test driver to use driver = parsed_test.driver_cls if not driver: if self.default_driver: driver = self.test_driver_map[self.default_driver] else: logger.error("missing driver for test '{}'".format(test_name)) return False # Finally run the driver instantiation try: instance = driver(self.env, test_env) instance.Fore = self.Fore instance.Style = self.Style instance.add_test(actions) except Exception as e: error_msg = str(e) error_msg += "\nTraceback:\n" error_msg += "\n".join(traceback.format_tb(sys.exc_info()[2])) logger.error(error_msg) return False return True def dump_testsuite_result(self) -> None: """Log a summary of test results. Subclasses are free to override this to do whatever is suitable for them. """ lines = ['Summary:'] # Display test count for each status, but only for status that have # at least one test. Sort them by status value, to get consistent # order. def sort_key(couple: Tuple[TestStatus, int]) -> Any: status, _ = couple return status.value stats = sorted( ((status, count) for status, count in self.report_index.status_counters.items() if count), key=sort_key) for status, count in stats: lines.append(' {}{: <12}{} {}'.format(status.color(self.colors), status.name, self.Style.RESET_ALL, count)) if not stats: lines.append(' <no test result>') logger.info('\n'.join(lines)) # Dump the comment file with open(os.path.join(self.output_dir, "comment"), "w") as f: self.write_comment_file(f) def collect_result(self, job: TestFragment) -> bool: """Run internal function. :param job: A job that is finished. """ assert self.main.args # Keep track of the number of consecutive failures seen so far if it # reaches the maximum number allowed, we must abort the testsuite. max_consecutive_failures = self.main.args.max_consecutive_failures consecutive_failures = 0 self.return_values[job.uid] = job.return_value while job.test_instance.result_queue: result, tb = job.test_instance.result_queue.pop() # The test results that reach this point are special: there were # serialized/deserialized through YAML, so the Log layer # disappeared. assert result.status is not None # Log the test result. If error output is requested and the test # failed unexpectedly, show the detailed logs. log_line = summary_line(result, self.colors, self.main.args.show_time_info) if (self.main.args.show_error_output and result.status not in (TestStatus.PASS, TestStatus.XFAIL, TestStatus.XPASS, TestStatus.SKIP)): def format_log(log: Log) -> str: return "\n" + str(log) + self.Style.RESET_ALL if result.diff: log_line += format_log(result.diff) else: log_line += format_log(result.log) logger.info(log_line) def indented_tb(tb: List[str]) -> str: return "".join(" {}".format(line) for line in tb) assert result.test_name not in self.report_index.entries, ( "cannot push twice results for {}" "\nFirst push happened at:" "\n{}" "\nThis one happened at:" "\n{}".format( result.test_name, indented_tb(self.result_tracebacks[result.test_name]), indented_tb(tb), )) with open(self.test_result_filename(result.test_name), "w") as fd: yaml.dump(result, fd) self.report_index.add_result(result) self.result_tracebacks[result.test_name] = tb # Update the number of consecutive failures, aborting the testsuite # if appropriate if result.status in (TestStatus.ERROR, TestStatus.FAIL): consecutive_failures += 1 if (max_consecutive_failures > 0 and consecutive_failures >= max_consecutive_failures): self.aborted_too_many_failures = True logger.error( "Too many consecutive failures, aborting the testsuite" ) raise KeyboardInterrupt else: consecutive_failures = 0 return False def setup_result_dir(self) -> None: """Create the output directory in which the results are stored.""" assert self.main.args if os.path.isdir(self.old_output_dir): rm(self.old_output_dir, True) if os.path.isdir(self.output_dir): mv(self.output_dir, self.old_output_dir) mkdir(self.output_dir) if self.main.args.dump_environ: with open(os.path.join(self.output_dir, "environ.sh"), "w") as f: for var_name in sorted(os.environ): f.write("export {}={}\n".format( var_name, quote_arg(os.environ[var_name]))) # Unlike the previous methods, the following ones are supposed to be # overriden. @property def tests_subdir(self) -> str: """ Return the subdirectory in which tests are looked for. The returned directory name is considered relative to the root testsuite directory (self.root_dir). """ raise NotImplementedError @property def test_driver_map(self) -> Dict[str, Type[TestDriver]]: """Return a map from test driver names to TestDriver subclasses. Test finders will be able to use this map to fetch the test drivers referenced in testcases. """ raise NotImplementedError @property def default_driver(self) -> Optional[str]: """Return the name of the default driver for testcases. When tests do not query a specific driver, the one associated to this name is used instead. If this property returns None, all tests are required to query a driver. """ raise NotImplementedError def test_name(self, test_dir: str) -> str: """Compute the test name given a testcase spec. This function can be overridden. By default it uses the name of the test directory. Note that the test name should be a valid filename (not dir seprators, or special characters such as ``:``, ...). """ raise NotImplementedError @property def test_finders(self) -> List[TestFinder]: """Return test finders to probe tests directories.""" raise NotImplementedError def add_options(self, parser: argparse.ArgumentParser) -> None: """Add testsuite specific switches. Subclasses can override this method to add their own testsuite command-line options. :param parser: Parser for command-line arguments. See <https://docs.python.org/3/library/argparse.html> for usage. """ raise NotImplementedError def set_up(self) -> None: """Execute operations before running the testsuite. Before running this, command-line arguments were parsed. After this returns, the testsuite will look for testcases. By default, this does nothing. Overriding this method allows testsuites to prepare the execution of the testsuite depending on their needs. For instance: * process testsuite-specific options; * initialize environment variables; * adjust self.env (object forwarded to test drivers). """ raise NotImplementedError def tear_down(self) -> None: """Execute operation when finalizing the testsuite. By default, this cleans the working (temporary) directory in which the tests were run. """ raise NotImplementedError def write_comment_file(self, comment_file: IO[str]) -> None: """Write the comment file's content. :param comment_file: File descriptor for the comment file. Overriding methods should only call its "write" method (or print to it). """ raise NotImplementedError @property def default_max_consecutive_failures(self) -> int: """Return the default maximum number of consecutive failures. In some cases, aborting the testsuite when there are just too many failures saves time and costs: the software to test/environment is too broken, there is no point to continue running the testsuite. This property must return the number of test failures (FAIL or ERROR) that trigger the abortion of the testuite. If zero, this behavior is disabled. """ raise NotImplementedError @property def default_failure_exit_code(self) -> int: """Return the default exit code when at least one test fails.""" raise NotImplementedError
def testsuite_main(self, args: Optional[List[str]] = None) -> int: """Main for the main testsuite script. :param args: Command line arguments. If None, use `sys.argv`. :return: The testsuite status code (0 for success, a positive for failure). """ self.main = Main(platform_args=True) # Add common options parser = self.main.argument_parser parser.add_argument( "-o", "--output-dir", metavar="DIR", default="./out", help="select output dir", ) parser.add_argument("-t", "--temp-dir", metavar="DIR", default=Env().tmp_dir) parser.add_argument( "-d", "--dev-temp", nargs="?", default=None, const="tmp", help="Unlike --temp-dir, use this very directory to store" " testsuite temporaries (i.e. no random subdirectory). Also" " automatically disable temp dir cleanup, to be developer" " friendly. If no directory is provided, use the local" " \"tmp\" directory") parser.add_argument( "--max-consecutive-failures", "-M", metavar="N", type=int, default=self.default_max_consecutive_failures, help="Number of test failures (FAIL or ERROR) that trigger the" " abortion of the testuite. If zero, this behavior is disabled. In" " some cases, aborting the testsuite when there are just too many" " failures saves time and costs: the software to test/environment" " is too broken, there is no point to continue running the" " testsuite.") parser.add_argument( "--keep-old-output-dir", default=False, action="store_true", help="This is default with this testsuite framework. The option" " is kept only to keep backward compatibility of invocation with" " former framework (gnatpython.testdriver)", ) parser.add_argument( "--disable-cleanup", dest="enable_cleanup", action="store_false", default=True, help="disable cleanup of working space", ) parser.add_argument( "-j", "--jobs", dest="jobs", type=int, metavar="N", default=Env().build.cpu.cores, help="Specify the number of jobs to run simultaneously", ) parser.add_argument( "--show-error-output", "-E", action="store_true", help="When testcases fail, display their output. This is for" " convenience for interactive use.", ) parser.add_argument( "--show-time-info", action="store_true", help="Display time information for test results, if available") parser.add_argument( "--dump-environ", dest="dump_environ", action="store_true", default=False, help="Dump all environment variables in a file named environ.sh," " located in the output directory (see --output-dir). This" " file can then be sourced from a Bourne shell to recreate" " the environement that existed when this testsuite was run" " to produce a given testsuite report.", ) parser.add_argument( "--xunit-output", dest="xunit_output", metavar="FILE", help="Output testsuite report to the given file in the standard" " XUnit XML format. This is useful to display results in" " continuous build systems such as Jenkins.", ) parser.add_argument( "--gaia-output", action="store_true", help="Output a GAIA-compatible testsuite report next to the YAML" " report.") parser.add_argument( "--truncate-logs", "-T", metavar="N", type=int, default=200, help="When outputs (for instance subprocess outputs) exceed 2*N" " lines, only include the first and last N lines in logs. This is" " necessary when storage for testsuite results have size limits," " and the useful information is generally either at the beginning" " or the end of such outputs. If 0, never truncate logs.") parser.add_argument( "--failure-exit-code", metavar="N", type=int, default=self.default_failure_exit_code, help="Exit code the testsuite must use when at least one test" " result shows a failure/error. By default, this is" f" {self.default_failure_exit_code}. This option is useful when" " running a testsuite in a continuous integration setup, as this" " can make the testing process stop when there is a regression.") parser.add_argument("sublist", metavar="tests", nargs="*", default=[], help="test") # Add user defined options self.add_options(parser) # Parse options self.main.parse_args(args) assert self.main.args is not None # If there is a chance for the logging to end up in a non-tty stream, # disable colors. If not, be user-friendly and automatically show error # outputs. if (self.main.args.log_file or not isatty(sys.stdout) or not isatty(sys.stderr)): enable_colors = False else: # interactive-only enable_colors = True self.main.args.show_error_output = True self.colors = ColorConfig(enable_colors) self.Fore = self.colors.Fore self.Style = self.colors.Style self.env = BaseEnv.from_env() self.env.enable_colors = enable_colors self.env.root_dir = self.root_dir self.env.test_dir = self.test_dir # At this stage compute commonly used paths Keep the working dir as # short as possible, to avoid the risk of having a path that's too long # (a problem often seen on Windows, or when using WRS tools that have # their own max path limitations). # # Note that we do make sure that working_dir is an absolute path, as we # are likely to be changing directories when running each test. A # relative path would no longer work under those circumstances. d = os.path.abspath(self.main.args.output_dir) self.output_dir = os.path.join(d, "new") self.old_output_dir = os.path.join(d, "old") if self.main.args.dev_temp: # Use a temporary directory for developers: make sure it is an # empty directory and disable cleanup to ease post-mortem # investigation. self.working_dir = os.path.abspath(self.main.args.dev_temp) rm(self.working_dir, recursive=True) mkdir(self.working_dir) self.main.args.enable_cleanup = False else: # If the temp dir is supposed to be randomized, we need to create a # subdirectory, so check that the parent directory exists first. if not os.path.isdir(self.main.args.temp_dir): logger.critical("temp dir '%s' does not exist", self.main.args.temp_dir) return 1 self.working_dir = tempfile.mkdtemp( "", "tmp", os.path.abspath(self.main.args.temp_dir)) # Create the new output directory that will hold the results and create # an index for it. self.setup_result_dir() self.report_index = ReportIndex(self.output_dir) # Store in global env: target information and common paths self.env.output_dir = self.output_dir self.env.working_dir = self.working_dir self.env.options = self.main.args # User specific startup self.set_up() # Retrieve the list of test self.has_error = False self.test_list = self.get_test_list(self.main.args.sublist) # Launch the mainloop self.total_test = len(self.test_list) self.run_test = 0 self.scheduler = Scheduler( job_provider=self.job_factory, tokens=self.main.args.jobs, # correct_result expects specifically TestFragment instances (a Job # subclass), while Scheduler only guarantees Job instances. # Test drivers are supposed to register only TestFragment # instances, so the following cast should be fine. collect=cast(Any, self.collect_result), ) actions = DAG() for parsed_test in self.test_list: if not self.add_test(actions, parsed_test): self.has_error = True actions.check() with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd: fd.write(actions.as_dot()) # Run the tests. Note that when the testsuite aborts because of too # many consecutive test failures, we still want to produce a report and # exit through regular ways, to catch KeyboardInterrupt exceptions, # which e3's scheduler uses to abort the execution loop, but only in # such cases. In other words, let the exception propagates if it's the # user that interrupted the testsuite. try: self.scheduler.run(actions) except KeyboardInterrupt: if not self.aborted_too_many_failures: # interactive-only raise self.report_index.write() self.dump_testsuite_result() if self.main.args.xunit_output: dump_xunit_report(self, self.main.args.xunit_output) if self.main.args.gaia_output: dump_gaia_report(self, self.output_dir) # Clean everything self.tear_down() # Return the appropriate status code: 1 when there is a framework # issue, the failure status code from the --failure-exit-code=N option # when there is a least one testcase failure, or 0. statuses = { s for s, count in self.report_index.status_counters.items() if count } if self.has_error: return 1 elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses: return self.main.args.failure_exit_code else: return 0