def run(self, pav_cfg, args): """Resolve the test configurations into individual tests and assign to schedulers. Have those schedulers kick off jobs to run the individual tests themselves. :param pav_cfg: The pavilion configuration. :param args: The parsed command line argument object. """ # 1. Resolve the test configs # - Get sched vars from scheduler. # - Compile variables. # mb_tracker = MultiBuildTracker() local_builds_only = getattr(args, 'local_builds_only', False) test_list = self._get_tests( pav_cfg, args, mb_tracker, build_only=self.BUILD_ONLY, local_builds_only=getattr(args, 'local_builds_only', False)) if test_list is None: return errno.EINVAL all_tests = test_list self.last_tests = all_tests if not all_tests: fprint("You must specify at least one test.", file=self.errfile) return errno.EINVAL series = TestSeries(pav_cfg, all_tests) self.last_series = series res = self.check_result_format(all_tests) if res != 0: self._complete_tests(all_tests) return res all_tests = [test for test in all_tests if not test.skipped] res = cmd_utils.build_local( tests=all_tests, max_threads=pav_cfg.build_threads, mb_tracker=mb_tracker, build_verbosity=args.build_verbosity, outfile=self.outfile, errfile=self.errfile) if res != 0: self._complete_tests(all_tests) return res self._complete_tests([test for test in all_tests if test.build_only and test.build_local]) wait = getattr(args, 'wait', None) report_status = getattr(args, 'status', False) if self.BUILD_ONLY and local_builds_only: non_local_build_tests = [test for test in all_tests if not test.build_local] if non_local_build_tests: fprint( "Skipping tests that are set to build on nodes: {}" .format([test.name for test in non_local_build_tests]), file=self.outfile, color=output.YELLOW) return 0 return series.run_tests( pav_cfg=pav_cfg, wait=wait, report_status=report_status, outfile=self.outfile, errfile=self.errfile )
def run_set(self, log=False, local_builds_only=False, build_only=False, build_verbosity=0, wait=None, run_cmd=None, rebuild=False): """Runs tests in set. """ mb_tracker = MultiBuildTracker(log=log) if not run_cmd: run_cmd = commands.get_command('run') try: new_conditions = {'only_if': self.only_if, 'not_if': self.not_if} # resolve configs test_configs = cmd_utils.get_test_configs( pav_cfg=self.pav_cfg, host=self.host, tests=self._test_names, modes=self.modes, overrides=self.series_obj.overrides, outfile=self.outfile, conditions=new_conditions, ) # Remove non-local builds when only doing local builds. if build_only and local_builds_only: locally_built_tests = [] for ptest in test_configs: if ptest.config['build']['on_nodes'].lower() != 'true': locally_built_tests.append(ptest) test_configs = locally_built_tests # configs->tests test_list = cmd_utils.configs_to_tests( pav_cfg=self.pav_cfg, proto_tests=test_configs, mb_tracker=mb_tracker, build_only=build_only, rebuild=rebuild, outfile=self.outfile, ) except (commands.CommandError, test_config.TestConfigError) as err: self.done = True output.fprint("Error resolving configs. \n{}".format(err.args[0]), file=self.errfile, color=output.RED) return errno.EINVAL if not test_list: self.done = True self.all_pass = True fprint("You must specify at least one test.", file=self.errfile) return errno.EINVAL all_tests = test_list run_cmd.last_tests = all_tests # assign tests to series and vice versa self.series_obj.add_tests(all_tests) for test_obj in all_tests: self.tests[test_obj.name] = test_obj run_cmd.last_series = self.series_obj # make sure result parsers are ok res = cmd_utils.check_result_format(all_tests, self.errfile) if res != 0: self.done = True cmd_utils.complete_tests(all_tests) return res all_tests = [test for test in all_tests if not test.skipped] # attempt to build res = cmd_utils.build_local(tests=all_tests, max_threads=self.pav_cfg.build_threads, mb_tracker=mb_tracker, build_verbosity=build_verbosity, outfile=self.outfile, errfile=self.errfile) if res != 0: self.done = True cmd_utils.complete_tests(all_tests) return res cmd_utils.complete_tests([ test for test in all_tests if test.build_only and test.build_local ]) if build_only and local_builds_only: non_local_build_tests = [ test for test in all_tests if not test.build_local ] if non_local_build_tests: fprint("Skipping tests that are not set to build on nodes. {}". format([test.name for test in non_local_build_tests]), file=self.outfile, color=output.YELLOW) return 0 # deal with simultaneous here if self.series_obj.config['simultaneous'] is None: res = self.series_obj.run_tests(tests=all_tests, wait=wait) return res else: simult = int(self.series_obj.config['simultaneous']) for test in all_tests: self.test_wait(simult) self.series_obj.run_tests(tests=[test])
def run_set(self): """Runs tests in set. """ mb_tracker = MultiBuildTracker(log=False) run_cmd = commands.get_command('run') # run.RunCommand._get_tests function try: new_conditions = { 'only_if': self.only_if, 'not_if': self.not_if } # resolve configs test_configs = cmd_utils.get_test_configs( pav_cfg=self.pav_cfg, host=self.host, tests=self._test_names, modes=self.modes, outfile=self.outfile, conditions=new_conditions, ) # configs->tests test_list = cmd_utils.configs_to_tests( pav_cfg=self.pav_cfg, proto_tests=test_configs, mb_tracker=mb_tracker, outfile=self.outfile, ) except (commands.CommandError, test_config.TestConfigError) as err: self.done = True output.fprint("Error resolving configs. \n{}".format(err.args[0]), file=self.errfile, color=output.RED) return None if not test_list: self.done = True self.all_pass = True return None all_tests = test_list run_cmd.last_tests = all_tests # assign tests to series and vice versa self.series_obj.add_tests(all_tests) for test_obj in all_tests: self.tests[test_obj.name] = test_obj run_cmd.last_series = self.series_obj # make sure result parsers are ok res = cmd_utils.check_result_format(all_tests, self.errfile) if res != 0: self.done = True cmd_utils.complete_tests(all_tests) return None # attempt to build res = cmd_utils.build_local( tests=all_tests, max_threads=self.pav_cfg.build_threads, mb_tracker=mb_tracker, outfile=self.outfile, errfile=self.errfile ) if res != 0: self.done = True cmd_utils.complete_tests(all_tests) return None # deal with simultaneous here if self.series_obj.config['simultaneous'] is None: self.series_obj.run_tests(tests=all_tests) else: simult = int(self.series_obj.config['simultaneous']) for test in all_tests: self.test_wait(simult) self.series_obj.run_tests(tests=[test])
def build_local(tests: List[TestRun], mb_tracker: MultiBuildTracker, max_threads: int = 4, build_verbosity: int = 0, outfile: TextIO = StringIO(), errfile: TextIO = StringIO()): """Build all tests that request for their build to occur on the kickoff host. :param tests: The list of tests to potentially build. :param max_threads: Maximum number of build threads to start. :param build_verbosity: How much info to print during building. 0 - Quiet, 1 - verbose, 2+ - very verbose :param mb_tracker: The tracker for all builds. :param outfile: Where to print user messages. :param errfile: Where to print user error messages. """ test_threads = [] # type: List[Union[threading.Thread, None]] remote_builds = [] cancel_event = threading.Event() # Generate new build names for each test that is rebuilding. # We do this here, even for non_local tests, because otherwise the # non-local tests can't tell what was built fresh either on a # front-end or by other tests rebuilding on nodes. for test in tests: if test.rebuild and test.builder.exists(): test.builder.deprecate() test.builder.rename_build() test.build_name = test.builder.name test.save_attributes() # We don't want to start threads that are just going to wait on a lock, # so we'll rearrange the builds so that the uniq build names go first. # We'll use this as a stack, so tests that should build first go at # the end of the list. build_order = [] # If we've seen a build name, the build can go later. seen_build_names = set() for test in tests: if not test.build_local: remote_builds.append(test) elif test.builder.name not in seen_build_names: build_order.append(test) seen_build_names.add(test.builder.name) else: build_order.insert(0, test) # Keep track of what the last message printed per build was. # This is for double build verbosity. message_counts = {test.id: 0 for test in tests} # Used to track which threads are for which tests. test_by_threads = {} if build_verbosity > 0: output.fprint(BUILD_STATUS_PREAMBLE.format(when='When', test_id='TestID', state_len=STATES.max_length, state='State'), 'Message', file=outfile, width=None) builds_running = 0 # Run and track <max_threads> build threads, giving output according # to the verbosity level. As threads finish, new ones are started until # either all builds complete or a build fails, in which case all tests # are aborted. while build_order or test_threads: # Start a new thread if we haven't hit our limit. if build_order and builds_running < max_threads: test = build_order.pop() test_thread = threading.Thread(target=test.build, args=(cancel_event, )) test_threads.append(test_thread) test_by_threads[test_thread] = test test_thread.start() # Check if all our threads are alive, and join those that aren't. for i in range(len(test_threads)): thread = test_threads[i] if not thread.is_alive(): thread.join() builds_running -= 1 test_threads[i] = None test = test_by_threads[thread] del test_by_threads[thread] # Only output test status after joining a thread. if build_verbosity == 1: notes = mb_tracker.get_notes(test.builder) when, state, msg = notes[-1] when = output.get_relative_timestamp(when) preamble = (BUILD_STATUS_PREAMBLE.format( when=when, test_id=test.id, state_len=STATES.max_length, state=state)) output.fprint(preamble, msg, wrap_indent=len(preamble), file=outfile, width=None) test_threads = [thr for thr in test_threads if thr is not None] if cancel_event.is_set(): for thread in test_threads: thread.join() for test in tests: if (test.status.current().state not in (STATES.BUILD_FAILED, STATES.BUILD_ERROR)): test.status.set( STATES.ABORTED, "Run aborted due to failures in other builds.") output.fprint("Build error while building tests. Cancelling runs.", color=output.RED, file=outfile, clear=True) output.fprint( "Failed builds are placed in <working_dir>/test_runs/" "<test_id>/build for the corresponding test run.", color=output.CYAN, file=outfile) for failed_build in mb_tracker.failures(): output.fprint( "Build error for test {f.test.name} (#{f.test.id}).". format(f=failed_build), file=errfile) output.fprint( "See test status file (pav cat {id} status) and/or " "the test build log (pav log build {id})".format( id=failed_build.test.id), file=errfile) return errno.EINVAL state_counts = mb_tracker.state_counts() if build_verbosity == 0: # Print a self-clearing one-liner of the counts of the # build statuses. parts = [] for state in sorted(state_counts.keys()): parts.append("{}: {}".format(state, state_counts[state])) line = ' | '.join(parts) output.fprint(line, end='\r', file=outfile, width=None, clear=True) elif build_verbosity > 1: for test in tests: seen = message_counts[test.id] msgs = mb_tracker.messages[test.builder][seen:] for when, state, msg in msgs: when = output.get_relative_timestamp(when) state = '' if state is None else state preamble = BUILD_STATUS_PREAMBLE.format( when=when, test_id=test.id, state_len=STATES.max_length, state=state) output.fprint(preamble, msg, wrap_indent=len(preamble), file=outfile, width=None) message_counts[test.id] += len(msgs) time.sleep(BUILD_SLEEP_TIME) if build_verbosity == 0: # Print a newline after our last status update. output.fprint(width=None, file=outfile) return 0