def _end_test_run(self, start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry): if initial_results is None: _log.error('No results generated') return test_run_results.RunDetails(exit_code=-1) # Some crash logs can take a long time to be written out so look # for new logs after the test run finishes. _log.debug("looking for new crash logs") self._look_for_new_crash_logs(initial_results, start_time) if retry_results: self._look_for_new_crash_logs(retry_results, start_time) _log.debug("summarizing results") summarized_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry) self._printer.print_results(end_time - start_time, initial_results, summarized_results) exit_code = -1 if not self._options.dry_run: self._port.print_leaks_summary() self._output_perf_metrics(end_time - start_time, initial_results) self._save_json_files(summarized_results, initial_results) results_path = self._filesystem.join(self._results_directory, "results.html") self._copy_results_html_file(results_path) if initial_results.keyboard_interrupted: exit_code = INTERRUPTED_EXIT_STATUS else: if self._options.show_results and (initial_results.unexpected_results_by_name or (self._options.full_results_html and initial_results.total_failures)): self._port.show_results_html_file(results_path) exit_code = self._port.exit_code_from_summarized_results(summarized_results) return test_run_results.RunDetails(exit_code, summarized_results, initial_results, retry_results, enabled_pixel_tests_in_retry)
def test_print_results(self): port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False)) printer, out = self.get_printer() initial_results = test_run_results_unittest.run_results(port) summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) details = test_run_results.RunDetails(summary['num_regressions'], summary, initial_results, None) printer.print_results(details) self.assertNotEmpty(out)
def test_print_results(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() initial_results = test_run_results_unittest.run_results(port) full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) failing_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False, only_include_failing=True) details = test_run_results.RunDetails(failing_summary['num_regressions'], full_summary, failing_summary, initial_results, None) printer.print_results(details) self.assertTrue(out.getvalue().find('but passed') != -1)
def print_expectations(self, args): aggregate_test_names = set() aggregate_tests_to_run = set() aggregate_tests_to_skip = set() tests_to_run_by_device = {} device_type_list = self._port.DEFAULT_DEVICE_TYPES or [self._port.DEVICE_TYPE] for device_type in device_type_list: """Run the tests and return a RunDetails object with the results.""" for_device_type = 'for {} '.format(device_type) if device_type else '' self._printer.write_update('Collecting tests {}...'.format(for_device_type)) try: paths, test_names = self._collect_tests(args, device_type=device_type) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update('Parsing expectations {}...'.format(for_device_type)) self._expectations[device_type] = test_expectations.TestExpectations(self._port, test_names, force_expectations_pass=self._options.force, device_type=device_type) self._expectations[device_type].parse_all_expectations() aggregate_test_names.update(test_names) tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names, device_type=device_type) aggregate_tests_to_skip.update(tests_to_skip) tests_to_run_by_device[device_type] = [test for test in tests_to_run if test not in aggregate_tests_to_run] aggregate_tests_to_run.update(tests_to_run) aggregate_tests_to_skip = aggregate_tests_to_skip - aggregate_tests_to_run self._printer.print_found(len(aggregate_test_names), len(aggregate_tests_to_run), self._options.repeat_each, self._options.iterations) test_col_width = len(max(aggregate_tests_to_run.union(aggregate_tests_to_skip), key=len)) + 1 self._print_expectations_for_subset(device_type_list[0], test_col_width, tests_to_run_by_device[device_type_list[0]], aggregate_tests_to_skip) for device_type in device_type_list[1:]: self._print_expectations_for_subset(device_type, test_col_width, tests_to_run_by_device[device_type]) return 0
def run(self, args): """Run the tests and return a RunDetails object with the results.""" self._printer.write_update("Collecting tests ...") try: paths, test_names = self._collect_tests(args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update("Parsing expectations ...") self._expectations = test_expectations.TestExpectations( self._port, test_names, force_expectations_pass=self._options.force) self._expectations.parse_all_expectations() tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) if not self._set_up_run(tests_to_run): return test_run_results.RunDetails(exit_code=-1) enabled_pixel_tests_in_retry = False try: initial_results = self._run_tests( tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations, int(self._options.child_processes), retrying=False) tests_to_retry = self._tests_to_retry( initial_results, include_crashes=self._port.should_retry_crashes()) # Don't retry failures when interrupted by user or failures limit exception. retry_failures = self._options.retry_failures and not ( initial_results.interrupted or initial_results.keyboard_interrupted) if retry_failures and tests_to_retry: enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed( ) _log.info('') _log.info("Retrying %s ..." % pluralize(len(tests_to_retry), "unexpected failure")) _log.info('') retry_results = self._run_tests(tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1, num_workers=1, retrying=True) if enabled_pixel_tests_in_retry: self._options.pixel_tests = False else: retry_results = None finally: self._clean_up_run() end_time = time.time() # Some crash logs can take a long time to be written out so look # for new logs after the test run finishes. _log.debug("looking for new crash logs") self._look_for_new_crash_logs(initial_results, start_time) if retry_results: self._look_for_new_crash_logs(retry_results, start_time) _log.debug("summarizing results") summarized_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry) results_including_passes = None if self._options.results_server_host: results_including_passes = test_run_results.summarize_results( self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=True, include_time_and_modifiers=True) self._printer.print_results(end_time - start_time, initial_results, summarized_results) exit_code = -1 if not self._options.dry_run: self._port.print_leaks_summary() self._upload_json_files(summarized_results, initial_results, results_including_passes, start_time, end_time) results_path = self._filesystem.join(self._results_directory, "results.html") self._copy_results_html_file(results_path) if initial_results.keyboard_interrupted: exit_code = INTERRUPTED_EXIT_STATUS else: if self._options.show_results and ( initial_results.unexpected_results_by_name or (self._options.full_results_html and initial_results.total_failures)): self._port.show_results_html_file(results_path) exit_code = self._port.exit_code_from_summarized_results( summarized_results) return test_run_results.RunDetails(exit_code, summarized_results, initial_results, retry_results, enabled_pixel_tests_in_retry)
def run(self, args): """Run the tests and return a RunDetails object with the results.""" start_time = time.time() self._printer.write_update("Collecting tests ...") running_all_tests = False try: paths, test_names, running_all_tests = self._collect_tests(args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails( exit_code=test_run_results.NO_TESTS_EXIT_STATUS) self._printer.write_update("Parsing expectations ...") self._expectations = test_expectations.TestExpectations( self._port, test_names) tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails( exit_code=test_run_results.NO_TESTS_EXIT_STATUS) exit_code = self._set_up_run(tests_to_run) if exit_code: return test_run_results.RunDetails(exit_code=exit_code) # Don't retry failures if an explicit list of tests was passed in. if self._options.retry_failures is None: should_retry_failures = len(paths) < len(test_names) else: should_retry_failures = self._options.retry_failures enabled_pixel_tests_in_retry = False try: self._start_servers(tests_to_run) num_workers = self._port.num_workers( int(self._options.child_processes)) initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations, num_workers) # Don't retry failures when interrupted by user or failures limit exception. should_retry_failures = should_retry_failures and not ( initial_results.interrupted or initial_results.keyboard_interrupted) tests_to_retry = self._tests_to_retry(initial_results) all_retry_results = [] if should_retry_failures and tests_to_retry: enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed( ) for retry_attempt in xrange(1, self._options.num_retries + 1): if not tests_to_retry: break _log.info('') _log.info( 'Retrying %s, attempt %d of %d...', grammar.pluralize('unexpected failure', len(tests_to_retry)), retry_attempt, self._options.num_retries) retry_results = self._run_tests( tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1, num_workers=num_workers, retry_attempt=retry_attempt) all_retry_results.append(retry_results) tests_to_retry = self._tests_to_retry(retry_results) if enabled_pixel_tests_in_retry: self._options.pixel_tests = False finally: self._stop_servers() self._clean_up_run() # Some crash logs can take a long time to be written out so look # for new logs after the test run finishes. self._printer.write_update("looking for new crash logs") self._look_for_new_crash_logs(initial_results, start_time) for retry_attempt_results in all_retry_results: self._look_for_new_crash_logs(retry_attempt_results, start_time) _log.debug("summarizing results") summarized_full_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry) summarized_failing_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry, only_include_failing=True) exit_code = summarized_failing_results['num_regressions'] if exit_code > test_run_results.MAX_FAILURES_EXIT_STATUS: _log.warning('num regressions (%d) exceeds max exit status (%d)', exit_code, test_run_results.MAX_FAILURES_EXIT_STATUS) exit_code = test_run_results.MAX_FAILURES_EXIT_STATUS if not self._options.dry_run: self._write_json_files(summarized_full_results, summarized_failing_results, initial_results, running_all_tests) if self._options.write_full_results_to: self._filesystem.copyfile( self._filesystem.join(self._results_directory, "full_results.json"), self._options.write_full_results_to) self._upload_json_files() results_path = self._filesystem.join(self._results_directory, "results.html") self._copy_results_html_file(results_path) if initial_results.keyboard_interrupted: exit_code = test_run_results.INTERRUPTED_EXIT_STATUS else: if initial_results.interrupted: exit_code = test_run_results.EARLY_EXIT_STATUS if self._options.show_results and ( exit_code or (self._options.full_results_html and initial_results.total_failures)): self._port.show_results_html_file(results_path) self._printer.print_results(time.time() - start_time, initial_results, summarized_failing_results) return test_run_results.RunDetails(exit_code, summarized_full_results, summarized_failing_results, initial_results, all_retry_results, enabled_pixel_tests_in_retry)
def run(self, args): """Runs the tests and return a RunDetails object with the results.""" start_time = time.time() self._printer.write_update('Collecting tests ...') running_all_tests = False if not args or any('external' in path for path in args): self._printer.write_update( 'Generating MANIFEST.json for web-platform-tests ...') WPTManifest.ensure_manifest(self._port.host) self._printer.write_update('Completed generating manifest.') self._printer.write_update('Collecting tests ...') try: paths, all_test_names, running_all_tests = self._collect_tests( args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails( exit_code=exit_codes.NO_TESTS_EXIT_STATUS) # Create a sorted list of test files so the subset chunk, # if used, contains alphabetically consecutive tests. if self._options.order == 'natural': all_test_names.sort(key=self._port.test_key) elif self._options.order == 'random': all_test_names.sort() random.Random(self._options.seed).shuffle(all_test_names) test_names, tests_in_other_chunks = self._finder.split_into_chunks( all_test_names) self._printer.write_update('Parsing expectations ...') self._expectations = test_expectations.TestExpectations( self._port, test_names) tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._expectations.remove_tests_from_expectations( tests_in_other_chunks) self._printer.print_found(len(all_test_names), len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails( exit_code=exit_codes.NO_TESTS_EXIT_STATUS) exit_code = self._set_up_run(tests_to_run) if exit_code: return test_run_results.RunDetails(exit_code=exit_code) # Don't retry failures if an explicit list of tests was passed in. if self._options.retry_failures is None: should_retry_failures = len(paths) < len(test_names) else: should_retry_failures = self._options.retry_failures try: self._start_servers(tests_to_run) if self._options.watch: run_results = self._run_test_loop(tests_to_run, tests_to_skip) else: run_results = self._run_test_once(tests_to_run, tests_to_skip, should_retry_failures) initial_results, all_retry_results, enabled_pixel_tests_in_retry = run_results finally: self._stop_servers() self._clean_up_run() # Some crash logs can take a long time to be written out so look # for new logs after the test run finishes. self._printer.write_update('Looking for new crash logs ...') self._look_for_new_crash_logs(initial_results, start_time) for retry_attempt_results in all_retry_results: self._look_for_new_crash_logs(retry_attempt_results, start_time) self._printer.write_update('Summarizing results ...') summarized_full_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry) summarized_failing_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry, only_include_failing=True) exit_code = summarized_failing_results['num_regressions'] if exit_code > exit_codes.MAX_FAILURES_EXIT_STATUS: _log.warning('num regressions (%d) exceeds max exit status (%d)', exit_code, exit_codes.MAX_FAILURES_EXIT_STATUS) exit_code = exit_codes.MAX_FAILURES_EXIT_STATUS if not self._options.dry_run: self._write_json_files(summarized_full_results, summarized_failing_results, initial_results, running_all_tests) self._upload_json_files() self._copy_results_html_file(self._results_directory, 'results.html') self._copy_results_html_file(self._results_directory, 'legacy-results.html') if initial_results.keyboard_interrupted: exit_code = exit_codes.INTERRUPTED_EXIT_STATUS else: if initial_results.interrupted: exit_code = exit_codes.EARLY_EXIT_STATUS if self._options.show_results and ( exit_code or initial_results.total_failures): self._port.show_results_html_file( self._filesystem.join(self._results_directory, 'results.html')) self._printer.print_results(time.time() - start_time, initial_results) return test_run_results.RunDetails(exit_code, summarized_full_results, summarized_failing_results, initial_results, all_retry_results, enabled_pixel_tests_in_retry)
def run(self, args): """Run the tests and return a RunDetails object with the results.""" self._printer.write_update("Collecting tests ...") try: paths, test_names = self._collect_tests(args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update("Parsing expectations ...") self._expectations = test_expectations.TestExpectations(self._port, test_names, force_expectations_pass=self._options.force) self._expectations.parse_all_expectations() tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) default_device_tests = [] # Look for tests with custom device requirements. custom_device_tests = defaultdict(list) for test_file in tests_to_run: custom_device = self._custom_device_for_test(test_file) if custom_device: custom_device_tests[custom_device].append(test_file) else: default_device_tests.append(test_file) if custom_device_tests: for device_class in custom_device_tests: _log.debug('{} tests use device {}'.format(len(custom_device_tests[device_class]), device_class)) initial_results = None retry_results = None enabled_pixel_tests_in_retry = False needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for test in tests_to_run) needs_web_platform_test_server = any(self._needs_web_platform_test(test) for test in tests_to_run) needs_websockets = any(self._is_websocket_test(test) for test in tests_to_run) self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow, needs_http=needs_http, needs_web_platform_test_server=needs_web_platform_test_server, needs_websockets=needs_websockets) self._printer.write_update("Checking build ...") if not self._port.check_build(): _log.error("Build check failed") return test_run_results.RunDetails(exit_code=-1) if self._options.clobber_old_results: self._clobber_old_results() # Create the output directory if it doesn't already exist. self._port.host.filesystem.maybe_make_directory(self._results_directory) if default_device_tests: _log.info('') _log.info("Running %s", pluralize(len(tests_to_run), "test")) _log.info('') if not self._set_up_run(tests_to_run): return test_run_results.RunDetails(exit_code=-1) initial_results, retry_results, enabled_pixel_tests_in_retry = self._run_test_subset(default_device_tests, tests_to_skip) # Only use a single worker for custom device classes self._options.child_processes = 1 for device_class in custom_device_tests: device_tests = custom_device_tests[device_class] if device_tests: _log.info('') _log.info('Running %s for %s', pluralize(len(device_tests), "test"), device_class) _log.info('') if not self._set_up_run(device_tests, device_class): return test_run_results.RunDetails(exit_code=-1) device_initial_results, device_retry_results, device_enabled_pixel_tests_in_retry = self._run_test_subset(device_tests, tests_to_skip) initial_results = initial_results.merge(device_initial_results) if initial_results else device_initial_results retry_results = retry_results.merge(device_retry_results) if retry_results else device_retry_results enabled_pixel_tests_in_retry |= device_enabled_pixel_tests_in_retry self._runner.stop_servers() end_time = time.time() return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry)
def run(self, args): """Run the tests and return a RunDetails object with the results.""" self._printer.write_update("Collecting tests ...") try: paths, test_names = self._collect_tests(args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update("Parsing expectations ...") self._expectations = test_expectations.TestExpectations( self._port, test_names, force_expectations_pass=self._options.force) self._expectations.parse_all_expectations() tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) default_device_tests = [] # Look for tests with custom device requirements. custom_device_tests = defaultdict(list) for test_file in tests_to_run: custom_device = self._custom_device_for_test(test_file) if custom_device: custom_device_tests[custom_device].append(test_file) else: default_device_tests.append(test_file) if custom_device_tests: for device_class in custom_device_tests: _log.debug('{} tests use device {}'.format( len(custom_device_tests[device_class]), device_class)) initial_results = None retry_results = None enabled_pixel_tests_in_retry = False if default_device_tests: _log.info('') _log.info("Running %s", pluralize(len(tests_to_run), "test")) _log.info('') if not self._set_up_run(tests_to_run): return test_run_results.RunDetails(exit_code=-1) initial_results, retry_results, enabled_pixel_tests_in_retry = self._run_test_subset( default_device_tests, tests_to_skip) for device_class in custom_device_tests: device_tests = custom_device_tests[device_class] if device_tests: _log.info('') _log.info('Running %s for %s', pluralize(len(device_tests), "test"), device_class) _log.info('') if not self._set_up_run(device_tests, device_class): return test_run_results.RunDetails(exit_code=-1) device_initial_results, device_retry_results, device_enabled_pixel_tests_in_retry = self._run_test_subset( device_tests, tests_to_skip) initial_results = initial_results.merge( device_initial_results ) if initial_results else device_initial_results retry_results = retry_results.merge( device_retry_results ) if retry_results else device_retry_results enabled_pixel_tests_in_retry |= device_enabled_pixel_tests_in_retry end_time = time.time() return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry)
def run(self, args): num_failed_uploads = 0 device_type_list = self._port.supported_device_types() try: tests_to_run_by_device, aggregate_tests_to_skip = self._collect_tests(args, device_type_list) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) aggregate_tests_to_run = set() # type: Set[Test] for v in tests_to_run_by_device.values(): aggregate_tests_to_run.update(v) skipped_tests_by_path = defaultdict(set) for test in aggregate_tests_to_skip: skipped_tests_by_path[test.test_path].add(test) # If a test is marked skipped, but was explicitly requested, run it anyways if self._options.skipped != 'always': for arg in args: if arg in skipped_tests_by_path: tests = skipped_tests_by_path[arg] tests_to_run_by_device[device_type_list[0]].extend(tests) aggregate_tests_to_run |= tests aggregate_tests_to_skip -= tests del skipped_tests_by_path[arg] aggregate_tests = aggregate_tests_to_run | aggregate_tests_to_skip self._printer.print_found(len(aggregate_tests), len(aggregate_tests_to_run), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to see if all tests we are running are skipped. if aggregate_tests == aggregate_tests_to_skip: # XXX: this is currently identical to the follow if, which likely isn't intended _log.error("All tests skipped.") return test_run_results.RunDetails(exit_code=0, skipped_all_tests=True) # Check to make sure we have no tests to run that are not skipped. if not aggregate_tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) self._printer.write_update("Checking build ...") if not self._port.check_build(): _log.error("Build check failed") return test_run_results.RunDetails(exit_code=-1) if self._options.clobber_old_results: self._clobber_old_results() # Create the output directory if it doesn't already exist. self._port.host.filesystem.maybe_make_directory(self._results_directory) needs_http = any(test.needs_http_server for tests in itervalues(tests_to_run_by_device) for test in tests) needs_web_platform_test_server = any(test.needs_wpt_server for tests in itervalues(tests_to_run_by_device) for test in tests) needs_websockets = any(test.needs_websocket_server for tests in itervalues(tests_to_run_by_device) for test in tests) self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, needs_http=needs_http, needs_web_platform_test_server=needs_web_platform_test_server, needs_websockets=needs_websockets) initial_results = None retry_results = None enabled_pixel_tests_in_retry = False max_child_processes_for_run = 1 child_processes_option_value = self._options.child_processes uploads = [] for device_type in device_type_list: self._options.child_processes = min(self._port.max_child_processes(device_type=device_type), int(child_processes_option_value or self._port.default_child_processes(device_type=device_type))) _log.info('') if not self._options.child_processes: _log.info('Skipping {} because {} is not available'.format(pluralize(len(tests_to_run_by_device[device_type]), 'test'), str(device_type))) _log.info('') continue max_child_processes_for_run = max(self._options.child_processes, max_child_processes_for_run) self._printer.print_baseline_search_path(device_type=device_type) _log.info(u'Running {}{}'.format(pluralize(len(tests_to_run_by_device[device_type]), 'test'), u' for {}'.format(device_type) if device_type else '')) _log.info('') start_time_for_device = time.time() if not tests_to_run_by_device[device_type]: continue test_inputs = [self._test_input_for_file(test, device_type=device_type) for test in tests_to_run_by_device[device_type]] if not self._set_up_run(test_inputs, device_type=device_type): return test_run_results.RunDetails(exit_code=-1) configuration = self._port.configuration_for_upload(self._port.target_host(0)) if not configuration.get('flavor', None): # The --result-report-flavor argument should override wk1/wk2 configuration['flavor'] = 'wk2' if self._options.webkit_test_runner else 'wk1' temp_initial_results, temp_retry_results, temp_enabled_pixel_tests_in_retry = self._run_test_subset(test_inputs, device_type=device_type) skipped_results = TestRunResults(self._expectations[device_type], len(aggregate_tests_to_skip)) for skipped_test in set(aggregate_tests_to_skip): skipped_result = test_results.TestResult(skipped_test.test_path) skipped_result.type = test_expectations.SKIP skipped_results.add(skipped_result, expected=True) temp_initial_results = temp_initial_results.merge(skipped_results) if self._options.report_urls: self._printer.writeln('\n') self._printer.write_update('Preparing upload data ...') upload = Upload( suite=self._options.suite or 'layout-tests', configuration=configuration, details=Upload.create_details(options=self._options), commits=self._port.commits_for_upload(), timestamp=start_time, run_stats=Upload.create_run_stats( start_time=start_time_for_device, end_time=time.time(), tests_skipped=temp_initial_results.remaining + temp_initial_results.expected_skips, ), results=self._results_to_upload_json_trie(self._expectations[device_type], temp_initial_results), ) for hostname in self._options.report_urls: self._printer.write_update('Uploading to {} ...'.format(hostname)) if not upload.upload(hostname, log_line_func=self._printer.writeln): num_failed_uploads += 1 else: uploads.append(upload) self._printer.writeln('Uploads completed!') initial_results = initial_results.merge(temp_initial_results) if initial_results else temp_initial_results retry_results = retry_results.merge(temp_retry_results) if retry_results else temp_retry_results enabled_pixel_tests_in_retry |= temp_enabled_pixel_tests_in_retry if (initial_results and (initial_results.interrupted or initial_results.keyboard_interrupted)) or \ (retry_results and (retry_results.interrupted or retry_results.keyboard_interrupted)): break # Used for final logging, max_child_processes_for_run is most relevant here. self._options.child_processes = max_child_processes_for_run self._runner.stop_servers() end_time = time.time() result = self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry) if self._options.report_urls and uploads: self._printer.writeln('\n') self._printer.write_update('Preparing to upload test archive ...') with self._filesystem.mkdtemp() as temp: archive = self._filesystem.join(temp, 'test-archive') shutil.make_archive(archive, 'zip', self._results_directory) for upload in uploads: for hostname in self._options.report_urls: self._printer.write_update('Uploading archive to {} ...'.format(hostname)) if not upload.upload_archive(hostname, self._filesystem.open_binary_file_for_reading(archive + '.zip'), log_line_func=self._printer.writeln): num_failed_uploads += 1 if num_failed_uploads: result.exit_code = -1 return result
def run(self, args): num_failed_uploads = 0 total_tests = set() aggregate_test_names = set() aggregate_tests = set() tests_to_run_by_device = {} device_type_list = self._port.supported_device_types() for device_type in device_type_list: """Run the tests and return a RunDetails object with the results.""" for_device_type = u'for {} '.format(device_type) if device_type else '' self._printer.write_update(u'Collecting tests {}...'.format(for_device_type)) try: paths, test_names = self._collect_tests(args, device_type=device_type) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update(u'Parsing expectations {}...'.format(for_device_type)) self._expectations[device_type] = test_expectations.TestExpectations(self._port, test_names, force_expectations_pass=self._options.force, device_type=device_type) self._expectations[device_type].parse_all_expectations() aggregate_test_names.update(test_names) tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names, device_type=device_type) total_tests.update(tests_to_run) total_tests.update(tests_to_skip) tests_to_run_by_device[device_type] = [test for test in tests_to_run if test not in aggregate_tests] aggregate_tests.update(tests_to_run) # If a test is marked skipped, but was explicitly requested, run it anyways if self._options.skipped != 'always': for arg in args: if arg in total_tests and arg not in aggregate_tests: tests_to_run_by_device[device_type_list[0]].append(arg) aggregate_tests.add(arg) tests_to_skip = total_tests - aggregate_tests self._printer.print_found(len(aggregate_test_names), len(aggregate_tests), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to make sure we're not skipping every test. if not sum([len(tests) for tests in itervalues(tests_to_run_by_device)]): _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for tests in itervalues(tests_to_run_by_device) for test in tests) needs_web_platform_test_server = any(self._needs_web_platform_test(test) for tests in itervalues(tests_to_run_by_device) for test in tests) needs_websockets = any(self._is_websocket_test(test) for tests in itervalues(tests_to_run_by_device) for test in tests) self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow, needs_http=needs_http, needs_web_platform_test_server=needs_web_platform_test_server, needs_websockets=needs_websockets) self._printer.write_update("Checking build ...") if not self._port.check_build(): _log.error("Build check failed") return test_run_results.RunDetails(exit_code=-1) if self._options.clobber_old_results: self._clobber_old_results() # Create the output directory if it doesn't already exist. self._port.host.filesystem.maybe_make_directory(self._results_directory) initial_results = None retry_results = None enabled_pixel_tests_in_retry = False max_child_processes_for_run = 1 child_processes_option_value = self._options.child_processes uploads = [] for device_type in device_type_list: self._runner._test_is_slow = lambda test_file: self._test_is_slow(test_file, device_type=device_type) self._options.child_processes = min(self._port.max_child_processes(device_type=device_type), int(child_processes_option_value or self._port.default_child_processes(device_type=device_type))) _log.info('') if not self._options.child_processes: _log.info('Skipping {} because {} is not available'.format(pluralize(len(tests_to_run_by_device[device_type]), 'test'), str(device_type))) _log.info('') continue max_child_processes_for_run = max(self._options.child_processes, max_child_processes_for_run) self._printer.print_baseline_search_path(device_type=device_type) _log.info(u'Running {}{}'.format(pluralize(len(tests_to_run_by_device[device_type]), 'test'), u' for {}'.format(device_type) if device_type else '')) _log.info('') start_time_for_device = time.time() if not tests_to_run_by_device[device_type]: continue if not self._set_up_run(tests_to_run_by_device[device_type], device_type=device_type): return test_run_results.RunDetails(exit_code=-1) configuration = self._port.configuration_for_upload(self._port.target_host(0)) if not configuration.get('flavor', None): # The --result-report-flavor argument should override wk1/wk2 configuration['flavor'] = 'wk2' if self._options.webkit_test_runner else 'wk1' temp_initial_results, temp_retry_results, temp_enabled_pixel_tests_in_retry = self._run_test_subset(tests_to_run_by_device[device_type], tests_to_skip, device_type=device_type) if self._options.report_urls: self._printer.writeln('\n') self._printer.write_update('Preparing upload data ...') upload = Upload( suite='layout-tests', configuration=configuration, details=Upload.create_details(options=self._options), commits=self._port.commits_for_upload(), timestamp=start_time, run_stats=Upload.create_run_stats( start_time=start_time_for_device, end_time=time.time(), tests_skipped=temp_initial_results.remaining + temp_initial_results.expected_skips, ), results=self._results_to_upload_json_trie(self._expectations[device_type], temp_initial_results), ) for hostname in self._options.report_urls: self._printer.write_update('Uploading to {} ...'.format(hostname)) if not upload.upload(hostname, log_line_func=self._printer.writeln): num_failed_uploads += 1 else: uploads.append(upload) self._printer.writeln('Uploads completed!') initial_results = initial_results.merge(temp_initial_results) if initial_results else temp_initial_results retry_results = retry_results.merge(temp_retry_results) if retry_results else temp_retry_results enabled_pixel_tests_in_retry |= temp_enabled_pixel_tests_in_retry if (initial_results and (initial_results.interrupted or initial_results.keyboard_interrupted)) or \ (retry_results and (retry_results.interrupted or retry_results.keyboard_interrupted)): break # Used for final logging, max_child_processes_for_run is most relevant here. self._options.child_processes = max_child_processes_for_run self._runner.stop_servers() end_time = time.time() result = self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry) if self._options.report_urls and uploads: self._printer.writeln('\n') self._printer.write_update('Preparing to upload test archive ...') with self._filesystem.mkdtemp() as temp: archive = self._filesystem.join(temp, 'test-archive') shutil.make_archive(archive, 'zip', self._results_directory) for upload in uploads: for hostname in self._options.report_urls: self._printer.write_update('Uploading archive to {} ...'.format(hostname)) if not upload.upload_archive(hostname, self._filesystem.open_binary_file_for_reading(archive + '.zip'), log_line_func=self._printer.writeln): num_failed_uploads += 1 if num_failed_uploads: result.exit_code = -1 return result
def run(self, args): """Run the tests and return a RunDetails object with the results.""" self._printer.write_update("Collecting tests ...") try: paths, test_names = self._collect_tests(args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update("Parsing expectations ...") self._expectations = test_expectations.TestExpectations(self._port, test_names) tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) if not self._set_up_run(tests_to_run): return test_run_results.RunDetails(exit_code=-1) # Don't retry failures if an explicit list of tests was passed in. if self._options.retry_failures is None: should_retry_failures = len(paths) < len(test_names) else: should_retry_failures = self._options.retry_failures start_time = time.time() enabled_pixel_tests_in_retry = False try: initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations, int(self._options.child_processes), retrying=False) tests_to_retry = self._tests_to_retry(initial_results, include_crashes=self._port.should_retry_crashes()) if should_retry_failures and tests_to_retry and not initial_results.interrupted: enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed() _log.info('') _log.info("Retrying %d unexpected failure(s) ..." % len(tests_to_retry)) _log.info('') retry_results = self._run_tests(tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1, num_workers=1, retrying=True) if enabled_pixel_tests_in_retry: self._options.pixel_tests = False else: retry_results = None finally: self._clean_up_run() end_time = time.time() # Some crash logs can take a long time to be written out so look # for new logs after the test run finishes. _log.debug("looking for new crash logs") self._look_for_new_crash_logs(initial_results, start_time) if retry_results: self._look_for_new_crash_logs(retry_results, start_time) _log.debug("summarizing results") summarized_full_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry) summarized_failing_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, only_include_failing=True) self._printer.print_results(end_time - start_time, initial_results, summarized_failing_results) exit_code = self._port.exit_code_from_summarized_results(summarized_failing_results) if not self._options.dry_run: self._write_json_files(summarized_full_results, summarized_failing_results, initial_results) self._upload_json_files() results_path = self._filesystem.join(self._results_directory, "results.html") self._copy_results_html_file(results_path) if self._options.show_results and (exit_code or (self._options.full_results_html and initial_results.total_failures)): self._port.show_results_html_file(results_path) return test_run_results.RunDetails(exit_code, summarized_full_results, summarized_failing_results, initial_results, retry_results, enabled_pixel_tests_in_retry)
def run(self, args): total_tests = set() aggregate_test_names = set() aggregate_tests = set() tests_to_run_by_device = {} device_type_list = self._port.supported_device_types() for device_type in device_type_list: """Run the tests and return a RunDetails object with the results.""" for_device_type = 'for {} '.format( device_type) if device_type else '' self._printer.write_update( 'Collecting tests {}...'.format(for_device_type)) try: paths, test_names = self._collect_tests( args, device_type=device_type) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update( 'Parsing expectations {}...'.format(for_device_type)) self._expectations[ device_type] = test_expectations.TestExpectations( self._port, test_names, force_expectations_pass=self._options.force, device_type=device_type) self._expectations[device_type].parse_all_expectations() aggregate_test_names.update(test_names) tests_to_run, tests_to_skip = self._prepare_lists( paths, test_names, device_type=device_type) total_tests.update(tests_to_run) total_tests.update(tests_to_skip) tests_to_run_by_device[device_type] = [ test for test in tests_to_run if test not in aggregate_tests ] aggregate_tests.update(tests_to_run) tests_to_skip = total_tests - aggregate_tests self._printer.print_found(len(aggregate_test_names), len(aggregate_tests), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to make sure we're not skipping every test. if not sum( [len(tests) for tests in tests_to_run_by_device.itervalues()]): _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for tests in tests_to_run_by_device.itervalues() for test in tests) needs_web_platform_test_server = any( self._needs_web_platform_test(test) for tests in tests_to_run_by_device.itervalues() for test in tests) needs_websockets = any( self._is_websocket_test(test) for tests in tests_to_run_by_device.itervalues() for test in tests) self._runner = LayoutTestRunner( self._options, self._port, self._printer, self._results_directory, self._test_is_slow, needs_http=needs_http, needs_web_platform_test_server=needs_web_platform_test_server, needs_websockets=needs_websockets) self._printer.write_update("Checking build ...") if not self._port.check_build(): _log.error("Build check failed") return test_run_results.RunDetails(exit_code=-1) if self._options.clobber_old_results: self._clobber_old_results() # Create the output directory if it doesn't already exist. self._port.host.filesystem.maybe_make_directory( self._results_directory) initial_results = None retry_results = None enabled_pixel_tests_in_retry = False max_child_processes_for_run = 1 child_processes_option_value = self._options.child_processes for device_type in device_type_list: self._runner._test_is_slow = lambda test_file: self._test_is_slow( test_file, device_type=device_type) self._options.child_processes = min( self._port.max_child_processes(device_type=device_type), int(child_processes_option_value or self._port.default_child_processes( device_type=device_type))) _log.info('') if not self._options.child_processes: _log.info('Skipping {} because {} is not available'.format( pluralize(len(tests_to_run_by_device[device_type]), 'test'), str(device_type))) _log.info('') continue max_child_processes_for_run = max(self._options.child_processes, max_child_processes_for_run) self._printer.print_baseline_search_path(device_type=device_type) _log.info('Running {}{}'.format( pluralize(len(tests_to_run_by_device[device_type]), 'test'), ' for {}'.format(str(device_type)) if device_type else '')) _log.info('') if not tests_to_run_by_device[device_type]: continue if not self._set_up_run(tests_to_run_by_device[device_type], device_type=device_type): return test_run_results.RunDetails(exit_code=-1) temp_initial_results, temp_retry_results, temp_enabled_pixel_tests_in_retry = self._run_test_subset( tests_to_run_by_device[device_type], tests_to_skip, device_type=device_type) initial_results = initial_results.merge( temp_initial_results ) if initial_results else temp_initial_results retry_results = retry_results.merge( temp_retry_results) if retry_results else temp_retry_results enabled_pixel_tests_in_retry |= temp_enabled_pixel_tests_in_retry # Used for final logging, max_child_processes_for_run is most relevant here. self._options.child_processes = max_child_processes_for_run self._runner.stop_servers() end_time = time.time() return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry)
def run(self, args): """Run the tests and return a RunDetails object with the results.""" self._printer.write_update("Collecting tests ...") try: paths, test_names = self._collect_tests(args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails(exit_code=-1) self._printer.write_update("Parsing expectations ...") self._expectations = test_expectations.TestExpectations( self._port, test_names, force_expectations_pass=self._options.force) self._expectations.parse_all_expectations() tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) start_time = time.time() # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails(exit_code=-1) # Look for tests with custom device requirements. test_device_mapping = defaultdict(list) for test_file in tests_to_run: test_device_mapping[self._custom_device_for_test(test_file) or self._port.DEFAULT_DEVICE_TYPE].append( test_file) # Order device types from most specific to least specific in the hopes that some of the more specific device # types will match the less specific device types. device_type_order = [] types_with_family = [] remaining_types = [] for device_type in test_device_mapping.iterkeys(): if device_type and device_type.hardware_family and device_type.hardware_type: device_type_order.append(device_type) elif device_type and device_type.hardware_family: types_with_family.append(device_type) else: remaining_types.append(device_type) device_type_order.extend(types_with_family + remaining_types) needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for test in tests_to_run) needs_web_platform_test_server = any( self._needs_web_platform_test(test) for test in tests_to_run) needs_websockets = any( self._is_websocket_test(test) for test in tests_to_run) self._runner = LayoutTestRunner( self._options, self._port, self._printer, self._results_directory, self._test_is_slow, needs_http=needs_http, needs_web_platform_test_server=needs_web_platform_test_server, needs_websockets=needs_websockets) self._printer.write_update("Checking build ...") if not self._port.check_build(): _log.error("Build check failed") return test_run_results.RunDetails(exit_code=-1) if self._options.clobber_old_results: self._clobber_old_results() # Create the output directory if it doesn't already exist. self._port.host.filesystem.maybe_make_directory( self._results_directory) initial_results = None retry_results = None enabled_pixel_tests_in_retry = False child_processes_option_value = self._options.child_processes while device_type_order: device_type = device_type_order[0] tests = test_device_mapping[device_type] del device_type_order[0] self._options.child_processes = min( self._port.max_child_processes(device_type=device_type), int(child_processes_option_value or self._port.default_child_processes( device_type=device_type))) _log.info('') if not self._options.child_processes: _log.info('Skipping {} because {} is not available'.format( pluralize(len(test_device_mapping[device_type]), 'test'), str(device_type))) _log.info('') continue # This loop looks for any less-specific device types which match the current device type index = 0 while index < len(device_type_order): if device_type_order[index] == device_type: tests.extend(test_device_mapping[device_type_order[index]]) # Remove devices types from device_type_order once tests associated with that type have been claimed. del device_type_order[index] else: index += 1 _log.info('Running {}{}'.format( pluralize(len(tests), 'test'), ' for {}'.format(str(device_type)) if device_type else '')) _log.info('') if not self._set_up_run(tests, device_type): return test_run_results.RunDetails(exit_code=-1) temp_initial_results, temp_retry_results, temp_enabled_pixel_tests_in_retry = self._run_test_subset( tests, tests_to_skip) initial_results = initial_results.merge( temp_initial_results ) if initial_results else temp_initial_results retry_results = retry_results.merge( temp_retry_results) if retry_results else temp_retry_results enabled_pixel_tests_in_retry |= temp_enabled_pixel_tests_in_retry self._runner.stop_servers() end_time = time.time() return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry)