def _aggregate_results_for_test(cls, test): subtest_results = cls._aggregate_results( test['tests']) if 'tests' in test else {} results = {} for metric_name, metric in iteritems(test.get('metrics', {})): if not isinstance(metric, list): results[metric_name] = {None: {}} for config_name, values in iteritems(metric): results[metric_name][None][ config_name] = cls._flatten_list(values) continue # Filter duplicate aggregators that could have arisen from merging JSONs. aggregator_list = list(set(metric)) results[metric_name] = {} for aggregator in aggregator_list: values_by_config_iteration = cls._subtest_values_by_config_iteration( subtest_results, metric_name, aggregator) for config_name, values_by_iteration in iteritems( values_by_config_iteration): results[metric_name].setdefault(aggregator, {}) results[metric_name][aggregator][config_name] = [ cls._aggregate_values(aggregator, values) for values in values_by_iteration ] return {'metrics': results, 'tests': subtest_results}
def _lint_subtest_results(cls, subtests, parent_test, parent_aggregator_list): iteration_groups_by_config = {} for test_name, test in iteritems(subtests): aggregator_list = None if 'metrics' not in test and 'tests' not in test: raise TypeError('"%s" does not contain metrics or tests' % test_name) if 'metrics' in test: metrics = test['metrics'] if not isinstance(metrics, dict): raise TypeError('The metrics in "%s" is not a dictionary' % test_name) for metric_name, metric in iteritems(metrics): if isinstance(metric, list): cls._lint_aggregator_list(test_name, metric_name, metric, parent_test, parent_aggregator_list) aggregator_list = metric elif isinstance(metric, dict): cls._lint_configuration(test_name, metric_name, metric, parent_test, parent_aggregator_list, iteration_groups_by_config) else: raise TypeError('"%s" metric of "%s" was not an aggregator list or a dictionary of configurations: %s' % (metric_name, test_name, str(metric))) if 'tests' in test: cls._lint_subtest_results(test['tests'], test_name, aggregator_list) elif aggregator_list: raise TypeError('"%s" requires aggregation but it has no subtests' % (test_name)) return iteration_groups_by_config
def _lint_configuration(cls, test_name, metric_name, configurations, parent_test, parent_aggregator_list, iteration_groups_by_config): # FIXME: Check that config_name is always "current". for config_name, values in iteritems(configurations): nested_list_count = [isinstance(value, list) for value in values].count(True) if nested_list_count not in [0, len(values)]: raise TypeError( '"%s" metric of "%s" had malformed values: %s' % (metric_name, test_name, json.dumps(values))) if nested_list_count: value_shape = [] for value_group in values: value_shape.append(len(value_group)) cls._lint_values(test_name, metric_name, value_group) else: value_shape = len(values) cls._lint_values(test_name, metric_name, values) iteration_groups_by_config.setdefault(metric_name, {}).setdefault( config_name, value_shape) if parent_aggregator_list and value_shape != iteration_groups_by_config[ metric_name][config_name]: raise TypeError( '"%s" metric of "%s" had a mismatching subtest values' % (metric_name, parent_test))
def _subtest_values_by_config_iteration(cls, subtest_results, metric_name, aggregator): values_by_config_iteration = {} for subtest_name, subtest in iteritems(subtest_results): results_for_metric = subtest['metrics'].get(metric_name, {}) if aggregator in results_for_metric: results_for_aggregator = results_for_metric.get(aggregator) elif None in results_for_metric: results_for_aggregator = results_for_metric.get(None) elif len(results_for_metric.keys()) == 1: results_for_aggregator = results_for_metric.get(list(results_for_metric.keys())[0]) else: results_for_aggregator = {} for config_name, values in iteritems(results_for_aggregator): values_by_config_iteration.setdefault(config_name, [[] for _ in values]) for iteration, value in enumerate(values): values_by_config_iteration[config_name][iteration].append(value) return values_by_config_iteration
def _do_post_tests_work(self, driver): additional_results = [] if not driver: return additional_results post_test_output = driver.do_post_tests_work() if post_test_output: for test_name, doc_list in iteritems(post_test_output.world_leaks_dict): additional_results.append(test_results.TestResult(test_name, [test_failures.FailureDocumentLeak(doc_list)])) return additional_results
def _print_tests_result_with_status(self, status, runner): mapping = runner.result_map_by_status(status) if mapping: self._stream.writeln(runner.NAME_FOR_STATUS[status]) self._stream.writeln('') need_newline = False for test, output in iteritems(mapping): need_newline = Manager._print_test_result( self._stream, test, output) if need_newline: self._stream.writeln('')
def _stats_trie(self, initial_results): def _worker_number(worker_name): return int(worker_name.split('/')[1]) if worker_name else -1 stats = {} for result in initial_results.results_by_name.values(): if result.type != test_expectations.SKIP: stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))} stats_trie = {} for name, value in iteritems(stats): json_results_generator.add_path_to_trie(name, value, stats_trie) return stats_trie
def convert_trie_to_flat_paths(trie, prefix=None): """Converts the directory structure in the given trie to flat paths, prepending a prefix to each.""" result = {} for name, data in iteritems(trie): if prefix: name = prefix + "/" + name if len(data) and not "results" in data: result.update(convert_trie_to_flat_paths(data, name)) else: result[name] = data return result
def _convert_json_to_current_version(self, results_json): """If the JSON does not match the current version, converts it to the current version and adds in the new version number. """ if self.VERSION_KEY in results_json: archive_version = results_json[self.VERSION_KEY] if archive_version == self.VERSION: return else: archive_version = 3 # version 3->4 if archive_version == 3: num_results = len(results_json.values()) for builder, results in iteritems(results_json): self._convert_tests_to_trie(results) results_json[self.VERSION_KEY] = self.VERSION
def run(self, tests, num_workers): if not tests: return self.printer.write_update('Sharding tests ...') shards = Runner._shard_tests(tests) original_level = server_process_logger.level server_process_logger.setLevel(logging.CRITICAL) try: self._num_workers = min(num_workers, len(shards)) with message_pool.get( self, lambda caller: _Worker(caller, self.port, shards), self._num_workers) as pool: pool.run(('test', shard) for shard, _ in iteritems(shards)) finally: server_process_logger.setLevel(original_level)
def _should_log_linter_warning(warning, files, cwd, host): abs_filename = host.filesystem.join(cwd, warning.filename) # Case 1, the line the warning was tied to is in our patch. if abs_filename in files and files[abs_filename] and warning.line_number in files[abs_filename]: return True for file, lines in iteritems(warning.related_files): abs_filename = host.filesystem.join(cwd, file) if abs_filename in files: # Case 2, a file associated with the warning is in our patch # Note that this will really only happen if you delete a test. if lines is None: return True # Case 3, a line associated with the warning is in our patch. for line in lines: if files[abs_filename] and line in files[abs_filename]: return True return False
def _shard_by_directory(self, test_inputs, num_workers): """Returns a lists of shards, each shard containing all the files in a directory. This is the default mode, and gets as much parallelism as we can while minimizing flakiness caused by inter-test dependencies.""" shards = [] tests_by_dir = {} # FIXME: Given that the tests are already sorted by directory, # we can probably rewrite this to be clearer and faster. for test_input in test_inputs: directory = self._split(test_input.test_name)[0] tests_by_dir.setdefault(directory, []) tests_by_dir[directory].append(test_input) for directory, test_inputs in iteritems(tests_by_dir): shard = TestShard(directory, test_inputs) shards.append(shard) # Sort the shards by directory name. shards.sort(key=lambda shard: shard.name) return shards
def run(self, tests, num_workers): if not tests: return self.printer.write_update('Sharding tests ...') shards = Runner._shard_tests(tests) original_level = server_process_logger.level server_process_logger.setLevel(logging.CRITICAL) try: if Runner.instance: raise RuntimeError('Cannot nest API test runners') Runner.instance = self self._num_workers = min(num_workers, len(shards)) devices = None if getattr(self.port, 'DEVICE_MANAGER', None): devices = dict( available_devices=self.port.DEVICE_MANAGER. AVAILABLE_DEVICES, initialized_devices=self.port.DEVICE_MANAGER. INITIALIZED_DEVICES, ) with TaskPool( workers=self._num_workers, setup=setup_shard, setupkwargs=dict(port=self.port, devices=devices), teardown=teardown_shard, ) as pool: for name, tests in iteritems(shards): pool.do(run_shard, name, *tests) pool.wait() finally: server_process_logger.setLevel(original_level) Runner.instance = None
def __init__(self, port_obj, output_dir, additional_dirs=None, port=None): """Args: port_obj: handle to the platform-specific routines output_dir: the absolute path to the layout test result directory """ http_server_base.HttpServerBase.__init__(self, port_obj) # We use the name "httpd" instead of "apache" to make our paths (e.g. the pid file: /tmp/WebKit/httpd.pid) # match old-run-webkit-tests: https://bugs.webkit.org/show_bug.cgi?id=63956 self._name = 'httpd' self._port = port if self._port is not None: self._mappings = [{'port': self._port}] else: self._mappings = [{'port': self.HTTP_SERVER_PORT}, {'port': self.ALTERNATIVE_HTTP_SERVER_PORT}, {'port': self.HTTPS_SERVER_PORT, 'sslcert': True}] self._output_dir = output_dir self._filesystem.maybe_make_directory(output_dir) self._pid_file = self._filesystem.join(self._runtime_path, '%s.pid' % self._name) if port_obj.host.platform.is_cygwin(): # Convert to MSDOS file naming: precompiledBuildbot = re.compile('^/home/buildbot') precompiledDrive = re.compile('^/cygdrive/[cC]') output_dir = precompiledBuildbot.sub("C:/cygwin/home/buildbot", output_dir) output_dir = precompiledDrive.sub("C:", output_dir) self.tests_dir = precompiledBuildbot.sub("C:/cygwin/home/buildbot", self.tests_dir) self.tests_dir = precompiledDrive.sub("C:", self.tests_dir) self._pid_file = self._filesystem.join("C:/xampp/apache/logs", '%s.pid' % self._name) mime_types_path = self._filesystem.join(self.tests_dir, "http", "conf", "mime.types") cert_file = self._filesystem.join(self.tests_dir, "http", "conf", "webkit-httpd.pem") access_log = self._filesystem.join(output_dir, "access_log.txt") error_log = self._filesystem.join(output_dir, "error_log.txt") document_root = self._filesystem.join(self.tests_dir, "http", "tests") php_ini_dir = self._filesystem.join(self.tests_dir, "http", "conf") if port_obj.get_option('http_access_log'): access_log = port_obj.get_option('http_access_log') if port_obj.get_option('http_error_log'): error_log = port_obj.get_option('http_error_log') # FIXME: We shouldn't be calling a protected method of _port_obj! executable = self._port_obj._path_to_apache() config_file_path = self._copy_apache_config_file(self.tests_dir, output_dir) start_cmd = [executable, '-f', config_file_path, '-C', 'DocumentRoot "%s"' % document_root, '-c', 'TypesConfig "%s"' % mime_types_path, '-c', 'PHPINIDir "%s"' % php_ini_dir, '-c', 'CustomLog "%s" common' % access_log, '-c', 'ErrorLog "%s"' % error_log, '-c', 'PidFile "%s"' % self._pid_file, '-k', "start"] for (alias, path) in self.aliases(): start_cmd.extend(['-c', 'Alias %s "%s"' % (alias, path)]) if not port_obj.host.platform.is_win(): start_cmd.extend(['-C', 'User "%s"' % os.environ.get("USERNAME", os.environ.get("USER", ""))]) enable_ipv6 = self._port_obj.http_server_supports_ipv6() # Perform part of the checks Apache's APR does when trying to listen to # a specific host/port. This allows us to avoid trying to listen to # IPV6 addresses when it fails on Apache. APR itself tries to call # getaddrinfo() again without AI_ADDRCONFIG if the first call fails # with EBADFLAGS, but that is not how it normally fails in our use # cases, so ignore that for now. # See https://bugs.webkit.org/show_bug.cgi?id=98602#c7 try: socket.getaddrinfo('::1', 0, 0, 0, 0, socket.AI_ADDRCONFIG) except: enable_ipv6 = False bind_address = '' if self._port_obj.get_option("http_all_interfaces") else '127.0.0.1:' for mapping in self._mappings: port = mapping['port'] start_cmd += ['-C', 'Listen %s%d' % (bind_address, port)] # We listen to both IPv4 and IPv6 loop-back addresses, but ignore # requests to 8000 from random users on network. # See https://bugs.webkit.org/show_bug.cgi?id=37104 if enable_ipv6: start_cmd += ['-C', 'Listen [::1]:%d' % port] if additional_dirs: for alias, path in iteritems(additional_dirs): start_cmd += ['-c', 'Alias %s "%s"' % (alias, path), # Disable CGI handler for additional dirs. '-c', '<Location %s>' % alias, '-c', 'RemoveHandler .cgi .pl', '-c', '</Location>'] stop_cmd = [executable, '-f', config_file_path, '-c', 'PidFile "%s"' % self._pid_file, '-k', "stop"] start_cmd.extend(['-c', 'SSLCertificateFile "%s"' % cert_file]) self._start_cmd = start_cmd self._stop_cmd = stop_cmd
def summarize_results(port_obj, expectations_by_type, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=False, include_time_and_modifiers=False): """Returns a dictionary containing a summary of the test runs, with the following fields: 'version': a version indicator 'fixable': The number of fixable tests (NOW - PASS) 'skipped': The number of skipped tests (NOW & SKIPPED) 'num_regressions': The number of non-flaky failures 'num_flaky': The number of flaky failures 'num_missing': The number of tests with missing results 'num_passes': The number of unexpected passes 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} 'date': the current date and time """ results = {} results['version'] = 4 tbe = initial_results.tests_by_expectation tbt = initial_results.tests_by_timeline results['fixable'] = len(tbt[test_expectations.NOW] - tbe[test_expectations.PASS]) results['skipped'] = len(tbt[test_expectations.NOW] & tbe[test_expectations.SKIP]) num_passes = 0 num_flaky = 0 num_missing = 0 num_regressions = 0 keywords = {} for expectation_string, expectation_enum in test_expectations.TestExpectations.EXPECTATIONS.items(): keywords[expectation_enum] = expectation_string.upper() for modifier_string, modifier_enum in test_expectations.TestExpectations.MODIFIERS.items(): keywords[modifier_enum] = modifier_string.upper() tests = {} other_crashes_dict = {} for test_name, result in iteritems(initial_results.results_by_name): # Note that if a test crashed in the original run, we ignore # whether or not it crashed when we retried it (if we retried it), # and always consider the result not flaky. pixel_tests_enabled = enabled_pixel_tests_in_retry or port_obj._options.pixel_tests or bool(result.reftest_type) # We're basically trying to find the first non-skip expectation, and use that expectation object for the remainder of the loop. # This works because tests are run on the first device type which won't skip them, regardless of other expectations, and never re-run. expected = 'SKIP' expectations = list(expectations_by_type.values())[0] for element in expectations_by_type.values(): test_expectation = element.filtered_expectations_for_test(test_name, pixel_tests_enabled, port_obj._options.world_leaks) expected = element.model().expectations_to_string(test_expectation) if expected != 'SKIP': expectations = element continue result_type = result.type actual = [keywords[result_type]] if result_type == test_expectations.SKIP: continue if result.is_other_crash: other_crashes_dict[test_name] = {} continue test_dict = {} if result.has_stderr: test_dict['has_stderr'] = True if result.reftest_type: test_dict.update(reftest_type=list(result.reftest_type)) if expectations.model().has_modifier(test_name, test_expectations.WONTFIX): test_dict['wontfix'] = True if result_type == test_expectations.PASS: num_passes += 1 # FIXME: include passing tests that have stderr output. if expected == 'PASS' and not include_passes: continue elif result_type == test_expectations.CRASH: if test_name in initial_results.unexpected_results_by_name: num_regressions += 1 test_dict['report'] = 'REGRESSION' elif result_type == test_expectations.MISSING: if test_name in initial_results.unexpected_results_by_name: num_missing += 1 test_dict['report'] = 'MISSING' elif test_name in initial_results.unexpected_results_by_name: if retry_results and test_name not in retry_results.unexpected_results_by_name: actual.extend(expectations.model().get_expectations_string(test_name).split(" ")) num_flaky += 1 test_dict['report'] = 'FLAKY' elif retry_results: retry_result_type = retry_results.unexpected_results_by_name[test_name].type if result_type != retry_result_type: if enabled_pixel_tests_in_retry and result_type == test_expectations.TEXT and (retry_result_type == test_expectations.IMAGE_PLUS_TEXT or retry_result_type == test_expectations.MISSING): if retry_result_type == test_expectations.MISSING: num_missing += 1 num_regressions += 1 test_dict['report'] = 'REGRESSION' else: num_flaky += 1 test_dict['report'] = 'FLAKY' actual.append(keywords[retry_result_type]) else: num_regressions += 1 test_dict['report'] = 'REGRESSION' else: num_regressions += 1 test_dict['report'] = 'REGRESSION' test_dict['expected'] = expected test_dict['actual'] = " ".join(actual) if include_time_and_modifiers: test_dict['time'] = round(1000 * result.test_run_time) # FIXME: Fix get_modifiers to return modifiers in new format. test_dict['modifiers'] = ' '.join(expectations.model().get_modifiers(test_name)).replace('BUGWK', 'webkit.org/b/') test_dict.update(_interpret_test_failures(result.failures)) if retry_results: retry_result = retry_results.unexpected_results_by_name.get(test_name) if retry_result: test_dict.update(_interpret_test_failures(retry_result.failures)) # Store test hierarchically by directory. e.g. # foo/bar/baz.html: test_dict # foo/bar/baz1.html: test_dict # # becomes # foo: { # bar: { # baz.html: test_dict, # baz1.html: test_dict # } # } parts = test_name.split('/') current_map = tests for i, part in enumerate(parts): if i == (len(parts) - 1): current_map[part] = test_dict break if part not in current_map: current_map[part] = {} current_map = current_map[part] results['tests'] = tests results['num_passes'] = num_passes results['num_flaky'] = num_flaky results['num_missing'] = num_missing results['num_regressions'] = num_regressions results['uses_expectations_file'] = port_obj.uses_test_expectations_file() results['interrupted'] = initial_results.interrupted # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?) results['layout_tests_dir'] = port_obj.layout_tests_dir() results['has_pretty_patch'] = port_obj.pretty_patch.pretty_patch_available() results['pixel_tests_enabled'] = port_obj.get_option('pixel_tests') results['other_crashes'] = other_crashes_dict results['date'] = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y") try: # We only use the svn revision for using trac links in the results.html file, # Don't do this by default since it takes >100ms. # FIXME: Do we really need to populate this both here and in the json_results_generator? if port_obj.get_option("builder_name"): port_obj.host.initialize_scm() results['revision'] = port_obj.host.scm().head_svn_revision() except Exception as e: _log.warn("Failed to determine svn revision for checkout (cwd: %s, webkit_base: %s), leaving 'revision' key blank in full_results.json.\n%s" % (port_obj._filesystem.getcwd(), port_obj.path_from_webkit_base(), e)) # Handle cases where we're running outside of version control. import traceback _log.debug('Failed to learn head svn revision:') _log.debug(traceback.format_exc()) results['revision'] = "" return results
def _prepare_config(self): base_conf_file = self._port_obj.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'layout_tests', 'servers', 'lighttpd.conf') out_conf_file = os.path.join(self._output_dir, 'lighttpd.conf') time_str = time.strftime("%d%b%Y-%H%M%S") access_file_name = "access.log-" + time_str + ".txt" access_log = os.path.join(self._output_dir, access_file_name) log_file_name = "error.log-" + time_str + ".txt" error_log = os.path.join(self._output_dir, log_file_name) if self._port_obj.get_option('http_access_log'): access_log = self._port_obj.get_option('http_access_log') if self._port_obj.get_option('http_error_log'): error_log = self._port_obj.get_option('http_error_log') # Write out the config base_conf = self._filesystem.read_text_file(base_conf_file) # FIXME: This should be re-worked so that this block can # use with open() instead of a manual file.close() call. f = self._filesystem.open_text_file_for_writing(out_conf_file) f.write(base_conf) # Write out our cgi handlers. Run perl through env so that it # processes the #! line and runs perl with the proper command # line arguments. Emulate apache's mod_asis with a cat cgi handler. f.write(('cgi.assign = ( ".cgi" => "/usr/bin/env",\n' ' ".pl" => "/usr/bin/env",\n' ' ".asis" => "/bin/cat",\n' ' ".php" => "%s" )\n\n') % self._port_obj._path_to_lighttpd_php()) # Setup log files f.write(('server.errorlog = "%s"\n' 'accesslog.filename = "%s"\n\n') % (error_log, access_log)) # Setup upload folders. Upload folder is to hold temporary upload files # and also POST data. This is used to support XHR layout tests that # does POST. f.write(('server.upload-dirs = ( "%s" )\n\n') % (self._output_dir)) # Setup a link to where the js test templates and media resources are stored. operator = "=" for alias in self.aliases(): f.write(('alias.url %s ( "%s" => "%s" )\n\n') % (operator, alias[0], alias[1])) operator = "+=" if self._additional_dirs: for alias, path in iteritems(self._additional_dirs): f.write(('alias.url += ( "%s" => "%s" )\n\n') % (alias, path)) # dump out of virtual host config at the bottom. if self._root: if self._port: # Have both port and root dir. mappings = [{'port': self._port, 'docroot': self._root}] else: # Have only a root dir - set the ports as for LayoutTests. # This is used in ui_tests to run http tests against a browser. # default set of ports as for LayoutTests but with a # specified root. mappings = [{'port': self.HTTP_SERVER_PORT, 'docroot': self._root}, {'port': self.ALTERNATIVE_HTTP_SERVER_PORT, 'docroot': self._root}, {'port': self.HTTPS_SERVER_PORT, 'docroot': self._root, 'sslcert': self._pem_file}] else: mappings = self.VIRTUALCONFIG bind_address = '' if self._port_obj.get_option('http_all_addresses') else '127.0.0.1' for mapping in mappings: ssl_setup = '' if 'sslcert' in mapping: ssl_setup = (' ssl.engine = "enable"\n' ' ssl.pemfile = "%s"\n' % mapping['sslcert']) f.write(('$SERVER["socket"] == "%s:%d" {\n' ' server.document-root = "%s"\n' + ssl_setup + '}\n\n') % (bind_address, mapping['port'], mapping['docroot'])) f.close() executable = self._port_obj._path_to_lighttpd() module_path = self._port_obj._path_to_lighttpd_modules() start_cmd = [executable, # Newly written config file '-f', os.path.join(self._output_dir, 'lighttpd.conf'), # Where it can find its module dynamic libraries '-m', module_path] if not self._run_background: start_cmd.append('-D') # Don't background. # Copy liblightcomp.dylib to /tmp/lighttpd/lib to work around the # bug that mod_alias.so loads it from the hard coded path. if self._port_obj.host.platform.is_mac(): tmp_module_path = '/tmp/lighttpd/lib' if not self._filesystem.exists(tmp_module_path): self._filesystem.maybe_make_directory(tmp_module_path) lib_file = 'liblightcomp.dylib' self._filesystem.copyfile(self._filesystem.join(module_path, lib_file), self._filesystem.join(tmp_module_path, lib_file)) self._start_cmd = start_cmd self._env = self._port_obj.setup_environ_for_server('lighttpd') self._mappings = mappings
def _aggregate_results(cls, tests): results = {} for test_name, test in iteritems(tests): results[test_name] = cls._aggregate_results_for_test(test) return results
def print_unexpected_results(self, summarized_results, enabled_pixel_tests_in_retry=False): passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results['actual'].split(" ") expected = results['expected'].split(" ") def is_expected(result): return (result in expected) or (result in ('AUDIO', 'TEXT', 'IMAGE+TEXT') and 'FAIL' in expected) if all(is_expected(actual_result) for actual_result in actual): # Don't print anything for tests that ran as expected. return if actual == ['PASS']: if 'CRASH' in expected: add_to_dict_of_lists(passes, 'Expected to crash, but passed', test) elif 'TIMEOUT' in expected: add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test) else: add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) elif enabled_pixel_tests_in_retry and ( actual == ['TEXT', 'IMAGE+TEXT'] or actual == ['TEXT', 'MISSING']): add_to_dict_of_lists(regressions, actual[0], test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results['actual'], test) resultsjsonparser.for_each_test(summarized_results['tests'], add_result) if len(passes) or len(flaky) or len(regressions): self._print("") if len(passes): for key, tests in iteritems(passes): self._print("%s: (%d)" % (key, len(tests))) tests.sort() for test in tests: self._print(" %s" % test) self._print("") self._print("") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in iteritems(flaky): result = TestExpectations.EXPECTATIONS[key.lower()] self._print("Unexpected flakiness: %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test( summarized_results['tests'], test) actual = result['actual'].split(" ") expected = result['expected'].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] # FIXME: clean this up once the old syntax is gone new_expectations_list = [ TestExpectationParser._inverted_expectation_tokens[exp] for exp in list(set(actual) | set(expected)) ] self._print(" %s [ %s ]" % (test, " ".join(new_expectations_list))) self._print("") self._print("") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in iteritems(regressions): result = TestExpectations.EXPECTATIONS[key.lower()] self._print("Regressions: Unexpected %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: self._print(" %s [ %s ]" % (test, TestExpectationParser. _inverted_expectation_tokens[key])) self._print("") if len(summarized_results['tests']) and self.debug_logging: self._print("%s" % ("-" * 78))
def result_map_by_status(self, status=None): map = {} for test_name, result in iteritems(self.results): if result[0] == status: map[test_name] = result[1] return map
def run(self, args, json_output=None): if json_output: json_output = self.host.filesystem.abspath(json_output) if not self.host.filesystem.isdir( self.host.filesystem.dirname(json_output) ) or self.host.filesystem.isdir(json_output): raise RuntimeError('Cannot write to {}'.format(json_output)) start_time = time.time() self._stream.write_update('Checking build ...') if not self._port.check_api_test_build( self._binaries_for_arguments(args)): _log.error('Build check failed') return Manager.FAILED_BUILD_CHECK self._initialize_devices() self._stream.write_update('Collecting tests ...') try: test_names = self._collect_tests(args) except ScriptError: self._stream.writeln('Failed to collect tests') return Manager.FAILED_COLLECT_TESTS self._stream.write_update('Found {} tests'.format(len(test_names))) if len(test_names) == 0: self._stream.writeln('No tests found') return Manager.FAILED_COLLECT_TESTS if self._port.get_option('dump'): for test in test_names: self._stream.writeln(test) return Manager.SUCCESS test_names = [ test for test in test_names for _ in range(self._options.repeat_each) ] if self._options.repeat_each != 1: _log.debug('Repeating each test {} times'.format( self._options.iterations)) try: _log.info('Running tests') runner = Runner(self._port, self._stream) for i in range(self._options.iterations): _log.debug('\nIteration {}'.format(i + 1)) runner.run( test_names, int(self._options.child_processes) if self._options.child_processes else self._port.default_child_processes()) except KeyboardInterrupt: # If we receive a KeyboardInterrupt, print results. self._stream.writeln('') end_time = time.time() successful = runner.result_map_by_status(runner.STATUS_PASSED) disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED)) _log.info('Ran {} tests of {} with {} successful'.format( len(runner.results) - disabled, len(test_names), len(successful))) result_dictionary = { 'Skipped': [], 'Failed': [], 'Crashed': [], 'Timedout': [], } self._stream.writeln('-' * 30) result = Manager.SUCCESS if len(successful) * self._options.repeat_each + disabled == len( test_names): self._stream.writeln('All tests successfully passed!') if json_output: self.host.filesystem.write_text_file( json_output, json.dumps(result_dictionary, indent=4)) else: self._stream.writeln('Test suite failed') self._stream.writeln('') skipped = [] for test in test_names: if test not in runner.results: skipped.append(test) result_dictionary['Skipped'].append({ 'name': test, 'output': None }) if skipped: self._stream.writeln('Skipped {} tests'.format(len(skipped))) self._stream.writeln('') if self._options.verbose: for test in skipped: self._stream.writeln(' {}'.format(test)) self._print_tests_result_with_status(runner.STATUS_FAILED, runner) self._print_tests_result_with_status(runner.STATUS_CRASHED, runner) self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner) for test, result in iteritems(runner.results): status_to_string = { runner.STATUS_FAILED: 'Failed', runner.STATUS_CRASHED: 'Crashed', runner.STATUS_TIMEOUT: 'Timedout', }.get(result[0]) if not status_to_string: continue result_dictionary[status_to_string].append({ 'name': test, 'output': result[1] }) if json_output: self.host.filesystem.write_text_file( json_output, json.dumps(result_dictionary, indent=4)) result = Manager.FAILED_TESTS if self._options.report_urls: self._stream.writeln('\n') self._stream.write_update('Preparing upload data ...') status_to_test_result = { runner.STATUS_PASSED: None, runner.STATUS_FAILED: Upload.Expectations.FAIL, runner.STATUS_CRASHED: Upload.Expectations.CRASH, runner.STATUS_TIMEOUT: Upload.Expectations.TIMEOUT, } upload = Upload( suite=self._options.suite or 'api-tests', configuration=self._port.configuration_for_upload( self._port.target_host(0)), details=Upload.create_details(options=self._options), commits=self._port.commits_for_upload(), run_stats=Upload.create_run_stats( start_time=start_time, end_time=end_time, tests_skipped=len(result_dictionary['Skipped']), ), results={ test: Upload.create_test_result( actual=status_to_test_result[result[0]]) for test, result in iteritems(runner.results) if result[0] in status_to_test_result }, ) for url in self._options.report_urls: self._stream.write_update('Uploading to {} ...'.format(url)) if not upload.upload(url, log_line_func=self._stream.writeln): result = Manager.FAILED_UPLOAD self._stream.writeln('Uploads completed!') return result