def __init__(self, port_obj, output_dir):
super(WPTServe, self).__init__(port_obj, output_dir)
# These ports must match wpt_support/wpt.config.json
http_port, http_alt_port, https_port = (8001, 8081, 8444)
ws_port, wss_port = (9001, 9444)
self._name = 'wptserve'
self._log_prefixes = ('access_log', 'error_log')
self._mappings = [{
'port': http_port
}, {
'port': http_alt_port
}, {
'port': https_port,
'sslcert': True
}, {
'port': ws_port
}, {
'port': wss_port,
'sslcert': True
}]
# TODO(burnik): We can probably avoid PID files for WPT in the future.
fs = self._filesystem
self._pid_file = fs.join(self._runtime_path, '%s.pid' % self._name)
finder = PathFinder(fs)
path_to_thirdparty = finder.path_from_tools_scripts(
'webkitpy', 'thirdparty')
path_to_wpt_support = finder.path_from_tools_scripts(
'webkitpy', 'thirdparty', 'wpt')
path_to_wpt_root = fs.join(path_to_wpt_support, 'wpt')
path_to_wpt_config = fs.join(path_to_wpt_support, 'wpt.config.json')
path_to_wpt_tests = fs.abspath(
fs.join(self._port_obj.layout_tests_dir(), 'external', 'wpt'))
path_to_ws_handlers = fs.join(path_to_wpt_tests, 'websockets',
'handlers')
serve_script = fs.join(path_to_wpt_root, 'serve')
start_cmd = [
self._port_obj.host.executable, '-u', serve_script, '--config',
path_to_wpt_config, '--doc_root', path_to_wpt_tests
]
# TODO(burnik): Merge with default start_cmd once we roll in websockets.
if self._port_obj.host.filesystem.exists(path_to_ws_handlers):
start_cmd += ['--ws_doc_root', path_to_ws_handlers]
self._stdout = self._stderr = self._executive.DEVNULL
# TODO(burnik): We should stop setting the CWD once WPT can be run without it.
self._cwd = path_to_wpt_root
self._env = port_obj.host.environ.copy()
self._env.update({'PYTHONPATH': path_to_thirdparty})
self._start_cmd = start_cmd
expiration_date = datetime.date(2025, 1, 4)
if datetime.date.today() > expiration_date - datetime.timedelta(30):
logging.getLogger(__name__).error(
'Pre-generated keys and certificates are going to be expired at %s.'
' Please re-generate them by following steps in %s/README.chromium.'
% (expiration_date.strftime('%b %d %Y'), path_to_wpt_support))
def generate_manifest(host, dest_path):
"""Generates MANIFEST.json on the specified directory."""
executive = host.executive
finder = PathFinder(host.filesystem)
wpt_exec_path = finder.path_from_tools_scripts('webkitpy',
'thirdparty', 'wpt',
'wpt', 'wpt')
cmd = [
'python', wpt_exec_path, 'manifest', '--work', '--tests-root',
dest_path
]
_log.debug('Running command: %s', ' '.join(cmd))
proc = executive.popen(cmd,
stdout=executive.PIPE,
stderr=executive.PIPE,
stdin=executive.PIPE)
out, err = proc.communicate('')
if proc.returncode:
_log.info('# ret> %d', proc.returncode)
if out:
_log.info(out)
if err:
_log.info(err)
host.exit(proc.returncode)
return proc.returncode, out
def do_POST(self):
json_raw_data = self.rfile.read(
int(self.headers.getheader('content-length')))
json_data = json.loads(json_raw_data)
test_list = ''
for each in json_data['tests']:
test_list += each + ' '
filesystem = FileSystem()
path_finder = PathFinder(filesystem)
script_dir = path_finder.path_from_tools_scripts()
executable_path = script_dir + '/run-webkit-tests'
cmd = 'python ' + executable_path + ' --no-show-results '
cmd += test_list
process = subprocess.Popen(cmd,
shell=True,
cwd=script_dir,
env=None,
stdout=subprocess.PIPE,
stderr=STDOUT)
self.send_response(200)
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Content-type', 'text/html')
self.end_headers()
while process.poll() is None:
html_output = '<br>' + str(process.stdout.readline())
self.wfile.write(html_output)
self.wfile.flush()
time.sleep(0.05)
process.wait()
def test_filter_transform_patch(self):
host = Host()
finder = PathFinder(host.filesystem)
resources_path = finder.path_from_tools_scripts('webkitpy', 'w3c', 'resources')
sample_patch = host.filesystem.read_text_file(host.filesystem.join(resources_path, 'sample.patch'))
expected_patch = host.filesystem.read_text_file(host.filesystem.join(resources_path, 'expected.patch'))
cl = GerritCL({'change_id': 1}, MockGerritAPI(None, None, None))
actual_patch = cl.filter_transform_patch(sample_patch)
self.assertEqual(actual_patch, expected_patch)
def run_pylint(self, path):
finder = PathFinder(FileSystem())
executive = Executive()
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join([
finder.path_from_tools_scripts(),
finder.path_from_blink_source('build', 'scripts'),
get_blinkpy_thirdparty_dir(),
get_blink_tools_dir(),
finder.path_from_blink_source('bindings', 'scripts'),
finder.path_from_chromium_base('build', 'android'),
finder.path_from_chromium_base('third_party', 'catapult', 'devil'),
finder.path_from_chromium_base('third_party', 'pymock'),
])
return executive.run_command([
sys.executable,
finder.path_from_depot_tools_base('pylint.py'),
'--output-format=parseable',
'--rcfile=' + finder.path_from_tools_scripts('webkitpy', 'pylintrc'),
path,
], env=env, error_handler=executive.ignore_error)
def generate_manifest(host, dest_path):
"""Generates MANIFEST.json on the specified directory."""
finder = PathFinder(host.filesystem)
wpt_exec_path = finder.path_from_tools_scripts('webkitpy',
'thirdparty', 'wpt',
'wpt', 'wpt')
cmd = [
'python', wpt_exec_path, 'manifest', '--work', '--tests-root',
dest_path
]
# ScriptError will be raised if the command fails.
host.executive.run_command(
cmd,
return_stderr=
True # This will also include stderr in the exception message.
)
class WPTExpectationsUpdater(object):
def __init__(self, host):
self.host = host
self.port = self.host.port_factory.get()
self.git_cl = GitCL(host)
self.finder = PathFinder(self.host.filesystem)
self.ports_with_no_results = set()
self.ports_with_all_pass = set()
def run(self, args=None):
"""Downloads text new baselines and adds test expectations lines."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v',
'--verbose',
action='store_true',
help='More verbose logging.')
args = parser.parse_args(args)
log_level = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=log_level, format='%(message)s')
issue_number = self.get_issue_number()
if issue_number == 'None':
_log.error('No issue on current branch.')
return 1
build_to_status = self.get_latest_try_jobs()
_log.debug('Latest try jobs: %r', build_to_status)
if not build_to_status:
_log.error('No try job information was collected.')
return 1
# The manifest may be used below to do check which tests are reference tests.
WPTManifest.ensure_manifest(self.host)
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if job_status.result == 'SUCCESS':
self.ports_with_all_pass.add(self.port_name(build))
port_results = self.get_failing_results_dict(build)
test_expectations = self.merge_dicts(test_expectations,
port_results)
# And then we merge results for different platforms that had the same results.
for test_name, platform_result in test_expectations.iteritems():
# platform_result is a dict mapping platforms to results.
test_expectations[test_name] = self.merge_same_valued_keys(
platform_result)
test_expectations = self.download_text_baselines(test_expectations)
test_expectation_lines = self.create_line_list(test_expectations)
self.write_to_test_expectations(test_expectation_lines)
return 0
def get_issue_number(self):
"""Returns current CL number. Can be replaced in unit tests."""
return self.git_cl.get_issue_number()
def get_latest_try_jobs(self):
"""Returns the latest finished try jobs as Build objects."""
return self.git_cl.latest_try_jobs(self._get_try_bots())
def get_failing_results_dict(self, build):
"""Returns a nested dict of failing test results.
Retrieves a full list of layout test results from a builder result URL.
Collects the builder name, platform and a list of tests that did not
run as expected.
Args:
build: A Build object.
Returns:
A dictionary with the structure: {
test-with-failing-result: {
'full-port-name': {
'expected': 'TIMEOUT',
'actual': 'CRASH',
'bug': 'crbug.com/11111'
}
}
}
If results could be fetched but none are failing,
this will return an empty dictionary.
"""
port_name = self.port_name(build)
if port_name in self.ports_with_all_pass:
# All tests passed, so there should be no failing results.
return {}
layout_test_results = self.host.buildbot.fetch_results(build)
if layout_test_results is None:
_log.warning('No results for build %s', build)
self.ports_with_no_results.add(self.port_name(build))
return {}
test_results = [
result
for result in layout_test_results.didnt_run_as_expected_results()
if not result.did_pass()
]
return self.generate_results_dict(self.port_name(build), test_results)
@memoized
def port_name(self, build):
return self.host.builders.port_name_for_builder_name(
build.builder_name)
def generate_results_dict(self, full_port_name, test_results):
"""Makes a dict with results for one platform.
Args:
full_port_name: The fully-qualified port name, e.g. "win-win10".
test_results: A list of LayoutTestResult objects.
Returns:
A dict mapping the full port name to a dict with the results for
the given test and platform.
"""
test_dict = {}
for result in test_results:
test_name = result.test_name()
if not self.port.is_wpt_test(test_name):
continue
test_dict[test_name] = {
full_port_name: {
'expected': result.expected_results(),
'actual': result.actual_results(),
'bug': 'crbug.com/626703'
}
}
return test_dict
def merge_dicts(self, target, source, path=None):
"""Recursively merges nested dictionaries.
Args:
target: First dictionary, which is updated based on source.
source: Second dictionary, not modified.
path: A list of keys, only used for making error messages.
Returns:
The updated target dictionary.
"""
path = path or []
for key in source:
if key in target:
if (isinstance(target[key], dict)) and isinstance(
source[key], dict):
self.merge_dicts(target[key], source[key],
path + [str(key)])
elif target[key] == source[key]:
pass
else:
raise ValueError(
'The key: %s already exist in the target dictionary.' %
'.'.join(path))
else:
target[key] = source[key]
return target
def merge_same_valued_keys(self, dictionary):
"""Merges keys in dictionary with same value.
Traverses through a dict and compares the values of keys to one another.
If the values match, the keys are combined to a tuple and the previous
keys are removed from the dict.
Args:
dictionary: A dictionary with a dictionary as the value.
Returns:
A new dictionary with updated keys to reflect matching values of keys.
Example: {
'one': {'foo': 'bar'},
'two': {'foo': 'bar'},
'three': {'foo': 'bar'}
}
is converted to a new dictionary with that contains
{('one', 'two', 'three'): {'foo': 'bar'}}
"""
merged_dict = {}
matching_value_keys = set()
keys = sorted(dictionary.keys())
while keys:
current_key = keys[0]
found_match = False
if current_key == keys[-1]:
merged_dict[current_key] = dictionary[current_key]
keys.remove(current_key)
break
for next_item in keys[1:]:
if dictionary[current_key] == dictionary[next_item]:
found_match = True
matching_value_keys.update([current_key, next_item])
if next_item == keys[-1]:
if found_match:
merged_dict[tuple(
matching_value_keys)] = dictionary[current_key]
keys = [
k for k in keys if k not in matching_value_keys
]
else:
merged_dict[current_key] = dictionary[current_key]
keys.remove(current_key)
matching_value_keys = set()
return merged_dict
def get_expectations(self, results, test_name=''):
"""Returns a set of test expectations to use based on results.
Returns a set of one or more test expectations based on the expected
and actual results of a given test name. This function is to decide
expectations for tests that could not be rebaselined.
Args:
results: A dictionary that maps one test to its results. Example:
{
'test_name': {
'expected': 'PASS',
'actual': 'FAIL',
'bug': 'crbug.com/11111'
}
}
test_name: The test name string (optional).
Returns:
A set of one or more test expectation strings with the first letter
capitalized. Example: set(['Failure', 'Timeout']).
"""
# If the result is MISSING, this implies that the test was not
# rebaselined and has an actual result but no baseline. We can't
# add a Missing expectation (this is not allowed), but no other
# expectation is correct.
# We also want to skip any new manual tests that are not automated;
# see crbug.com/708241 for context.
if (results['actual'] == 'MISSING'
or '-manual.' in test_name and results['actual'] == 'TIMEOUT'):
return {'Skip'}
expectations = set()
failure_types = ('TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO')
other_types = ('TIMEOUT', 'CRASH', 'PASS')
for actual in results['actual'].split():
if actual in failure_types:
expectations.add('Failure')
if actual in other_types:
expectations.add(actual.capitalize())
return expectations
def create_line_list(self, merged_results):
"""Creates list of test expectations lines.
Traverses through the given |merged_results| dictionary and parses the
value to create one test expectations line per key.
Args:
merged_results: A dictionary with the format:
{
'test_name': {
'platform': {
'expected: 'PASS',
'actual': 'FAIL',
'bug': 'crbug.com/11111'
}
}
}
Returns:
A list of test expectations lines with the format:
['BUG_URL [PLATFORM(S)] TEST_NAME [EXPECTATION(S)]']
"""
line_list = []
for test_name, port_results in sorted(merged_results.iteritems()):
if not self.port.is_wpt_test(test_name):
_log.warning(
'Non-WPT test "%s" unexpectedly passed to create_line_list.',
test_name)
continue
for port_names, results in sorted(port_results.iteritems()):
line_list.append(
self._create_line(test_name, port_names, results))
return line_list
def _create_line(self, test_name, port_names, results):
"""Constructs and returns a test expectation line string."""
port_names = self.tuple_or_value_to_list(port_names)
# The set of ports with no results is assumed to have have no
# overlap with the set of port names passed in here.
assert (set(port_names) & self.ports_with_no_results) == set()
# The ports with no results are generally ports of builders that
# failed, maybe for unrelated reasons. At this point, we add ports
# with no results to the list of platforms because we're guessing
# that this new expectation might be cross-platform and should
# also apply to any ports that we weren't able to get results for.
port_names.extend(self.ports_with_no_results)
specifier_part = self.specifier_part(port_names, test_name)
line_parts = [results['bug']]
if specifier_part:
line_parts.append(specifier_part)
line_parts.append(test_name)
line_parts.append('[ %s ]' %
' '.join(self.get_expectations(results, test_name)))
return ' '.join(line_parts)
def specifier_part(self, port_names, test_name):
"""Returns the specifier part for a new test expectations line.
Args:
port_names: A list of full port names that the line should apply to.
test_name: The test name for the expectation line.
Returns:
The specifier part of the new expectation line, e.g. "[ Mac ]".
This will be an empty string if the line should apply to all platforms.
"""
specifiers = []
for name in sorted(port_names):
specifiers.append(
self.host.builders.version_specifier_for_port_name(name))
specifiers.extend(self.skipped_specifiers(test_name))
specifiers = self.simplify_specifiers(
specifiers, self.port.configuration_specifier_macros())
if not specifiers:
return ''
return '[ %s ]' % ' '.join(specifiers)
@staticmethod
def tuple_or_value_to_list(tuple_or_value):
"""Converts a tuple to a list, and a string value to a one-item list."""
if isinstance(tuple_or_value, tuple):
return list(tuple_or_value)
return [tuple_or_value]
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
specifiers = []
for port in self.all_try_builder_ports():
if port.skips_test(test_name):
specifiers.append(
self.host.builders.version_specifier_for_port_name(
port.name()))
return specifiers
@memoized
def all_try_builder_ports(self):
"""Returns a list of Port objects for all try builders."""
return [
self.host.port_factory.get_from_builder_name(name)
for name in self._get_try_bots()
]
@staticmethod
def simplify_specifiers(specifiers, configuration_specifier_macros): # pylint: disable=unused-argument
"""Converts some collection of specifiers to an equivalent and maybe shorter list.
The input strings are all case-insensitive, but the strings in the
return value will all be capitalized.
Args:
specifiers: A collection of lower-case specifiers.
configuration_specifier_macros: A dict mapping "macros" for
groups of specifiers to lists of specific specifiers. In
practice, this is a dict mapping operating systems to
supported versions, e.g. {"win": ["win7", "win10"]}.
Returns:
A shortened list of specifiers. For example, ["win7", "win10"]
would be converted to ["Win"]. If the given list covers all
supported platforms, then an empty list is returned.
This list will be sorted and have capitalized specifier strings.
"""
specifiers = {specifier.lower() for specifier in specifiers}
for macro_specifier, version_specifiers in configuration_specifier_macros.iteritems(
):
macro_specifier = macro_specifier.lower()
version_specifiers = {
specifier.lower()
for specifier in version_specifiers
}
if version_specifiers.issubset(specifiers):
specifiers -= version_specifiers
specifiers.add(macro_specifier)
if specifiers == {
macro.lower()
for macro in configuration_specifier_macros.keys()
}:
return []
return sorted(specifier.capitalize() for specifier in specifiers)
def write_to_test_expectations(self, line_list):
"""Writes the given lines to the TestExpectations file.
The place in the file where the new lines are inserted is after a marker
comment line. If this marker comment line is not found, then everything
including the marker line is appended to the end of the file.
Args:
line_list: A list of lines to add to the TestExpectations file.
"""
if not line_list:
_log.info('No lines to write to TestExpectations.')
return
_log.info('Lines to write to TestExpectations:')
for line in line_list:
_log.info(' %s', line)
expectations_file_path = self.port.path_to_generic_test_expectations_file(
)
file_contents = self.host.filesystem.read_text_file(
expectations_file_path)
line_list = [
line for line in line_list if
self._test_name_from_expectation_string(line) not in file_contents
]
if not line_list:
return
marker_comment_index = file_contents.find(MARKER_COMMENT)
if marker_comment_index == -1:
file_contents += '\n%s\n' % MARKER_COMMENT
file_contents += '\n'.join(line_list)
else:
end_of_marker_line = (file_contents[marker_comment_index:].find(
'\n')) + marker_comment_index
file_contents = file_contents[:end_of_marker_line + 1] + '\n'.join(
line_list) + file_contents[end_of_marker_line:]
self.host.filesystem.write_text_file(expectations_file_path,
file_contents)
@staticmethod
def _test_name_from_expectation_string(expectation_string):
return TestExpectationLine.tokenize_line(
filename='', expectation_string=expectation_string,
line_number=0).name
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
Invokes `webkit-patch rebaseline-cl` in order to download new baselines
(-expected.txt files) for testharness.js tests that did not crash or
time out. Then, the platform-specific test is removed from the overall
failure test dictionary and the resulting dictionary is returned.
Args:
test_results: A dict mapping test name to platform to test results.
Returns:
An updated test_results dictionary which should only contain
test failures for tests that couldn't be rebaselined.
"""
tests_to_rebaseline, test_results = self.get_tests_to_rebaseline(
test_results)
if not tests_to_rebaseline:
_log.info('No tests to rebaseline.')
return test_results
_log.info('Tests to rebaseline:')
for test in tests_to_rebaseline:
_log.info(' %s', test)
webkit_patch = self.finder.path_from_tools_scripts('webkit-patch')
self.host.executive.run_command([
'python',
webkit_patch,
'rebaseline-cl',
'--verbose',
'--no-trigger-jobs',
'--fill-missing',
] + tests_to_rebaseline)
return test_results
def get_tests_to_rebaseline(self, test_results):
"""Returns a list of tests to download new baselines for.
Creates a list of tests to rebaseline depending on the tests' platform-
specific results. In general, this will be non-ref tests that failed
due to a baseline mismatch (rather than crash or timeout).
Args:
test_results: A dictionary of failing test results, mapping tests
to platforms to result dicts.
Returns:
A pair: A set of tests to be rebaselined, and a modified copy of
the test results dictionary. The tests to be rebaselined should
include testharness.js tests that failed due to a baseline mismatch.
"""
new_test_results = copy.deepcopy(test_results)
tests_to_rebaseline = set()
for test_path in test_results:
for platform, result in test_results[test_path].iteritems():
if self.can_rebaseline(test_path, result):
del new_test_results[test_path][platform]
tests_to_rebaseline.add(test_path)
return sorted(tests_to_rebaseline), new_test_results
def can_rebaseline(self, test_path, result):
if self.is_reference_test(test_path):
return False
if result['actual'] in ('CRASH', 'TIMEOUT', 'MISSING'):
return False
return True
def is_reference_test(self, test_path):
"""Checks whether a given file is a testharness.js test."""
return bool(self.port.reference_files(test_path))
def _get_try_bots(self):
return self.host.builders.all_try_builder_names()
class Bisector(object):
def __init__(self, tests, is_debug):
self.executive = Executive()
self.tests = tests
self.expected_failure = tests[-1]
self.is_debug = is_debug
self.path_finder = PathFinder(FileSystem())
def bisect(self):
if self.test_fails_in_isolation():
self.buckets = [Bucket([self.expected_failure])]
print '%s fails when run in isolation.' % self.expected_failure
self.print_result()
return 0
if not self.test_fails(self.tests):
_log.error('%s does not fail', self.expected_failure)
return 1
# Split the list of test into buckets. Each bucket has at least one test required to cause
# the expected failure at the end. Split buckets in half until there are only buckets left
# with one item in them.
self.buckets = [Bucket(self.tests[:-1]), Bucket([self.expected_failure])]
while not self.is_done():
self.print_progress()
self.split_largest_bucket()
self.print_result()
self.verify_non_flaky()
return 0
def test_fails_in_isolation(self):
return self.test_bucket_list_fails([Bucket([self.expected_failure])])
def verify_non_flaky(self):
print 'Verifying the failure is not flaky by running 10 times.'
count_failures = 0
for _ in range(0, 10):
if self.test_bucket_list_fails(self.buckets):
count_failures += 1
print 'Failed %d/10 times' % count_failures
def print_progress(self):
count = 0
for bucket in self.buckets:
count += len(bucket.tests)
print '%d tests left, %d buckets' % (count, len(self.buckets))
def print_result(self):
tests = []
for bucket in self.buckets:
tests += bucket.tests
extra_args = ' --debug' if self.is_debug else ''
print 'run-webkit-tests%s --child-processes=1 --order=none %s' % (extra_args, ' '.join(tests))
def is_done(self):
for bucket in self.buckets:
if bucket.size() > 1:
return False
return True
def split_largest_bucket(self):
index = 0
largest_index = 0
largest_size = 0
for bucket in self.buckets:
if bucket.size() > largest_size:
largest_index = index
largest_size = bucket.size()
index += 1
bucket_to_split = self.buckets[largest_index]
halfway_point = int(largest_size / 2)
first_half = Bucket(bucket_to_split.tests[:halfway_point])
second_half = Bucket(bucket_to_split.tests[halfway_point:])
buckets_before = self.buckets[:largest_index]
buckets_after = self.buckets[largest_index + 1:]
# Do the second half first because it tends to be faster because the http tests are front-loaded and slow.
new_buckets = buckets_before + [second_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
new_buckets = buckets_before + [first_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
self.buckets = buckets_before + [first_half, second_half] + buckets_after
def test_bucket_list_fails(self, buckets):
tests = []
for bucket in buckets:
tests += bucket.tests
return self.test_fails(tests)
def test_fails(self, tests):
extra_args = ['--debug'] if self.is_debug else []
path_to_run_webkit_tests = self.path_finder.path_from_tools_scripts('run-webkit-tests')
output = self.executive.popen(
[path_to_run_webkit_tests, '--child-processes', '1', '--order', 'none', '--no-retry',
'--no-show-results', '--verbose'] + extra_args + tests, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
failure_string = self.expected_failure + ' failed'
if failure_string in output.stderr.read():
return True
return False
