def __init__(self):
super(RebaselineCL, self).__init__(options=[
optparse.make_option(
'--issue',
type='int',
default=None,
help=
'Rietveld issue number; if none given, this will be obtained via `git cl issue`.'
),
optparse.make_option(
'--dry-run',
action='store_true',
default=False,
help=
'Dry run mode; list actions that would be performed but do not do anything.'
),
optparse.make_option(
'--only-changed-tests',
action='store_true',
default=False,
help=
'Only download new baselines for tests that are changed in the CL.'
),
optparse.make_option('--no-trigger-jobs',
dest='trigger_jobs',
action='store_false',
default=True,
help='Do not trigger any try jobs.'),
self.no_optimize_option,
self.results_directory_option,
])
self.rietveld = Rietveld(Web())
def run(self, args=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v', '--verbose', action='store_true', help='More verbose logging.')
args = parser.parse_args(args)
log_level = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=log_level, format='%(message)s')
issue_number = self.get_issue_number()
if issue_number == 'None':
_log.error('No issue on current branch.')
return 1
rietveld = Rietveld(self.host.web)
builds = rietveld.latest_try_jobs(issue_number, self.get_try_bots())
_log.debug('Latest try jobs: %r', builds)
if not builds:
_log.error('No try job information was collected.')
return 1
test_expectations = {}
for build in builds:
platform_results = self.get_failing_results_dict(build)
test_expectations = self.merge_dicts(test_expectations, platform_results)
for test_name, platform_result in test_expectations.iteritems():
test_expectations[test_name] = self.merge_same_valued_keys(platform_result)
test_expectations = self.get_expected_txt_files(test_expectations)
test_expectation_lines = self.create_line_list(test_expectations)
self.write_to_test_expectations(test_expectation_lines)
return 0
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
web = MockWeb(urls={
'https://codereview.chromium.org/api/11112222': json.dumps({
'patchsets': [1, 2],
}),
'https://codereview.chromium.org/api/11112222/2': json.dumps({
'try_job_results': [
{
'builder': 'MOCK Try Win',
'buildnumber': 5000,
'result': 0,
},
{
'builder': 'MOCK Try Mac',
'buildnumber': 4000,
'result': 0,
},
],
'files': {
'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html': {'status': 'M'},
'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html': {'status': 'M'},
},
}),
})
self.tool.builders = BuilderList({
"MOCK Try Win": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Release"],
"is_try_builder": True,
},
"MOCK Try Linux": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"],
"is_try_builder": True,
},
})
self.command.rietveld = Rietveld(web)
self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), json.dumps({
'failures': [
'fast/dom/prototype-newtest.html',
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
port = self.mac_port
tests = [
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
]
for test in tests:
self._write(port.host.filesystem.join(port.layout_tests_dir(), test), 'contents')
def test_latest_try_jobs_with_patchset(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(
rietveld.latest_try_jobs(11112222,
('bar-builder', 'other-builder'),
patchset_number=2),
[Build('bar-builder', 50)])
def test_filter_latest_jobs_higher_build_last(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(
rietveld._filter_latest_builds(
[Build('foo', 3),
Build('bar', 5),
Build('foo', 5)]),
[Build('bar', 5), Build('foo', 5)])
def test_latest_try_jobs_http_error(self):
def raise_error(_):
raise urllib2.URLError('Some request error message')
web = self.mock_web()
web.get_binary = raise_error
rietveld = Rietveld(web)
self.assertEqual(rietveld.latest_try_jobs(11112222, ('bar-builder',)), [])
self.assertLog(['ERROR: Request failed to URL: https://codereview.chromium.org/api/11112222\n'])
def __init__(self, log_executive=False):
self.wakeup_event = threading.Event()
self.bugs = MockBugzilla()
self.buildbot = MockBuildBot()
self.executive = MockExecute(should_log=log_executive)
self._irc = None
self.user = MockUser()
self._scm = MockSCM()
self._checkout = MockCheckout()
self.status_server = MockStatusServer()
self.irc_password = "******"
self.codereview = Rietveld(self.executive)
def __init__(self, log_executive=False):
self.wakeup_event = threading.Event()
self.bugs = MockBugzilla()
self.buildbot = MockBuildBot()
self.executive = MockExecute()
if log_executive:
self.executive.run_and_throw_if_fail = lambda args: log("MOCK run_and_throw_if_fail: %s" % args)
self._irc = None
self.user = MockUser()
self._scm = MockSCM()
self._checkout = MockCheckout()
self.status_server = MockStatusServer()
self.irc_password = "******"
self.codereview = Rietveld(self.executive)
def __init__(self, path):
MultiCommandTool.__init__(self)
self._path = path
self.wakeup_event = threading.Event()
self.bugs = Bugzilla()
self.buildbot = BuildBot()
self.executive = Executive()
self._irc = None
self.user = User()
self._scm = None
self._checkout = None
self.status_server = StatusServer()
self.codereview = Rietveld(self.executive)
def test_url_for_issue(self):
rietveld = Rietveld(Mock())
self.assertEqual(rietveld.url_for_issue(34223),
"https://wkrietveld.appspot.com/34223")
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
web = MockWeb(urls={
'https://codereview.chromium.org/api/11112222': json.dumps({
'patchsets': [1, 2],
}),
'https://codereview.chromium.org/api/11112222/2': json.dumps({
'try_job_results': [
{
'builder': 'MOCK Try Win',
'buildnumber': 5000,
'result': 0,
},
{
'builder': 'MOCK Try Mac',
'buildnumber': 4000,
'result': 0,
},
],
'files': {
'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html': {'status': 'M'},
'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html': {'status': 'M'},
},
}),
})
self.tool.builders = BuilderList({
"MOCK Try Win": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Release"],
"is_try_builder": True,
},
"MOCK Try Linux": {
"port_name": "test-linux-trusty",
"specifiers": ["Trusty", "Release"],
"is_try_builder": True,
},
})
self.command.rietveld = Rietveld(web)
layout_test_results = LayoutTestResults({
'tests': {
'fast': {
'dom': {
'prototype-inheritance.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True,
},
'prototype-banana.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True,
},
'prototype-taco.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True,
},
'prototype-chocolate.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'prototype-crashy.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True,
},
'prototype-newtest.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True,
'is_missing_text': True,
},
'prototype-slowtest.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True,
},
}
},
'svg': {
'dynamic-updates': {
'SVGFEDropShadowElement-dom-stdDeviation-attr.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'has_stderr': True,
'is_unexpected': True,
}
}
}
}
})
for build in [Build('MOCK Try Win', 5000), Build('MOCK Try Mac', 4000)]:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), json.dumps({
'failures': [
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'fast/dom/prototype-slowtest.html',
'fast/dom/prototype-taco.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
port = self.mac_port
tests = [
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
]
for test in tests:
self._write(port.host.filesystem.join(port.layout_tests_dir(), test), 'contents')
def test_changed_files(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(rietveld.changed_files(11112222),
['some/path/bar.html', 'some/path/foo.cc'])
def test_changed_files_no_results(self):
rietveld = Rietveld(self.mock_web())
self.assertIsNone(rietveld.changed_files(11113333))
def test_filter_latest_jobs_empty(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(rietveld._filter_latest_builds([]), [])
def test_latest_try_jobs_no_relevant_builders(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(rietveld.latest_try_jobs(11112222, ('foo', 'bar')),
[])
def test_latest_try_jobs_non_json_response(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(rietveld.latest_try_jobs(11113333, ('bar-builder', )),
[])
self.assertLog(['ERROR: Invalid JSON: my non-JSON contents\n'])
def test_latest_try_jobs(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(
rietveld.latest_try_jobs(11112222,
('bar-builder', 'other-builder')),
[Build('bar-builder', 60)])
def test_filter_latest_jobs_no_build_number(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(
rietveld._filter_latest_builds(
[Build('foo', 3), Build('bar'),
Build('bar')]), [Build('bar'), Build('foo', 3)])
