def test_fetch_results_with_weird_step_name(self):
fetcher = TestResultsFetcher()
fetcher.builders.step_name_for_builder = lambda builder: None
fetcher.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=123&'
'callback=ADD_RESULTS&builder=builder&name=full_results.json':
b'ADD_RESULTS(' + json.dumps(
[{
"TestType": "blink_web_tests on Intel GPU (with patch)"
}, {
"TestType": "base_unittests (with patch)"
}]).encode('utf8', 'replace') + b');',
'https://test-results.appspot.com/data/layout_results/builder/123/'
'blink_web_tests%20on%20Intel%20GPU%20%28with%20patch%29/'
'layout-test-results/failing_results.json':
json.dumps({
'passed': True
}).encode('utf8', 'replace')
})
results = fetcher.fetch_results(Build('builder', 123))
self.assertEqual(
results._results, # pylint: disable=protected-access
{'passed': True})
self.assertLog([])
def test_latest_try_jobs_failures(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "FAILURE",
"builder": {
"builder": "builder-a"
},
"number": 100
},
{
"status": "INFRA_FAILURE",
"builder": {
"builder": "builder-b"
},
"number": 200
}
]
}"""
}])
self.assertEqual(
git_cl.latest_try_jobs(builder_names=['builder-a', 'builder-b']), {
Build('builder-a', 100): TryJobStatus('COMPLETED', 'FAILURE'),
Build('builder-b', 200): TryJobStatus('COMPLETED', 'FAILURE'),
})
def test_try_job_results_with_other_builder(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "FAILURE",
"builder": {
"builder": "builder-a"
},
"number": 100,
"tags": [
{"key": "user_agent", "value": "cq"}
]
}
]
}"""
}])
# We ignore builders that we explicitly don't care about;
# so if we only care about other-builder, not builder-a,
# then no exception is raised.
self.assertEqual(
git_cl.try_job_results(builder_names=['other-builder']), {})
def test_wait_for_try_jobs_done(self):
host = MockHost()
host.executive = MockExecutive(output='lgtm')
git_cl = GitCL(host)
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "FAILURE",
"builder": {
"builder": "some-builder"
},
"number": 100
}
]
}"""
}])
self.assertEqual(
git_cl.wait_for_try_jobs(),
CLStatus(status='lgtm',
try_job_results={
Build('some-builder', 100):
TryJobStatus('COMPLETED', 'FAILURE'),
}))
self.assertEqual(host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_wait_for_try_jobs_cl_closed(self):
host = MockHost()
host.executive = MockExecutive(output='closed')
git_cl = GitCL(host)
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "STARTED",
"builder": {
"builder": "some-builder"
}
}
]
}"""
}])
self.assertEqual(
git_cl.wait_for_try_jobs(),
CLStatus(
status='closed',
try_job_results={
Build('some-builder', None): TryJobStatus('STARTED', None),
},
))
self.assertEqual(host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_fetch_results_with_weird_step_name(self):
buildbot = BuildBot()
buildbot.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=123&'
'callback=ADD_RESULTS&builder=builder&name=full_results.json':
'ADD_RESULTS(%s);' %
(json.dumps([{
"TestType":
"webkit_layout_tests on Intel GPU (with patch)"
}, {
"TestType": "base_unittests (with patch)"
}])),
'https://test-results.appspot.com/data/layout_results/builder/123/'
'webkit_layout_tests%20on%20Intel%20GPU%20%28with%20patch%29/'
'layout-test-results/failing_results.json':
json.dumps({'passed': True}),
})
results = buildbot.fetch_results(Build('builder', 123))
self.assertEqual(
results._results,
{ # pylint: disable=protected-access
'passed': True
})
self.assertLog([])
def test_wait_for_try_jobs_timeout(self):
host = MockHost()
git_cl = GitCL(host)
response = {
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "STARTED",
"builder": {
"builder": "some-builder"
},
"number": 100
}
]
}"""
}
# Specify the same response 10 times to ensure each poll gets ones.
git_cl._host.web = MockWeb(responses=[response] * 10)
self.assertIsNone(git_cl.wait_for_try_jobs())
self.assertEqual(
host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n'
'Waiting for try jobs. 600 seconds passed.\n'
'Waiting for try jobs. 1800 seconds passed.\n'
'Waiting for try jobs. 3000 seconds passed.\n'
'Waiting for try jobs. 4200 seconds passed.\n'
'Waiting for try jobs. 5400 seconds passed.\n'
'Waiting for try jobs. 6600 seconds passed.\n'
'Timed out waiting for try jobs.\n')
def __init__(self,
log_executive=False,
web=None,
git=None,
os_name=None,
os_version=None,
time_return_val=123):
super(MockHost, self).__init__(
log_executive=log_executive,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self._add_base_manifest_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._git = git
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Linux (dbg)': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Debug']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release'],
'is_try_builder': True,
},
'fake_blink_try_linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'fake_blink_try_win': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'android_blink_rel': {
'bucket': 'master.tryserver.chromium.android',
'port_name': 'android-kitkat',
'specifiers': ['KitKat', 'Release'],
'is_try_builder': True,
},
})
def test_fetch_web_test_results_with_no_results_fetched(self):
fetcher = TestResultsFetcher()
fetcher.web = MockWeb()
results = fetcher.fetch_web_test_results(fetcher.results_url('B'))
self.assertIsNone(results)
self.assertLog([
'DEBUG: Got 404 response from:\n'
'https://test-results.appspot.com/data/layout_results/B/results/layout-test-results/failing_results.json\n'
])
def test_fetch_web_test_results_with_no_results_fetched(self):
buildbot = BuildBot()
buildbot.web = MockWeb()
results = buildbot.fetch_web_test_results(buildbot.results_url('B'))
self.assertIsNone(results)
self.assertLog([
'DEBUG: Got 404 response from:\n'
'https://test-results.appspot.com/data/layout_results/B/results/layout-test-results/failing_results.json\n'
])
def test_fetch_webdriver_test_results_with_no_results(self):
fetcher = TestResultsFetcher()
fetcher.web = MockWeb()
results = fetcher.fetch_webdriver_test_results(Build('bar-rel', 123),
'foo.chrome')
self.assertIsNone(results)
self.assertLog([
'DEBUG: Got 404 response from:\n'
'https://test-results.appspot.com/testfile?buildnumber=123&'
'master=foo.chrome&builder=bar-rel&'
'testtype=webdriver_tests_suite+%28with+patch%29&name=full_results.json\n'
])
def test_fetch_webdriver_test_results_with_no_results(self):
buildbot = BuildBot()
buildbot.web = MockWeb()
results = buildbot.fetch_webdriver_test_results(
Build('linux-rel', 123), 'tryserver.chromium.linux')
self.assertIsNone(results)
self.assertLog([
'DEBUG: Got 404 response from:\n'
'https://test-results.appspot.com/testfile?buildnumber=123&'
'master=tryserver.chromium.linux&builder=linux-rel&'
'testtype=webdriver_tests_suite+%28with+patch%29&name=full_results.json\n'
])
def test_get_step_name(self):
fetcher = TestResultsFetcher()
fetcher.web = MockWeb(urls={
'https://test-results.appspot.com/testfile?buildnumber=5&'
'callback=ADD_RESULTS&builder=foo&name=full_results.json':
'ADD_RESULTS(%s);' % (json.dumps(
[{"TestType": "webkit_layout_tests (with patch)"},
{"TestType": "not_site_per_process_webkit_layout_tests (with patch)"},
{"TestType": "webkit_layout_tests (retry with patch)"},
{"TestType": "base_unittests (with patch)"}]))
})
step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
self.assertEqual(step_name, 'webkit_layout_tests (with patch)')
self.assertLog([])
def test_fetch_webdriver_results_success(self):
fetcher = TestResultsFetcher()
fetcher.web = MockWeb(urls={
'https://test-results.appspot.com/testfile?buildnumber=123&'
'master=foo.chrome&builder=bar-rel&'
'testtype=webdriver_tests_suite+%28with+patch%29&'
'name=full_results.json':
json.dumps({'passed': True}),
})
results = fetcher.fetch_webdriver_test_results(
Build('bar-rel', 123), 'foo.chrome')
self.assertEqual(results._results, { # pylint: disable=protected-access
'passed': True
})
self.assertLog([])
def test_try_job_results(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "SUCCESS",
"builder": {
"builder": "builder-a"
},
"number": 111,
"tags": [
{"key": "user_agent", "value": "cq"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "builder-b"
},
"number": 222
},
{
"status": "INFRA_FAILURE",
"builder": {
"builder": "builder-c"
},
"number": 333
}
]
}"""
}])
self.assertEqual(
git_cl.try_job_results(issue_number=None),
{
Build('builder-a', 111):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('builder-b', 222):
TryJobStatus('SCHEDULED', None),
# INFRA_FAILURE is mapped to FAILURE for this build.
Build('builder-c', 333):
TryJobStatus('COMPLETED', 'FAILURE'),
})
def test_latest_try_jobs(self):
# Here we have multiple builds with the same name, but we only take the
# latest one (based on build number).
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "SUCCESS",
"builder": {
"builder": "builder-b"
},
"number": 100
},
{
"status": "SUCCESS",
"builder": {
"builder": "builder-b"
},
"number": 90
},
{
"status": "SCHEDULED",
"builder": {
"builder": "builder-a"
}
},
{
"status": "SUCCESS",
"builder": {
"builder": "builder-c"
},
"number": 123
}
]
}"""
}])
self.assertEqual(
git_cl.latest_try_jobs(builder_names=['builder-a', 'builder-b']), {
Build('builder-a'): TryJobStatus('SCHEDULED'),
Build('builder-b', 100): TryJobStatus('COMPLETED', 'SUCCESS'),
})
def test_fetch_full_results_webdriver(self):
buildbot = BuildBot()
buildbot.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=321&'
'master=tryserver.chromium.linux&builder=foo&'
'testtype=webdriver_tests_suite+%28with+patch%29&'
'name=full_results.json':
json.dumps({'passed': True}),
})
results = buildbot.fetch_full_results(
Build('foo', 321), master='tryserver.chromium.linux')
self.assertEqual(
results._results,
{ # pylint: disable=protected-access
'passed': True
})
self.assertLog([])
def test_get_step_name_for_wpt(self):
fetcher = TestResultsFetcher()
fetcher.builders.step_name_for_builder = lambda builder: None
fetcher.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=5&'
'callback=ADD_RESULTS&builder=foo&name=full_results.json':
b'ADD_RESULTS(' +
(json.dumps([{
"TestType": "wpt_tests_suite (with patch)"
}, {
"TestType": "wpt_tests_suite (retry with patch)"
}, {
"TestType": "base_unittests (with patch)"
}])).encode('utf8', 'replace') + b');'
})
step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
self.assertEqual(step_name, 'wpt_tests_suite (with patch)')
self.assertLog([])
def test_latest_try_jobs_started(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "STARTED",
"builder": {
"builder": "builder-a"
},
"number": 100
}
]
}"""
}])
self.assertEqual(git_cl.latest_try_jobs(builder_names=['builder-a']),
{Build('builder-a', 100): TryJobStatus('STARTED')})
def test_wait_for_try_jobs_no_results_not_considered_finished(self):
host = MockHost()
git_cl = GitCL(host)
# git_cl.fetch_raw_try_job_results = lambda **_: []
response = {
'status_code': 200,
'body': SEARCHBUILDS_RESPONSE_PREFIX + b"{}"
}
# Specify the same response 10 times to ensure each poll gets ones.
git_cl._host.web = MockWeb(responses=[response] * 10)
self.assertIsNone(git_cl.wait_for_try_jobs())
self.assertEqual(
host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n'
'Waiting for try jobs. 600 seconds passed.\n'
'Waiting for try jobs. 1800 seconds passed.\n'
'Waiting for try jobs. 3000 seconds passed.\n'
'Waiting for try jobs. 4200 seconds passed.\n'
'Waiting for try jobs. 5400 seconds passed.\n'
'Waiting for try jobs. 6600 seconds passed.\n'
'Timed out waiting for try jobs.\n')
def test_try_job_results_skip_experimental_cq(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "SUCCESS",
"builder": {
"builder": "builder-a"
},
"number": 111,
"tags": [
{"key": "user_agent", "value": "cq"}
]
},
{
"status": "SUCCESS",
"builder": {
"builder": "builder-b"
},
"number": 222,
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "true"}
]
}
]
}"""
}])
self.assertEqual(
# Only one build appears - builder-b is ignored because it is
# experimental.
git_cl.try_job_results(issue_number=None, cq_only=True),
{
Build('builder-a', 111): TryJobStatus('COMPLETED', 'SUCCESS'),
})
def test_latest_try_jobs_cq_only(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-a"
},
"tags": [
{"key": "user_agent", "value": "cq"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-b"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "false"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-c"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "false"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-a-experimental"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "true"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-b-experimental"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "true"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "other-a"
},
"tags": [
{"key": "user_agent", "value": "git_cl_try"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "other-b"
},
"tags": [
{"key": "user_agent", "value": "git_cl_try"},
{"key": "cq_experimental", "value": "false"}
]
}
]
}"""
}])
self.assertEqual(
git_cl.latest_try_jobs(cq_only=True), {
Build('cq-a'): TryJobStatus('SCHEDULED'),
Build('cq-b'): TryJobStatus('SCHEDULED'),
Build('cq-c'): TryJobStatus('SCHEDULED'),
})
