def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
# TODO(robertma): Even though we have mocked the builder list (and hence
# all_port_names), we are still relying on the knowledge of currently
# configured ports and their fallback order. Ideally, we should improve
# MockPortFactory and use it.
self.host.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'Fake Test Mac10.11': {
'port_name': 'mac-mac10.11',
'specifiers': ['Mac10.11', 'Release']
},
'Fake Test Mac10.10': {
'port_name': 'mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
})
# Note: this is a pre-assumption of the tests in this file. If this
# assertion fails, port configurations are likely changed, and the
# tests need to be adjusted accordingly.
self.assertEqual(sorted(self.host.port_factory.all_port_names()),
['linux-trusty', 'mac-mac10.10', 'mac-mac10.11', 'mac-mac10.12', 'win-win10'])
def _assert_expectations(self, test_data, expectations_string,
only_ignore_very_flaky):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky),
expectations_string)
def fake_builder_list(self):
return BuilderList({
'Dummy builder name': {
'port_name': 'dummy-port',
'specifiers': []
},
})
def test_release_builders(self):
# Testing private method - pylint: disable=protected-access
self.tool.builders = BuilderList({
"MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["Mac10.10", "Release"]},
"MOCK Mac10.11 (dbg)": {"port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Debug"]},
})
self.assertEqual(self.command._release_builders(), ['MOCK Mac10.10'])
def fake_builder_list(self):
return BuilderList({
"Dummy builder name": {
"port_name": "dummy-port",
"specifiers": []
},
})
def test_all_results_by_path(self):
test_data = {
'tests': {
'foo': {
'multiple_pass.html': {'results': [[4, 'P'], [1, 'P'], [2, 'P']]},
'fail.html': {'results': [[2, 'Z']]},
'all_types.html': {
'results': [[2, 'A'], [1, 'C'], [2, 'F'], [1, 'I'], [1, 'O'], [1, 'N'], [1, 'P'], [1, 'T'],
[1, 'Y'], [10, 'X'], [1, 'Z'], [1, 'K']]
},
'not_run.html': {'results': []},
}
}
}
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'))
results_by_path = expectations.all_results_by_path()
expected_output = {
'foo/multiple_pass.html': ['PASS'],
'foo/fail.html': ['IMAGE+TEXT'],
'foo/all_types.html': ['AUDIO', 'CRASH', 'IMAGE', 'IMAGE+TEXT', 'LEAK', 'MISSING', 'PASS', 'TEXT', 'TIMEOUT']
}
self.assertEqual(results_by_path, expected_output)
def setUp(self):
super(TestAutoRebaseline, self).setUp()
self.tool.builders = BuilderList({
'MOCK Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
'MOCK Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release']
},
'MOCK Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release']
},
'MOCK Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'MOCK Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release']
},
'MOCK Win7 (dbg)': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Debug']
},
})
self.command.latest_revision_processed_on_all_bots = lambda: 9000
self.command.bot_revision_data = lambda git: [{
'builder': 'MOCK Win7',
'revision': '9000'
}]
def sample_builder_list():
return BuilderList({
'Blink A': {
'port_name': 'port-a',
'specifiers': ['A', 'Release']
},
'Blink B': {
'port_name': 'port-b',
'specifiers': ['B', 'Release']
},
'Blink B (dbg)': {
'port_name': 'port-b',
'specifiers': ['B', 'Debug']
},
'Blink C (dbg)': {
'port_name': 'port-c',
'specifiers': ['C', 'Release']
},
'Try A': {
'port_name': 'port-a',
'specifiers': ['A', 'Release'],
"is_try_builder": True
},
'Try B': {
'port_name': 'port-b',
'specifiers': ['B', 'Release'],
"is_try_builder": True
},
})
def test_run_commit_queue_for_cl_only_checks_non_blink_bots(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
host.builders = BuilderList({
'fakeos_blink_rel': {
'port_name': 'test-fakeos',
'specifiers': ['FakeOS', 'Release'],
'is_try_builder': True,
}
})
importer = TestImporter(host)
importer.git_cl = MockGitCL(host,
results={
Build('fakeos_blink_rel', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
])
def setUp(self):
super(UpdateW3CTestExpectationsTest, self).setUp()
self.host = MockHost()
self.mock_dict_one = {
'fake/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/626703'},
'two': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/626703'}
}
}
self.mock_dict_two = {
'imported/fake/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/626703'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/626703'},
'three': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/626703'}
}
}
self.mock_dict_three = {
'imported/fake/test/path.html': {
'four': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/626703'}}
}
self.mock_dict_four = {
'imported/fake/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/626703'}
}
}
self.host.builders = BuilderList({
'mac': {'port_name': 'test-mac'},
})
def mock_host(self):
"""Returns a mock host with fake values set up for testing."""
host = MockHost()
host.port_factory = MockPortFactory(host)
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
'MOCK Try Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
})
# Write a dummy manifest file, describing what tests exist.
host.filesystem.write_text_file(
host.port_factory.get().layout_tests_dir() +
'/external/wpt/MANIFEST.json',
json.dumps({
'items': {
'reftest': {
'reftest.html':
[['/reftest.html', [['/reftest-ref.html', '==']], {}]]
},
'testharness': {
'test/path.html': [['/test/path.html', {}]],
'test/zzzz.html': [['/test/zzzz.html', {}]],
},
'manual': {
'x-manual.html': [['/x-manual.html', {}]],
},
},
}))
return host
def setUp(self):
super(TestAutoRebaseline, self).setUp()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"]
},
"MOCK Mac10.11": {
"port_name": "test-mac-mac10.11",
"specifiers": ["Mac10.11", "Release"]
},
"MOCK Precise": {
"port_name": "test-linux-precise",
"specifiers": ["Precise", "Release"]
},
"MOCK Trusty": {
"port_name": "test-linux-trusty",
"specifiers": ["Trusty", "Release"]
},
"MOCK Win7": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Release"]
},
"MOCK Win7 (dbg)": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Debug"]
},
})
self.command.latest_revision_processed_on_all_bots = lambda: 9000
self.command.bot_revision_data = lambda scm: [{
"builder": "MOCK Win7",
"revision": "9000"
}]
def __init__(self,
log_executive=False,
executive_throws_when_run=None,
initialize_scm_by_default=True,
web=None,
scm=None,
os_name=None,
os_version=None,
time_return_val=123):
MockSystemHost.__init__(self,
log_executive,
executive_throws_when_run,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._scm = scm
# FIXME: we should never initialize the SCM by default, since the real
# object doesn't either. This has caused at least one bug (see bug 89498).
if initialize_scm_by_default:
self.initialize_scm()
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList(BUILDERS)
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
web = MockWeb(urls={
'https://codereview.chromium.org/api/11112222': json.dumps({
'patchsets': [1, 2],
}),
'https://codereview.chromium.org/api/11112222/2': json.dumps({
'try_job_results': [
{
'builder': 'MOCK Try Win',
'buildnumber': 5000,
'result': 0,
},
{
'builder': 'MOCK Try Mac',
'buildnumber': 4000,
'result': 0,
},
],
'files': {
'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html': {'status': 'M'},
'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html': {'status': 'M'},
},
}),
})
self.tool.builders = BuilderList({
"MOCK Try Win": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Release"],
"is_try_builder": True,
},
"MOCK Try Linux": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"],
"is_try_builder": True,
},
})
self.command.rietveld = Rietveld(web)
self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), json.dumps({
'failures': [
'fast/dom/prototype-newtest.html',
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
port = self.mac_port
tests = [
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
]
for test in tests:
self._write(port.host.filesystem.join(port.layout_tests_dir(), test), 'contents')
def setUp(self):
super(UpdateW3CTestExpectationsTest, self).setUp()
self.host = MockHost()
self.host.builders = BuilderList({
'mac': {
'port_name': 'test-mac'
},
})
def _define_builders(self, builders_dict):
"""Defines the available builders for the test.
Args:
builders_dict: A dictionary containing builder names to their
attributes, see BuilderList.__init__ for the format.
"""
self._host.builders = BuilderList(builders_dict)
def mock_host(self):
super(WPTExpectationsUpdaterTest, self).setUp()
host = MockHost()
host.port_factory = MockPortFactory(host)
host.builders = BuilderList({
'MOCK Try Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
'MOCK Try Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
})
host.filesystem.write_text_file(
host.port_factory.get().layout_tests_dir() +
'/external/wpt/MANIFEST.json',
json.dumps({
'items': {
'reftest': {
'reftest.html':
[['/reftest.html', [['/reftest-ref.html', '==']], {}]]
},
'testharness': {
'test/path.html': [['/test/path.html', {}]],
'test/zzzz.html': [['/test/zzzz.html', {}]],
},
'manual': {
'x-manual.html': [['/x-manual.html', {}]],
},
},
}))
return host
def test_get_from_builder_name(self):
host = MockHost()
host.builders = BuilderList({
'My Fake Mac10.12 Builder': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release'],
}
})
self.assertEqual(
factory.PortFactory(host).get_from_builder_name(
'My Fake Mac10.12 Builder').name(), 'mac-mac10.12')
def fake_builders_list():
return BuilderList({
'foo-builder': {
'port_name': 'dummy-port',
'specifiers': ['Linux', 'Release']
},
'bar-builder': {
'port_name': 'dummy-port',
'specifiers': ['Mac', 'Debug']
},
})
def test_execute_stuck_on_alternate_rebaseline_branch(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Win7": {"port_name": "test-win-win7", "specifiers": ["Win7", "Release"]},
})
old_branch_name = self.tool.scm().current_branch_or_ref
try:
self.command.tree_status = lambda: 'open'
self.tool.scm().current_branch_or_ref = lambda: 'auto-rebaseline-alt-temporary-branch'
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
['git', 'cl', 'upload', '-f'],
['git', 'pull'],
['git', 'cl', 'land', '-f', '-v'],
['git', 'config', 'branch.auto-rebaseline-temporary-branch.rietveldissue'],
['git', 'cl', 'set_close'],
])
self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Mac Win10 ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
finally:
self.tool.scm().current_branch_or_ref = old_branch_name
def test_release_builders(self):
# Testing private method - pylint: disable=protected-access
self.tool.builders = BuilderList({
'MOCK Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
'MOCK Mac10.11 (dbg)': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Debug']
},
})
self.assertEqual(self.command._release_builders(), ['MOCK Mac10.10'])
def _assert_is_flaky(self, results_string, should_be_flaky, only_ignore_very_flaky, expected=None):
results_json = self._results_json_from_test_data({})
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'))
results_entry = self._results_from_string(results_string)
if expected:
results_entry[bot_test_expectations.ResultsJSON.EXPECTATIONS_KEY] = expected
num_actual_results = len(expectations._flaky_types_in_results(results_entry, only_ignore_very_flaky))
if should_be_flaky:
self.assertGreater(num_actual_results, 1)
else:
self.assertLessEqual(num_actual_results, 1)
def test_execute_test_passes_everywhere(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["Mac10.10", "Release"]},
"MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"]},
})
self.command.tree_status = lambda: 'open'
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
['git', 'cl', 'upload', '-f'],
['git', 'pull'],
['git', 'cl', 'land', '-f', '-v'],
['git', 'config', 'branch.auto-rebaseline-temporary-branch.rietveldissue'],
['git', 'cl', 'set_close'],
])
# The mac ports should both be removed since they're the only ones in builders._exact_matches.
self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
def setUp(self):
host = MockWebKitPatch()
host.port_factory = MockPortFactory(host)
port = host.port_factory.get()
base_dir = port.layout_tests_dir()
host.filesystem.write_text_file(base_dir + '/a/x.html', '<html>')
host.filesystem.write_text_file(base_dir + '/a/y.html', '<html>')
host.filesystem.write_text_file(base_dir + '/a/z.html', '<html>')
host.builders = BuilderList({
'MOCK Mac10.12': {'port_name': 'test-mac-mac10.12', 'specifiers': ['Mac10.12', 'Release']},
'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']},
'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']},
})
self.host = host
def test_rebaseline_test_passes_everywhere(self):
test_port = self.tool.port_factory.get('test')
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(
Build(builder),
LayoutTestResults({
'tests': {
'fast': {
'dom': {
'prototype-taco.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
'Dummy test contents')
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
'MOCK Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
'MOCK Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release']
},
})
self.command.execute(self.options(), [], self.tool)
self.assertEqual(self.tool.executive.calls, [])
# The mac ports should both be removed since they're the only ones in the builder list.
self.assertEqual(
self.tool.filesystem.read_text_file(
test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ]
""")
def test_rebaseline_test_passes_everywhere(self):
test_port = self.tool.port_factory.get('test')
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(
Build(builder),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
"Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"]
},
"MOCK Mac10.11": {
"port_name": "test-mac-mac10.11",
"specifiers": ["Mac10.11", "Release"]
},
})
self.command.execute(self.options(), [], self.tool)
self.assertEqual(self.tool.executive.calls, [])
# The mac ports should both be removed since they're the only ones in the builder list.
self.assertEqual(
self.tool.filesystem.read_text_file(
test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ]
""")
def test_execute_git_cl_hangs(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
# Have prototype-chocolate only fail on "MOCK Mac10.11".
self.tool.buildbot.set_results(Build('MOCK Mac10.11'), LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "PASS",
"actual": "PASS TEXT",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
self.tool.builders = BuilderList({
"MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"]},
})
self.command.SECONDS_BEFORE_GIVING_UP = 0
self.command.tree_status = lambda: 'open'
self.tool.executive = MockExecutive()
self.tool.executive.calls = []
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt',
'--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-taco.html']],
[['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt',
'--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-taco.html']],
[['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html']],
['git', 'cl', 'upload', '-f'],
])
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
self._git = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.buildbot = BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, git, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self.builders = BuilderList.load_default_builder_list(self.filesystem)
def _basic_execute_test(self, expected_executive_calls, auth_refresh_token_json=None, commit_author=None, dry_run=False):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["Mac10.10", "Release"]},
"MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"]},
})
self.command.tree_status = lambda: 'open'
self._execute_with_mock_options(auth_refresh_token_json=auth_refresh_token_json,
commit_author=commit_author, dry_run=dry_run)
self.assertEqual(self.tool.executive.calls, expected_executive_calls)
# The mac ports should both be removed since they're the only ones in builders._exact_matches.
self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
def __init__(self,
log_executive=False,
web=None,
git=None,
os_name=None,
os_version=None,
time_return_val=123):
super(MockHost, self).__init__(log_executive=log_executive,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self._add_base_manifest_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._git = git
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Linux (dbg)': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Debug']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'Fake Test Linux Try Bot': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
})
