def bisect(self):
# this is a slot so it will be called in the thread
self.started.emit()
try:
Bisector.bisect(self, *self._bisect_args)
except MozRegressionError:
self._finish_on_exception(None)
def bisect(self):
# this is a slot so it will be called in the thread
self.started.emit()
try:
Bisector.bisect(self, *self._bisect_args)
except MozRegressionError:
self._finish_on_exception(None)
def setUp(self):
self.handler = MagicMock(find_fix=False, ensure_good_and_bad=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(),
self.test_runner,
Mock(),
dl_in_background=False)
self.bisector.download_background = False
def setUp(self):
self.handler = MagicMock(find_fix=False, ensure_good_and_bad=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(), self.test_runner, Mock(), dl_in_background=False)
self.bisector.download_background = False
# shim for py2.7
if not hasattr(self, "assertRaisesRegex"):
self.assertRaisesRegex = self.assertRaisesRegexp
def __init__(self, good, bad, platform, warning, warning_limit, warning_re,
ignore_lines, required_test):
init_logger()
self.use_nightly = True
try:
self.good = parse_date(good)
self.bad = parse_date(bad)
except DateFormatError:
# This hopefully a revision range. We can bypass nightly and
# go directly to InboundHandler. That itself is a bit of a misnomer,
# it will still bisect m-c builds, but by changeset range, not date
# range.
self.use_nightly = False
self.good = good
self.bad = bad
self.ignore_lines = ignore_lines
self.test_runner = WarningTestRunner(warning,
platform,
ignore_lines=ignore_lines,
warning_re=warning_re,
warning_limit=warning_limit,
required_test=required_test)
# Convert the platform to a mozregression friendly version.
# Also avoid overwriting the os module by *not* using |os| for a
# variable name.
(_os, bits) = re.match(r'([a-zA-Z]+)-?([0-9]+)?', platform).groups()
if not bits or bits not in (32, 64):
bits = 32
# windows7-32
# windows7-32-vm
# win32
# win64
if '64' in platform:
bits = 64
if _os.startswith('win'):
_os = 'win'
print("_os = %s bits = %s" % (_os, bits))
# TODO(ER): We might be able to ditch this.
self.fetch_config = create_config('firefox', _os, int(bits))
# Hardcode to m-c for now.
self.fetch_config.set_repo('mozilla-central')
self.fetch_config.set_build_type('debug')
class FakeDownloadManager:
def focus_download(self, foo):
pass
dm = FakeDownloadManager()
self.bisector = Bisector(self.fetch_config, self.test_runner, dm,
False, None)
def bisect_further(self):
assert self.bisection
self.started.emit()
handler = self.bisection.handler
try:
nhandler = InboundHandler(find_fix=self.bisection.handler.find_fix)
Bisector.bisect(self, nhandler, handler.good_revision,
handler.bad_revision)
except MozRegressionError:
self._finish_on_exception(None)
def bisect_further(self):
assert self.bisection
self.started.emit()
handler = self.bisection.handler
try:
nhandler = InboundHandler(find_fix=self.bisection.handler.find_fix)
Bisector.bisect(self, nhandler, handler.good_revision,
handler.bad_revision)
except MozRegressionError:
self._finish_on_exception(None)
def bisect_further(self):
assert self.bisection
self.started.emit()
handler = self.bisection.handler
try:
nhandler = InboundHandler(find_fix=self.bisection.handler.find_fix)
Bisector.bisect(self, nhandler, handler.good_revision,
handler.bad_revision, expand=DEFAULT_EXPAND,
interrupt=self.should_stop.is_set)
except MozRegressionError:
self._finish_on_exception(None)
except StopIteration:
self.finished.emit(None, Bisection.USER_EXIT)
def __init__(self, fetch_config, persist=None):
QObject.__init__(self)
Bisector.__init__(self, fetch_config, GuiTestRunner(), persist=persist)
self.download_manager = GuiBuildDownloadManager(self.download_dir)
self.bisection = None
self.mid = None
self.build_infos = None
self._bisect_args = None
self.error = None
self.download_manager.download_finished.connect(
self._build_dl_finished)
self.test_runner.evaluate_finished.connect(self._evaluate_finished)
def __init__(self, fetch_config, persist=None):
QObject.__init__(self)
Bisector.__init__(self, fetch_config, GuiTestRunner(), persist=persist)
self.download_manager = GuiBuildDownloadManager(self.download_dir)
self.bisection = None
self.mid = None
self.build_infos = None
self._bisect_args = None
self.error = None
self.download_manager.download_finished.connect(
self._build_dl_finished)
self.test_runner.evaluate_finished.connect(self._evaluate_finished)
def nightlies_to_inbound(self):
"""
Call this when going from nightlies to inbound.
"""
assert self.bisection
self.started.emit()
try:
# first we need to find the changesets
first, last = self.bisection.handler.find_inbound_changesets()
# create the inbound handler, and go with that
handler = InboundHandler(find_fix=self.bisection.handler.find_fix)
Bisector.bisect(self, handler, first, last)
except MozRegressionError:
self._finish_on_exception(None)
def nightlies_to_inbound(self):
"""
Call this when going from nightlies to inbound.
"""
assert self.bisection
self.started.emit()
try:
# first we need to find the changesets
first, last = self.bisection.handler.find_inbound_changesets()
# create the inbound handler, and go with that
handler = InboundHandler(find_fix=self.bisection.handler.find_fix)
Bisector.bisect(self, handler, first, last)
except MozRegressionError:
self._finish_on_exception(None)
def __init__(self, fetch_config, test_runner, download_manager, download_in_background=True):
QObject.__init__(self)
Bisector.__init__(self, fetch_config, test_runner, download_manager)
self.bisection = None
self.mid = None
self.build_infos = None
self._bisect_args = None
self.error = None
self._next_build_index = None
self.download_in_background = download_in_background
self.index_promise = None
self._persist_files = ()
self.should_stop = threading.Event()
self.download_manager.download_finished.connect(self._build_dl_finished)
self.test_runner.evaluate_finished.connect(self._evaluate_finished)
def __init__(self, fetch_config, test_runner, download_manager,
download_in_background=True):
QObject.__init__(self)
Bisector.__init__(self, fetch_config, test_runner, download_manager)
self.bisection = None
self.mid = None
self.build_infos = None
self._bisect_args = None
self.error = None
self._next_build_index = None
self.download_in_background = download_in_background
self.index_promise = None
self.download_manager.download_finished.connect(
self._build_dl_finished)
self.test_runner.evaluate_finished.connect(self._evaluate_finished)
def setUp(self):
self.handler = MagicMock(find_fix=False, ensure_good_and_bad=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(), self.test_runner,
Mock(),
dl_in_background=False)
self.bisector.download_background = False
def bisect_further(self):
assert self.bisection
self.started.emit()
handler = self.bisection.handler
try:
nhandler = InboundHandler(find_fix=self.bisection.handler.find_fix)
Bisector.bisect(self,
nhandler,
handler.good_revision,
handler.bad_revision,
expand=DEFAULT_EXPAND,
interrupt=self.should_stop.is_set)
except MozRegressionError:
self._finish_on_exception(None)
except StopIteration:
self.finished.emit(None, Bisection.USER_EXIT)
def bisector(self):
if self._bisector is None:
self._bisector = Bisector(
self.fetch_config,
self.test_runner,
self.build_download_manager,
dl_in_background=self.options.background_dl)
return self._bisector
def __init__(self, good, bad, platform, warning,
warning_limit, warning_re, ignore_lines,
required_test):
init_logger()
self.use_nightly = True
try:
self.good = parse_date(good)
self.bad = parse_date(bad)
except DateFormatError:
# This hopefully a revision range. We can bypass nightly and
# go directly to InboundHandler. That itself is a bit of a misnomer,
# it will still bisect m-c builds, but by changeset range, not date
# range.
self.use_nightly = False
self.good = good
self.bad = bad
self.ignore_lines = ignore_lines
self.test_runner = WarningTestRunner(
warning, platform,
ignore_lines=ignore_lines,
warning_re=warning_re,
warning_limit=warning_limit,
required_test=required_test)
# Convert the platform to a mozregression friendly version.
# Also avoid overwriting the os module by *not* using |os| for a
# variable name.
(_os, bits) = re.match(r'([a-zA-Z]+)-?([0-9]+)?', platform).groups()
if not bits or bits not in (32, 64):
bits = 32
# windows7-32
# windows7-32-vm
# win32
# win64
if '64' in platform:
bits = 64
if _os.startswith('win'):
_os = 'win'
print "_os = %s bits = %s" % (_os, bits)
# TODO(ER): We might be able to ditch this.
self.fetch_config = create_config('firefox', _os, int(bits))
# Hardcode to m-c for now.
self.fetch_config.set_repo('mozilla-central')
self.fetch_config.set_build_type('debug')
class FakeDownloadManager:
def focus_download(self, foo):
pass
dm = FakeDownloadManager()
self.bisector = Bisector(self.fetch_config, self.test_runner, dm, False, None)
def bisector(self):
if self._bisector is None:
self._bisector = Bisector(
self.fetch_config,
self.test_runner,
self.build_download_manager,
dl_in_background=self.options.background_dl,
approx_chooser=(None if self.options.approx_policy != 'auto'
else ApproxPersistChooser(7)),
)
return self._bisector
class TestBisector(unittest.TestCase):
def setUp(self):
self.handler = MagicMock(find_fix=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(), self.test_runner,
dl_in_background=False)
self.bisector.download_background = False
@patch("mozregression.bisector.BuildDownloadManager")
def test__bisect_no_data(self, dl):
build_data = MyBuildData()
result = self.bisector._bisect(self.handler, build_data)
# test that handler methods where called
self.handler.set_build_data.assert_called_with(build_data)
self.handler.no_data.assert_called_once_with()
# check return code
self.assertEqual(result, Bisection.NO_DATA)
@patch("mozregression.bisector.BuildDownloadManager")
def test__bisect_finished(self, dl):
build_data = MyBuildData([1])
result = self.bisector._bisect(self.handler, build_data)
# test that handler methods where called
self.handler.set_build_data.assert_called_with(build_data)
self.handler.finished.assert_called_once_with()
# check return code
self.assertEqual(result, Bisection.FINISHED)
@patch("mozregression.bisector.BuildDownloadManager")
def do__bisect(self, build_data, verdicts, dl):
iter_verdict = iter(verdicts)
def evaluate(build_info, allow_back=False):
return iter_verdict.next(), {
'application_changeset': 'unused',
'application_repository': 'unused'
}
self.test_runner.evaluate = Mock(side_effect=evaluate)
result = self.bisector._bisect(self.handler, build_data)
return {
'result': result,
}
def test__bisect_case1(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# we answered bad
call(MyBuildData([3, 4])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_data.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_case1_hunt_fix(self):
self.handler.find_fix = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([1, 2, 3])),
# we answered bad
call(MyBuildData([2, 3])),
])
# ensure that we called the handler's methods
self.assertEqual(self.handler.initialize.mock_calls, [call()]*3)
self.handler.build_good. \
assert_called_once_with(2, MyBuildData([1, 2, 3]))
self.handler.build_bad.assert_called_once_with(1, MyBuildData([2, 3]))
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_case2(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3]), ['r', 's'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3])),
# we asked for a retry
call(MyBuildData([1, 2, 3])),
# we skipped one
call(MyBuildData([1, 3])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_retry.assert_called_with(1)
self.handler.build_skip.assert_called_with(1)
self.assertTrue(self.handler.build_data.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_with_back(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]),
['g', 'back', 'b', 'g'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# oups! let's go back
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered bad this time
call(MyBuildData([1, 2, 3])),
# then good
call(MyBuildData([2, 3])),
])
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_user_exit(self):
test_result = self.do__bisect(MyBuildData(range(20)), ['e'])
# check that set_build_data was called
self.handler.set_build_data.\
assert_has_calls([call(MyBuildData(range(20)))])
# ensure that we called the handler's method
self.handler.initialize.assert_called_once_with()
self.handler.user_exit.assert_called_with(10)
# user exit
self.assertEqual(test_result['result'], Bisection.USER_EXIT)
def test__bisect_with_background_download(self):
self.bisector.dl_in_background = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
call(MyBuildData([1, 2, 3, 4, 5])), # first call
call(MyBuildData([3, 4, 5])), # download backgound
call(MyBuildData([1, 2, 3, 4, 5])), # put back the right data
call(MyBuildData([1, 2, 3])), # download backgound
call(MyBuildData([1, 2, 3, 4, 5])), # put back the right data
call(MyBuildData([3, 4, 5])), # we answered good
call(MyBuildData([4, 5])), # download backgound
call(MyBuildData([3, 4, 5])), # put back the right data
call(MyBuildData([3, 4])), # download backgound
call(MyBuildData([3, 4, 5])), # put back the right data
call(MyBuildData([3, 4])) # we answered bad
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_data.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect(self, _bisect):
_bisect.return_value = 1
build_data = Mock()
build_data_class = Mock(return_value=build_data)
self.handler.build_data_class = build_data_class
result = self.bisector.bisect(self.handler, 'g', 'b', s=1)
build_data_class.assert_called_with(self.bisector.fetch_config,
'g', 'b', s=1)
self.assertFalse(build_data.reverse.called)
_bisect.assert_called_with(self.handler, build_data)
self.assertEqual(result, 1)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect_reverse(self, _bisect):
build_data = Mock()
build_data_class = Mock(return_value=build_data)
self.handler.build_data_class = build_data_class
self.handler.find_fix = True
self.bisector.bisect(self.handler, 'g', 'b', s=1)
build_data_class.assert_called_with(self.bisector.fetch_config,
'b', 'g', s=1)
_bisect.assert_called_with(self.handler, build_data)
class TestBisector(unittest.TestCase):
def setUp(self):
self.handler = MagicMock(find_fix=False, ensure_good_and_bad=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(), self.test_runner,
Mock(),
dl_in_background=False)
self.bisector.download_background = False
def test__bisect_no_data(self):
build_range = MyBuildData()
result = self.bisector._bisect(self.handler, build_range)
# test that handler methods where called
self.handler.set_build_range.assert_called_with(build_range)
self.handler.no_data.assert_called_once_with()
# check return code
self.assertEqual(result, Bisection.NO_DATA)
def test__bisect_finished(self):
build_range = MyBuildData([1])
result = self.bisector._bisect(self.handler, build_range)
# test that handler methods where called
self.handler.set_build_range.assert_called_with(build_range)
self.handler.finished.assert_called_once_with()
# check return code
self.assertEqual(result, Bisection.FINISHED)
def do__bisect(self, build_range, verdicts):
iter_verdict = iter(verdicts)
def evaluate(build_info, allow_back=False):
verdict = iter_verdict.next()
if isinstance(verdict, Exception):
raise verdict
return verdict
self.test_runner.evaluate = Mock(side_effect=evaluate)
result = self.bisector._bisect(self.handler, build_range)
return {
'result': result,
}
def test_ensure_good_bad_invalid(self):
self.handler.ensure_good_and_bad = True
with self.assertRaisesRegexp(MozRegressionError,
"expected to be good"):
self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['b'])
with self.assertRaisesRegexp(MozRegressionError,
"expected to be bad"):
self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'g'])
def test_ensure_good_bad(self):
self.handler.ensure_good_and_bad = True
data = MyBuildData([1, 2, 3, 4, 5])
self.do__bisect(data,
['s', 'r', 'g', 'b', 'e'])
self.test_runner.evaluate.assert_has_calls([
call(data[0]), # tested good (then skip)
call(data[0]), # tested good (then retry)
call(data[0]), # tested good
call(data[-1]), # tested bad
])
self.assertEqual(
self.bisector.download_manager.download_in_background.call_count,
0
)
def test_ensure_good_bad_with_bg_dl(self):
self.handler.ensure_good_and_bad = True
self.bisector.dl_in_background = True
data = MyBuildData([1, 2, 3, 4, 5])
self.do__bisect(data,
['s', 'r', 'g', 'e'])
self.test_runner.evaluate.assert_has_calls([
call(data[0]), # tested good (then skip)
call(data[0]), # tested good (then retry)
call(data[0]), # tested good
call(data[-1]), # tested bad
])
self.bisector.download_manager.download_in_background.assert_has_calls(
[call(data[-1]), # bad in backgound
call(data[data.mid_point()])] # and mid build
)
def test_ensure_good_bad_with_find_fix(self):
self.handler.ensure_good_and_bad = True
self.handler.find_fix = True
data = MyBuildData([1, 2, 3, 4, 5])
self.do__bisect(data, ['g', 'e'])
self.test_runner.evaluate.assert_has_calls([
call(data[-1]), # tested good (then skip)
call(data[0]), # tested bad
])
def test__bisect_case1(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# we answered bad
call(MyBuildData([3, 4])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_with_launcher_exception(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]),
['g', LauncherError("err")])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# launcher exception, equivalent to a skip
call(MyBuildData([3, 5])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_skip.assert_called_with(1)
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_case1_hunt_fix(self):
self.handler.find_fix = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([1, 2, 3])),
# we answered bad
call(MyBuildData([2, 3])),
])
# ensure that we called the handler's methods
self.assertEqual(self.handler.initialize.mock_calls, [call()]*3)
self.handler.build_good. \
assert_called_once_with(2, MyBuildData([1, 2, 3]))
self.handler.build_bad.assert_called_once_with(1, MyBuildData([2, 3]))
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_case2(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3]), ['r', 's'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3])),
# we asked for a retry
call(MyBuildData([1, 2, 3])),
# we skipped one
call(MyBuildData([1, 3])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_retry.assert_called_with(1)
self.handler.build_skip.assert_called_with(1)
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_with_back(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]),
['g', 'back', 'b', 'g'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# oups! let's go back
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered bad this time
call(MyBuildData([1, 2, 3])),
# then good
call(MyBuildData([2, 3])),
])
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_user_exit(self):
test_result = self.do__bisect(MyBuildData(range(20)), ['e'])
# check that set_build_range was called
self.handler.set_build_range.\
assert_has_calls([call(MyBuildData(range(20)))])
# ensure that we called the handler's method
self.handler.initialize.assert_called_once_with()
self.handler.user_exit.assert_called_with(10)
# user exit
self.assertEqual(test_result['result'], Bisection.USER_EXIT)
def test__bisect_with_background_download(self):
self.bisector.dl_in_background = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
call(MyBuildData([1, 2, 3, 4, 5])), # first call
call(MyBuildData([3, 4, 5])), # we answered good
call(MyBuildData([3, 4])) # we answered bad
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect(self, _bisect):
_bisect.return_value = 1
build_range = Mock()
create_range = Mock(return_value=build_range)
self.handler.create_range = create_range
result = self.bisector.bisect(self.handler, 'g', 'b', s=1)
create_range.assert_called_with(self.bisector.fetch_config,
'g', 'b', s=1)
self.assertFalse(build_range.reverse.called)
_bisect.assert_called_with(self.handler, build_range)
self.assertEqual(result, 1)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect_reverse(self, _bisect):
build_range = Mock()
create_range = Mock(return_value=build_range)
self.handler.create_range = create_range
self.handler.find_fix = True
self.bisector.bisect(self.handler, 'g', 'b', s=1)
create_range.assert_called_with(self.bisector.fetch_config,
'b', 'g', s=1)
_bisect.assert_called_with(self.handler, build_range)
class WarningBisector(object):
def __init__(self, good, bad, platform, warning,
warning_limit, warning_re, ignore_lines,
required_test):
init_logger()
self.use_nightly = True
try:
self.good = parse_date(good)
self.bad = parse_date(bad)
except DateFormatError:
# This hopefully a revision range. We can bypass nightly and
# go directly to InboundHandler. That itself is a bit of a misnomer,
# it will still bisect m-c builds, but by changeset range, not date
# range.
self.use_nightly = False
self.good = good
self.bad = bad
self.ignore_lines = ignore_lines
self.test_runner = WarningTestRunner(
warning, platform,
ignore_lines=ignore_lines,
warning_re=warning_re,
warning_limit=warning_limit,
required_test=required_test)
# Convert the platform to a mozregression friendly version.
# Also avoid overwriting the os module by *not* using |os| for a
# variable name.
(_os, bits) = re.match(r'([a-zA-Z]+)-?([0-9]+)?', platform).groups()
if not bits or bits not in (32, 64):
bits = 32
# windows7-32
# windows7-32-vm
# win32
# win64
if '64' in platform:
bits = 64
if _os.startswith('win'):
_os = 'win'
print "_os = %s bits = %s" % (_os, bits)
# TODO(ER): We might be able to ditch this.
self.fetch_config = create_config('firefox', _os, int(bits))
# Hardcode to m-c for now.
self.fetch_config.set_repo('mozilla-central')
self.fetch_config.set_build_type('debug')
class FakeDownloadManager:
def focus_download(self, foo):
pass
dm = FakeDownloadManager()
self.bisector = Bisector(self.fetch_config, self.test_runner, dm, False, None)
def bisect(self):
if self.use_nightly:
result = self.bisect_nightly()
else:
result = self.bisect_inbound(self.good, self.bad)
(good, bad) = result
if self.test_runner.check_for_move(self.fetch_config.repo, good):
print "You should probably try bisecting again from the good revision"
print "Done bisecting I guess"
return result
def bisect_nightly(self):
handler = NightlyHandler(ensure_good_and_bad=True)
result = self.bisector.bisect(handler, self.good, self.bad)
if result == Bisection.FINISHED:
print "Got as far as we can go bisecting nightlies..."
handler.print_range()
print "Switching bisection method to taskcluster"
result = self.bisect_inbound(handler.good_revision, handler.bad_revision)
else:
# TODO(ER): maybe this should be an exception...
result = (None, None)
return result
def bisect_inbound(self, good_rev, bad_rev):
# Remember, InboundHandler is just a changeset based bisector. It will
# still potentially bisect m-c first.
handler = InboundHandler()
result = self.bisector.bisect(handler, good_rev, bad_rev, expand=0)
if result == Bisection.FINISHED:
print "No more m-c revisions :("
handler.print_range()
# Try switching over to the integration branch.
if len(handler.build_range) == 2:
result = handler.handle_merge()
if result:
branch, good_rev, bad_rev = result
self.fetch_config.set_repo(branch)
return self.bisect_inbound(good_rev, bad_rev)
return (handler.good_revision, handler.bad_revision)
def setUp(self):
self.handler = Mock(find_fix=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(), self.test_runner)
class TestBisector(unittest.TestCase):
def setUp(self):
self.handler = Mock(find_fix=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(), self.test_runner)
def test__bisect_no_data(self):
build_data = MyBuildData()
result = self.bisector._bisect(self.handler, build_data)
# test that handler methods where called
self.handler.set_build_data.assert_called_with(build_data)
self.handler.no_data.assert_called_once_with()
# check return code
self.assertEqual(result, Bisector.NO_DATA)
def test__bisect_finished(self):
build_data = MyBuildData([1])
result = self.bisector._bisect(self.handler, build_data)
# test that handler methods where called
self.handler.set_build_data.assert_called_with(build_data)
self.handler.finished.assert_called_once_with()
# check return code
self.assertEqual(result, Bisector.FINISHED)
def do__bisect(self, build_data, verdicts):
iter_verdict = iter(verdicts)
def evaluate(build_info, allow_back=False):
return iter_verdict.next(), {
'application_changeset': 'unused',
'application_repository': 'unused'
}
self.test_runner.evaluate = Mock(side_effect=evaluate)
result = self.bisector._bisect(self.handler, build_data)
return {
'result': result,
}
def test__bisect_case1(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# we answered bad
call(MyBuildData([3, 4])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_data.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisector.FINISHED)
def test__bisect_case1_hunt_fix(self):
self.handler.find_fix = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([1, 2, 3])),
# we answered bad
call(MyBuildData([2, 3])),
])
# ensure that we called the handler's methods
self.assertEqual(self.handler.initialize.mock_calls, [call()]*3)
self.handler.build_good. \
assert_called_once_with(2, MyBuildData([1, 2, 3]))
self.handler.build_bad.assert_called_once_with(1, MyBuildData([2, 3]))
# bisection is finished
self.assertEqual(test_result['result'], Bisector.FINISHED)
def test__bisect_case2(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3]), ['r', 's'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3])),
# we asked for a retry
call(MyBuildData([1, 2, 3])),
# we skipped one
call(MyBuildData([1, 3])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_retry.assert_called_with(1)
self.handler.build_skip.assert_called_with(1)
self.assertTrue(self.handler.build_data.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisector.FINISHED)
def test__bisect_with_back(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]),
['g', 'back', 'b', 'g'])
# check that set_build_data was called
self.handler.set_build_data.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# oups! let's go back
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered bad this time
call(MyBuildData([1, 2, 3])),
# then good
call(MyBuildData([2, 3])),
])
# bisection is finished
self.assertEqual(test_result['result'], Bisector.FINISHED)
def test__bisect_user_exit(self):
test_result = self.do__bisect(MyBuildData(range(20)), ['e'])
# check that set_build_data was called
self.handler.set_build_data.\
assert_has_calls([call(MyBuildData(range(20)))])
# ensure that we called the handler's method
self.handler.initialize.assert_called_once_with()
self.handler.user_exit.assert_called_with(10)
# user exit
self.assertEqual(test_result['result'], Bisector.USER_EXIT)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect(self, _bisect):
_bisect.return_value = 1
build_data = Mock()
build_data_class = Mock(return_value=build_data)
self.handler.build_data_class = build_data_class
result = self.bisector.bisect(self.handler, 'g', 'b', s=1)
build_data_class.assert_called_with(self.bisector.fetch_config,
'g', 'b', s=1)
self.assertFalse(build_data.reverse.called)
_bisect.assert_called_with(self.handler, build_data)
self.assertEqual(result, 1)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect_reverse(self, _bisect):
build_data = Mock()
build_data_class = Mock(return_value=build_data)
self.handler.build_data_class = build_data_class
self.handler.find_fix = True
self.bisector.bisect(self.handler, 'g', 'b', s=1)
build_data_class.assert_called_with(self.bisector.fetch_config,
'b', 'g', s=1)
_bisect.assert_called_with(self.handler, build_data)
class TestBisector(unittest.TestCase):
def setUp(self):
self.handler = MagicMock(find_fix=False, ensure_good_and_bad=False)
self.test_runner = Mock()
self.bisector = Bisector(Mock(),
self.test_runner,
Mock(),
dl_in_background=False)
self.bisector.download_background = False
def test__bisect_no_data(self):
build_range = MyBuildData()
result = self.bisector._bisect(self.handler, build_range)
# test that handler methods where called
self.handler.set_build_range.assert_called_with(build_range)
self.handler.no_data.assert_called_once_with()
# check return code
self.assertEqual(result, Bisection.NO_DATA)
def test__bisect_finished(self):
build_range = MyBuildData([1])
result = self.bisector._bisect(self.handler, build_range)
# test that handler methods where called
self.handler.set_build_range.assert_called_with(build_range)
self.handler.finished.assert_called_once_with()
# check return code
self.assertEqual(result, Bisection.FINISHED)
def do__bisect(self, build_range, verdicts):
iter_verdict = iter(verdicts)
def evaluate(build_info, allow_back=False):
verdict = next(iter_verdict)
if isinstance(verdict, Exception):
raise verdict
return verdict
self.test_runner.evaluate = Mock(side_effect=evaluate)
result = self.bisector._bisect(self.handler, build_range)
return {
'result': result,
}
def test_ensure_good_bad_invalid(self):
self.handler.ensure_good_and_bad = True
with self.assertRaisesRegexp(MozRegressionError,
"expected to be good"):
self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['b'])
with self.assertRaisesRegexp(MozRegressionError, "expected to be bad"):
self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'g'])
def test_ensure_good_bad(self):
self.handler.ensure_good_and_bad = True
data = MyBuildData([1, 2, 3, 4, 5])
self.do__bisect(data, ['s', 'r', 'g', 'b', 'e'])
self.test_runner.evaluate.assert_has_calls([
call(data[0]), # tested good (then skip)
call(data[0]), # tested good (then retry)
call(data[0]), # tested good
call(data[-1]), # tested bad
])
self.assertEqual(
self.bisector.download_manager.download_in_background.call_count,
0)
def test_ensure_good_bad_with_bg_dl(self):
self.handler.ensure_good_and_bad = True
self.bisector.dl_in_background = True
data = MyBuildData([1, 2, 3, 4, 5])
self.do__bisect(data, ['s', 'r', 'g', 'e'])
self.test_runner.evaluate.assert_has_calls([
call(data[0]), # tested good (then skip)
call(data[0]), # tested good (then retry)
call(data[0]), # tested good
call(data[-1]), # tested bad
])
self.bisector.download_manager.download_in_background.assert_has_calls(
[
call(data[-1]), # bad in backgound
call(data[data.mid_point()])
] # and mid build
)
def test_ensure_good_bad_with_find_fix(self):
self.handler.ensure_good_and_bad = True
self.handler.find_fix = True
data = MyBuildData([1, 2, 3, 4, 5])
self.do__bisect(data, ['g', 'e'])
self.test_runner.evaluate.assert_has_calls([
call(data[-1]), # tested good (then skip)
call(data[0]), # tested bad
])
def test__bisect_case1(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# we answered bad
call(MyBuildData([3, 4])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_with_launcher_exception(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]),
['g', LauncherError("err")])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# launcher exception, equivalent to a skip
call(MyBuildData([3, 5])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_skip.assert_called_with(1)
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_case1_hunt_fix(self):
self.handler.find_fix = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([1, 2, 3])),
# we answered bad
call(MyBuildData([2, 3])),
])
# ensure that we called the handler's methods
self.assertEqual(self.handler.initialize.mock_calls, [call()] * 3)
self.handler.build_good. \
assert_called_once_with(2, MyBuildData([1, 2, 3]))
self.handler.build_bad.assert_called_once_with(1, MyBuildData([2, 3]))
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_case2(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3]), ['r', 's'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3])),
# we asked for a retry
call(MyBuildData([1, 2, 3])),
# we skipped one
call(MyBuildData([1, 3])),
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_retry.assert_called_with(1)
self.handler.build_skip.assert_called_with(1)
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_with_back(self):
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]),
['g', 'back', 'b', 'g'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
# first call
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered good
call(MyBuildData([3, 4, 5])),
# oups! let's go back
call(MyBuildData([1, 2, 3, 4, 5])),
# we answered bad this time
call(MyBuildData([1, 2, 3])),
# then good
call(MyBuildData([2, 3])),
])
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
def test__bisect_user_exit(self):
test_result = self.do__bisect(MyBuildData(list(range(20))), ['e'])
# check that set_build_range was called
self.handler.set_build_range.\
assert_has_calls([call(MyBuildData(list(range(20))))])
# ensure that we called the handler's method
self.handler.initialize.assert_called_once_with()
self.handler.user_exit.assert_called_with(10)
# user exit
self.assertEqual(test_result['result'], Bisection.USER_EXIT)
def test__bisect_with_background_download(self):
self.bisector.dl_in_background = True
test_result = self.do__bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
# check that set_build_range was called
self.handler.set_build_range.assert_has_calls([
call(MyBuildData([1, 2, 3, 4, 5])), # first call
call(MyBuildData([3, 4, 5])), # we answered good
call(MyBuildData([3, 4])) # we answered bad
])
# ensure that we called the handler's methods
self.handler.initialize.assert_called_with()
self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
self.assertTrue(self.handler.build_range.ensure_limits_called)
# bisection is finished
self.assertEqual(test_result['result'], Bisection.FINISHED)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect(self, _bisect):
_bisect.return_value = 1
build_range = Mock()
create_range = Mock(return_value=build_range)
self.handler.create_range = create_range
result = self.bisector.bisect(self.handler, 'g', 'b', s=1)
create_range.assert_called_with(self.bisector.fetch_config,
'g',
'b',
s=1)
self.assertFalse(build_range.reverse.called)
_bisect.assert_called_with(self.handler, build_range)
self.assertEqual(result, 1)
@patch('mozregression.bisector.Bisector._bisect')
def test_bisect_reverse(self, _bisect):
build_range = Mock()
create_range = Mock(return_value=build_range)
self.handler.create_range = create_range
self.handler.find_fix = True
self.bisector.bisect(self.handler, 'g', 'b', s=1)
create_range.assert_called_with(self.bisector.fetch_config,
'b',
'g',
s=1)
_bisect.assert_called_with(self.handler, build_range)
class WarningBisector(object):
def __init__(self, good, bad, platform, warning, warning_limit, warning_re,
ignore_lines, required_test):
init_logger()
self.use_nightly = True
try:
self.good = parse_date(good)
self.bad = parse_date(bad)
except DateFormatError:
# This hopefully a revision range. We can bypass nightly and
# go directly to InboundHandler. That itself is a bit of a misnomer,
# it will still bisect m-c builds, but by changeset range, not date
# range.
self.use_nightly = False
self.good = good
self.bad = bad
self.ignore_lines = ignore_lines
self.test_runner = WarningTestRunner(warning,
platform,
ignore_lines=ignore_lines,
warning_re=warning_re,
warning_limit=warning_limit,
required_test=required_test)
# Convert the platform to a mozregression friendly version.
# Also avoid overwriting the os module by *not* using |os| for a
# variable name.
(_os, bits) = re.match(r'([a-zA-Z]+)-?([0-9]+)?', platform).groups()
if not bits or bits not in (32, 64):
bits = 32
# windows7-32
# windows7-32-vm
# win32
# win64
if '64' in platform:
bits = 64
if _os.startswith('win'):
_os = 'win'
print("_os = %s bits = %s" % (_os, bits))
# TODO(ER): We might be able to ditch this.
self.fetch_config = create_config('firefox', _os, int(bits))
# Hardcode to m-c for now.
self.fetch_config.set_repo('mozilla-central')
self.fetch_config.set_build_type('debug')
class FakeDownloadManager:
def focus_download(self, foo):
pass
dm = FakeDownloadManager()
self.bisector = Bisector(self.fetch_config, self.test_runner, dm,
False, None)
def bisect(self):
if self.use_nightly:
result = self.bisect_nightly()
else:
result = self.bisect_inbound(self.good, self.bad)
(good, bad) = result
if self.test_runner.check_for_move(self.fetch_config.repo, good):
print(
"You should probably try bisecting again from the good revision"
)
print("Done bisecting I guess")
return result
def bisect_nightly(self):
handler = NightlyHandler(ensure_good_and_bad=True)
result = self.bisector.bisect(handler, self.good, self.bad)
if result == Bisection.FINISHED:
print("Got as far as we can go bisecting nightlies...")
handler.print_range()
print("Switching bisection method to taskcluster")
result = self.bisect_inbound(handler.good_revision,
handler.bad_revision)
else:
# TODO(ER): maybe this should be an exception...
result = (None, None)
return result
def bisect_inbound(self, good_rev, bad_rev):
# Remember, InboundHandler is just a changeset based bisector. It will
# still potentially bisect m-c first.
handler = InboundHandler()
result = self.bisector.bisect(handler, good_rev, bad_rev, expand=0)
if result == Bisection.FINISHED:
print("No more m-c revisions :(")
handler.print_range()
# Try switching over to the integration branch.
if len(handler.build_range) == 2:
result = handler.handle_merge()
if result:
branch, good_rev, bad_rev = result
self.fetch_config.set_repo(branch)
return self.bisect_inbound(good_rev, bad_rev)
return (handler.good_revision, handler.bad_revision)
