def _MakeBuildbucketBisectJob(bisect_job):
"""Creates a bisect job object that the buildbucket service can use.
Args:
bisect_job: The entity (try_job.TryJob) off of which to create the
buildbucket job.
Returns:
A buildbucket_job.BisectJob object populated with the necessary attributes
to pass it to the buildbucket service to start the job.
"""
config = bisect_job.GetConfigDict()
if bisect_job.job_type != 'bisect':
raise request_handler.InvalidInputError(
'Recipe only supports bisect jobs at this time.')
if not bisect_job.master_name.startswith('ChromiumPerf'):
raise request_handler.InvalidInputError(
'Recipe is only implemented on for tests run on chromium.perf '
'(and chromium.perf.fyi).')
return buildbucket_job.BisectJob(
good_revision=config['good_revision'],
bad_revision=config['bad_revision'],
test_command=config['command'],
metric=config['metric'],
repeats=config['repeat_count'],
timeout_minutes=config['max_time_minutes'],
truncate=config['truncate_percent'],
bug_id=bisect_job.bug_id,
gs_bucket='chrome-perf',
original_bot_name=config['original_bot_name'],
)
def _EditEntity(self):
"""Edits an existing entity according to the request parameters."""
name = self.request.get('edit-name')
if not name:
raise request_handler.InvalidInputError('No name given.')
entity = self._model_class.get_by_id(name)
if not entity:
raise request_handler.InvalidInputError(
'Entity "%s" does not exist, cannot edit.' % name)
self._UpdateAndReportResults(entity)
def _ShowAlertsForKeys(self, keys):
"""Show alerts for |keys|.
Query for anomalies with overlapping revision. The |keys|
parameter for group_report is a comma-separated list of urlsafe strings
for Keys for Anomaly entities. (Each key corresponds to an alert)
Args:
keys: Comma-separated list of urlsafe strings for Anomaly keys.
"""
urlsafe_keys = keys.split(',')
try:
keys = [ndb.Key(urlsafe=k) for k in urlsafe_keys]
# Errors that can be thrown here include ProtocolBufferDecodeError
# in google.net.proto.ProtocolBuffer. We want to catch any errors here
# because they're almost certainly urlsafe key decoding errors.
except Exception:
raise request_handler.InvalidInputError(
'Invalid Anomaly key given.')
requested_anomalies = utils.GetMulti(keys)
for i, anomaly_entity in enumerate(requested_anomalies):
if anomaly_entity is None:
raise request_handler.InvalidInputError(
'No Anomaly found for key %s.' % urlsafe_keys[i])
if not requested_anomalies:
raise request_handler.InvalidInputError('No anomalies found.')
sheriff_key = requested_anomalies[0].sheriff
min_range = utils.MinimumAlertRange(requested_anomalies)
if min_range:
query = anomaly.Anomaly.query(
anomaly.Anomaly.sheriff == sheriff_key)
query = query.order(-anomaly.Anomaly.timestamp)
anomalies = query.fetch(limit=_QUERY_LIMIT)
# Filter out anomalies that have been marked as invalid or ignore.
# Include all anomalies with an overlapping revision range that have
# been associated with a bug, or are not yet triaged.
anomalies = [
a for a in anomalies if a.bug_id is None or a.bug_id > 0
]
anomalies = _GetOverlaps(anomalies, min_range[0], min_range[1])
# Make sure alerts in specified param "keys" are included.
key_set = {a.key for a in anomalies}
for anomaly_entity in requested_anomalies:
if anomaly_entity.key not in key_set:
anomalies.append(anomaly_entity)
else:
anomalies = requested_anomalies
self._ShowAlerts(anomalies)
def _AddEntity(self):
"""Adds adds a new entity according to the request parameters."""
name = self.request.get('add-name')
if not name:
raise request_handler.InvalidInputError(
'No name given when adding new ')
if self._model_class.get_by_id(name):
raise request_handler.InvalidInputError(
'Entity "%s" already exists, cannot add.' % name)
entity = self._model_class(id=name)
self._UpdateAndReportResults(entity)
def _GetAndValidateConfigContents(self):
"""Returns a config dict if one could be gotten, or None otherwise."""
config = self.request.get('config')
if not config:
raise request_handler.InvalidInputError(
'No config contents given.')
try:
config_dict = json.loads(config)
except (ValueError, TypeError) as json_parse_error:
raise request_handler.InvalidInputError(str(json_parse_error))
if not isinstance(config_dict, dict):
raise request_handler.InvalidInputError('Config was not a dict.')
return config_dict
def _MakeBuildbucketBisectJob(bisect_job):
"""Creates a bisect job object that the buildbucket service can use.
Args:
bisect_job: The entity (try_job.TryJob) off of which to create the
buildbucket job.
Returns:
A buildbucket_job.BisectJob object populated with the necessary attributes
to pass it to the buildbucket service to start the job.
"""
config = bisect_job.GetConfigDict()
if bisect_job.job_type not in ['bisect', 'bisect-fyi']:
raise request_handler.InvalidInputError(
'Recipe only supports bisect jobs at this time.')
# Recipe bisect supports 'perf' and 'return_code' test types only.
# TODO (prasadv): Update bisect form on dashboard to support test_types.
test_type = 'perf'
if config.get('bisect_mode') == 'return_code':
test_type = config['bisect_mode']
return buildbucket_job.BisectJob(
try_job_id=bisect_job.key.id(),
good_revision=config['good_revision'],
bad_revision=config['bad_revision'],
test_command=config['command'],
metric=config['metric'],
repeats=config['repeat_count'],
timeout_minutes=config['max_time_minutes'],
bug_id=bisect_job.bug_id,
gs_bucket='chrome-perf',
recipe_tester_name=config['recipe_tester_name'],
test_type=test_type,
required_initial_confidence=config.get('required_initial_confidence'))
def post(self):
"""Updates the user-selected anomaly threshold configuration.
Request parameters:
add-edit: Either 'add' if adding a new config, or 'edit'.
add-name: A new anomaly config name, if adding one.
edit-name: An existing anomaly config name, if editing one.
patterns: Newline-separated list of test path patterns to monitor.
Depending on the specific sub-class, this will also take other
parameters for specific properties of the entity being edited.
"""
try:
edit_type = self.request.get('add-edit')
if edit_type == 'add':
self._AddEntity()
elif edit_type == 'edit':
self._EditEntity()
else:
raise request_handler.InvalidInputError(
'Invalid value for add-edit.')
except request_handler.InvalidInputError as error:
message = str(
error) + ' Model class: ' + self._model_class.__name__
self.RenderHtml('result.html', {'errors': [message]})
def post(self):
"""Returns dynamic data for /group_report with some set of alerts.
The set of alerts is determined by the keys, bug ID or revision given.
Request parameters:
keys: A comma-separated list of urlsafe Anomaly keys (optional).
bug_id: A bug number on the Chromium issue tracker (optional).
rev: A revision number (optional).
Outputs:
JSON for the /group_report page XHR request.
"""
keys = self.request.get('keys')
bug_id = self.request.get('bug_id')
rev = self.request.get('rev')
try:
if bug_id:
self._ShowAlertsWithBugId(bug_id)
elif keys:
self._ShowAlertsForKeys(keys)
elif rev:
self._ShowAlertsAroundRevision(rev)
else:
# TODO(qyearsley): Instead of just showing an error here, show a form
# where the user can input a bug ID or revision.
raise request_handler.InvalidInputError(
'No anomalies specified.')
except request_handler.InvalidInputError as error:
self.response.out.write(json.dumps({'error': str(error)}))
def testPost_RunCount2_ExceptionInPerformBisect_CustomMetricNotTicked(
self, mock_tick, mock_perform_bisect):
mock_perform_bisect.side_effect = request_handler.InvalidInputError()
try_job.TryJob(
bug_id=111, status='failed',
last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8),
run_count=2).put()
self.testapp.post('/auto_bisect')
self.assertEqual(0, mock_tick.call_count)
def _ShowAlertsWithBugId(self, bug_id):
"""Show alerts for |bug_id|.
Args:
bug_id: A bug ID (as an int or string). Could be also be a pseudo-bug ID,
such as -1 or -2 indicating invalid or ignored.
"""
if not _IsInt(bug_id):
raise request_handler.InvalidInputError('Invalid bug ID "%s".' %
bug_id)
bug_id = int(bug_id)
anomaly_query = anomaly.Anomaly.query(anomaly.Anomaly.bug_id == bug_id)
anomalies = anomaly_query.fetch(limit=_DISPLAY_LIMIT)
stoppage_alert_query = stoppage_alert.StoppageAlert.query(
stoppage_alert.StoppageAlert.bug_id == bug_id)
stoppage_alerts = stoppage_alert_query.fetch(limit=_DISPLAY_LIMIT)
self._ShowAlerts(anomalies + stoppage_alerts, bug_id)
def _ShowAlertsAroundRevision(self, rev):
"""Shows a alerts whose revision range includes the given revision.
Args:
rev: A revision number, as a string.
"""
if not _IsInt(rev):
raise request_handler.InvalidInputError('Invalid rev "%s".' % rev)
rev = int(rev)
# We can't make a query that has two inequality filters on two different
# properties (start_revision and end_revision). Therefore we first query
# Anomaly entities based on one of these, then filter the resulting list.
anomaly_query = anomaly.Anomaly.query(anomaly.Anomaly.end_revision >= rev)
anomaly_query = anomaly_query.order(anomaly.Anomaly.end_revision)
anomalies = anomaly_query.fetch(limit=_QUERY_LIMIT)
anomalies = [a for a in anomalies if a.start_revision <= rev]
stoppage_alert_query = stoppage_alert.StoppageAlert.query(
stoppage_alert.StoppageAlert.end_revision == rev)
stoppage_alerts = stoppage_alert_query.fetch(limit=_DISPLAY_LIMIT)
self._ShowAlerts(anomalies + stoppage_alerts)
def _ValidatePatterns(test_path_patterns):
"""Raises an exception if any test path patterns are invalid."""
for pattern in test_path_patterns:
if not _IsValidTestPathPattern(pattern):
raise request_handler.InvalidInputError(
'Invalid test path pattern: "%s"' % pattern)
class StartNewBisectForBugTest(testing_common.TestCase):
def setUp(self):
super(StartNewBisectForBugTest, self).setUp()
self.SetCurrentUser('*****@*****.**')
namespaced_stored_object.Set(
start_try_job._TESTER_DIRECTOR_MAP_KEY,
{
'ChromiumPerf': {
'linux_perf_tester': 'linux_perf_bisector',
'win64_nv_tester': 'linux_perf_bisector',
}
})
@mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
def testStartNewBisectForBug_StartsBisect(self, mock_perform_bisect):
testing_common.AddTests(
['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
anomaly.Anomaly(
bug_id=111, test=test_key,
start_revision=300100, end_revision=300200,
median_before_anomaly=100, median_after_anomaly=200).put()
auto_bisect.StartNewBisectForBug(111)
job = try_job.TryJob.query(try_job.TryJob.bug_id == 111).get()
mock_perform_bisect.assert_called_once_with(job)
def testStartNewBisectForBug_RevisionTooLow_ReturnsError(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
anomaly.Anomaly(
bug_id=222, test=test_key,
start_revision=1200, end_revision=1250,
median_before_anomaly=100, median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(222)
self.assertEqual({'error': 'Invalid "good" revision: 1199.'}, result)
@mock.patch.object(
auto_bisect.start_try_job, 'PerformBisect',
mock.MagicMock(side_effect=request_handler.InvalidInputError(
'Some reason')))
def testStartNewBisectForBug_InvalidInputErrorRaised_ReturnsError(self):
testing_common.AddTests(['Foo'], ['bar'], {'sunspider': {'score': {}}})
test_key = utils.TestKey('Foo/bar/sunspider/score')
anomaly.Anomaly(
bug_id=345, test=test_key,
start_revision=300100, end_revision=300200,
median_before_anomaly=100, median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(345)
self.assertEqual({'error': 'Some reason'}, result)
@mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
def testStartNewBisectForBug_WithDefaultRevs_StartsBisect(
self, mock_perform_bisect):
testing_common.AddTests(
['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
testing_common.AddRows(
'ChromiumPerf/linux-release/sunspider/score',
{
1199: {
'a_default_rev': 'r_foo',
'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857'
},
1250: {
'a_default_rev': 'r_foo',
'r_foo': 'fc34e5346446854637311ad7793a95d56e314042'
}
})
anomaly.Anomaly(
bug_id=333, test=test_key,
start_revision=1200, end_revision=1250,
median_before_anomaly=100, median_after_anomaly=200).put()
auto_bisect.StartNewBisectForBug(333)
job = try_job.TryJob.query(try_job.TryJob.bug_id == 333).get()
mock_perform_bisect.assert_called_once_with(job)
def testStartNewBisectForBug_UnbisectableTest_ReturnsError(self):
testing_common.AddTests(['V8'], ['x86'], {'v8': {'sunspider': {}}})
# The test suite "v8" is in the black-list of test suite names.
test_key = utils.TestKey('V8/x86/v8/sunspider')
anomaly.Anomaly(
bug_id=444, test=test_key,
start_revision=155000, end_revision=155100,
median_before_anomaly=100, median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(444)
self.assertEqual({'error': 'Could not select a test.'}, result)
