def testGetAnomalyConfigDict(self):
testing_common.AddTests(['M'], ['b'], {'foo': {'bar': {}}})
test = utils.TestKey('M/b/foo/bar').get()
# The sample test has no overridden config.
self.assertEqual({}, anomaly_config.GetAnomalyConfigDict(test))
# Override the config for the test added above.
# The overridden config is set in the pre-put hook of the Test.
my_config = {
'_comment': 'Very particular segment sizes.',
'max_window_size': 721,
'min_segment_size': 123,
}
my_patterns = [test.test_path]
anomaly_config.AnomalyConfig(config=my_config,
patterns=my_patterns).put()
test.put()
# The sample test now has an overridden config which is used.
# Extraneous "comment" keys are ignored.
expected = {
'max_window_size': 721,
'min_segment_size': 123,
}
self.assertEqual(expected, anomaly_config.GetAnomalyConfigDict(test))
def _GetAnomalyConfigNameAndDict(self, test):
"""Gets the anomaly threshold dict to use and its name.
Args:
test: A Test entity.
Returns:
A (name, config dict) pair.
Raises:
ValueError: The user-specified dict couldn't be parsed.
"""
# Get the anomaly config name and config dict based on the test.
config_name = 'Default config'
if test.overridden_anomaly_config:
config_name = test.overridden_anomaly_config.string_id()
config_dict = anomaly_config.GetAnomalyConfigDict(test)
# If the user specified a config, then use that.
input_config_json = self.request.get('config')
if input_config_json:
try:
config_dict = json.loads(input_config_json)
except ValueError:
raise QueryParameterError('Invalid JSON.')
config_name = 'Custom config'
return config_name, config_dict
def _UpdateInvalidAndConfirmedAnomalyRevs(test_bench):
"""Updates TestBench entity with invalid and confirmed anomalies revs."""
# Start rev for getting Anomalies should be at min_segment_size.
test = test_bench.test.get()
config_dict = anomaly_config.GetAnomalyConfigDict(test)
min_segment_size = config_dict.get('min_segment_size')
start_index = min(min_segment_size, len(test_bench.data_series)) - 1
start_rev = test_bench.data_series[start_index][0]
query = anomaly.Anomaly.query(anomaly.Anomaly.test == test_bench.test)
anomalies = query.fetch()
anomalies.sort(key=lambda a: a.end_revision)
anomalies = [
a for a in anomalies
if a.end_revision >= start_rev and not a.is_improvement
]
test_bench.invalid_anomaly_revs = [
_GetRevsAroundRev(test_bench.data_series, a.end_revision)
for a in anomalies if a.bug_id == -1
]
test_bench.confirmed_anomaly_revs = [
_GetRevsAroundRev(test_bench.data_series, a.end_revision)
for a in anomalies if a.bug_id > 0
]
def _IsAnomalyRecovered(anomaly_entity):
"""Checks whether an Anomaly has recovered.
An Anomaly will be considered "recovered" if there's a change point in
the series after the Anomaly with roughly equal magnitude and opposite
direction.
Args:
anomaly_entity: The original regression Anomaly.
Returns:
True if the Anomaly should be marked as recovered, False otherwise.
"""
test = anomaly_entity.test.get()
config = anomaly_config.GetAnomalyConfigDict(test)
max_num_rows = config.get('max_window_size',
find_anomalies.DEFAULT_NUM_POINTS)
rows = [
r for r in find_anomalies.GetRowsToAnalyze(test, max_num_rows)
if r.revision > anomaly_entity.end_revision
]
change_points = find_anomalies.FindChangePointsForTest(rows, config)
delta_anomaly = (anomaly_entity.median_after_anomaly -
anomaly_entity.median_before_anomaly)
for change in change_points:
delta_change = change.median_after - change.median_before
if (_IsOppositeDirection(delta_anomaly, delta_change)
and _IsApproximatelyEqual(delta_anomaly, -delta_change)):
logging.debug('Anomaly %s recovered; recovery change point %s.',
anomaly_entity.key, change.AsDict())
return True
return False
def _IsAlertRecovered(self, alert_entity):
test = alert_entity.GetTestMetadataKey().get()
if not test:
logging.error(
'TestMetadata %s not found for Anomaly %s, deleting test.',
utils.TestPath(alert_entity.GetTestMetadataKey()),
alert_entity)
return False
config = anomaly_config.GetAnomalyConfigDict(test)
max_num_rows = config.get('max_window_size',
find_anomalies.DEFAULT_NUM_POINTS)
rows = [
r for r in find_anomalies.GetRowsToAnalyze(test, max_num_rows)
if r.revision > alert_entity.end_revision
]
change_points = find_anomalies.FindChangePointsForTest(rows, config)
delta_anomaly = (alert_entity.median_after_anomaly -
alert_entity.median_before_anomaly)
for change in change_points:
delta_change = change.median_after - change.median_before
if (self._IsOppositeDirection(delta_anomaly, delta_change) and
self._IsApproximatelyEqual(delta_anomaly, -delta_change)):
logging.debug(
'Anomaly %s recovered; recovery change point %s.',
alert_entity.key, change.AsDict())
return True
return False
def _ProcessTest(test_key):
"""Processes a test to find new anomalies.
Args:
test_key: The ndb.Key for a TestMetadata.
"""
test = yield test_key.get_async()
config = anomaly_config.GetAnomalyConfigDict(test)
max_num_rows = config.get('max_window_size', DEFAULT_NUM_POINTS)
rows = yield GetRowsToAnalyzeAsync(test, max_num_rows)
# If there were no rows fetched, then there's nothing to analyze.
if not rows:
# In some cases (e.g. if some points are deleted) it might be possible
# that last_alerted_revision is incorrect. In this case, reset it.
highest_rev = yield _HighestRevision(test_key)
if test.last_alerted_revision > highest_rev:
logging.error(
'last_alerted_revision %d is higher than highest rev %d '
'for test %s; setting last_alerted_revision to None.',
test.last_alerted_revision, highest_rev, test.test_path)
test.last_alerted_revision = None
yield test.put_async()
logging.error('No rows fetched for %s', test.test_path)
raise ndb.Return(None)
sheriff = yield _GetSheriffForTest(test)
if not sheriff:
logging.error('No sheriff for %s', test_key)
raise ndb.Return(None)
# Get anomalies and check if they happen in ref build also.
change_points = FindChangePointsForTest(rows, config)
change_points = yield _FilterAnomaliesFoundInRef(change_points, test_key,
len(rows))
anomalies = [_MakeAnomalyEntity(c, test, rows) for c in change_points]
# If no new anomalies were found, then we're done.
if not anomalies:
return
logging.info('Found at least one anomaly in: %s', test.test_path)
# Update the last_alerted_revision property of the test.
test.last_alerted_revision = anomalies[-1].end_revision
yield test.put_async()
yield alert_group.GroupAlertsAsync(anomalies,
utils.TestSuiteName(test.key),
'Anomaly')
# TODO(simonhatch): email_sheriff.EmailSheriff() isn't a tasklet yet, so this
# code will run serially.
# Email sheriff about any new regressions.
for anomaly_entity in anomalies:
if (anomaly_entity.bug_id is None and not anomaly_entity.is_improvement
and not sheriff.summarize):
email_sheriff.EmailSheriff(sheriff, test, anomaly_entity)
yield ndb.put_multi_async(anomalies)
def _GetConfigDict(self, test):
"""Gets the name of the anomaly threshold dict to use."""
input_config_json = self.request.get('config')
if not input_config_json:
return anomaly_config.GetAnomalyConfigDict(test)
try:
return json.loads(input_config_json)
except ValueError:
raise QueryParameterError('Invalid JSON.')
def testGetAnomalyConfigDict_OverriddenConfigNotFound(
self, mock_logging_warning):
testing_common.AddTests(['M'], ['b'], {'foo': {'bar': {}}})
test = utils.TestKey('M/b/foo/bar').get()
test.overridden_anomaly_config = ndb.Key('AnomalyConfig', 'Non-existent')
self.assertEqual({}, anomaly_config.GetAnomalyConfigDict(test))
mock_logging_warning.assert_called_once_with(
'No AnomalyConfig fetched from key %s for test %s',
ndb.Key('AnomalyConfig', 'Non-existent'), 'M/b/foo/bar')
self.assertIsNone(test.key.get().overridden_anomaly_config)
def _FilterAnomaliesFoundInRef(change_points, test_key, num_rows):
"""Filters out the anomalies that match the anomalies in ref build.
Background about ref build tests: Variation in test results can be caused
by changes in Chrome or changes in the test-running environment. The ref
build results are results from a reference (stable) version of Chrome run
in the same environment. If an anomaly happens in the ref build results at
the same time as an anomaly happened in the test build, that suggests that
the variation was caused by a change in the test-running environment, and
can be ignored.
Args:
change_points: ChangePoint objects returned by FindChangePoints.
test_key: ndb.Key of monitored TestMetadata.
num_rows: Number of Rows that were analyzed from the test. When fetching
the ref build Rows, we need not fetch more than |num_rows| rows.
Returns:
A copy of |change_points| possibly with some entries filtered out.
Any entries in |change_points| whose end revision matches that of
an anomaly found in the corresponding ref test will be filtered out.
"""
# Get anomalies for ref build.
ref_test = _CorrespondingRefTest(test_key)
if not ref_test:
raise ndb.Return(change_points[:])
ref_config = anomaly_config.GetAnomalyConfigDict(ref_test)
ref_rows = yield GetRowsToAnalyzeAsync(ref_test, num_rows)
ref_change_points = FindChangePointsForTest(ref_rows, ref_config)
if not ref_change_points:
raise ndb.Return(change_points[:])
# We need to still alert on benchmark_duration, even if the ref moves since
# that can signal some blow-up in cycle time. If we decide to expand this
# to a greater set of metrics, we should move this to something more
# generic like stored_object.
test_path = utils.TestPath(test_key)
if test_path.split('/')[-1] == 'benchmark_duration':
raise ndb.Return(change_points[:])
change_points_filtered = []
for c in change_points:
# Log information about what anomaly got filtered and what did not.
if not _IsAnomalyInRef(c, ref_change_points):
# TODO(qyearsley): Add test coverage. See catapult:#1346.
logging.info(
'Nothing was filtered out for test %s, and revision %s',
test_path, c.x_value)
change_points_filtered.append(c)
else:
logging.info('Filtering out anomaly for test %s, and revision %s',
test_path, c.x_value)
raise ndb.Return(change_points_filtered)
def testProcessTest_RefineAnomalyPlacement_MinSize0Max2Elements(self):
testing_common.AddTests(['ChromiumPerf'], ['linux-perf'],
{'sizes': {
'method_count': {}
}})
test = utils.TestKey(
('ChromiumPerf/linux-perf/sizes/method_count')).get()
test_container_key = utils.GetTestContainerKey(test.key)
custom_config = {
'max_window_size': 10,
'min_absolute_change': 50,
'min_relative_change': 0,
'min_segment_size': 0,
}
anomaly_config.AnomalyConfig(config=custom_config,
patterns=[test.test_path]).put()
test.UpdateSheriff()
test.put()
self.assertEqual(custom_config,
anomaly_config.GetAnomalyConfigDict(test))
sample_data = [
(6990, 100),
(6991, 100),
(6992, 100),
(6993, 100),
(6994, 100),
(6995, 100),
(6996, 100),
(6997, 100),
(6998, 100),
(6999, 100),
(7000, 100),
(7001, 155),
(7002, 155),
(7003, 155),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
sheriff.Sheriff(email='*****@*****.**',
id='sheriff',
patterns=[test.test_path]).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.UP, new_anomalies[0].direction)
self.assertEqual(7001, new_anomalies[0].start_revision)
self.assertEqual(7001, new_anomalies[0].end_revision)
def _FilterAnomaliesFoundInRef(change_points, test_key, num_rows):
"""Filters out the anomalies that match the anomalies in ref build.
Background about ref build tests: Variation in test results can be caused
by changes in Chrome or changes in the test-running environment. The ref
build results are results from a reference (stable) version of Chrome run
in the same environment. If an anomaly happens in the ref build results at
the same time as an anomaly happened in the test build, that suggests that
the variation was caused by a change in the test-running environment, and
can be ignored.
Args:
change_points: ChangePoint objects returned by FindChangePoints.
test_key: ndb.Key of monitored Test.
num_rows: Number of Rows that were analyzed from the Test. When fetching
the ref build Rows, we need not fetch more than |num_rows| rows.
Returns:
A copy of |change_points| possibly with some entries filtered out.
Any entries in |change_points| whose end revision matches that of
an anomaly found in the corresponding ref test will be filtered out.
"""
# Get anomalies for ref build.
ref_test = _CorrespondingRefTest(test_key)
if not ref_test:
return change_points[:]
ref_config = anomaly_config.GetAnomalyConfigDict(ref_test)
ref_rows = GetRowsToAnalyze(ref_test, num_rows)
ref_change_points = FindChangePointsForTest(ref_rows, ref_config)
if not ref_change_points:
return change_points[:]
change_points_filtered = []
test_path = utils.TestPath(test_key)
for c in change_points:
# Log information about what anomaly got filtered and what did not.
if not _IsAnomalyInRef(c, ref_change_points):
# TODO(qyearsley): Add test coverage. See catapult:#1346.
logging.info(
'Nothing was filtered out for test %s, and revision %s',
test_path, c.x_value)
change_points_filtered.append(c)
else:
logging.info('Filtering out anomaly for test %s, and revision %s',
test_path, c.x_value)
return change_points_filtered
def FindMagnitudeBetweenCommits(test_key, start_commit, end_commit):
start_commit = _GitHashToCommitPosition(start_commit)
end_commit = _GitHashToCommitPosition(end_commit)
test = test_key.get()
num_points = anomaly_config.GetAnomalyConfigDict(test).get(
'min_segment_size', find_change_points.MIN_SEGMENT_SIZE)
start_rows = graph_data.GetRowsForTestBeforeAfterRev(
test_key, start_commit, num_points, 0)
end_rows = graph_data.GetRowsForTestBeforeAfterRev(test_key, end_commit, 0,
num_points)
if not start_rows or not end_rows:
return None
median_before = math_utils.Median([r.value for r in start_rows])
median_after = math_utils.Median([r.value for r in end_rows])
return median_after - median_before
def _RunBaseAlertProcessing(test_bench):
"""Runs base alert processing simulation on TestBench entity.
This function runs the current find_change_points.FindChangePoints
implementation and saves the revisions around the found anomalies to
a TestBench entity.
Args:
test_bench: A TestBench entity.
"""
test = test_bench.test.get()
config_dict = anomaly_config.GetAnomalyConfigDict(test)
change_points = debug_alert.SimulateAlertProcessing(
test_bench.data_series, **config_dict)
test_bench.base_anomaly_revs = [
_GetRevsAroundRev(test_bench.data_series, change_point.x_value)
for change_point in change_points if _IsRegression(change_point, test)
]
def RunFindChangePoints(
test, series, find_change_points_func=find_change_points.FindChangePoints,
**kwargs):
"""Runs an change-point-finding function on part of a data series.
This function will be repeatedly called by SimulateAlertProcessingPipeline
in the bench_find_change_points module with the same TestMetadata entity but
with more and more points added to the end.
This is meant to imitate the current behavior of FindChangePoints on the perf
dashboard.
Args:
test: A graph_data.TestMetadata entity.
series: A list of ordered (x, y) pairs.
find_change_points_func: A function that has the same interface as
find_change_points.FindChangePoints.
**kwargs: Extra parameters to add to the anomaly config dict.
Returns:
A list of objects with the property x_value.
"""
# The anomaly threshold config dictionary determines how many points are
# analyzed and how far apart alerts should be, as well as other thresholds.
config = anomaly_config.GetAnomalyConfigDict(test)
config.update(kwargs)
# |series| contains all data so far in the TestMetadata, but typically when
# a test is processed (in find_anomalies.ProcessTest) only the last "window"
# of points is looked at. This window size depends on the test. To get the
# same behavior as the current default, we take only the last window.
series = _GetLastWindow(series, config.get('max_window_size'))
if len(series) < 2:
return []
# Find anomalies for the requested test.
change_points = find_change_points_func(series, **config)
return _RemoveKnownAnomalies(test, change_points)
def _IsAnomalyRecovered(anomaly_entity):
"""Checks whether anomaly has recovered.
We have the measurements for the segment before the anomaly. If we take
the measurements for the latest segment after the anomaly, we can find if
the anomaly recovered.
Args:
anomaly_entity: The original regression anomaly.
Returns:
A tuple (is_anomaly_recovered, measurements), where is_anomaly_recovered
is True if anomaly has recovered, and measurements is dictionary
of name to value of measurements used to evaluate if anomaly recovered.
measurements is None if anomaly has not recovered.
"""
# 1. Check if the Anomaly entity has std_dev_before_anomaly and
# window_end_revision properties which we're using to decide whether or
# not it is recovered.
if (anomaly_entity.std_dev_before_anomaly is None or
anomaly_entity.window_end_revision is None):
return False, None
test = anomaly_entity.test.get()
config = anomaly_config.GetAnomalyConfigDict(test)
latest_rows = find_anomalies.GetRowsToAnalyze(
test, anomaly_entity.segment_size_after)
latest_values = [row.value for row in latest_rows
if row.revision > anomaly_entity.window_end_revision]
# 2. Segment size filter.
if len(latest_values) < anomaly_entity.segment_size_after:
return False, None
median_before = anomaly_entity.median_before_anomaly
median_after = math_utils.Median(latest_values)
std_dev_before = anomaly_entity.std_dev_before_anomaly
std_dev_after = math_utils.StandardDeviation(latest_values)
multiple_of_std_dev = config.get('multiple_of_std_dev',
_DEFAULT_MULTIPLE_OF_STD_DEV)
min_relative_change = config.get('min_relative_change',
_DEFAULT_MIN_RELATIVE_CHANGE)
min_absolute_change = config.get('min_absolute_change',
_DEFAULT_MIN_ABSOLUTE_CHANGE)
# If no improvement direction is provided, use absolute changes.
if test.improvement_direction == anomaly.UNKNOWN:
absolute_change = abs(median_after - median_before)
relative_change = abs(_RelativeChange(median_before, median_after))
else:
if test.improvement_direction == anomaly.UP:
direction = -1
else:
direction = 1
absolute_change = direction * (median_after - median_before)
relative_change = direction * _RelativeChange(median_before, median_after)
measurements = {
'segment_size_after': anomaly_entity.segment_size_after,
'window_end_revision': anomaly_entity.window_end_revision,
'median_before': median_before,
'median_after': median_after,
'std_dev_before': std_dev_before,
'std_dev_after': std_dev_after,
'multiple_of_std_dev': multiple_of_std_dev,
'min_relative_change': min_relative_change,
'min_absolute_change': min_absolute_change,
'absolute_change': absolute_change,
'relative_change': relative_change,
}
# 3. If it's an improvement, return.
if absolute_change <= 0:
return True, measurements
# 4. Absolute change filter.
if min_absolute_change > 0 and absolute_change >= min_absolute_change:
return False, None
# 5. Relative change filter.
if relative_change >= min_relative_change:
return False, None
# 6. Standard deviation filter.
min_std_dev = min(std_dev_before, std_dev_after)
if absolute_change > min_std_dev:
return False, None
return True, measurements
