def testZeroMedian_ResultProperties(self):
nums = [3.4, 8, 100.2, 78, 3, -4, 12, 3.14, 1024]
zeroed_nums = find_change_points._ZeroMedian(nums)
# The output of _ZeroMedian has the same standard deviation as the input.
self.assertEqual(math_utils.StandardDeviation(nums),
math_utils.StandardDeviation(zeroed_nums))
# Also, the median of the output is always zero.
self.assertEqual(0, math_utils.Median(zeroed_nums))
def _PassesThresholds(
values, split_index, min_segment_size, min_absolute_change,
min_relative_change, min_steppiness, multiple_of_std_dev):
"""Checks whether a point in a series appears to be an change point.
Args:
values: A list of numbers.
split_index: An index in the list of numbers.
min_segment_size: Threshold for size of segments before or after a point.
min_absolute_change: Minimum absolute median change threshold.
min_relative_change: Minimum relative median change threshold.
min_steppiness: Threshold for how similar to a step a change point must be.
multiple_of_std_dev: Threshold for change as multiple of std. deviation.
Returns:
True if it passes all of the thresholds, False otherwise.
"""
left, right = values[:split_index], values[split_index:]
left_median, right_median = math_utils.Median(left), math_utils.Median(right)
# 1. Segment size filter.
if len(left) < min_segment_size or len(right) < min_segment_size:
return False
# 2. Absolute change filter.
absolute_change = abs(left_median - right_median)
if absolute_change < min_absolute_change:
return False
# 3. Relative change filter.
relative_change = _RelativeChange(left_median, right_median)
if relative_change < min_relative_change:
return False
# 4. Multiple of standard deviation filter.
# The left and right sides are independently normalized so that their medians
# are zero, then the two sides are concatenated together. This, in effect,
# removes the step so that the step itself doesn't affect the std. dev.
min_std_dev = min(
math_utils.StandardDeviation(left),
math_utils.StandardDeviation(right))
if absolute_change < multiple_of_std_dev * min_std_dev:
return False
# 5. Steppiness filter.
steppiness = find_step.Steppiness(values, split_index)
if steppiness < min_steppiness:
return False
# Passed all filters!
return True
def testNormalize_ResultMeanIsZeroAndStdDevIsOne(self): # When a data series is normalized, it is guaranteed that the result # should have a mean of 0.0 and a standard deviation and variance of 1.0. _, y_values = zip(*_QUITE_STEPPISH) normalized = find_step._Normalize(y_values) self.assertAlmostEqual(0.0, math_utils.Mean(normalized)) self.assertAlmostEqual(1.0, math_utils.StandardDeviation(normalized))
def MakeChangePoint(series, split_index):
"""Makes a ChangePoint object for the given series at the given point.
Args:
series: A list of (x, y) pairs.
split_index: Index of the first point after the split.
Returns:
A ChangePoint object.
"""
assert 0 <= split_index < len(series)
x_values, y_values = zip(*series)
left, right = y_values[:split_index], y_values[split_index:]
left_median, right_median = math_utils.Median(left), math_utils.Median(
right)
ttest_results = ttest.WelchsTTest(left, right)
return ChangePoint(
x_value=x_values[split_index],
median_before=left_median,
median_after=right_median,
size_before=len(left),
size_after=len(right),
window_start=x_values[0],
window_end=x_values[-1], # inclusive bound
relative_change=math_utils.RelativeChange(left_median, right_median),
std_dev_before=math_utils.StandardDeviation(left),
t_statistic=ttest_results.t,
degrees_of_freedom=ttest_results.df,
p_value=ttest_results.p)
def post(self):
"""Task queue task to get stats before/after a revision of a single Test.
Request parameters:
revision: A central revision to look around.
num_around: The number of points before and after the given revision.
test_key: The urlsafe string of a Test key.
parent_key: The urlsafe string of a StatContainer key.
"""
datastore_hooks.SetPrivilegedRequest()
revision = int(self.request.get('revision'))
num_around = int(self.request.get('num_around'), 10)
test_key = ndb.Key(urlsafe=self.request.get('test_key'))
container_key = ndb.Key(urlsafe=self.request.get('parent_key'))
# Get the Rows and values before and starting from the given revision.
before_revs = graph_data.Row.query(
graph_data.Row.parent_test == test_key,
graph_data.Row.revision < revision).order(
-graph_data.Row.revision).fetch(limit=num_around)
before_vals = [b.value for b in before_revs]
after_revs = graph_data.Row.query(
graph_data.Row.parent_test == test_key,
graph_data.Row.revision >= revision).order(
graph_data.Row.revision).fetch(limit=num_around)
after_vals = [a.value for a in after_revs]
# There may be no Row at the particular revision requested; if so, we use
# the first revision after the given revision.
actual_revision = None
if after_vals:
actual_revision = after_revs[0].revision
test = test_key.get()
improvement_direction = self._ImprovementDirection(test)
median_before = math_utils.Median(before_vals)
median_after = math_utils.Median(after_vals)
mean_before = math_utils.Median(before_vals)
mean_after = math_utils.Median(after_vals)
details = {
'test_path': utils.TestPath(test_key),
'improvement_direction': improvement_direction,
'actual_revision': actual_revision,
'median_before': '%.2f' % median_before,
'median_after': '%.2f' % median_after,
'median_percent_improved': self._PercentImproved(
median_before, median_after, improvement_direction),
'mean_before': '%.2f' % mean_before,
'mean_after': '%.2f' % mean_after,
'mean_percent_improved': self._PercentImproved(
mean_before, mean_after, improvement_direction),
'std': '%.2f' % math_utils.StandardDeviation(before_vals + after_vals),
}
new_stat = IndividualStat(parent=container_key, details=details)
new_stat.put()
def _ExtractValueAndError(trace):
"""Returns the value and measure of error from a chartjson trace dict.
Args:
trace: A dict that has one "result" from a performance test, e.g. one
"value" in a Telemetry test, with the keys "trace_type", "value", etc.
Returns:
A pair (value, error) where |value| is a float and |error| is some measure
of variance used to show error bars; |error| could be None.
Raises:
BadRequestError: Data format was invalid.
"""
trace_type = trace.get('type')
if trace_type == 'scalar':
value = trace.get('value')
if value is None and trace.get('none_value_reason'):
return float('nan'), 0
try:
return float(value), 0
except:
raise BadRequestError('Expected scalar value, got: %r' % value)
if trace_type == 'list_of_scalar_values':
values = trace.get('values')
if not isinstance(values, list) and values is not None:
# Something else (such as a single scalar, or string) was given.
raise BadRequestError('Expected list of scalar values, got: %r' %
values)
if not values or None in values:
# None was included or values is None; this is not an error if there
# is a reason.
if trace.get('none_value_reason'):
return float('nan'), float('nan')
raise BadRequestError('Expected list of scalar values, got: %r' %
values)
if not all(_IsNumber(v) for v in values):
raise BadRequestError('Non-number found in values list: %r' %
values)
value = math_utils.Mean(values)
std = trace.get('std')
if std is not None:
error = std
else:
error = math_utils.StandardDeviation(values)
return value, error
if trace_type == 'histogram':
return _GeomMeanAndStdDevFromHistogram(trace)
raise BadRequestError('Invalid value type in chart object: %r' %
trace_type)
def _FlattenTrace(test_suite_name, chart_name, trace_name, trace,
is_ref=False, tracing_links=None, benchmark_description=''):
"""Takes a trace dict from dashboard JSON and readies it for display.
Traces can be either scalars or lists; if scalar we take the value directly;
if list we average the values and compute their standard deviation. We also
extract fields that are normally part of v0 row dicts that are uploaded
using add_point but are actually part of traces in the v1.0 format.
Args:
test_suite_name: The name of the test suite (benchmark).
chart_name: The name of the chart to which this trace belongs.
trace_name: The name of the passed trace.
trace: A trace dict extracted from a dashboard JSON chart.
is_ref: A boolean which indicates whether this trace comes from a
reference build.
tracing_links: A dictionary mapping trace names to about:tracing trace
urls in cloud storage
benchmark_description: A string documenting the benchmark suite to which
this trace belongs.
Returns:
A dict containing units, value, and error for this trace.
Raises:
BadRequestError: The data wasn't valid.
"""
if '@@' in chart_name:
tir_label, chart_name = chart_name.split('@@')
chart_name = chart_name + '/' + tir_label
trace_type = trace.get('type')
if trace_type == 'scalar':
value = trace.get('value')
if value is None:
if trace.get('none_value_reason'):
value = float('nan')
else:
# TODO(qyearsley): Add test coverage. See http://crbug.com/447432
raise BadRequestError('Expected scalar value, got: ' + value)
error = 0
elif trace_type == 'list_of_scalar_values':
values = trace.get('values')
if not values or None in values:
if trace.get('none_value_reason'):
value = float('nan')
error = float('nan')
else:
raise BadRequestError('Expected list of scalar values, got: ' + values)
else:
value = math_utils.Mean(values)
std = trace.get('std')
if std is not None:
error = std
else:
error = math_utils.StandardDeviation(values)
elif trace_type == 'histogram':
value, error = _GeomMeanAndStdDevFromHistogram(trace)
elif trace_type is not None:
raise BadRequestError('Invalid value type in chart object: ' + trace_type)
else:
raise BadRequestError('No trace type provided.')
# If there is a link to an about:tracing trace in cloud storage for this
# test trace_name, cache it.
tracing_uri = None
if (tracing_links and
trace_name in tracing_links and
'cloud_url' in tracing_links[trace_name]):
tracing_uri = tracing_links[trace_name]['cloud_url'].replace('\\/', '/')
trace_name = _EscapeName(trace_name)
if trace_name == 'summary':
subtest_name = chart_name
else:
subtest_name = chart_name + '/' + trace_name
name = test_suite_name + '/' + subtest_name
if trace_name == 'summary' and is_ref:
name += '/ref'
elif trace_name != 'summary' and is_ref:
name += '_ref'
row_dict = {
'test': name,
'value': value,
'error': error,
'units': trace['units'],
'tracing_uri': tracing_uri,
'benchmark_description': benchmark_description,
}
if 'improvement_direction' in trace:
improvement_direction_str = trace['improvement_direction']
if improvement_direction_str is None:
raise BadRequestError('improvement_direction must not be None')
row_dict['higher_is_better'] = _ImprovementDirectionToHigherIsBetter(
improvement_direction_str)
return row_dict
def testStandardDeviation_UsesPopulationStandardDeviation(self): self.assertAlmostEqual(2.5, math.sqrt(6.25)) self.assertAlmostEqual(2.5, math_utils.StandardDeviation([-3, 0, 1, 4]))
def testStandardDeviation_OneValue_ReturnsZero(self): self.assertEqual(0.0, math_utils.StandardDeviation([4.3]))
def testStandardDeviation_EmptyInput_ReturnsNan(self): self.assertTrue(math.isnan(math_utils.StandardDeviation([])))
def StdDevOfTwoNormalizedSides(index):
left, right = values[:index], values[index:]
return math_utils.StandardDeviation(
_ZeroMedian(left) + _ZeroMedian(right))
def _IsAnomalyRecovered(anomaly_entity):
"""Checks whether anomaly has recovered.
We have the measurements for the segment before the anomaly. If we take
the measurements for the latest segment after the anomaly, we can find if
the anomaly recovered.
Args:
anomaly_entity: The original regression anomaly.
Returns:
A tuple (is_anomaly_recovered, measurements), where is_anomaly_recovered
is True if anomaly has recovered, and measurements is dictionary
of name to value of measurements used to evaluate if anomaly recovered.
measurements is None if anomaly has not recovered.
"""
# 1. Check if the Anomaly entity has std_dev_before_anomaly and
# window_end_revision properties which we're using to decide whether or
# not it is recovered.
if (anomaly_entity.std_dev_before_anomaly is None or
anomaly_entity.window_end_revision is None):
return False, None
test = anomaly_entity.test.get()
config = anomaly_config.GetAnomalyConfigDict(test)
latest_rows = find_anomalies.GetRowsToAnalyze(
test, anomaly_entity.segment_size_after)
latest_values = [row.value for row in latest_rows
if row.revision > anomaly_entity.window_end_revision]
# 2. Segment size filter.
if len(latest_values) < anomaly_entity.segment_size_after:
return False, None
median_before = anomaly_entity.median_before_anomaly
median_after = math_utils.Median(latest_values)
std_dev_before = anomaly_entity.std_dev_before_anomaly
std_dev_after = math_utils.StandardDeviation(latest_values)
multiple_of_std_dev = config.get('multiple_of_std_dev',
_DEFAULT_MULTIPLE_OF_STD_DEV)
min_relative_change = config.get('min_relative_change',
_DEFAULT_MIN_RELATIVE_CHANGE)
min_absolute_change = config.get('min_absolute_change',
_DEFAULT_MIN_ABSOLUTE_CHANGE)
# If no improvement direction is provided, use absolute changes.
if test.improvement_direction == anomaly.UNKNOWN:
absolute_change = abs(median_after - median_before)
relative_change = abs(_RelativeChange(median_before, median_after))
else:
if test.improvement_direction == anomaly.UP:
direction = -1
else:
direction = 1
absolute_change = direction * (median_after - median_before)
relative_change = direction * _RelativeChange(median_before, median_after)
measurements = {
'segment_size_after': anomaly_entity.segment_size_after,
'window_end_revision': anomaly_entity.window_end_revision,
'median_before': median_before,
'median_after': median_after,
'std_dev_before': std_dev_before,
'std_dev_after': std_dev_after,
'multiple_of_std_dev': multiple_of_std_dev,
'min_relative_change': min_relative_change,
'min_absolute_change': min_absolute_change,
'absolute_change': absolute_change,
'relative_change': relative_change,
}
# 3. If it's an improvement, return.
if absolute_change <= 0:
return True, measurements
# 4. Absolute change filter.
if min_absolute_change > 0 and absolute_change >= min_absolute_change:
return False, None
# 5. Relative change filter.
if relative_change >= min_relative_change:
return False, None
# 6. Standard deviation filter.
min_std_dev = min(std_dev_before, std_dev_after)
if absolute_change > min_std_dev:
return False, None
return True, measurements
