def testZeroMedian_ResultProperties(self):
nums = [3.4, 8, 100.2, 78, 3, -4, 12, 3.14, 1024]
zeroed_nums = find_change_points._ZeroMedian(nums)
# The output of _ZeroMedian has the same standard deviation as the input.
self.assertEqual(math_utils.StandardDeviation(nums),
math_utils.StandardDeviation(zeroed_nums))
# Also, the median of the output is always zero.
self.assertEqual(0, math_utils.Median(zeroed_nums))
def _PassesThresholds(values, split_index, min_segment_size,
min_absolute_change, min_relative_change, min_steppiness,
multiple_of_std_dev):
"""Checks whether a point in a series appears to be an change point.
Args:
values: A list of numbers.
split_index: An index in the list of numbers.
min_segment_size: Threshold for size of segments before or after a point.
min_absolute_change: Minimum absolute median change threshold.
min_relative_change: Minimum relative median change threshold.
min_steppiness: Threshold for how similar to a step a change point must be.
multiple_of_std_dev: Threshold for change as multiple of std. deviation.
Returns:
A tuple of (bool, string) where the bool indicates whether the split index
passes the thresholds and the string being the reason it did not.
"""
left, right = values[:split_index], values[split_index:]
left_median, right_median = math_utils.Median(left), math_utils.Median(
right)
# 1. Segment size filter.
if len(left) < min_segment_size or len(right) < min_segment_size:
return (False, 'min_segment_size')
# 2. Absolute change filter.
absolute_change = abs(left_median - right_median)
if absolute_change < min_absolute_change:
return (False, 'min_absolute_change')
# 3. Relative change filter.
relative_change = math_utils.RelativeChange(left_median, right_median)
if relative_change < min_relative_change:
return (False, 'min_relative_change')
# 4. Multiple of standard deviation filter.
min_std_dev = min(math_utils.StandardDeviation(left),
math_utils.StandardDeviation(right))
if absolute_change < multiple_of_std_dev * min_std_dev:
return (False, 'min_std_dev')
# 5. Steppiness filter.
steppiness = find_step.Steppiness(values, split_index)
if steppiness < min_steppiness:
return (False, 'min_steppiness')
# Passed all filters!
return (True, 'passed')
def MakeChangePoint(series, split_index):
"""Makes a ChangePoint object for the given series at the given point.
Args:
series: A list of (x, y) pairs.
split_index: Index of the first point after the split.
Returns:
A ChangePoint object.
"""
assert 0 <= split_index < len(series)
x_values, y_values = zip(*series)
left, right = y_values[:split_index], y_values[split_index:]
left_median, right_median = math_utils.Median(left), math_utils.Median(
right)
ttest_results = ttest.WelchsTTest(left, right)
return ChangePoint(
x_value=x_values[split_index],
median_before=left_median,
median_after=right_median,
size_before=len(left),
size_after=len(right),
window_start=x_values[0],
window_end=x_values[-1], # inclusive bound
relative_change=math_utils.RelativeChange(left_median, right_median),
std_dev_before=math_utils.StandardDeviation(left),
t_statistic=ttest_results.t,
degrees_of_freedom=ttest_results.df,
p_value=ttest_results.p)
def testNormalize_ResultMeanIsZeroAndStdDevIsOne(self):
# When a data series is normalized, it is guaranteed that the result
# should have a mean of 0.0 and a standard deviation and variance of 1.0.
_, y_values = zip(*_QUITE_STEPPISH)
normalized = find_step._Normalize(y_values)
self.assertAlmostEqual(0.0, math_utils.Mean(normalized))
self.assertAlmostEqual(1.0, math_utils.StandardDeviation(normalized))
def testGetSimilarHistoricalTimings_Same(self):
now = datetime.datetime.now()
self._RecordTiming(
{
'configuration': 'linux',
'benchmark': 'foo',
'story': 'bar1'
}, now - datetime.timedelta(minutes=1), now)
median = math_utils.Median([i for i in range(0, 10)])
std_dev = math_utils.StandardDeviation([i for i in range(0, 10)])
p90 = math_utils.Percentile([i for i in range(0, 10)], 0.9)
for i in range(0, 10):
j = self._RecordTiming(
{
'configuration': 'linux',
'benchmark': 'foo',
'story': 'bar2'
}, now - datetime.timedelta(seconds=i), now)
timings, tags = timing_record.GetSimilarHistoricalTimings(j)
self.assertEqual(['try', 'linux', 'foo', 'bar2'], tags)
self.assertClose(median, timings[0].total_seconds())
self.assertClose(std_dev, timings[1].total_seconds())
self.assertClose(p90, timings[2].total_seconds())
def _ExtractValueAndError(trace):
"""Returns the value and measure of error from a chartjson trace dict.
Args:
trace: A dict that has one "result" from a performance test, e.g. one
"value" in a Telemetry test, with the keys "trace_type", "value", etc.
Returns:
A pair (value, error) where |value| is a float and |error| is some measure
of variance used to show error bars; |error| could be None.
Raises:
BadRequestError: Data format was invalid.
"""
trace_type = trace.get('type')
if trace_type == 'scalar':
value = trace.get('value')
if value is None and trace.get('none_value_reason'):
return float('nan'), 0
try:
return float(value), 0
except:
raise BadRequestError('Expected scalar value, got: %r' % value)
if trace_type == 'list_of_scalar_values':
values = trace.get('values')
if not isinstance(values, list) and values is not None:
# Something else (such as a single scalar, or string) was given.
raise BadRequestError('Expected list of scalar values, got: %r' %
values)
if not values or None in values:
# None was included or values is None; this is not an error if there
# is a reason.
if trace.get('none_value_reason'):
return float('nan'), float('nan')
raise BadRequestError('Expected list of scalar values, got: %r' %
values)
if not all(_IsNumber(v) for v in values):
raise BadRequestError('Non-number found in values list: %r' %
values)
value = math_utils.Mean(values)
std = trace.get('std')
if std is not None:
error = std
else:
error = math_utils.StandardDeviation(values)
return value, error
if trace_type == 'histogram':
return _GeomMeanAndStdDevFromHistogram(trace)
raise BadRequestError('Invalid value type in chart object: %r' %
trace_type)
def _Estimate(tags, completed_before=None):
records = _QueryTimingRecords(tags, completed_before)
if not records:
if tags:
return _Estimate(tags[:-1])
return None
times = [(r.completed - r.started).total_seconds() for r in records]
median = math_utils.Median(times)
std_dev = math_utils.StandardDeviation(times)
p90 = math_utils.Percentile(times, 0.9)
timings = Timings(datetime.timedelta(seconds=median),
datetime.timedelta(seconds=std_dev),
datetime.timedelta(seconds=p90))
return EstimateResult(timings, tags)
def StdDevOfTwoNormalizedSides(index):
left, right = values[:index], values[index:]
return math_utils.StandardDeviation(
_ZeroMedian(left) + _ZeroMedian(right))
def testStandardDeviation_UsesPopulationStandardDeviation(self):
self.assertAlmostEqual(2.5, math.sqrt(6.25))
self.assertAlmostEqual(2.5, math_utils.StandardDeviation([-3, 0, 1,
4]))
def testStandardDeviation_OneValue_ReturnsZero(self):
self.assertEqual(0.0, math_utils.StandardDeviation([4.3]))
def testStandardDeviation_EmptyInput_ReturnsNan(self):
self.assertTrue(math.isnan(math_utils.StandardDeviation([])))
def _Normalize(values):
"""Makes a series with the same shape but with variance = 1, mean = 0."""
mean = math_utils.Mean(values)
zeroed = [x - mean for x in values]
stddev = math_utils.StandardDeviation(zeroed)
return [math_utils.Divide(x, stddev) for x in zeroed]