def testZeroMedian_ResultProperties(self): nums = [3.4, 8, 100.2, 78, 3, -4, 12, 3.14, 1024] zeroed_nums = find_change_points._ZeroMedian(nums) # The output of _ZeroMedian has the same standard deviation as the input. self.assertEqual(math_utils.StandardDeviation(nums), math_utils.StandardDeviation(zeroed_nums)) # Also, the median of the output is always zero. self.assertEqual(0, math_utils.Median(zeroed_nums))
def _PassesThresholds(values, split_index, min_segment_size, min_absolute_change, min_relative_change, min_steppiness, multiple_of_std_dev): """Checks whether a point in a series appears to be an change point. Args: values: A list of numbers. split_index: An index in the list of numbers. min_segment_size: Threshold for size of segments before or after a point. min_absolute_change: Minimum absolute median change threshold. min_relative_change: Minimum relative median change threshold. min_steppiness: Threshold for how similar to a step a change point must be. multiple_of_std_dev: Threshold for change as multiple of std. deviation. Returns: A tuple of (bool, string) where the bool indicates whether the split index passes the thresholds and the string being the reason it did not. """ left, right = values[:split_index], values[split_index:] left_median, right_median = math_utils.Median(left), math_utils.Median( right) # 1. Segment size filter. if len(left) < min_segment_size or len(right) < min_segment_size: return (False, 'min_segment_size') # 2. Absolute change filter. absolute_change = abs(left_median - right_median) if absolute_change < min_absolute_change: return (False, 'min_absolute_change') # 3. Relative change filter. relative_change = math_utils.RelativeChange(left_median, right_median) if relative_change < min_relative_change: return (False, 'min_relative_change') # 4. Multiple of standard deviation filter. min_std_dev = min(math_utils.StandardDeviation(left), math_utils.StandardDeviation(right)) if absolute_change < multiple_of_std_dev * min_std_dev: return (False, 'min_std_dev') # 5. Steppiness filter. steppiness = find_step.Steppiness(values, split_index) if steppiness < min_steppiness: return (False, 'min_steppiness') # Passed all filters! return (True, 'passed')
def MakeChangePoint(series, split_index): """Makes a ChangePoint object for the given series at the given point. Args: series: A list of (x, y) pairs. split_index: Index of the first point after the split. Returns: A ChangePoint object. """ assert 0 <= split_index < len(series) x_values, y_values = zip(*series) left, right = y_values[:split_index], y_values[split_index:] left_median, right_median = math_utils.Median(left), math_utils.Median( right) ttest_results = ttest.WelchsTTest(left, right) return ChangePoint( x_value=x_values[split_index], median_before=left_median, median_after=right_median, size_before=len(left), size_after=len(right), window_start=x_values[0], window_end=x_values[-1], # inclusive bound relative_change=math_utils.RelativeChange(left_median, right_median), std_dev_before=math_utils.StandardDeviation(left), t_statistic=ttest_results.t, degrees_of_freedom=ttest_results.df, p_value=ttest_results.p)
def testNormalize_ResultMeanIsZeroAndStdDevIsOne(self): # When a data series is normalized, it is guaranteed that the result # should have a mean of 0.0 and a standard deviation and variance of 1.0. _, y_values = zip(*_QUITE_STEPPISH) normalized = find_step._Normalize(y_values) self.assertAlmostEqual(0.0, math_utils.Mean(normalized)) self.assertAlmostEqual(1.0, math_utils.StandardDeviation(normalized))
def testGetSimilarHistoricalTimings_Same(self): now = datetime.datetime.now() self._RecordTiming( { 'configuration': 'linux', 'benchmark': 'foo', 'story': 'bar1' }, now - datetime.timedelta(minutes=1), now) median = math_utils.Median([i for i in range(0, 10)]) std_dev = math_utils.StandardDeviation([i for i in range(0, 10)]) p90 = math_utils.Percentile([i for i in range(0, 10)], 0.9) for i in range(0, 10): j = self._RecordTiming( { 'configuration': 'linux', 'benchmark': 'foo', 'story': 'bar2' }, now - datetime.timedelta(seconds=i), now) timings, tags = timing_record.GetSimilarHistoricalTimings(j) self.assertEqual(['try', 'linux', 'foo', 'bar2'], tags) self.assertClose(median, timings[0].total_seconds()) self.assertClose(std_dev, timings[1].total_seconds()) self.assertClose(p90, timings[2].total_seconds())
def _ExtractValueAndError(trace): """Returns the value and measure of error from a chartjson trace dict. Args: trace: A dict that has one "result" from a performance test, e.g. one "value" in a Telemetry test, with the keys "trace_type", "value", etc. Returns: A pair (value, error) where |value| is a float and |error| is some measure of variance used to show error bars; |error| could be None. Raises: BadRequestError: Data format was invalid. """ trace_type = trace.get('type') if trace_type == 'scalar': value = trace.get('value') if value is None and trace.get('none_value_reason'): return float('nan'), 0 try: return float(value), 0 except: raise BadRequestError('Expected scalar value, got: %r' % value) if trace_type == 'list_of_scalar_values': values = trace.get('values') if not isinstance(values, list) and values is not None: # Something else (such as a single scalar, or string) was given. raise BadRequestError('Expected list of scalar values, got: %r' % values) if not values or None in values: # None was included or values is None; this is not an error if there # is a reason. if trace.get('none_value_reason'): return float('nan'), float('nan') raise BadRequestError('Expected list of scalar values, got: %r' % values) if not all(_IsNumber(v) for v in values): raise BadRequestError('Non-number found in values list: %r' % values) value = math_utils.Mean(values) std = trace.get('std') if std is not None: error = std else: error = math_utils.StandardDeviation(values) return value, error if trace_type == 'histogram': return _GeomMeanAndStdDevFromHistogram(trace) raise BadRequestError('Invalid value type in chart object: %r' % trace_type)
def _Estimate(tags, completed_before=None): records = _QueryTimingRecords(tags, completed_before) if not records: if tags: return _Estimate(tags[:-1]) return None times = [(r.completed - r.started).total_seconds() for r in records] median = math_utils.Median(times) std_dev = math_utils.StandardDeviation(times) p90 = math_utils.Percentile(times, 0.9) timings = Timings(datetime.timedelta(seconds=median), datetime.timedelta(seconds=std_dev), datetime.timedelta(seconds=p90)) return EstimateResult(timings, tags)
def StdDevOfTwoNormalizedSides(index): left, right = values[:index], values[index:] return math_utils.StandardDeviation( _ZeroMedian(left) + _ZeroMedian(right))
def testStandardDeviation_UsesPopulationStandardDeviation(self): self.assertAlmostEqual(2.5, math.sqrt(6.25)) self.assertAlmostEqual(2.5, math_utils.StandardDeviation([-3, 0, 1, 4]))
def testStandardDeviation_OneValue_ReturnsZero(self): self.assertEqual(0.0, math_utils.StandardDeviation([4.3]))
def testStandardDeviation_EmptyInput_ReturnsNan(self): self.assertTrue(math.isnan(math_utils.StandardDeviation([])))
def _Normalize(values): """Makes a series with the same shape but with variance = 1, mean = 0.""" mean = math_utils.Mean(values) zeroed = [x - mean for x in values] stddev = math_utils.StandardDeviation(zeroed) return [math_utils.Divide(x, stddev) for x in zeroed]