def test_compute_bounds_stddevs_from_mean(self,
comparison,
std_devs,
expected_bounds,
inclusive=False):
# mean = 3, stddev = ~1.414
value_history = range(1, 6)
class _FakeMetricStore:
def get_metric_history(*args, **kwargs):
return [
bigquery_client.MetricHistoryRow('', '',
datetime.datetime.now(),
'', v)
for v in value_history
]
assertion = metrics_pb2.Assertion(
std_devs_from_mean=metrics_pb2.Assertion.StdDevsFromMean(
comparison=metrics_pb2.Assertion.Comparison.Value(comparison),
std_devs=std_devs,
),
inclusive_bounds=inclusive,
)
collector = base.BaseCollector(
metrics_pb2.TestCompletedEvent(benchmark_id="test_benchmark"),
None, _FakeMetricStore())
bounds = collector.compute_bounds("metric_key", assertion)
self.assertSequenceAlmostEqual(
dataclasses.astuple(bounds),
# EQUAL is always inclusive
dataclasses.astuple(
utils.Bounds(*expected_bounds, inclusive
or comparison == 'EQUAL')),
places=3)
def test_compute_bounds_percent_difference_with_mean_value(
self, comparison, pct_diff, expected_bounds, inclusive=False):
# mean = 3
value_history = range(1, 6)
class _FakeMetricStore:
def get_metric_history(*args, **kwargs):
return [
bigquery_client.MetricHistoryRow('', '',
datetime.datetime.now(),
'', v)
for v in value_history
]
assertion = metrics_pb2.Assertion(
percent_difference=metrics_pb2.Assertion.PercentDifference(
comparison=metrics_pb2.Assertion.Comparison.Value(comparison),
use_historical_mean=True,
percent=pct_diff,
),
inclusive_bounds=inclusive,
)
collector = base.BaseCollector(
metrics_pb2.TestCompletedEvent(benchmark_id="test_benchmark"),
None, _FakeMetricStore())
bounds = collector.compute_bounds("metric_key", assertion)
self.assertSequenceAlmostEqual(
dataclasses.astuple(bounds),
dataclasses.astuple(utils.Bounds(*expected_bounds, inclusive)),
places=3)
def test_compute_bounds_within_bounds(self,
lower,
upper,
expected_bounds,
inclusive=False):
assertion = metrics_pb2.Assertion(
within_bounds=metrics_pb2.Assertion.WithinBounds(
lower_bound=lower,
upper_bound=upper,
),
inclusive_bounds=inclusive,
)
collector = base.BaseCollector(
metrics_pb2.TestCompletedEvent(benchmark_id="test_benchmark"),
None)
bounds = collector.compute_bounds("metric_key", assertion)
self.assertEqual(bounds, utils.Bounds(*expected_bounds, inclusive))
def test_compute_bounds_percent_difference_with_target_value(
self,
comparison,
target,
pct_diff,
expected_bounds,
inclusive=False):
assertion = metrics_pb2.Assertion(
percent_difference=metrics_pb2.Assertion.PercentDifference(
comparison=metrics_pb2.Assertion.Comparison.Value(comparison),
value=target,
percent=pct_diff,
),
inclusive_bounds=inclusive,
)
collector = base.BaseCollector(
metrics_pb2.TestCompletedEvent(benchmark_id="test_benchmark"),
None)
bounds = collector.compute_bounds("metric_key", assertion)
self.assertEqual(bounds, utils.Bounds(*expected_bounds, inclusive))
def test_min_time(self, window, timestamp, expected_min):
start_time = timestamp_pb2.Timestamp()
start_time.FromDatetime(datetime.datetime(2021, 2, 16, 0, 0, 0))
min_timestamp = timestamp_pb2.Timestamp()
if timestamp:
min_timestamp.FromDatetime(timestamp)
time_window = duration_pb2.Duration()
if window:
time_window.FromTimedelta(window)
assertion = metrics_pb2.Assertion(
std_devs_from_mean=metrics_pb2.Assertion.StdDevsFromMean(
comparison=metrics_pb2.Assertion.Comparison.WITHIN,
std_devs=1,
),
time_window=time_window,
min_timestamp=min_timestamp,
)
metric_store = bigquery_client.BigQueryMetricStore(
'fake_dataset', 'fake_project')
metric_store.get_metric_history = mock.Mock(return_value=[])
collector = base.BaseCollector(
metrics_pb2.TestCompletedEvent(
benchmark_id="test_benchmark",
start_time=start_time,
), None, metric_store)
collector.compute_bounds("metric_key", assertion)
metric_store.get_metric_history.assert_called_with(
benchmark_id="test_benchmark",
metric_key="metric_key",
min_time=expected_min)
def test_compute_bounds_fixed_value(self,
comparison,
threshold_value,
expected_bounds,
inclusive=False):
assertion = metrics_pb2.Assertion(
fixed_value=metrics_pb2.Assertion.FixedValue(
comparison=metrics_pb2.Assertion.Comparison.Value(comparison),
value=threshold_value,
),
inclusive_bounds=inclusive,
)
collector = base.BaseCollector(
metrics_pb2.TestCompletedEvent(benchmark_id="test_benchmark"),
None)
bounds = collector.compute_bounds("metric_key", assertion)
self.assertSequenceAlmostEqual(
dataclasses.astuple(bounds),
# EQUAL is always inclusive
dataclasses.astuple(
utils.Bounds(*expected_bounds, inclusive
or comparison == 'EQUAL')),
places=3)