def check_precision(mc: MockMetricsCollector, func: Callable, minimum_precision: float, maximum_overhead: float, looper=None): if asyncio.iscoroutinefunction(func): async def bench(): for _ in range(TIMING_ITER_COUNT): await func() start = time.perf_counter() looper.loop.run_until_complete(bench()) else: start = time.perf_counter() for _ in range(TIMING_ITER_COUNT): func() overhead = (time.perf_counter() - start) / TIMING_ITER_COUNT - TIMING_FUNC_DURATION mc.flush_accumulated() precision = abs(mc.events[0].avg - TIMING_FUNC_DURATION) assert len(mc.events) == 1 assert mc.events[0].name == TIMING_METRIC_NAME assert mc.events[0].count == TIMING_ITER_COUNT assert precision < minimum_precision, \ "Expected precision {}, actual {} ms".format(1000 * minimum_precision, 1000 * precision) assert 0 < overhead < maximum_overhead, \ "Expected overhead {}, actual {} ms".format(1000 * maximum_overhead, 1000 * overhead)
def test_metrics_collector_resets_accumulated_after_flush(): mc = MockMetricsCollector() mc.acc_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0) mc.flush_accumulated() mc.acc_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 2.0) mc.flush_accumulated() assert len(mc.events) == 2 assert mc.events[0] == (MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0) assert mc.events[1] == (MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 2.0)
def test_metrics_collector_measures_time(): mc = MockMetricsCollector() with mc.measure_time(MetricsName.LOOPER_RUN_TIME_SPENT): time.sleep(0.1) assert len(mc.events) == 0 mc.flush_accumulated() assert len(mc.events) == 1 assert mc.events[0][0] == MetricsName.LOOPER_RUN_TIME_SPENT assert abs(mc.events[0][1] - 0.1) < 0.001 # we want at least 1 ms precision
def test_async_measure_time_decorator(looper): mc = MockMetricsCollector() class Example: def __init__(self, metrics): self.metrics = metrics self.data = 2 @async_measure_time(TIMING_METRIC_NAME) async def slow_add(self, a, b): await asyncio.sleep(TIMING_FUNC_DURATION) return self.data + a + b e = Example(mc) async def f(): await e.slow_add(1, 3) # We want at least 5 ms precision and no more than 5 ms overhead check_precision(mc, f, minimum_precision=0.005, maximum_overhead=0.005, looper=looper) # Check that decorated async function works correctly e = Example(mc) r = looper.loop.run_until_complete(e.slow_add(1, 3)) assert r == 6
def test_metrics_collector_separates_different_events(): mc = MockMetricsCollector() mc.acc_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0) mc.acc_event(MetricsName.BACKUP_ORDERED_BATCH_SIZE, 2.0) mc.flush_accumulated() assert len(mc.events) == 2 assert (MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0) in mc.events assert (MetricsName.BACKUP_ORDERED_BATCH_SIZE, 2.0) in mc.events
def test_metrics_collector_measures_time(): mc = MockMetricsCollector() def f(): with mc.measure_time(TIMING_METRIC_NAME): time.sleep(TIMING_FUNC_DURATION) # We want at least 0.5 ms precision and no more than 1 ms overhead check_precision(mc, f, minimum_precision=0.0005, maximum_overhead=0.001)
def test_measure_time_decorator(): class Example: def __init__(self, metrics): self.metrics = metrics self.data = 2 @measure_time(MetricsName.LOOPER_RUN_TIME_SPENT) def slow_add(self, a, b): time.sleep(0.1) return self.data + a + b mc = MockMetricsCollector() e = Example(mc) r = e.slow_add(1, 3) assert len(mc.events) == 0 assert r == 6 mc.flush_accumulated() assert len(mc.events) == 1 assert mc.events[0][0] == MetricsName.LOOPER_RUN_TIME_SPENT assert abs(mc.events[0][1] - 0.1) < 0.001 # we want at least 1 ms precision
def test_async_measure_time_decorator(looper): class Example: def __init__(self, metrics): self.metrics = metrics self.data = 2 @async_measure_time(MetricsName.LOOPER_RUN_TIME_SPENT) async def slow_add(self, a, b): await asyncio.sleep(0.1) return self.data + a + b mc = MockMetricsCollector() e = Example(mc) r = looper.loop.run_until_complete(e.slow_add(1, 3)) assert len(mc.events) == 0 assert r == 6 mc.flush_accumulated() assert len(mc.events) == 1 assert mc.events[0][0] == MetricsName.LOOPER_RUN_TIME_SPENT assert abs(mc.events[0][1] - 0.1) < 0.005 # we want at least 5 ms precision
def test_metrics_collector_adds_events_when_flushing_accumulated(): mc = MockMetricsCollector() mc.acc_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0) mc.flush_accumulated() assert len(mc.events) == 1 assert mc.events[0] == (MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0)
def test_metrics_collector_accumulate_same_events_into_one(): mc = MockMetricsCollector() mc.acc_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 3.0) mc.acc_event(MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 2.0) mc.flush_accumulated() assert len(mc.events) == 1 assert mc.events[0] == (MetricsName.BACKUP_THREE_PC_BATCH_SIZE, 5.0)
def test_measure_time_decorator(): mc = MockMetricsCollector() class Example: def __init__(self, metrics): self.metrics = metrics self.data = 2 @measure_time(TIMING_METRIC_NAME) def slow_add(self, a, b): time.sleep(TIMING_FUNC_DURATION) return self.data + a + b # We want at least 0.5 ms precision and no more than 1 ms overhead e = Example(mc) check_precision(mc, lambda: e.slow_add(1, 3), minimum_precision=0.0005, maximum_overhead=0.001) # Check that decorated function works correctly e = Example(mc) r = e.slow_add(1, 3) assert r == 6
def test_metrics_collector_dont_add_events_when_flushing_empty(): mc = MockMetricsCollector() mc.flush_accumulated() assert mc.events == []
def test_metrics_collector_dont_add_events_when_accumulating(): mc = MockMetricsCollector() mc.add_event(gen_metrics_name(), 3.0) assert mc.events == []