def test_update(): vals = [1.0, 2.0, 3] tracker = VarianceTracker() for v in vals: tracker.update(v) assert tracker.mean == pytest.approx(2.0, 1e-6) assert tracker.count == len(vals) # Note: this is a population variance estimate (normalized by N-1) assert tracker.variance() == pytest.approx(1.0, 1e-6)
def test_merge_simple_trackers(): x1 = VarianceTracker() x1.update(1.0) x2 = VarianceTracker() x2.update(2.0) x2.update(3.0) merged = x1.merge(x2) assert merged.mean == pytest.approx(2.0, 1e-4) assert merged.variance() == pytest.approx(1.0, 1e-4) assert merged.count == 3
def __init__( self, variance: VarianceTracker = None, floats: FloatTracker = None, ints: IntTracker = None, theta_sketch: ThetaSketch = None, histogram: datasketches.kll_floats_sketch = None, frequent_numbers: dsketch.FrequentNumbersSketch = None, ): # Our own trackers if variance is None: variance = VarianceTracker() if floats is None: floats = FloatTracker() if ints is None: ints = IntTracker() if theta_sketch is None: theta_sketch = ThetaSketch() if histogram is None: histogram = datasketches.kll_floats_sketch(DEFAULT_HIST_K) if frequent_numbers is None: frequent_numbers = dsketch.FrequentNumbersSketch() self.variance = variance self.floats = floats self.ints = ints self.theta_sketch = theta_sketch self.histogram = histogram self.frequent_numbers = frequent_numbers
def from_protobuf(message: NumbersMessage): """ Load from a protobuf message Returns ------- number_tracker : NumberTracker """ theta = None if message.theta is not None and len(message.theta) > 0: theta = ThetaSketch.deserialize(message.theta) elif message.compact_theta is not None and len(message.compact_theta) > 0: theta = ThetaSketch.deserialize(message.compact_theta) opts = dict( theta_sketch=theta, variance=VarianceTracker.from_protobuf(message.variance), histogram=dsketch.deserialize_kll_floats_sketch(message.histogram), frequent_numbers=dsketch.FrequentNumbersSketch.from_protobuf( message.frequent_numbers ), ) if message.HasField("doubles"): opts["floats"] = FloatTracker.from_protobuf(message.doubles) if message.HasField("longs"): opts["ints"] = IntTracker.from_protobuf(message.longs) return NumberTracker(**opts)
def from_protobuf(message: NumbersMessage): """ Load from a protobuf message Returns ------- number_tracker : NumberTracker """ theta = None if message.compact_theta is not None and len( message.compact_theta) > 0: theta = ThetaSketch.deserialize(message.compact_theta) elif message.theta is not None and len(message.theta) > 0: logger.warning( "Possible missing data. Non-compact theta sketches are no longer supported" ) opts = dict( theta_sketch=theta, variance=VarianceTracker.from_protobuf(message.variance), histogram=dsketch.deserialize_kll_floats_sketch(message.histogram), ) if message.HasField("doubles"): opts["floats"] = FloatTracker.from_protobuf(message.doubles) if message.HasField("longs"): opts["ints"] = IntTracker.from_protobuf(message.longs) return NumberTracker(**opts)
def test_empty_merge_full(): x1 = VarianceTracker() vals = list(range(10)) for v in vals: x1.update(v) var = x1.variance() mean = x1.mean count = x1.count x2 = VarianceTracker().merge(x1) assert var == x2.variance() assert mean == x2.mean assert count == x2.count assert x1.variance() == x2.variance() assert x1.mean == x2.mean assert x1.count == x2.count
def test_protobuf_roundtrip_matches(): x1 = VarianceTracker() for i in range(10): x1.update(i) proto = x1.to_protobuf() roundtrip = VarianceTracker.from_protobuf(proto) assert x1.count == roundtrip.count assert x1.variance() == roundtrip.variance() assert x1.mean == roundtrip.mean
def test_merge_bigger(): x1 = VarianceTracker() for i in range(10): x1.update(i) assert x1.variance() == pytest.approx(9.1667, 1e-4) assert x1.count == 10 assert x1.mean == pytest.approx(4.5, 1e-7) merged = x1.merge(x1.copy()) assert merged.variance() == pytest.approx(8.684, 1e-4) assert merged.count == 20 assert merged.mean == pytest.approx(4.5, 1e-7)
def test_copy(): x1 = VarianceTracker() for i in range(10): x1.update(i) var = x1.variance() count = x1.count mean = x1.mean x2 = x1.copy() assert x2.variance() == var assert x2.count == count assert x2.mean == mean for i in range(3): x1.update(i) assert x1.variance() != var assert x1.count == count + 3 assert x1.mean != mean assert x2.variance() == var assert x2.mean == mean assert x2.count == count
def test_single_value(): tracker = VarianceTracker() tracker.update(1.0) assert tracker.count == 1 assert tracker.variance() == 0