def test_merge_floattrackers_should_addup():
first = IntTracker()
vals1 = [1, 2, 3]
for val in vals1:
first.update(val)
assert first.count == len(vals1)
assert first.max == max(vals1)
assert first.min == min(vals1)
assert first.sum == sum(vals1)
second = IntTracker()
vals2 = [4, 5, 6]
for val in vals2:
second.update(val)
assert second.count == len(vals2)
assert second.max == max(vals2)
assert second.min == min(vals2)
assert second.sum == sum(vals2)
all_vals = vals1 + vals2
merge_first = first.merge(second)
assert merge_first.count == len(all_vals)
assert merge_first.max == max(all_vals)
assert merge_first.min == min(all_vals)
assert merge_first.sum == sum(all_vals)
merge_second = second.merge(first)
assert merge_second.__dict__ == merge_first.__dict__
def test_values_are_min_max():
first = IntTracker()
vals1 = [1, 2, 3]
for val in vals1:
first.update(val)
assert first.count == len(vals1)
assert first.max == max(vals1)
assert first.min == min(vals1)
assert first.sum == sum(vals1)
def from_protobuf(message: NumbersMessage):
"""
Load from a protobuf message
Returns
-------
number_tracker : NumberTracker
"""
theta = None
if message.theta is not None and len(message.theta) > 0:
theta = ThetaSketch.deserialize(message.theta)
elif message.compact_theta is not None and len(message.compact_theta) > 0:
theta = ThetaSketch.deserialize(message.compact_theta)
opts = dict(
theta_sketch=theta,
variance=VarianceTracker.from_protobuf(message.variance),
histogram=dsketch.deserialize_kll_floats_sketch(message.histogram),
frequent_numbers=dsketch.FrequentNumbersSketch.from_protobuf(
message.frequent_numbers
),
)
if message.HasField("doubles"):
opts["floats"] = FloatTracker.from_protobuf(message.doubles)
if message.HasField("longs"):
opts["ints"] = IntTracker.from_protobuf(message.longs)
return NumberTracker(**opts)
def __init__(
self,
variance: VarianceTracker = None,
floats: FloatTracker = None,
ints: IntTracker = None,
theta_sketch: ThetaSketch = None,
histogram: datasketches.kll_floats_sketch = None,
frequent_numbers: dsketch.FrequentNumbersSketch = None,
):
# Our own trackers
if variance is None:
variance = VarianceTracker()
if floats is None:
floats = FloatTracker()
if ints is None:
ints = IntTracker()
if theta_sketch is None:
theta_sketch = ThetaSketch()
if histogram is None:
histogram = datasketches.kll_floats_sketch(DEFAULT_HIST_K)
if frequent_numbers is None:
frequent_numbers = dsketch.FrequentNumbersSketch()
self.variance = variance
self.floats = floats
self.ints = ints
self.theta_sketch = theta_sketch
self.histogram = histogram
self.frequent_numbers = frequent_numbers
def from_protobuf(message: NumbersMessage):
"""
Load from a protobuf message
Returns
-------
number_tracker : NumberTracker
"""
theta = None
if message.compact_theta is not None and len(
message.compact_theta) > 0:
theta = ThetaSketch.deserialize(message.compact_theta)
elif message.theta is not None and len(message.theta) > 0:
logger.warning(
"Possible missing data. Non-compact theta sketches are no longer supported"
)
opts = dict(
theta_sketch=theta,
variance=VarianceTracker.from_protobuf(message.variance),
histogram=dsketch.deserialize_kll_floats_sketch(message.histogram),
)
if message.HasField("doubles"):
opts["floats"] = FloatTracker.from_protobuf(message.doubles)
if message.HasField("longs"):
opts["ints"] = IntTracker.from_protobuf(message.longs)
return NumberTracker(**opts)