def _digest_from_protobuf(self, protobuf):
# Ensure that the input protobuf is indeed a protobuf object
if not isinstance(protobuf, TDigest_instance):
raise TypeError("Error: tried to decode a "
"non protobuf object into a TDigest")
digest_dict = {}
digest_dict["K"] = protobuf.K
digest_dict["delta"] = protobuf.delta
centroid_list = []
for centroid in protobuf.centroids:
current_centroid = {}
current_centroid["c"] = centroid.c
current_centroid["m"] = centroid.m
centroid_list.append(current_centroid)
digest_dict["centroids"] = centroid_list
digest = TDigest()
digest.update_from_dict(digest_dict)
return digest
def test_ints(self):
t = TDigest()
t.batch_update([1, 2, 3])
assert abs(t.percentile(50) - 2) < 0.0001
t = TDigest()
x = [1, 2, 2, 2, 2, 2, 2, 2, 3]
t.batch_update(x)
assert t.percentile(50) == 2
assert sum([c.count for c in t.C.values()]) == len(x)
def test_uniform(self):
t = TDigest()
x = random.random(size=10000)
t.batch_update(x)
assert abs(t.percentile(50) - 0.5) < 0.01
assert abs(t.percentile(10) - 0.1) < 0.01
assert abs(t.percentile(90) - 0.9) < 0.01
assert abs(t.percentile(1) - 0.01) < 0.005
assert abs(t.percentile(99) - 0.99) < 0.005
assert abs(t.percentile(0.1) - 0.001) < 0.001
assert abs(t.percentile(99.9) - 0.999) < 0.001
def test_trimmed_mean(self, percentile_range, data_size):
p1 = percentile_range[0]
p2 = percentile_range[1]
t = TDigest()
x = random.random(size=data_size)
t.batch_update(x)
tm_actual = t.trimmed_mean(p1, p2)
tm_expected = x[
bitwise_and(x >= percentile(x, p1), x <= percentile(x, p2))
].mean()
testing.assert_allclose(tm_actual, tm_expected, rtol=0.01, atol=0.01)
def test_extreme_percentiles_return_min_and_max(self, empty_tdigest):
t = TDigest()
data = random.randn(10000)
t.batch_update(data)
assert t.percentile(100.0) == data.max()
assert t.percentile(0) == data.min()
assert t.percentile(0.1) > data.min()
assert t.percentile(0.999) < data.max()
def test_trimmed_mean_corner_cases(self):
td = TDigest()
mean = td.trimmed_mean(0, 100)
assert mean == 0
td.update(1)
mean = td.trimmed_mean(0, 100)
assert mean == 1
td.update(1000)
mean = td.trimmed_mean(0, 100)
assert mean == 500.5
def test_data_comes_in_sorted_does_not_blow_up(self, empty_tdigest):
t = TDigest()
for x in range(10000):
t.update(x, 1)
assert len(t) < 5000
t = TDigest()
t.batch_update(range(10000))
assert len(t) < 1000
def rebook(self, excludes=[]):
"""
Force reset of all tdigests
"""
self._rebooked = True
for n, x in self:
if n in excludes:
continue
self._set(n, TDigest())
pass
def test_trimmed_mean_negative(self):
td = TDigest()
for i in range(100):
td.update(random.random())
for i in range(10):
td.update(i * 100)
mean = td.trimmed_mean(1, 99)
assert mean >= 0
def book(self, algname, name):
name_ = "."
name_ = name_.join([algname, name])
self.__logger.info("Booking %s", name_)
# TODO
# Explore more options for correctly initializing h1
value = self._get(name)
if value is not None:
self.__logger.error("TDigest already exists %s", name_)
else:
try:
h = TDigest()
except Exception:
self.__logger.error("TDigest fails to book")
raise
self[name_] = h
def empty_tdigest():
return TDigest()
def test_percentile_at_border_returns_an_intermediate_value(self, empty_tdigest):
data = [62.0, 202.0, 1415.0, 1433.0]
t = TDigest()
t.batch_update(data)
assert t.percentile(25) == 132.0
def test_negative_extreme_percentile_is_still_positive(self, empty_tdigest):
# Test https://github.com/CamDavidsonPilon/tdigest/issues/16
t = TDigest()
t.batch_update([62.0, 202.0, 1415.0, 1433.0])
print(t.percentile(26))
assert t.percentile(26) > 0