def test_quantiles_merge_accumulators(self):
# This test exercises merging multiple buffers and approximation accuracy.
# The max_num_elements is set to a small value to trigger buffers collapse
# and interpolation. Under the conditions below, buffer_size=125 and
# num_buffers=4, so we're only allowed to keep half of the input values.
num_accumulators = 100
num_quantiles = 5
eps = 0.01
max_num_elements = 1000
combine_fn = ApproximateQuantilesCombineFn.create(
num_quantiles, eps, max_num_elements)
combine_fn_weighted = ApproximateQuantilesCombineFn.create(
num_quantiles, eps, max_num_elements, weighted=True)
data = list(range(1000))
weights = list(reversed(range(1000)))
step = math.ceil(len(data) / num_accumulators)
accumulators = []
accumulators_weighted = []
for i in range(num_accumulators):
accumulator = combine_fn.create_accumulator()
accumulator_weighted = combine_fn_weighted.create_accumulator()
for element, weight in zip(data[i*step:(i+1)*step],
weights[i*step:(i+1)*step]):
accumulator = combine_fn.add_input(accumulator, element)
accumulator_weighted = combine_fn_weighted.add_input(
accumulator_weighted, (element, weight))
accumulators.append(accumulator)
accumulators_weighted.append(accumulator_weighted)
accumulator = combine_fn.merge_accumulators(accumulators)
accumulator_weighted = combine_fn_weighted.merge_accumulators(
accumulators_weighted)
quantiles = combine_fn.extract_output(accumulator)
quantiles_weighted = combine_fn_weighted.extract_output(
accumulator_weighted)
# In fact, the final accuracy is much higher than eps, but we test for a
# minimal accuracy here.
for q, actual_q in zip(quantiles, [0, 249, 499, 749, 999]):
self.assertAlmostEqual(q, actual_q, delta=max_num_elements * eps)
for q, actual_q in zip(quantiles_weighted, [0, 133, 292, 499, 999]):
self.assertAlmostEqual(q, actual_q, delta=max_num_elements * eps)
def test_correctness(self, epsilon, maxInputSize, *args):
"""
Verify that buffers are correct according to the two constraint equations.
"""
combine_fn = ApproximateQuantilesCombineFn.create(
num_quantiles=10, max_num_elements=maxInputSize, epsilon=epsilon)
b = combine_fn._num_buffers
k = combine_fn._buffer_size
n = maxInputSize
self.assertLessEqual((b - 2) * (1 << (b - 2)) + 0.5, (epsilon * n),
'(b-2)2^(b-2) + 1/2 <= eN')
self.assertGreaterEqual((k * 2)**(b - 1), n, 'k2^(b-1) >= N')
def test_efficiency(
self, epsilon, maxInputSize, expectedNumBuffers, expectedBufferSize):
"""
Verify the buffers are efficiently calculated according to the reference
table values.
"""
combine_fn = ApproximateQuantilesCombineFn.create(
num_quantiles=10, max_num_elements=maxInputSize, epsilon=epsilon)
self.assertEqual(
expectedNumBuffers, combine_fn._num_buffers, "Number of buffers")
self.assertEqual(expectedBufferSize, combine_fn._buffer_size, "Buffer size")