def test_insert_lookup_with_random_seed(self):
b = BloomFilter(length=15, random_seed=2)
self.assertNotIn(1, b)
b.add(1)
self.assertIn(1, b)
self.assertNotIn(2, b)
def test_multi_insertion_two_hash(self):
# Due to hash collisions, the estimate could be 1 or 2.
# We test the distribution.
estimates = []
for i in range(1000):
b1 = BloomFilter(length=100, num_hashes=2, random_seed=i)
b1.add(1)
b1.add(2)
estimates.append(UnionEstimator.estimate_cardinality(b1))
self.assertAlmostEqual(np.mean(estimates), 1.941, delta=0.04)
def test_not_compatible_different_lengths(self):
b1 = BloomFilter(length=10, random_seed=2)
b2 = BloomFilter(length=15, random_seed=2)
with self.assertRaises(AssertionError):
b1.assert_compatible(b2)
with self.assertRaises(AssertionError):
b2.assert_compatible(b1)
def test_factory(self):
factory = BloomFilter.get_sketch_factory(length=15)
b = factory(2)
b.add(1)
self.assertIn(1, b)
self.assertNotIn(2, b)
def test_bit_flip(self):
b = BloomFilter(length=100000, random_seed=4)
epsilon = math.log(3) # Equivalent to flipping 1/4 of the bits.
noiser = BlipNoiser(epsilon, np.random.RandomState(4))
noised = noiser(b)
average_bits = np.mean(noised.sketch)
# By central limit theorem, the average_bits should be roughly following
# N(0.25, 0.0014**2). So we set the absolute error to 0.01 which
# is even larger than 6.5-sigma, which means that if we test it daily,
# the expected failure will happen every 34 million years.
self.assertAlmostEqual(average_bits, 0.25, delta=0.01)
# Similarly, we blip 1 bits.
b.sketch = np.bitwise_xor(b.sketch, 1)
noised = noiser(b)
average_bits = np.mean(noised.sketch)
self.assertAlmostEqual(average_bits, 0.75, delta=0.01)
def test_cardinality_estimation(self):
b1 = BloomFilter(length=10, random_seed=3)
b1.sketch[0] = 1
b2 = BloomFilter(length=10, random_seed=3)
b2.sketch[1] = 1
cardinality = UnionEstimator()([b1, b2])[0]
self.assertEqual(cardinality, 2)
def test_bit_flip(self):
rs = np.random.RandomState()
b = BloomFilter(length=10)
b.add(1)
b.add(2)
noiser = FixedProbabilityBitFlipNoiser(probability=1.0,
random_state=rs)
# Since p(flip) = 1 we are computing the inverse
inverse = noiser(b)
# Now do it again, which should result in a copy of the original filter b.
inverse_of_inverse = noiser(inverse)
result = b.sketch + inverse.sketch
expected = np.ones(b.sketch.shape)
self.assertTrue(np.array_equal(expected, result),
'expected {} != {}'.format(expected, result))
self.assertTrue(
np.array_equal(b.sketch, inverse_of_inverse.sketch),
'expected {} == {}'.format(inverse_of_inverse.sketch, b.sketch))
def test_union_of_two(self):
b1 = BloomFilter(length=10, random_seed=5)
b1.sketch[0] = 1
b2 = BloomFilter(length=10, random_seed=5)
b2.sketch[1] = 1
union = UnionEstimator().union_sketches([b1, b2])
self.assertEqual(UnionEstimator.estimate_cardinality(union), 2)
expected = np.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0], dtype=np.int32)
np.testing.assert_equal(union.sketch, expected,
'The union sketch is no correct.')
def test_compatible(self):
b1 = BloomFilter(length=15, random_seed=2)
b2 = BloomFilter(length=15, random_seed=2)
self.assertTrue(b1.assert_compatible(b2))
self.assertTrue(b2.assert_compatible(b1))
def test_raise_error_with_full_bloom_filter(self):
b1 = BloomFilter(length=2, random_seed=3)
b1.add_ids(range(10))
with self.assertRaises(ValueError):
_ = UnionEstimator.estimate_cardinality(b1)
def test_multi_insertion_same_element(self):
b1 = BloomFilter(length=10, random_seed=3)
b1.add(1)
b1.add(1)
self.assertEqual(UnionEstimator.estimate_cardinality(b1), 1)
def test_single_insertion(self):
b1 = BloomFilter(length=10, random_seed=3)
b1.add(1)
self.assertEqual(UnionEstimator.estimate_cardinality(b1), 1)
def setUp(self):
super(InteroperabilityTest, self).setUp()
self.number_of_trials = 2
self.universe_size = 2000
self.set_size_list = [5, 7, 9]
self.large_set_size = 6
self.small_set_size = 3
self.sketch_size = 128
self.number_of_sets = 3
self.set_size = 50
self.num_large_sets = 1
self.num_small_sets = 3
self.order = set_generator.ORDER_RANDOM
self.user_activity_association = (
set_generator.USER_ACTIVITY_ASSOCIATION_INDEPENDENT)
self.shared_prop = 0.2
self.num_bloom_filter_hashes = 2
self.exponential_bloom_filter_decay_rate = 10
self.noiser_epsilon = np.log(3)
self.noiser_flip_probability = .25
self.set_random_state = np.random.RandomState(42)
self.sketch_random_state = np.random.RandomState(137)
self.noise_random_state = np.random.RandomState(3)
# non-noised estimators
estimator_config_cascading_legions = SketchEstimatorConfig(
name='cascading_legions',
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator())
estimator_config_bloom_filter = SketchEstimatorConfig(
name='bloom_filter-union_estimator',
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator())
estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig(
name='log_bloom_filter-first_moment_log',
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(method='log'))
estimator_config_exponential_bloom_filter = SketchEstimatorConfig(
name='exp_bloom_filter-first_moment_exp',
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(method='exp'))
estimator_config_voc = SketchEstimatorConfig(
name='vector_of_counts-sequential',
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator())
estimator_config_exact = SketchEstimatorConfig(
name='exact_set-lossless',
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator())
estimator_config_hll = SketchEstimatorConfig(
name='hyper_log_log',
sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(self.sketch_size),
estimator=HllCardinality())
config_list = [
estimator_config_exact,
estimator_config_cascading_legions,
estimator_config_bloom_filter,
estimator_config_logarithmic_bloom_filter,
estimator_config_exponential_bloom_filter,
estimator_config_voc,
estimator_config_hll,
]
self.name_to_non_noised_estimator_config = {
config.name: config for config in config_list
}
# noised estimators
noised_estimator_config_cascading_legions = SketchEstimatorConfig(
name='cascading_legions',
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator(),
sketch_noiser=Noiser(self.noiser_flip_probability))
noised_estimator_config_bloom_filter = SketchEstimatorConfig(
name='bloom_filter-union_estimator',
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator(),
sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state))
noised_estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig(
name='log_bloom_filter-first_moment_log',
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(
method='log',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state))
noised_estimator_config_exponential_bloom_filter = SketchEstimatorConfig(
name='exp_bloom_filter-first_moment_exp',
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(
method='exp',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state))
noised_estimator_config_voc = SketchEstimatorConfig(
name='vector_of_counts-sequential',
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator(),
sketch_noiser=LaplaceNoiser())
noised_estimator_config_exact = SketchEstimatorConfig(
name='exact_set-lossless',
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator(),
sketch_noiser=AddRandomElementsNoiser(1, self.noise_random_state))
noised_config_list = [
noised_estimator_config_exact,
noised_estimator_config_cascading_legions,
noised_estimator_config_bloom_filter,
noised_estimator_config_logarithmic_bloom_filter,
noised_estimator_config_exponential_bloom_filter,
noised_estimator_config_voc,
]
self.name_to_noised_estimator_config = {
config.name: config for config in noised_config_list
}
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
estimator_config_cascading_legions = SketchEstimatorConfig(
name='cascading-legions',
sketch_factory=CascadingLegions.get_sketch_factory(
FLAGS.sketch_size, FLAGS.sketch_size),
estimator=Estimator())
estimator_config_bloom_filter = SketchEstimatorConfig(
name='bloom_filter-union_estimator',
sketch_factory=BloomFilter.get_sketch_factory(
FLAGS.sketch_size, FLAGS.num_bloom_filter_hashes),
estimator=UnionEstimator())
estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig(
name='log_bloom_filter-first_moment_log',
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
FLAGS.sketch_size),
estimator=FirstMomentEstimator(method='log'))
estimator_config_exponential_bloom_filter = SketchEstimatorConfig(
name='exp_bloom_filter-first_moment_exp',
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
FLAGS.sketch_size, FLAGS.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(method='exp'))
estimator_config_voc = SketchEstimatorConfig(
name='vector_of_counts-sequential',
sketch_factory=VectorOfCounts.get_sketch_factory(FLAGS.sketch_size),
estimator=SequentialEstimator())
estimator_config_hll = SketchEstimatorConfig(
name='hll++',
sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(FLAGS.sketch_size),
estimator=HllCardinality())
estimator_config_exact = SketchEstimatorConfig(
name='exact_set-lossless',
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator())
estimator_config_list = [
estimator_config_bloom_filter,
estimator_config_logarithmic_bloom_filter,
estimator_config_exponential_bloom_filter,
estimator_config_cascading_legions,
estimator_config_exact,
estimator_config_hll,
estimator_config_voc,
]
name_to_estimator_config = {
'bloom_filter': estimator_config_bloom_filter,
'logarithmic_bloom_filter': estimator_config_logarithmic_bloom_filter,
'exponential_bloom_filter': estimator_config_exponential_bloom_filter,
'cascading_legions': estimator_config_cascading_legions,
'exact_set': estimator_config_exact,
'hll++': estimator_config_hll,
'vector_of_counts': estimator_config_voc,
}
set_generator_factory = (
set_generator.IndependentSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=FLAGS.universe_size,
num_sets=FLAGS.number_of_sets,
set_size=FLAGS.set_size))
for estimator_method_config in estimator_config_list:
print(f'Calculations for {estimator_method_config.name}')
set_rs = np.random.RandomState(1)
sketch_rs = np.random.RandomState(1)
simulator = Simulator(
num_runs=FLAGS.number_of_trials,
set_generator_factory=set_generator_factory,
sketch_estimator_config=estimator_method_config,
set_random_state=set_rs,
sketch_random_state=sketch_rs)
_, agg_data = simulator.run_all_and_aggregate()
print(f'Aggregate Statistics for {estimator_method_config.name}')
print(agg_data)
def setUp(self):
super(InteroperabilityTest, self).setUp()
self.number_of_trials = 2
self.universe_size = 2000
self.set_size = 5
self.large_set_size = 6
self.small_set_size = 3
self.sketch_size = 64
self.number_of_sets = 2
self.num_large_sets = 1
self.num_small_sets = 3
self.order = set_generator.ORDER_RANDOM
self.user_activity_association = (
set_generator.USER_ACTIVITY_ASSOCIATION_INDEPENDENT)
self.shared_prop = 0.2
self.num_bloom_filter_hashes = 2
self.exponential_bloom_filter_decay_rate = 10
self.noiser_epsilon = np.log(3)
self.noiser_flip_probability = .25
self.set_random_state = np.random.RandomState(42)
self.sketch_random_state = np.random.RandomState(137)
self.noise_random_state = np.random.RandomState(3)
# non-noised estimators
estimator_config_cascading_legions = EstimatorConfig(
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_bloom_filter = EstimatorConfig(
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_logarithmic_bloom_filter = EstimatorConfig(
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(method='log'),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_exponential_bloom_filter = EstimatorConfig(
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(method='exp'),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_geometric_bloom_filter = EstimatorConfig(
sketch_factory=GeometricBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=GeometricUnionEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_voc = EstimatorConfig(
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_exact = EstimatorConfig(
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_hll = EstimatorConfig(
sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(
self.sketch_size),
estimator=HllCardinality(),
sketch_noiser=None,
estimate_noiser=None)
self.name_to_non_noised_estimator_config = {
'exact_set': estimator_config_exact,
'cascading_legions': estimator_config_cascading_legions,
'bloom_filter': estimator_config_bloom_filter,
'logarithmic_bloom_filter':
estimator_config_logarithmic_bloom_filter,
'exponential_bloom_filter':
estimator_config_exponential_bloom_filter,
'geometric_bloom_filter': estimator_config_geometric_bloom_filter,
'vector_of_counts': estimator_config_voc,
'hll': estimator_config_hll,
}
# noised estimators
noised_estimator_config_cascading_legions = EstimatorConfig(
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator(),
sketch_noiser=Noiser(self.noiser_flip_probability),
estimate_noiser=None)
noised_estimator_config_bloom_filter = EstimatorConfig(
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator(),
sketch_noiser=BlipNoiser(self.noiser_epsilon,
self.noise_random_state),
estimate_noiser=None)
noised_estimator_config_logarithmic_bloom_filter = EstimatorConfig(
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(
method='log',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=None,
estimate_noiser=None)
noised_estimator_config_exponential_bloom_filter = EstimatorConfig(
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(
method='exp',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=None,
estimate_noiser=None)
noised_estimator_config_geometric_bloom_filter = EstimatorConfig(
sketch_factory=GeometricBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=GeometricUnionEstimator(),
sketch_noiser=BlipNoiser(self.noiser_epsilon,
self.noise_random_state),
estimate_noiser=None)
noised_estimator_config_voc = EstimatorConfig(
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator(),
sketch_noiser=LaplaceNoiser(),
estimate_noiser=None)
noised_estimator_config_exact = EstimatorConfig(
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator(),
sketch_noiser=AddRandomElementsNoiser(1, self.noise_random_state),
estimate_noiser=None)
self.name_to_noised_estimator_config = {
'exact_set': noised_estimator_config_exact,
'cascading_legions': noised_estimator_config_cascading_legions,
'bloom_filter': noised_estimator_config_bloom_filter,
'logarithmic_bloom_filter':
noised_estimator_config_logarithmic_bloom_filter,
'exponential_bloom_filter':
noised_estimator_config_exponential_bloom_filter,
'geometric_bloom_filter':
noised_estimator_config_geometric_bloom_filter,
'vector_of_counts': noised_estimator_config_voc,
}