def test_insert_lookup_with_random_seed(self): b = BloomFilter(length=15, random_seed=2) self.assertNotIn(1, b) b.add(1) self.assertIn(1, b) self.assertNotIn(2, b)
def test_multi_insertion_two_hash(self): # Due to hash collisions, the estimate could be 1 or 2. # We test the distribution. estimates = [] for i in range(1000): b1 = BloomFilter(length=100, num_hashes=2, random_seed=i) b1.add(1) b1.add(2) estimates.append(UnionEstimator.estimate_cardinality(b1)) self.assertAlmostEqual(np.mean(estimates), 1.941, delta=0.04)
def test_not_compatible_different_lengths(self): b1 = BloomFilter(length=10, random_seed=2) b2 = BloomFilter(length=15, random_seed=2) with self.assertRaises(AssertionError): b1.assert_compatible(b2) with self.assertRaises(AssertionError): b2.assert_compatible(b1)
def test_factory(self): factory = BloomFilter.get_sketch_factory(length=15) b = factory(2) b.add(1) self.assertIn(1, b) self.assertNotIn(2, b)
def test_bit_flip(self): b = BloomFilter(length=100000, random_seed=4) epsilon = math.log(3) # Equivalent to flipping 1/4 of the bits. noiser = BlipNoiser(epsilon, np.random.RandomState(4)) noised = noiser(b) average_bits = np.mean(noised.sketch) # By central limit theorem, the average_bits should be roughly following # N(0.25, 0.0014**2). So we set the absolute error to 0.01 which # is even larger than 6.5-sigma, which means that if we test it daily, # the expected failure will happen every 34 million years. self.assertAlmostEqual(average_bits, 0.25, delta=0.01) # Similarly, we blip 1 bits. b.sketch = np.bitwise_xor(b.sketch, 1) noised = noiser(b) average_bits = np.mean(noised.sketch) self.assertAlmostEqual(average_bits, 0.75, delta=0.01)
def test_cardinality_estimation(self): b1 = BloomFilter(length=10, random_seed=3) b1.sketch[0] = 1 b2 = BloomFilter(length=10, random_seed=3) b2.sketch[1] = 1 cardinality = UnionEstimator()([b1, b2])[0] self.assertEqual(cardinality, 2)
def test_bit_flip(self): rs = np.random.RandomState() b = BloomFilter(length=10) b.add(1) b.add(2) noiser = FixedProbabilityBitFlipNoiser(probability=1.0, random_state=rs) # Since p(flip) = 1 we are computing the inverse inverse = noiser(b) # Now do it again, which should result in a copy of the original filter b. inverse_of_inverse = noiser(inverse) result = b.sketch + inverse.sketch expected = np.ones(b.sketch.shape) self.assertTrue(np.array_equal(expected, result), 'expected {} != {}'.format(expected, result)) self.assertTrue( np.array_equal(b.sketch, inverse_of_inverse.sketch), 'expected {} == {}'.format(inverse_of_inverse.sketch, b.sketch))
def test_union_of_two(self): b1 = BloomFilter(length=10, random_seed=5) b1.sketch[0] = 1 b2 = BloomFilter(length=10, random_seed=5) b2.sketch[1] = 1 union = UnionEstimator().union_sketches([b1, b2]) self.assertEqual(UnionEstimator.estimate_cardinality(union), 2) expected = np.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0], dtype=np.int32) np.testing.assert_equal(union.sketch, expected, 'The union sketch is no correct.')
def test_compatible(self): b1 = BloomFilter(length=15, random_seed=2) b2 = BloomFilter(length=15, random_seed=2) self.assertTrue(b1.assert_compatible(b2)) self.assertTrue(b2.assert_compatible(b1))
def test_raise_error_with_full_bloom_filter(self): b1 = BloomFilter(length=2, random_seed=3) b1.add_ids(range(10)) with self.assertRaises(ValueError): _ = UnionEstimator.estimate_cardinality(b1)
def test_multi_insertion_same_element(self): b1 = BloomFilter(length=10, random_seed=3) b1.add(1) b1.add(1) self.assertEqual(UnionEstimator.estimate_cardinality(b1), 1)
def test_single_insertion(self): b1 = BloomFilter(length=10, random_seed=3) b1.add(1) self.assertEqual(UnionEstimator.estimate_cardinality(b1), 1)
def setUp(self): super(InteroperabilityTest, self).setUp() self.number_of_trials = 2 self.universe_size = 2000 self.set_size_list = [5, 7, 9] self.large_set_size = 6 self.small_set_size = 3 self.sketch_size = 128 self.number_of_sets = 3 self.set_size = 50 self.num_large_sets = 1 self.num_small_sets = 3 self.order = set_generator.ORDER_RANDOM self.user_activity_association = ( set_generator.USER_ACTIVITY_ASSOCIATION_INDEPENDENT) self.shared_prop = 0.2 self.num_bloom_filter_hashes = 2 self.exponential_bloom_filter_decay_rate = 10 self.noiser_epsilon = np.log(3) self.noiser_flip_probability = .25 self.set_random_state = np.random.RandomState(42) self.sketch_random_state = np.random.RandomState(137) self.noise_random_state = np.random.RandomState(3) # non-noised estimators estimator_config_cascading_legions = SketchEstimatorConfig( name='cascading_legions', sketch_factory=CascadingLegions.get_sketch_factory( self.sketch_size, self.sketch_size), estimator=Estimator()) estimator_config_bloom_filter = SketchEstimatorConfig( name='bloom_filter-union_estimator', sketch_factory=BloomFilter.get_sketch_factory( self.sketch_size, self.num_bloom_filter_hashes), estimator=UnionEstimator()) estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig( name='log_bloom_filter-first_moment_log', sketch_factory=LogarithmicBloomFilter.get_sketch_factory( self.sketch_size), estimator=FirstMomentEstimator(method='log')) estimator_config_exponential_bloom_filter = SketchEstimatorConfig( name='exp_bloom_filter-first_moment_exp', sketch_factory=ExponentialBloomFilter.get_sketch_factory( self.sketch_size, self.exponential_bloom_filter_decay_rate), estimator=FirstMomentEstimator(method='exp')) estimator_config_voc = SketchEstimatorConfig( name='vector_of_counts-sequential', sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size), estimator=SequentialEstimator()) estimator_config_exact = SketchEstimatorConfig( name='exact_set-lossless', sketch_factory=ExactSet.get_sketch_factory(), estimator=LosslessEstimator()) estimator_config_hll = SketchEstimatorConfig( name='hyper_log_log', sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(self.sketch_size), estimator=HllCardinality()) config_list = [ estimator_config_exact, estimator_config_cascading_legions, estimator_config_bloom_filter, estimator_config_logarithmic_bloom_filter, estimator_config_exponential_bloom_filter, estimator_config_voc, estimator_config_hll, ] self.name_to_non_noised_estimator_config = { config.name: config for config in config_list } # noised estimators noised_estimator_config_cascading_legions = SketchEstimatorConfig( name='cascading_legions', sketch_factory=CascadingLegions.get_sketch_factory( self.sketch_size, self.sketch_size), estimator=Estimator(), sketch_noiser=Noiser(self.noiser_flip_probability)) noised_estimator_config_bloom_filter = SketchEstimatorConfig( name='bloom_filter-union_estimator', sketch_factory=BloomFilter.get_sketch_factory( self.sketch_size, self.num_bloom_filter_hashes), estimator=UnionEstimator(), sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state)) noised_estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig( name='log_bloom_filter-first_moment_log', sketch_factory=LogarithmicBloomFilter.get_sketch_factory( self.sketch_size), estimator=FirstMomentEstimator( method='log', denoiser=SurrealDenoiser( probability=self.noiser_flip_probability)), sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state)) noised_estimator_config_exponential_bloom_filter = SketchEstimatorConfig( name='exp_bloom_filter-first_moment_exp', sketch_factory=ExponentialBloomFilter.get_sketch_factory( self.sketch_size, self.exponential_bloom_filter_decay_rate), estimator=FirstMomentEstimator( method='exp', denoiser=SurrealDenoiser( probability=self.noiser_flip_probability)), sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state)) noised_estimator_config_voc = SketchEstimatorConfig( name='vector_of_counts-sequential', sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size), estimator=SequentialEstimator(), sketch_noiser=LaplaceNoiser()) noised_estimator_config_exact = SketchEstimatorConfig( name='exact_set-lossless', sketch_factory=ExactSet.get_sketch_factory(), estimator=LosslessEstimator(), sketch_noiser=AddRandomElementsNoiser(1, self.noise_random_state)) noised_config_list = [ noised_estimator_config_exact, noised_estimator_config_cascading_legions, noised_estimator_config_bloom_filter, noised_estimator_config_logarithmic_bloom_filter, noised_estimator_config_exponential_bloom_filter, noised_estimator_config_voc, ] self.name_to_noised_estimator_config = { config.name: config for config in noised_config_list }
def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') estimator_config_cascading_legions = SketchEstimatorConfig( name='cascading-legions', sketch_factory=CascadingLegions.get_sketch_factory( FLAGS.sketch_size, FLAGS.sketch_size), estimator=Estimator()) estimator_config_bloom_filter = SketchEstimatorConfig( name='bloom_filter-union_estimator', sketch_factory=BloomFilter.get_sketch_factory( FLAGS.sketch_size, FLAGS.num_bloom_filter_hashes), estimator=UnionEstimator()) estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig( name='log_bloom_filter-first_moment_log', sketch_factory=LogarithmicBloomFilter.get_sketch_factory( FLAGS.sketch_size), estimator=FirstMomentEstimator(method='log')) estimator_config_exponential_bloom_filter = SketchEstimatorConfig( name='exp_bloom_filter-first_moment_exp', sketch_factory=ExponentialBloomFilter.get_sketch_factory( FLAGS.sketch_size, FLAGS.exponential_bloom_filter_decay_rate), estimator=FirstMomentEstimator(method='exp')) estimator_config_voc = SketchEstimatorConfig( name='vector_of_counts-sequential', sketch_factory=VectorOfCounts.get_sketch_factory(FLAGS.sketch_size), estimator=SequentialEstimator()) estimator_config_hll = SketchEstimatorConfig( name='hll++', sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(FLAGS.sketch_size), estimator=HllCardinality()) estimator_config_exact = SketchEstimatorConfig( name='exact_set-lossless', sketch_factory=ExactSet.get_sketch_factory(), estimator=LosslessEstimator()) estimator_config_list = [ estimator_config_bloom_filter, estimator_config_logarithmic_bloom_filter, estimator_config_exponential_bloom_filter, estimator_config_cascading_legions, estimator_config_exact, estimator_config_hll, estimator_config_voc, ] name_to_estimator_config = { 'bloom_filter': estimator_config_bloom_filter, 'logarithmic_bloom_filter': estimator_config_logarithmic_bloom_filter, 'exponential_bloom_filter': estimator_config_exponential_bloom_filter, 'cascading_legions': estimator_config_cascading_legions, 'exact_set': estimator_config_exact, 'hll++': estimator_config_hll, 'vector_of_counts': estimator_config_voc, } set_generator_factory = ( set_generator.IndependentSetGenerator. get_generator_factory_with_num_and_size( universe_size=FLAGS.universe_size, num_sets=FLAGS.number_of_sets, set_size=FLAGS.set_size)) for estimator_method_config in estimator_config_list: print(f'Calculations for {estimator_method_config.name}') set_rs = np.random.RandomState(1) sketch_rs = np.random.RandomState(1) simulator = Simulator( num_runs=FLAGS.number_of_trials, set_generator_factory=set_generator_factory, sketch_estimator_config=estimator_method_config, set_random_state=set_rs, sketch_random_state=sketch_rs) _, agg_data = simulator.run_all_and_aggregate() print(f'Aggregate Statistics for {estimator_method_config.name}') print(agg_data)
def setUp(self): super(InteroperabilityTest, self).setUp() self.number_of_trials = 2 self.universe_size = 2000 self.set_size = 5 self.large_set_size = 6 self.small_set_size = 3 self.sketch_size = 64 self.number_of_sets = 2 self.num_large_sets = 1 self.num_small_sets = 3 self.order = set_generator.ORDER_RANDOM self.user_activity_association = ( set_generator.USER_ACTIVITY_ASSOCIATION_INDEPENDENT) self.shared_prop = 0.2 self.num_bloom_filter_hashes = 2 self.exponential_bloom_filter_decay_rate = 10 self.noiser_epsilon = np.log(3) self.noiser_flip_probability = .25 self.set_random_state = np.random.RandomState(42) self.sketch_random_state = np.random.RandomState(137) self.noise_random_state = np.random.RandomState(3) # non-noised estimators estimator_config_cascading_legions = EstimatorConfig( sketch_factory=CascadingLegions.get_sketch_factory( self.sketch_size, self.sketch_size), estimator=Estimator(), sketch_noiser=None, estimate_noiser=None) estimator_config_bloom_filter = EstimatorConfig( sketch_factory=BloomFilter.get_sketch_factory( self.sketch_size, self.num_bloom_filter_hashes), estimator=UnionEstimator(), sketch_noiser=None, estimate_noiser=None) estimator_config_logarithmic_bloom_filter = EstimatorConfig( sketch_factory=LogarithmicBloomFilter.get_sketch_factory( self.sketch_size), estimator=FirstMomentEstimator(method='log'), sketch_noiser=None, estimate_noiser=None) estimator_config_exponential_bloom_filter = EstimatorConfig( sketch_factory=ExponentialBloomFilter.get_sketch_factory( self.sketch_size, self.exponential_bloom_filter_decay_rate), estimator=FirstMomentEstimator(method='exp'), sketch_noiser=None, estimate_noiser=None) estimator_config_geometric_bloom_filter = EstimatorConfig( sketch_factory=GeometricBloomFilter.get_sketch_factory( self.sketch_size), estimator=GeometricUnionEstimator(), sketch_noiser=None, estimate_noiser=None) estimator_config_voc = EstimatorConfig( sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size), estimator=SequentialEstimator(), sketch_noiser=None, estimate_noiser=None) estimator_config_exact = EstimatorConfig( sketch_factory=ExactSet.get_sketch_factory(), estimator=LosslessEstimator(), sketch_noiser=None, estimate_noiser=None) estimator_config_hll = EstimatorConfig( sketch_factory=HyperLogLogPlusPlus.get_sketch_factory( self.sketch_size), estimator=HllCardinality(), sketch_noiser=None, estimate_noiser=None) self.name_to_non_noised_estimator_config = { 'exact_set': estimator_config_exact, 'cascading_legions': estimator_config_cascading_legions, 'bloom_filter': estimator_config_bloom_filter, 'logarithmic_bloom_filter': estimator_config_logarithmic_bloom_filter, 'exponential_bloom_filter': estimator_config_exponential_bloom_filter, 'geometric_bloom_filter': estimator_config_geometric_bloom_filter, 'vector_of_counts': estimator_config_voc, 'hll': estimator_config_hll, } # noised estimators noised_estimator_config_cascading_legions = EstimatorConfig( sketch_factory=CascadingLegions.get_sketch_factory( self.sketch_size, self.sketch_size), estimator=Estimator(), sketch_noiser=Noiser(self.noiser_flip_probability), estimate_noiser=None) noised_estimator_config_bloom_filter = EstimatorConfig( sketch_factory=BloomFilter.get_sketch_factory( self.sketch_size, self.num_bloom_filter_hashes), estimator=UnionEstimator(), sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state), estimate_noiser=None) noised_estimator_config_logarithmic_bloom_filter = EstimatorConfig( sketch_factory=LogarithmicBloomFilter.get_sketch_factory( self.sketch_size), estimator=FirstMomentEstimator( method='log', denoiser=SurrealDenoiser( probability=self.noiser_flip_probability)), sketch_noiser=None, estimate_noiser=None) noised_estimator_config_exponential_bloom_filter = EstimatorConfig( sketch_factory=ExponentialBloomFilter.get_sketch_factory( self.sketch_size, self.exponential_bloom_filter_decay_rate), estimator=FirstMomentEstimator( method='exp', denoiser=SurrealDenoiser( probability=self.noiser_flip_probability)), sketch_noiser=None, estimate_noiser=None) noised_estimator_config_geometric_bloom_filter = EstimatorConfig( sketch_factory=GeometricBloomFilter.get_sketch_factory( self.sketch_size), estimator=GeometricUnionEstimator(), sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state), estimate_noiser=None) noised_estimator_config_voc = EstimatorConfig( sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size), estimator=SequentialEstimator(), sketch_noiser=LaplaceNoiser(), estimate_noiser=None) noised_estimator_config_exact = EstimatorConfig( sketch_factory=ExactSet.get_sketch_factory(), estimator=LosslessEstimator(), sketch_noiser=AddRandomElementsNoiser(1, self.noise_random_state), estimate_noiser=None) self.name_to_noised_estimator_config = { 'exact_set': noised_estimator_config_exact, 'cascading_legions': noised_estimator_config_cascading_legions, 'bloom_filter': noised_estimator_config_bloom_filter, 'logarithmic_bloom_filter': noised_estimator_config_logarithmic_bloom_filter, 'exponential_bloom_filter': noised_estimator_config_exponential_bloom_filter, 'geometric_bloom_filter': noised_estimator_config_geometric_bloom_filter, 'vector_of_counts': noised_estimator_config_voc, }