def test_clip_empty_vector_of_count(self, epsilon, clip_threshold,
expected):
sketch = VectorOfCounts(num_buckets=2, random_seed=0)
sketch.stats = np.array([2, 2])
pairwise_estimator = PairwiseEstimator(clip=True,
epsilon=epsilon,
clip_threshold=clip_threshold)
res = pairwise_estimator.clip_empty_vector_of_count(sketch)
np.testing.assert_array_equal(res.stats, expected)
def _create_sketches(this_stats, that_stats):
sketches = []
for stats in [this_stats, that_stats]:
sketch = None
if stats is not None:
sketch = VectorOfCounts(num_buckets=2, random_seed=1)
sketch.stats = np.array(stats)
sketches.append(sketch)
return sketches
def test_merge_no_clip(self):
sketch_list = []
for _ in range(2):
sketch = VectorOfCounts(num_buckets=2, random_seed=2)
sketch.add_ids([1])
sketch_list.append(sketch)
pairwise_estimator = PairwiseEstimator()
merged = pairwise_estimator.merge(sketch_list[0], sketch_list[1])
np.testing.assert_array_equal(np.sort(merged.stats), np.array([0,
1.5]))
def test_estimate_cardinality_no_clip(self):
sketch_list = []
for _ in range(3):
sketch = VectorOfCounts(num_buckets=2, random_seed=3)
sketch.add_ids([1])
sketch_list.append(sketch)
estimator = SequentialEstimator()
result = estimator(sketch_list)[0]
actual = 1.75
self.assertEqual(result, actual)
def test_has_full_intersection(self):
pairwise_estimator = PairwiseEstimator()
this = VectorOfCounts(num_buckets=64, random_seed=2)
this.add_ids(range(100))
that = VectorOfCounts(num_buckets=64, random_seed=2)
that.add_ids(range(100))
intersection_cardinality = pairwise_estimator._intersection(this, that)
self.assertTrue(
pairwise_estimator.has_full_intersection(intersection_cardinality,
this, that))
def test_has_zero_intersection(self):
pairwise_estimator = PairwiseEstimator()
this = VectorOfCounts(num_buckets=64, random_seed=2)
this.add_ids(range(100))
# Clip relies on hypothesis testing and hence requires a minimum size
that = VectorOfCounts(num_buckets=64, random_seed=2)
that.add_ids(range(100, 200))
intersection_cardinality = pairwise_estimator._intersection(this, that)
self.assertTrue(
pairwise_estimator.has_zero_intersection(intersection_cardinality,
this, that))
def test_assert_compatible_not_vector_of_count(self):
sketch = VectorOfCounts(num_buckets=4, random_seed=2)
estimator = PairwiseEstimator()
with self.assertRaises(AssertionError):
estimator.assert_compatible(sketch, [])
with self.assertRaises(AssertionError):
estimator.assert_compatible([], sketch)
def test_add_ids_random_state(self):
sketch1 = VectorOfCounts(num_buckets=8, random_seed=0)
sketch1.add_ids(range(4))
sketch2 = VectorOfCounts(num_buckets=8, random_seed=0)
sketch2.add_ids(range(4))
np.testing.assert_array_equal(
sketch1.stats,
sketch2.stats,
err_msg='Two VoC are not the same with the same random seed.')
def test_identity(self):
noiser = IdentityNoiser()
sketch_original = VectorOfCounts(num_buckets=8, random_seed=1)
sketch_noised = noiser(sketch_original)
np.testing.assert_array_equal(
sketch_original.stats,
sketch_noised.stats,
err_msg='IdentityNoiser should not change the sketch.')
def test_get_std_of_intersection(self, epsilon, intersection_cardinality,
expected):
this_sketch = VectorOfCounts(num_buckets=4, random_seed=0)
this_sketch.stats = np.array([2, 2, 0, 0])
that_sketch = VectorOfCounts(num_buckets=4, random_seed=0)
that_sketch.stats = np.array([2, 0, 2, 0])
pairwise_estimator = PairwiseEstimator(clip=True, epsilon=epsilon)
res = pairwise_estimator._get_std_of_intersection(
intersection_cardinality, this_sketch, that_sketch)
self.assertAlmostEqual(res, expected, 2)
def test_evaluate_closeness_to_a_value(self, epsilon,
intersection_cardinality,
value_to_compare_with, expected):
this_sketch = VectorOfCounts(num_buckets=4, random_seed=0)
this_sketch.stats = np.array([2, 2, 0, 0])
that_sketch = VectorOfCounts(num_buckets=4, random_seed=0)
that_sketch.stats = np.array([2, 0, 2, 0])
pairwise_estimator = PairwiseEstimator(clip=True, epsilon=epsilon)
res = pairwise_estimator.evaluate_closeness_to_a_value(
intersection_cardinality, value_to_compare_with, this_sketch,
that_sketch)
self.assertAlmostEqual(res, expected, 2)
def test_estimate_cardinality_with_clip(self):
base_sketch = VectorOfCounts(num_buckets=64, random_seed=3)
base_sketch.add_ids(range(100))
sketch_list_a = [base_sketch]
sketch_list_b = [base_sketch]
for _ in range(3):
empty_sketch = VectorOfCounts(num_buckets=64, random_seed=3)
sketch_list_a.append(empty_sketch) # add empty sketch
sketch_list_b.append(base_sketch) # add same sketch
estimator = SequentialEstimator(clip=True)
result_a = estimator(sketch_list_a)
result_b = estimator(sketch_list_b)
self.assertEqual(result_a, base_sketch.cardinality(),
msg='Fail to detect the no-intersection case.')
self.assertEqual(result_b, base_sketch.cardinality(),
msg='Fail to detect the full-intersection case.')
def test_assert_compatible_not_same_hash_function(self):
sketch1 = VectorOfCounts(num_buckets=4, random_seed=1)
sketch2 = VectorOfCounts(num_buckets=4, random_seed=2)
estimator = PairwiseEstimator()
with self.assertRaises(AssertionError):
estimator.assert_compatible(sketch1, sketch2)
def test_assert_compatible_not_equal_length(self):
sketch1 = VectorOfCounts(num_buckets=4, random_seed=2)
sketch2 = VectorOfCounts(num_buckets=8, random_seed=2)
estimator = PairwiseEstimator()
with self.assertRaises(AssertionError):
estimator.assert_compatible(sketch1, sketch2)
def test_laplace(self):
noiser = LaplaceNoiser()
sketch_original = VectorOfCounts(num_buckets=8, random_seed=1)
sketch_noised = noiser(sketch_original)
for o, n in zip(sketch_original.stats, sketch_noised.stats):
self.assertNotEqual(o, n)
def test_add_ids_multiple_times(self):
sketch = VectorOfCounts(num_buckets=8, random_seed=0)
sketch.add_ids([1])
with self.assertRaises(AssertionError):
sketch.add_ids([1])
def test_add_ids(self):
sketch = VectorOfCounts(num_buckets=8, random_seed=0)
sketch.add_ids([1])
self.assertEqual(sketch.cardinality(), 1)
def setUp(self):
super(InteroperabilityTest, self).setUp()
self.number_of_trials = 2
self.universe_size = 2000
self.set_size = 5
self.large_set_size = 6
self.small_set_size = 3
self.sketch_size = 64
self.number_of_sets = 2
self.num_large_sets = 1
self.num_small_sets = 3
self.order = set_generator.ORDER_RANDOM
self.user_activity_association = (
set_generator.USER_ACTIVITY_ASSOCIATION_INDEPENDENT)
self.shared_prop = 0.2
self.num_bloom_filter_hashes = 2
self.exponential_bloom_filter_decay_rate = 10
self.noiser_epsilon = np.log(3)
self.noiser_flip_probability = .25
self.set_random_state = np.random.RandomState(42)
self.sketch_random_state = np.random.RandomState(137)
self.noise_random_state = np.random.RandomState(3)
# non-noised estimators
estimator_config_cascading_legions = EstimatorConfig(
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_bloom_filter = EstimatorConfig(
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_logarithmic_bloom_filter = EstimatorConfig(
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(method='log'),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_exponential_bloom_filter = EstimatorConfig(
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(method='exp'),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_geometric_bloom_filter = EstimatorConfig(
sketch_factory=GeometricBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=GeometricUnionEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_voc = EstimatorConfig(
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_exact = EstimatorConfig(
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator(),
sketch_noiser=None,
estimate_noiser=None)
estimator_config_hll = EstimatorConfig(
sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(
self.sketch_size),
estimator=HllCardinality(),
sketch_noiser=None,
estimate_noiser=None)
self.name_to_non_noised_estimator_config = {
'exact_set': estimator_config_exact,
'cascading_legions': estimator_config_cascading_legions,
'bloom_filter': estimator_config_bloom_filter,
'logarithmic_bloom_filter':
estimator_config_logarithmic_bloom_filter,
'exponential_bloom_filter':
estimator_config_exponential_bloom_filter,
'geometric_bloom_filter': estimator_config_geometric_bloom_filter,
'vector_of_counts': estimator_config_voc,
'hll': estimator_config_hll,
}
# noised estimators
noised_estimator_config_cascading_legions = EstimatorConfig(
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator(),
sketch_noiser=Noiser(self.noiser_flip_probability),
estimate_noiser=None)
noised_estimator_config_bloom_filter = EstimatorConfig(
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator(),
sketch_noiser=BlipNoiser(self.noiser_epsilon,
self.noise_random_state),
estimate_noiser=None)
noised_estimator_config_logarithmic_bloom_filter = EstimatorConfig(
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(
method='log',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=None,
estimate_noiser=None)
noised_estimator_config_exponential_bloom_filter = EstimatorConfig(
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(
method='exp',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=None,
estimate_noiser=None)
noised_estimator_config_geometric_bloom_filter = EstimatorConfig(
sketch_factory=GeometricBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=GeometricUnionEstimator(),
sketch_noiser=BlipNoiser(self.noiser_epsilon,
self.noise_random_state),
estimate_noiser=None)
noised_estimator_config_voc = EstimatorConfig(
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator(),
sketch_noiser=LaplaceNoiser(),
estimate_noiser=None)
noised_estimator_config_exact = EstimatorConfig(
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator(),
sketch_noiser=AddRandomElementsNoiser(1, self.noise_random_state),
estimate_noiser=None)
self.name_to_noised_estimator_config = {
'exact_set': noised_estimator_config_exact,
'cascading_legions': noised_estimator_config_cascading_legions,
'bloom_filter': noised_estimator_config_bloom_filter,
'logarithmic_bloom_filter':
noised_estimator_config_logarithmic_bloom_filter,
'exponential_bloom_filter':
noised_estimator_config_exponential_bloom_filter,
'geometric_bloom_filter':
noised_estimator_config_geometric_bloom_filter,
'vector_of_counts': noised_estimator_config_voc,
}
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
estimator_config_cascading_legions = SketchEstimatorConfig(
name='cascading-legions',
sketch_factory=CascadingLegions.get_sketch_factory(
FLAGS.sketch_size, FLAGS.sketch_size),
estimator=Estimator())
estimator_config_bloom_filter = SketchEstimatorConfig(
name='bloom_filter-union_estimator',
sketch_factory=BloomFilter.get_sketch_factory(
FLAGS.sketch_size, FLAGS.num_bloom_filter_hashes),
estimator=UnionEstimator())
estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig(
name='log_bloom_filter-first_moment_log',
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
FLAGS.sketch_size),
estimator=FirstMomentEstimator(method='log'))
estimator_config_exponential_bloom_filter = SketchEstimatorConfig(
name='exp_bloom_filter-first_moment_exp',
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
FLAGS.sketch_size, FLAGS.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(method='exp'))
estimator_config_voc = SketchEstimatorConfig(
name='vector_of_counts-sequential',
sketch_factory=VectorOfCounts.get_sketch_factory(FLAGS.sketch_size),
estimator=SequentialEstimator())
estimator_config_hll = SketchEstimatorConfig(
name='hll++',
sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(FLAGS.sketch_size),
estimator=HllCardinality())
estimator_config_exact = SketchEstimatorConfig(
name='exact_set-lossless',
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator())
estimator_config_list = [
estimator_config_bloom_filter,
estimator_config_logarithmic_bloom_filter,
estimator_config_exponential_bloom_filter,
estimator_config_cascading_legions,
estimator_config_exact,
estimator_config_hll,
estimator_config_voc,
]
name_to_estimator_config = {
'bloom_filter': estimator_config_bloom_filter,
'logarithmic_bloom_filter': estimator_config_logarithmic_bloom_filter,
'exponential_bloom_filter': estimator_config_exponential_bloom_filter,
'cascading_legions': estimator_config_cascading_legions,
'exact_set': estimator_config_exact,
'hll++': estimator_config_hll,
'vector_of_counts': estimator_config_voc,
}
set_generator_factory = (
set_generator.IndependentSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=FLAGS.universe_size,
num_sets=FLAGS.number_of_sets,
set_size=FLAGS.set_size))
for estimator_method_config in estimator_config_list:
print(f'Calculations for {estimator_method_config.name}')
set_rs = np.random.RandomState(1)
sketch_rs = np.random.RandomState(1)
simulator = Simulator(
num_runs=FLAGS.number_of_trials,
set_generator_factory=set_generator_factory,
sketch_estimator_config=estimator_method_config,
set_random_state=set_rs,
sketch_random_state=sketch_rs)
_, agg_data = simulator.run_all_and_aggregate()
print(f'Aggregate Statistics for {estimator_method_config.name}')
print(agg_data)
def test_get_std_of_sketch_sum(self, epsilon, expected):
sketch = VectorOfCounts(num_buckets=2, random_seed=0)
sketch.stats = np.array([2, 2])
pairwise_estimator = PairwiseEstimator(clip=True, epsilon=epsilon)
res = pairwise_estimator._get_std_of_sketch_sum(sketch)
self.assertEqual(res, expected)
def test_merge_with_clip(self):
this_sketch = VectorOfCounts(num_buckets=64, random_seed=2)
this_sketch.add_ids(range(100))
# First test no intersection
that_sketch = VectorOfCounts(num_buckets=64, random_seed=2)
that_sketch.add_ids(range(100, 200))
pairwise_estimator = PairwiseEstimator(clip=True)
merged = pairwise_estimator.merge(this_sketch, that_sketch)
np.testing.assert_array_equal(
x=merged.stats,
y=this_sketch.stats + that_sketch.stats,
err_msg='Fail to detect the no-intersection case.')
# Then test full intersection
that_sketch = VectorOfCounts(num_buckets=64, random_seed=2)
that_sketch.add_ids(range(100))
merged = pairwise_estimator.merge(this_sketch, that_sketch)
np.testing.assert_array_equal(
x=merged.stats,
y=this_sketch.stats,
err_msg='Fail to detect the full-intersection case.')
def setUp(self):
super(InteroperabilityTest, self).setUp()
self.number_of_trials = 2
self.universe_size = 2000
self.set_size_list = [5, 7, 9]
self.large_set_size = 6
self.small_set_size = 3
self.sketch_size = 128
self.number_of_sets = 3
self.set_size = 50
self.num_large_sets = 1
self.num_small_sets = 3
self.order = set_generator.ORDER_RANDOM
self.user_activity_association = (
set_generator.USER_ACTIVITY_ASSOCIATION_INDEPENDENT)
self.shared_prop = 0.2
self.num_bloom_filter_hashes = 2
self.exponential_bloom_filter_decay_rate = 10
self.noiser_epsilon = np.log(3)
self.noiser_flip_probability = .25
self.set_random_state = np.random.RandomState(42)
self.sketch_random_state = np.random.RandomState(137)
self.noise_random_state = np.random.RandomState(3)
# non-noised estimators
estimator_config_cascading_legions = SketchEstimatorConfig(
name='cascading_legions',
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator())
estimator_config_bloom_filter = SketchEstimatorConfig(
name='bloom_filter-union_estimator',
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator())
estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig(
name='log_bloom_filter-first_moment_log',
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(method='log'))
estimator_config_exponential_bloom_filter = SketchEstimatorConfig(
name='exp_bloom_filter-first_moment_exp',
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(method='exp'))
estimator_config_voc = SketchEstimatorConfig(
name='vector_of_counts-sequential',
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator())
estimator_config_exact = SketchEstimatorConfig(
name='exact_set-lossless',
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator())
estimator_config_hll = SketchEstimatorConfig(
name='hyper_log_log',
sketch_factory=HyperLogLogPlusPlus.get_sketch_factory(self.sketch_size),
estimator=HllCardinality())
config_list = [
estimator_config_exact,
estimator_config_cascading_legions,
estimator_config_bloom_filter,
estimator_config_logarithmic_bloom_filter,
estimator_config_exponential_bloom_filter,
estimator_config_voc,
estimator_config_hll,
]
self.name_to_non_noised_estimator_config = {
config.name: config for config in config_list
}
# noised estimators
noised_estimator_config_cascading_legions = SketchEstimatorConfig(
name='cascading_legions',
sketch_factory=CascadingLegions.get_sketch_factory(
self.sketch_size, self.sketch_size),
estimator=Estimator(),
sketch_noiser=Noiser(self.noiser_flip_probability))
noised_estimator_config_bloom_filter = SketchEstimatorConfig(
name='bloom_filter-union_estimator',
sketch_factory=BloomFilter.get_sketch_factory(
self.sketch_size, self.num_bloom_filter_hashes),
estimator=UnionEstimator(),
sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state))
noised_estimator_config_logarithmic_bloom_filter = SketchEstimatorConfig(
name='log_bloom_filter-first_moment_log',
sketch_factory=LogarithmicBloomFilter.get_sketch_factory(
self.sketch_size),
estimator=FirstMomentEstimator(
method='log',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state))
noised_estimator_config_exponential_bloom_filter = SketchEstimatorConfig(
name='exp_bloom_filter-first_moment_exp',
sketch_factory=ExponentialBloomFilter.get_sketch_factory(
self.sketch_size, self.exponential_bloom_filter_decay_rate),
estimator=FirstMomentEstimator(
method='exp',
denoiser=SurrealDenoiser(
probability=self.noiser_flip_probability)),
sketch_noiser=BlipNoiser(self.noiser_epsilon, self.noise_random_state))
noised_estimator_config_voc = SketchEstimatorConfig(
name='vector_of_counts-sequential',
sketch_factory=VectorOfCounts.get_sketch_factory(self.sketch_size),
estimator=SequentialEstimator(),
sketch_noiser=LaplaceNoiser())
noised_estimator_config_exact = SketchEstimatorConfig(
name='exact_set-lossless',
sketch_factory=ExactSet.get_sketch_factory(),
estimator=LosslessEstimator(),
sketch_noiser=AddRandomElementsNoiser(1, self.noise_random_state))
noised_config_list = [
noised_estimator_config_exact,
noised_estimator_config_cascading_legions,
noised_estimator_config_bloom_filter,
noised_estimator_config_logarithmic_bloom_filter,
noised_estimator_config_exponential_bloom_filter,
noised_estimator_config_voc,
]
self.name_to_noised_estimator_config = {
config.name: config for config in noised_config_list
}
