def test_cuckoo_overflow(self): # Insert items to this cuckoo filter cuckoo = CuckooFilter(self.TOTAL_ITEMS, bits_per_item=32, use_redis=False) i = 0 try: for i in range(100): cuckoo.add(i) except NotEnoughSpaceException: print("stop in {0}".format(i)) # Check if previously inserted items are in the filter, expected # true for all items num_inserted = i for i in range(num_inserted): self.assertEquals(cuckoo.contain(i), True, msg="Not found {0}".format(i)) # Check non-existing items, a few false positives expected total_queries = 0 false_queries = 0 for i in range(2 * self.TOTAL_ITEMS): if cuckoo.contain(i): false_queries += 1 total_queries += 1 false_positive_rate = 100.0 * false_queries / total_queries
def single_CF(item_num, fp, exp_block_num=6): """ """ capacity = item_num single_table_length = int(capacity / 4 / exp_block_num) single_capacity = single_table_length * 0.9375 * 4 false_positive = fp single_false_positive = 1 - (1.0 - false_positive)**(single_capacity / capacity) fingerprint_size_double = math.ceil( math.log(8.0 / single_false_positive, 2)) dcf_fingerprint_size = fingerprint_size_double single_cf = CuckooFilter(capacity=single_table_length, fingerprint_size=dcf_fingerprint_size) return [single_cf]
def dcf_insert(data, sketch): cur = len(sketch) - 1 # while sketch[cur].size == 0 and cur > 1: # cur -= 1 for i in range(len(data)): result = sketch[cur].insert(data[i]) if result != "yes": cur += 1 if cur >= len(sketch): single_cf = CuckooFilter( capacity=sketch[0].capacity, fingerprint_size=sketch[0].fingerprint_size) sketch.append(0) sketch[cur] = single_cf sketch[cur].buckets[result[0]].insert(result[1]) sketch[cur].size += 1 else: sketch[cur].buckets[result[0]].insert(result[1]) sketch[cur].size += 1 return True
def Dynamic_CF(item_num, fp, exp_block_num=6): """ """ capacity = item_num single_table_length = int(capacity / 4 / exp_block_num) single_capacity = single_table_length * 0.9375 * 4 false_positive = fp single_false_positive = 1 - (1.0 - false_positive)**(single_capacity / capacity) fingerprint_size_double = math.ceil( math.log(8.0 / single_false_positive, 2)) # if fingerprint_size_double > 0 and fingerprint_size_double <= 4: # dcf_fingerprint_size = 4 # elif fingerprint_size_double > 4 and fingerprint_size_double <= 8: # dcf_fingerprint_size = 8 # elif fingerprint_size_double > 8 and fingerprint_size_double <= 12: # dcf_fingerprint_size = 12 # elif fingerprint_size_double > 12 and fingerprint_size_double <= 16: # dcf_fingerprint_size = 16 # elif fingerprint_size_double > 16 and fingerprint_size_double <= 24: # dcf_fingerprint_size = 16 # elif fingerprint_size_double > 24 and fingerprint_size_double <= 32: # dcf_fingerprint_size = 16 # else: # print("fingerprint out of range!!!") # dcf_fingerprint_size = 16 dcf_fingerprint_size = fingerprint_size_double DCF_list = [None] * exp_block_num for i in range(exp_block_num): DCF_list[i] = CuckooFilter(capacity=single_table_length, fingerprint_size=dcf_fingerprint_size) return DCF_list
from cuckoofilter import CuckooFilter from random import randint import timeit cuckoo_redis = CuckooFilter(1000, bits_per_item=32, use_redis=True) print("Add 2, 200, and 300") cuckoo_redis.add(2) cuckoo_redis.add(200) cuckoo_redis.add(300) print("2 is present? {0}".format(cuckoo_redis.contain(2))) print("200 is present? {0}".format(cuckoo_redis.contain(2))) cuckoo_redis.delete(2) print("Delete 2, it should be false? {0}".format(cuckoo_redis.contain(2))) cuckoo_redis.delete(200) print("Delete 200, it should be false? {0}".format(cuckoo_redis.contain(200))) cuckoo_redis.delete(999) print("in redis") cuckoo_redis = CuckooFilter(1000, bits_per_item=32, use_redis=True) print( timeit.Timer(lambda: cuckoo_redis.add(randint(0, 1000))).timeit( number=200)) cuckoo = CuckooFilter(1000, bits_per_item=32, use_redis=False) print("in memory (not storing at all the value!)") print(timeit.Timer(lambda: cuckoo.add(randint(0, 1000))).timeit(number=200))
def setUp(self): self.capacity = 1000 self.fingerprint_size = 2 self.cuckoofilter = CuckooFilter(self.capacity, self.fingerprint_size)
class CuckooFilterTest(unittest.TestCase): def setUp(self): self.capacity = 1000 self.fingerprint_size = 2 self.cuckoofilter = CuckooFilter(self.capacity, self.fingerprint_size) def test_cuckoo_filter_creation(self): # test to cuckoo filter creation self.assertIsNotNone(self.cuckoofilter) # check that it uses the input arguments. self.assertEqual(self.capacity, self.cuckoofilter.get_capacity()) def test_insertions(self): # insert a string into the cuckoo filter initial_size = self.cuckoofilter.get_size() self.cuckoofilter.insert("James") self.assertEqual(initial_size + 1, self.cuckoofilter.get_size()) def test_deletions(self): self.cuckoofilter.insert("New") self.assertTrue(self.cuckoofilter.remove("New")) self.assertFalse("New" in self.cuckoofilter) # test deleting an element that was not inserted self.assertFalse(self.cuckoofilter.remove("Not inserted")) def test_containing(self): self.cuckoofilter.insert("finally") self.assertTrue("finally" in self.cuckoofilter) def test_that_cuckoo_filter_fills_up(self): with self.assertRaises(Exception) as context: for i in range(10 * self.capacity): self.cuckoofilter.insert(str(i)) self.assertTrue( "CuckooFilter has filled up!" in str(context.exception))
delete_result[x].append(y) elif x == tem + 1: delete_result.append([]) delete_result[x].append(y) tem += 1 else: distance = x - tem for j in range(distance - 1): delete_result.append(["non"]) tem += 1 delete_result.append([]) delete_result[x].append(y) tem += 1 ##过滤掉到达但未离开的元素 CF_filtrate = CuckooFilter(capacity=2 ** 21, bucket_size=4, fingerprint_size=18) # CF_mur_filter = CuckooFilter_mur(capacity=2 ** 21, bucket_size=4, fingerprint_size=18) for i in range(len(delete_result)): if delete_result[i] == "non": print("non appear in delete") else: for j in range(len(delete_result[i])): CF_filtrate.insert(delete_result[i][j]) # CF_mur_filter.insert(delete_result[i][j]) count = 0 for i in range(len(insert_result)): if insert_result[i] == "non": print("non appear in insert") else: for j in range(len(insert_result[i])):
def test_hash(self): cuckoo = CuckooFilter(self.TOTAL_ITEMS, bits_per_item=32, use_redis=False) i, h = cuckoo._generate_index_tag_hash(0) self.assertEquals(h, 213)
def disabled_test_cuckoo_redis(self): cuckoo = CuckooFilter(self.TOTAL_ITEMS, bits_per_item=32, use_redis=True) cuckoo.add(2) cuckoo.add(200) self.assertTrue(cuckoo.contain(2)) self.assertTrue(cuckoo.contain(200)) cuckoo.delete(2) self.assertFalse(cuckoo.contain(2))
def test_cuckoo_insert(items, n): c = CuckooFilter(n, 2) for item in items: c.insert(item) return c