def test_cuckoo_overflow(self):
# Insert items to this cuckoo filter
cuckoo = CuckooFilter(self.TOTAL_ITEMS,
bits_per_item=32,
use_redis=False)
i = 0
try:
for i in range(100):
cuckoo.add(i)
except NotEnoughSpaceException:
print("stop in {0}".format(i))
# Check if previously inserted items are in the filter, expected
# true for all items
num_inserted = i
for i in range(num_inserted):
self.assertEquals(cuckoo.contain(i),
True,
msg="Not found {0}".format(i))
# Check non-existing items, a few false positives expected
total_queries = 0
false_queries = 0
for i in range(2 * self.TOTAL_ITEMS):
if cuckoo.contain(i):
false_queries += 1
total_queries += 1
false_positive_rate = 100.0 * false_queries / total_queries
def single_CF(item_num, fp, exp_block_num=6):
"""
"""
capacity = item_num
single_table_length = int(capacity / 4 / exp_block_num)
single_capacity = single_table_length * 0.9375 * 4
false_positive = fp
single_false_positive = 1 - (1.0 - false_positive)**(single_capacity /
capacity)
fingerprint_size_double = math.ceil(
math.log(8.0 / single_false_positive, 2))
dcf_fingerprint_size = fingerprint_size_double
single_cf = CuckooFilter(capacity=single_table_length,
fingerprint_size=dcf_fingerprint_size)
return [single_cf]
def dcf_insert(data, sketch):
cur = len(sketch) - 1
# while sketch[cur].size == 0 and cur > 1:
# cur -= 1
for i in range(len(data)):
result = sketch[cur].insert(data[i])
if result != "yes":
cur += 1
if cur >= len(sketch):
single_cf = CuckooFilter(
capacity=sketch[0].capacity,
fingerprint_size=sketch[0].fingerprint_size)
sketch.append(0)
sketch[cur] = single_cf
sketch[cur].buckets[result[0]].insert(result[1])
sketch[cur].size += 1
else:
sketch[cur].buckets[result[0]].insert(result[1])
sketch[cur].size += 1
return True
def Dynamic_CF(item_num, fp, exp_block_num=6):
"""
"""
capacity = item_num
single_table_length = int(capacity / 4 / exp_block_num)
single_capacity = single_table_length * 0.9375 * 4
false_positive = fp
single_false_positive = 1 - (1.0 - false_positive)**(single_capacity /
capacity)
fingerprint_size_double = math.ceil(
math.log(8.0 / single_false_positive, 2))
# if fingerprint_size_double > 0 and fingerprint_size_double <= 4:
# dcf_fingerprint_size = 4
# elif fingerprint_size_double > 4 and fingerprint_size_double <= 8:
# dcf_fingerprint_size = 8
# elif fingerprint_size_double > 8 and fingerprint_size_double <= 12:
# dcf_fingerprint_size = 12
# elif fingerprint_size_double > 12 and fingerprint_size_double <= 16:
# dcf_fingerprint_size = 16
# elif fingerprint_size_double > 16 and fingerprint_size_double <= 24:
# dcf_fingerprint_size = 16
# elif fingerprint_size_double > 24 and fingerprint_size_double <= 32:
# dcf_fingerprint_size = 16
# else:
# print("fingerprint out of range!!!")
# dcf_fingerprint_size = 16
dcf_fingerprint_size = fingerprint_size_double
DCF_list = [None] * exp_block_num
for i in range(exp_block_num):
DCF_list[i] = CuckooFilter(capacity=single_table_length,
fingerprint_size=dcf_fingerprint_size)
return DCF_list
from cuckoofilter import CuckooFilter
from random import randint
import timeit
cuckoo_redis = CuckooFilter(1000, bits_per_item=32, use_redis=True)
print("Add 2, 200, and 300")
cuckoo_redis.add(2)
cuckoo_redis.add(200)
cuckoo_redis.add(300)
print("2 is present? {0}".format(cuckoo_redis.contain(2)))
print("200 is present? {0}".format(cuckoo_redis.contain(2)))
cuckoo_redis.delete(2)
print("Delete 2, it should be false? {0}".format(cuckoo_redis.contain(2)))
cuckoo_redis.delete(200)
print("Delete 200, it should be false? {0}".format(cuckoo_redis.contain(200)))
cuckoo_redis.delete(999)
print("in redis")
cuckoo_redis = CuckooFilter(1000, bits_per_item=32, use_redis=True)
print(
timeit.Timer(lambda: cuckoo_redis.add(randint(0, 1000))).timeit(
number=200))
cuckoo = CuckooFilter(1000, bits_per_item=32, use_redis=False)
print("in memory (not storing at all the value!)")
print(timeit.Timer(lambda: cuckoo.add(randint(0, 1000))).timeit(number=200))
def setUp(self):
self.capacity = 1000
self.fingerprint_size = 2
self.cuckoofilter = CuckooFilter(self.capacity, self.fingerprint_size)
class CuckooFilterTest(unittest.TestCase):
def setUp(self):
self.capacity = 1000
self.fingerprint_size = 2
self.cuckoofilter = CuckooFilter(self.capacity, self.fingerprint_size)
def test_cuckoo_filter_creation(self):
# test to cuckoo filter creation
self.assertIsNotNone(self.cuckoofilter)
# check that it uses the input arguments.
self.assertEqual(self.capacity, self.cuckoofilter.get_capacity())
def test_insertions(self):
# insert a string into the cuckoo filter
initial_size = self.cuckoofilter.get_size()
self.cuckoofilter.insert("James")
self.assertEqual(initial_size + 1, self.cuckoofilter.get_size())
def test_deletions(self):
self.cuckoofilter.insert("New")
self.assertTrue(self.cuckoofilter.remove("New"))
self.assertFalse("New" in self.cuckoofilter)
# test deleting an element that was not inserted
self.assertFalse(self.cuckoofilter.remove("Not inserted"))
def test_containing(self):
self.cuckoofilter.insert("finally")
self.assertTrue("finally" in self.cuckoofilter)
def test_that_cuckoo_filter_fills_up(self):
with self.assertRaises(Exception) as context:
for i in range(10 * self.capacity):
self.cuckoofilter.insert(str(i))
self.assertTrue(
"CuckooFilter has filled up!" in str(context.exception))
delete_result[x].append(y)
elif x == tem + 1:
delete_result.append([])
delete_result[x].append(y)
tem += 1
else:
distance = x - tem
for j in range(distance - 1):
delete_result.append(["non"])
tem += 1
delete_result.append([])
delete_result[x].append(y)
tem += 1
##过滤掉到达但未离开的元素
CF_filtrate = CuckooFilter(capacity=2 ** 21, bucket_size=4, fingerprint_size=18)
# CF_mur_filter = CuckooFilter_mur(capacity=2 ** 21, bucket_size=4, fingerprint_size=18)
for i in range(len(delete_result)):
if delete_result[i] == "non":
print("non appear in delete")
else:
for j in range(len(delete_result[i])):
CF_filtrate.insert(delete_result[i][j])
# CF_mur_filter.insert(delete_result[i][j])
count = 0
for i in range(len(insert_result)):
if insert_result[i] == "non":
print("non appear in insert")
else:
for j in range(len(insert_result[i])):
def test_hash(self):
cuckoo = CuckooFilter(self.TOTAL_ITEMS,
bits_per_item=32,
use_redis=False)
i, h = cuckoo._generate_index_tag_hash(0)
self.assertEquals(h, 213)
def disabled_test_cuckoo_redis(self):
cuckoo = CuckooFilter(self.TOTAL_ITEMS,
bits_per_item=32,
use_redis=True)
cuckoo.add(2)
cuckoo.add(200)
self.assertTrue(cuckoo.contain(2))
self.assertTrue(cuckoo.contain(200))
cuckoo.delete(2)
self.assertFalse(cuckoo.contain(2))
def test_cuckoo_insert(items, n):
c = CuckooFilter(n, 2)
for item in items:
c.insert(item)
return c