def test_add(self): aset = AddressSet(self.TABLE_LEN) addr = "".join(chr(b) for b in xrange(20)) self.assertNotIn(addr, aset) aset.add(addr) self.assertIn(addr, aset) self.assertEqual(len(aset), 1)
def test_collision(self): aset = AddressSet(self.TABLE_LEN) # the last HASH_BYTES (1) bytes are the "hash", and only the next BYTES_PER_ADDR (8) rightmost bytes are stored addr1 = "".join(chr(b) for b in xrange(20)) addr2 = addr1.replace(chr(20 - self.HASH_BYTES - self.BYTES_PER_ADDR), "\0") # the leftmost byte that's stored self.collision_tester(aset, addr1, addr2)
def test_collision_fail(self): aset = AddressSet(self.TABLE_LEN) # the last 1 (HASH_BYTES) bytes are the "hash", and only the next 8 (BYTES_PER_ADDR) rightmost bytes are stored addr1 = "".join(chr(b) for b in xrange(20)) addr2 = addr1.replace(chr(20 - self.HASH_BYTES - self.BYTES_PER_ADDR - 1), "\0") # the rightmost byte not stored self.assertRaises(unittest.TestCase.failureException, self.collision_tester, aset, addr1, addr2) self.assertEqual(len(aset), 1)
def test_false_positives(self): aset = AddressSet(131072, bytes_per_addr=5) # reduce bytes_per_addr to increase failure probability rand_byte_count = aset._hash_bytes + aset._bytes_per_addr nonrand_prefix = (20 - rand_byte_count) * "\0" for i in xrange(aset._max_len): aset.add(nonrand_prefix + "".join(chr(random.randrange(256)) for i in xrange(rand_byte_count))) for i in xrange(524288): self.assertNotIn( nonrand_prefix + "".join(chr(random.randrange(256)) for i in xrange(rand_byte_count)), aset)
def test_file(self): aset = AddressSet(self.TABLE_LEN) addr = "".join(chr(b) for b in xrange(20)) aset.add(addr) dbfile = tempfile.TemporaryFile() aset.tofile(dbfile) dbfile.seek(0) aset = AddressSet.fromfile(dbfile) self.assertTrue(dbfile.closed) # should be closed by AddressSet in read-only mode self.assertIn(addr, aset) self.assertEqual(len(aset), 1)
def test_false_positives(self): aset = AddressSet(1024, bytes_per_addr=8) rand_byte_count = aset._hash_bytes + aset._bytes_per_addr nonrand_prefix = (20 - rand_byte_count) * "\0" for i in range(aset._max_len): aset.add(nonrand_prefix + "".join( chr(random.randrange(256)) for i in range(rand_byte_count))) for i in range(8192): self.assertNotIn( nonrand_prefix + "".join( chr(random.randrange(256)) for i in range(rand_byte_count)), aset)
def test_pickle_mmap(self): aset = AddressSet(self.TABLE_LEN) addr = "".join(chr(b) for b in xrange(20)) aset.add(addr) dbfile = tempfile.NamedTemporaryFile(delete=False) try: aset.tofile(dbfile) dbfile.seek(0) aset = AddressSet.fromfile(dbfile) # now it's an mmap pickled = pickle.dumps(aset, protocol=pickle.HIGHEST_PROTOCOL) aset.close() # also closes the file aset = pickle.loads(pickled) self.assertIn(addr, aset) self.assertEqual(len(aset), 1) finally: aset.close() dbfile.close() os.remove(dbfile.name)
def test_file_update(self): aset = AddressSet(self.TABLE_LEN) dbfile = tempfile.NamedTemporaryFile(delete=False) try: aset.tofile(dbfile) dbfile.seek(0) aset = AddressSet.fromfile(dbfile, mmap_access=mmap.ACCESS_WRITE) addr = "".join(chr(b) for b in xrange(20)) aset.add(addr) aset.close() self.assertTrue(dbfile.closed) dbfile = open(dbfile.name, "rb") aset = AddressSet.fromfile(dbfile) self.assertIn(addr, aset) self.assertEqual(len(aset), 1) finally: aset.close() dbfile.close() os.remove(dbfile.name)
def test_null(self): aset = AddressSet(self.TABLE_LEN) addr = 20 * "\0" aset.add(addr) self.assertNotIn(addr, aset) self.assertEqual(len(aset), 0)
class TestAddressSet(unittest.TestCase): HASH_BYTES = 1 TABLE_LEN = 2**(8 * HASH_BYTES) BYTES_PER_ADDR = AddressSet(1)._bytes_per_addr def test_add(self): aset = AddressSet(self.TABLE_LEN) addr = "".join(chr(b) for b in xrange(20)) self.assertNotIn(addr, aset) aset.add(addr) self.assertIn(addr, aset) self.assertEqual(len(aset), 1) def collision_tester(self, aset, addr1, addr2): aset.add(addr1) self.assertIn(addr1, aset) self.assertNotIn(addr2, aset) self.assertEqual(len(aset), 1) aset.add(addr2) self.assertIn(addr1, aset) self.assertIn(addr2, aset) self.assertEqual(len(aset), 2) return aset # def test_collision(self): aset = AddressSet(self.TABLE_LEN) # the last HASH_BYTES (1) bytes are the "hash", and only the next BYTES_PER_ADDR (8) rightmost bytes are stored addr1 = "".join(chr(b) for b in xrange(20)) addr2 = addr1.replace(chr(20 - self.HASH_BYTES - self.BYTES_PER_ADDR), "\0") # the leftmost byte that's stored self.collision_tester(aset, addr1, addr2) # def test_collision_fail(self): aset = AddressSet(self.TABLE_LEN) # the last 1 (HASH_BYTES) bytes are the "hash", and only the next 8 (BYTES_PER_ADDR) rightmost bytes are stored addr1 = "".join(chr(b) for b in xrange(20)) addr2 = addr1.replace( chr(20 - self.HASH_BYTES - self.BYTES_PER_ADDR - 1), "\0") # the rightmost byte not stored self.assertRaises(unittest.TestCase.failureException, self.collision_tester, aset, addr1, addr2) self.assertEqual(len(aset), 1) def test_null(self): aset = AddressSet(self.TABLE_LEN) addr = 20 * "\0" aset.add(addr) self.assertNotIn(addr, aset) self.assertEqual(len(aset), 0) # very unlikely to fail; if it does, there's probably a significant problem def test_false_positives(self): aset = AddressSet( 131072, bytes_per_addr=5 ) # reduce bytes_per_addr to increase failure probability rand_byte_count = aset._hash_bytes + aset._bytes_per_addr nonrand_prefix = (20 - rand_byte_count) * "\0" for i in xrange(aset._max_len): aset.add(nonrand_prefix + "".join( chr(random.randrange(256)) for i in xrange(rand_byte_count))) for i in xrange(524288): self.assertNotIn( nonrand_prefix + "".join( chr(random.randrange(256)) for i in xrange(rand_byte_count)), aset) def test_file(self): aset = AddressSet(self.TABLE_LEN) addr = "".join(chr(b) for b in xrange(20)) aset.add(addr) dbfile = tempfile.TemporaryFile() aset.tofile(dbfile) dbfile.seek(0) aset = AddressSet.fromfile(dbfile) self.assertTrue( dbfile.closed) # should be closed by AddressSet in read-only mode self.assertIn(addr, aset) self.assertEqual(len(aset), 1) def test_file_update(self): aset = AddressSet(self.TABLE_LEN) dbfile = tempfile.NamedTemporaryFile(delete=False) try: aset.tofile(dbfile) dbfile.seek(0) aset = AddressSet.fromfile(dbfile, mmap_access=mmap.ACCESS_WRITE) addr = "".join(chr(b) for b in xrange(20)) aset.add(addr) aset.close() self.assertTrue(dbfile.closed) dbfile = open(dbfile.name, "rb") aset = AddressSet.fromfile(dbfile) self.assertIn(addr, aset) self.assertEqual(len(aset), 1) finally: aset.close() dbfile.close() os.remove(dbfile.name) def test_pickle_mmap(self): aset = AddressSet(self.TABLE_LEN) addr = "".join(chr(b) for b in xrange(20)) aset.add(addr) dbfile = tempfile.NamedTemporaryFile(delete=False) try: aset.tofile(dbfile) dbfile.seek(0) aset = AddressSet.fromfile(dbfile) # now it's an mmap pickled = pickle.dumps(aset, protocol=pickle.HIGHEST_PROTOCOL) aset.close() # also closes the file aset = pickle.loads(pickled) self.assertIn(addr, aset) self.assertEqual(len(aset), 1) finally: aset.close() dbfile.close() os.remove(dbfile.name)