def test_non_membership_comp(self): """ Ensure that non-elements of the skip list are not in it, comparing the results to the authenticated skip list. """ for i in xrange(0, 10 * self.num_iters): elems = map(IntElem, self.generate_elems(0, 25, 5)) esl = EmbeddedSkipList.new(elems, IntElem(-201), IntElem(201), coin=HashCoin(), conn_info=None) asl = AuthSkipList.new(elems, IntElem(-201), IntElem(201), coin=HashCoin()) bad_elem = IntElem(random.randint(-100, 100)) while bad_elem in elems: bad_elem = IntElem(random.randint(-100, 100)) bad_elems = [IntElem(-150), IntElem(150), bad_elem] for elem in bad_elems: evisited, eclosest = esl.root.search(elem) avisited, aclosest = asl.root.search(elem) new = [] for n, f in evisited: new.append((n, f)) evisited = new eelems, eproof = esl.do_query(evisited, eclosest, elem) aelems, aproof = asl.do_query(avisited, aclosest, elem) efound = eelems[0] == elem afound = aelems[0] == elem eproof = [ AuthNode._hash(x.serialize()) for x in reversed(eelems) ] + eproof aproof = [ AuthNode._hash(x.serialize()) for x in reversed(aelems) ] + aproof self.assertEqual(efound, afound) self.assertTrue(not efound) self.assertEqual(AuthSkipList.verify(aproof), asl.root.label) self.assertEqual(EmbeddedSkipList.verify(eproof), esl.root.label) # Make sure the embedded one isn't cheating somehow self.assertEqual(esl.root.label, asl.root.label)
def test_verify_in(self): """ Ensure that proofs of membership validate. """ for i in range(0, self.num_iters): elems = gen_elems(-100, 100, 50) sl = AuthSkipList.new(elems, IntElem(-101), IntElem(101)) for elem in elems: found, proof = sl.contains(elem) self.assertEqual(AuthSkipList.verify(proof), sl.root.label)
def skiplist_test(self): """ Make sure the different coins can be used to guarantee the same skiplist. """ for n in self.sizes[:-1]: for iters in range(self.num_iters): max_elem = n * 100 min_elem = -1 * max_elem lower = IntElem(min_elem-1) upper = IntElem(max_elem+1) elems = [IntElem(random.randint(min_elem, max_elem)) for i in range(n)] rand = random.randint(0,1000) random.seed(rand) prefix = [random.randint(0,1) == 1 for i in range(10 * n)] random.seed(rand) sl_base = AuthSkipList.new( elems, lower, upper, BaseCoin()) random.seed(rand) sl_recd = AuthSkipList.new( elems, lower, upper, RecordedCoin()) sl_pref = AuthSkipList.new( elems, lower, upper, PrefixCoin(prefix)) sl_rpre = AuthSkipList.new( elems, lower, upper, RecordedPrefixCoin(prefix)) sl_seed = AuthSkipList.new( elems, lower, upper, SeededCoin(rand)) self.assertEqual(sl_seed.to_list_of_lists(), sl_rpre.to_list_of_lists()) self.assertEqual(sl_rpre.to_list_of_lists(), sl_pref.to_list_of_lists()) self.assertEqual(sl_pref.to_list_of_lists(), sl_recd.to_list_of_lists()) self.assertEqual(sl_recd.to_list_of_lists(), sl_base.to_list_of_lists()) self.assertEqual(sl_seed.root.label, sl_rpre.root.label) self.assertEqual(sl_rpre.root.label, sl_pref.root.label) self.assertEqual(sl_pref.root.label, sl_recd.root.label) self.assertEqual(sl_recd.root.label, sl_base.root.label)
def test_comparison_to_auth(self): """ Test embedded list and authenticated list side-by-side to see where the former goes wrong. Mostly here for debugging. """ for i in xrange(0, self.num_iters): elems = map(IntElem, self.generate_elems(0, 50, 10)) esl = EmbeddedSkipList.new(elems, IntElem(-1), IntElem(101), coin=HashCoin(), conn_info=None) asl = AuthSkipList.new(elems, IntElem(-1), IntElem(101), coin=HashCoin()) for elem in elems: evisited, eclosest = esl.root.search(elem) avisited, aclosest = asl.root.search(elem) new = [] for n, f in evisited: new.append((n, f)) evisited = new eelems, eproof = esl.do_query(evisited, eclosest, elem) aelems, aproof = asl.do_query(avisited, aclosest, elem) efound = eelems[0] == elem afound = aelems[0] == elem eproof = [ AuthNode._hash(x.serialize()) for x in reversed(eelems) ] + eproof aproof = [ AuthNode._hash(x.serialize()) for x in reversed(aelems) ] + aproof # Make sure the embedded one isn't cheating somehow self.assertEqual(esl.root.label, asl.root.label) self.assertEqual(AuthSkipList.verify(aproof), asl.root.label) self.assertEqual(EmbeddedSkipList.verify(eproof), esl.root.label) # Make sure the embedded one isn't cheating somehow self.assertEqual(esl.root.label, asl.root.label)
def test_stringify(self): for i in range(0, self.num_iters): elems = [IntElem(random.randint(0, 100000000)) for i in range(0, 1000)] elems.sort() sl = AuthSkipList.new(elems, IntElem(-1), IntElem(100000001)) lower = random.choice(elems[1:200]) upper = random.choice(elems[500:-1]) vo = SkipListVO.range_query(sl, lower, upper) self.assertTrue(isinstance(vo, SkipListVO)) serialized = vo.serialize() new_vo = SkipListVO.deserialize(serialized, IntElem) print 'vo:' for l in vo.to_list_of_lists(): print l print print new_vo.root print 'new_vo:' for l in new_vo.to_list_of_lists(): print l self.assertEqual(vo, new_vo)
def test_different_single_insert(self): for i in range(0, self.num_iters): print print i orig_seed = random.randint(0, 10000000) elems = [IntElem(random.randint(0, 100)) for i in range(0, 5)] elems.sort() sl = AuthSkipList.new(elems, IntElem(elems[0].key-1), IntElem(elems[-1].key+1), SeededCoin(orig_seed)) new_elem = IntElem(random.randint(elems[1].key, elems[-2].key)) min_ne = elems[1] max_ne = elems[-2] vo = SkipListVO.range_query(sl, min_ne, max_ne) try: ret_elems = vo.verify(min_ne, max_ne, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' %e.msg) seed = random.randint(0, 1000000) vo.coin = SeededCoin(seed) sl.coin = SeededCoin(seed) self.assertEqual(vo.root.label, sl.root.label) vo.insert(new_elem) sl.insert(new_elem) self.assertEqual(sl.root.label, vo.root.label)
def test_multiple_inserts(self): for i in range(0, self.num_iters): orig_seed = random.randint(0, 10000000) elems = [IntElem(random.randint(0, 100000000)) for i in range(0, 100)] elems.sort() sl = AuthSkipList.new(elems, IntElem(elems[0].key-1), IntElem(elems[-1].key+1), SeededCoin(orig_seed)) new_elems = [IntElem(random.randint(elems[1].key, elems[-2].key)) for i in range(0, 10)] min_ne = min(new_elems) max_ne = max(new_elems) vo = SkipListVO.range_query(sl, min_ne, max_ne) try: ret_elems = vo.verify(min_ne, max_ne, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' %e.msg) seed = random.randint(0, 1000000) vo.coin = SeededCoin(seed) sl.coin = SeededCoin(seed) self.assertEqual(vo.root.label, sl.root.label) for elem in new_elems: vo.insert(elem) sl.insert(elem) self.assertEqual(sl.root.label, vo.root.label, 'failed on index %d' %new_elems.index(elem)) self.assertEqual(sl.root.label, vo.root.label)
def test_range(self): """ Ensure that range queries work correctly """ for i in range(0, self.num_iters): elems = [] # hacked up to avoid repeated elements in elems i = 0 while i < 5: x = IntElem(random.randint(-50, 50)) if x not in elems: i = i + 1 elems.append(x) sl = AuthSkipList.new(elems, IntElem(-101), IntElem(101)) lower = random.choice(sorted(elems)[0:2]) upper = random.choice(sorted(elems)[3:5]) vo = SkipListVO.range_query(sl, lower, upper) try: elems = vo.verify(lower, upper, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' %e.msg) self.assertEqual(elems, [e for e in sl.to_list_of_lists()[-1] if lower <= e <= upper])
def test_verified_insert(self): """ Test that verified insert actually allows the client to compute the new root label. """ lower_bound = -1000 upper_bound = 1000 num_elems = 10 num_new_elems = 150 for i in range(0, self.num_iters): elems = gen_elems(lower_bound, upper_bound, num_elems) elems = set(elems) sl = AuthSkipList.new(elems, IntElem(lower_bound-1), IntElem(upper_bound+1)) old_label = sl.root.label new_elems = gen_elems(lower_bound, upper_bound, num_new_elems) new_elems = [elem for elem in new_elems if elem not in elems] for elem in new_elems: print 'inserting elem %d' %elem.key print 'into list %s' %str(sl.to_list_of_lists()) ret_elems, proof, proof_diff = sl.insert_with_diff(elem) print 'result list: %s' %str(sl.to_list_of_lists()) self.assertEqual( AuthSkipList.verify( [AuthNode._hash(e.serialize()) for e in reversed(ret_elems)] + proof), old_label) new_proof = AuthSkipList.update_query( ret_elems, proof, proof_diff, elem) x, qproof = sl.contains(elem) self.assertTrue(x, 'Claims just-inserted element is not in list') self.assertEqual(AuthSkipList.verify(qproof), sl.root.label) np = AuthSkipList.verify(new_proof) print 'Root label: %s' %str(sl.root.label) print 'Recv label: %s' %str(np) self.assertEqual(np, sl.root.label) old_label = sl.root.label
def test_range_query(self): """ Check that range queries work """ lower_bound = -100000 upper_bound = 100000 num_elems = 1000 for i in range(0, self.num_iters): elems = gen_elems(lower_bound, upper_bound, num_elems) for x in elems: if elems.count(x) > 1: elems.remove(x) sl = AuthSkipList.new(elems, IntElem(lower_bound-1), IntElem(upper_bound+1)) for j in range(0, self.num_iters): lower = IntElem(random.randint(lower_bound/5, upper_bound/100)) upper = IntElem(random.randint(lower.key+1000, upper_bound/5)) proofs = sl._range_query(lower, upper) received = AuthSkipList._verify_range_query( proofs, lower, upper, sl.root.label) bottom = sl.root while bottom.down: bottom = bottom.down expected = [] while bottom.right and bottom.elem <= upper: if bottom.elem >= lower: expected.append(bottom.elem) bottom = bottom.right self.assertTrue(all([lower <= x <= upper for x in expected])) self.assertTrue(sorted(expected) == expected) print 'Range query from %d to %d' %(lower.key, upper.key) print "Expected length: %d" %len(expected) print "Received length: %d" %len(received) self.assertEqual(expected, received)
def test_valid(self): """ Ensure that we only create valid skip lists """ for i in range(0, self.num_iters): elems = gen_elems(0, 1000000000, self.size) sl = AuthSkipList.new(elems, IntElem(-1), IntElem(1000000001)) try: sl.valid() except InvalidSkipListException as e: self.assertTrue(False, 'Error: %s' %e.msg)
def test_vo_insert(self): seed = str(time.time()) random.seed(seed) for i in range(0, self.num_iters): elems = [IntElem(random.randint(0, 100000000)) for i in range(0, self.size)] num_elems = self.size elems.sort() for i in range(len(elems)-1): if elems[i] == elems[i+1]: elems[i+1] = elems[i+1] + 1 orig_elems = elems min_elem = IntElem(-1) max_elem = IntElem(100000001) sl = AuthSkipList.new(elems, min_elem, max_elem, RecordedCoin()) left = IntElem(random.randint(5, 6000)) right = IntElem(random.randint(left.key+100, 9000)) vo = SkipListVO.range_query( sl, left, right, PrefixCoin([])) try: elems = vo.verify(left, right, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' %e.msg) self.assertEqual(elems, [e for e in orig_elems if left<=e<=right]) elem = IntElem(random.randint(left.key+1, right.key-1)) while elem in orig_elems: elem = IntElem(random.randint(left.key+1, right.key-1)) bisect.insort(orig_elems, elem) sl.coin.read() sl.insert(elem) vo.coin.extend(sl.coin.read()) vo.insert(elem) try: elems = vo.verify(left, right, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' %e.msg) self.assertEqual(sl.root.label, vo.root.label) self.assertEqual(elems, [e for e in orig_elems if left<=e<=right])
def test_membership(self): """ Ensure that every element of the skip list is reflected as in it. """ for i in range(0, self.num_iters): elems = gen_elems(0, 100000000, self.size) sl = AuthSkipList.new(elems, IntElem(-1), IntElem(1000000001)) for elem in elems: found, proof = sl.contains(elem) self.assertTrue(found, 'skip list claims not to have element %d' %elem.key)
def setUp(self): random.seed(int(time.time())) # Setup for small, hand-made tests self.elems = map(IntElem, [0, 5, 6, 10]) self.sl = AuthSkipList.new(self.elems, IntElem(self.elems[0].key-1), IntElem(self.elems[-1].key+1)) self.lower = IntElem(1) self.upper = IntElem(9) self.vo = SkipListVO.range_query(self.sl, self.lower, self.upper)
def test_non_membership(self): """ Ensure that non-elements of the skip list are not in it """ for i in range(0, 10 * self.num_iters): elems = gen_elems(0, 25, 5) sl = AuthSkipList.new(elems, IntElem(-201), IntElem(201)) found, proof = sl.contains(IntElem(-150)) verification = AuthSkipList.verify(proof) self.assertEqual(verification, sl.root.label) found, proof = sl.contains(IntElem(150)) self.assertEqual(AuthSkipList.verify(proof), sl.root.label) bad_elem = IntElem(random.randint(-100, 100)) while bad_elem in elems: bad_elem = IntElem(random.randint(-100, 100)) found, proof = sl.contains(bad_elem) self.assertEqual(AuthSkipList.verify(proof), sl.root.label, 'proof of absence of %d fails to verify' %bad_elem.key)
def test_remote_hash_query(self): for i in range(0, self.num_iters): elems = map(IntElem, self.generate_elems()) elems.sort() sl = AuthSkipList.new(elems, IntElem(-1), IntElem(1000000001), HashCoin()) with HashSocketSLClient.new(elems, IntElem(-1), IntElem(1000000001), IntElem, self.host, self.port) as client: for j in range(0, self.num_iters): elem = random.choice(elems) good_result = sl.contains(elem) test_result = client.query(elem) self.assertEqual(good_result, test_result)
def test_big_range_query(self): """ Check that range queries work """ lower_bound = -100000 upper_bound = 100000 num_elems = 1000 for i in range(0, self.num_iters): elems = [random.randint(lower_bound, upper_bound) for i in range(num_elems)] elems = map(IntElem, set(elems)) sl = AuthSkipList.new(elems, IntElem(lower_bound-1), IntElem(upper_bound+1)) for j in range(0, self.num_iters): lower = IntElem(random.randint(lower_bound/5, upper_bound/100)) upper = IntElem(random.randint(lower.key + 1000, upper_bound/5)) vo = SkipListVO.range_query(sl, lower, upper) try: elems = vo.verify(lower, upper, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' %e.msg) expected = [e for e in sl.to_list_of_lists()[-1] if lower <= e <= upper] if len(elems) != len(expected): print 'num returned elems:', len(elems) print 'num expected elems:', len(expected) for x, y in zip(elems, expected): print x, print '---', print y self.assertEqual(elems, expected)
def test_remote_range_query_2(self): for i in range(0, self.num_iters): orig_seed = random.randint(0, 10000000) elems = map(IntElem, self.generate_elems()) elems.sort() sl = AuthSkipList.new(elems, IntElem(elems[0].key - 1), IntElem(elems[-1].key + 1), SeededCoin(orig_seed)) with SeededSocketSLClient.new(elems, IntElem(elems[0].key - 1), IntElem(elems[-1].key + 1), orig_seed, IntElem, self.host, self.port) as client: lower = random.choice(elems[1:self.size / 3]) upper = random.choice(elems[2 * self.size / 3:-1]) localvo = SkipListVO.range_query(sl, lower, upper) othervo = client.range_query(lower, upper) try: ret_elems = localvo.verify(lower, upper, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' % e.msg) self.assertEqual(ret_elems, [x for x in elems if lower <= x <= upper]) try: ret_elems = othervo.verify(lower, upper, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' % e.msg) self.assertEqual(ret_elems, [x for x in elems if lower <= x <= upper]) self.assertEqual(localvo, othervo)
def test_remote_range_query(self): total_time = 0 for i in range(0, self.num_iters): elems = map(IntElem, self.generate_elems()) elems.sort() seed = random.randint(0, 100000) sl = AuthSkipList.new(elems, IntElem(-1), IntElem(1000000001), SeededCoin(seed)) start = time.time() with SeededSocketSLClient.new(elems, IntElem(-1), IntElem(1000000001), seed, IntElem, self.host, self.port) as client: ## Get valid lower & upper bounds with enough room between ## them (about a third of the list) for a substantial range ## query lower = random.choice(elems[1:self.size / 3]) upper = random.choice(elems[2 * self.size / 3:-1]) self.assertEqual(SkipListVO.range_query(sl, lower, upper), client.range_query(lower, upper)) end = time.time() total_time = total_time + (end - start) print 'Average time taken: %s' % str(total_time / self.num_iters)
def handle(self): # Initialize the skiplist # Can have a persistent skiplist once batch insertion has been # implemented. coin_type = _recv(self.rfile, 'handler') if coin_type == 'SEED': raw_seed = _recv(self.rfile, 'handler') try: seed = int(raw_seed) except ValueError: raise SkipListServerException( 'Non-number %s passed in for seed value' % raw_seed) coin = SeededCoin(seed) elif coin_type == 'PREFIX': prefix = [p == 'True' for p in _recv_iterate(self.rfile, 'ELEMS')] coin = PrefixCoin(prefix) elif coin_type == 'HASH': coin = HashCoin() else: raise SkipListServerException('Invalid coin type %s' % coin_type) elems = [ self.server.elemClass.deserialize(elem) for elem in _recv_iterate(self.rfile, 'END_ELEMS') ] # Protocol: send the lower bound, then the upper bound, then the entire # list of elements to insert at first sl = AuthSkipList.new(elems[2:], elems[0], elems[1], coin) while True: task = self.rfile.readline().strip() if task == 'QUERY': raw_elem = _recv(self.rfile, 'handler') elem = self.server.elemClass.deserialize(raw_elem) proof = sl.contains(elem) sproof = SLHandler.serialize_query_result(proof) _send(self.wfile, sproof, 'handler') elif task == 'RANGE_QUERY': raw_lower, raw_upper = _recvi(self.rfile, 2, 'handler') lower, upper = (self.server.elemClass.deserialize(raw_lower), self.server.elemClass.deserialize(raw_upper)) vo = SkipListVO.range_query(sl, lower, upper) _send(self.wfile, vo.serialize(), 'handler') elif task == 'INSERT': for elem in _recv_iterate(self.rfile, 'END_INSERT'): sl.insert(self.server.elemClass.deserialize(elem)) elif task == 'REPREFIX': prefix = [ int(y) for y in _recv_iterate(self.rfile, 'END_REPREFIX') ] sl.coin.extend(prefix) elif task == 'RESEED': seed = int(_recv(self.rfile)) sl.coin.reseed(seed) elif task == 'PING': _send(self.wfile, 'PONG') elif task == 'QUIT': return else: raise Exception('Unknown task %s' % task)
def test_remote_batch_insert_helper(self): for i in range(0, self.num_iters): orig_seed = random.randint(0, 10000000) elems = map(IntElem, self.generate_elems()) elems.sort() sl = AuthSkipList.new(elems, IntElem(elems[0].key - 1), IntElem(elems[-1].key + 1), SeededCoin(orig_seed)) new_elems = [ IntElem(random.randint(elems[1].key, elems[-2].key)) for i in range(0, 2) ] min_ne = min(new_elems) max_ne = max(new_elems) while min_ne == max_ne: new_elems = [ IntElem(random.randint(elems[1].key, elems[-2].key)) for i in range(0, 2) ] min_ne = min(new_elems) max_ne = max(new_elems) vo = SkipListVO.range_query(sl, min_ne, max_ne) try: ret_elems = vo.verify(min_ne, max_ne, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' % e.msg) seed = random.randint(0, 1000000) vo.coin = SeededCoin(seed) sl.coin = SeededCoin(seed) self.assertEqual(vo.root.label, sl.root.label) orig_label = sl.root.label for elem in new_elems: vo.insert(elem) sl.insert(elem) self.assertEqual(sl.root.label, vo.root.label, 'failed on index %d' % new_elems.index(elem)) self.assertEqual(sl.root.label, vo.root.label) with SeededSocketSLClient.new(elems, IntElem(elems[0].key - 1), IntElem(elems[-1].key + 1), orig_seed, IntElem, self.host, self.port) as client: try: new_root = client.batch_insert(orig_label, new_elems, min_ne, max_ne, seed) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' % e.msg) self.assertEqual(new_root, vo.root.label) self.assertEqual(sl.root.label, new_root) self.assertEqual( len(sl.to_list_of_lists()[-1]) - 2, len(elems) + len(new_elems)) for elem in new_elems: proof = client.query(elem) self.assertEqual(proof, sl.contains(elem)) self.assertEqual(AuthSkipList.verify(proof[1]), sl.root.label)
def test_remote_batch_insert_hash_coin(self): for i in range(0, self.num_iters): elems = map(IntElem, self.generate_elems()) elems.sort() sl = AuthSkipList.new(elems, IntElem(elems[0].key - 1), IntElem(elems[-1].key + 1), HashCoin()) old_root = sl.root.label self.assertEqual(len(sl.to_list_of_lists()[-1]) - 2, self.size) ## Generate a small number (in this case, n/10) of random elements ## between the minimum and maximum elements (not including the ## boundary elements) to insert. new_elems = [ IntElem(random.randint(elems[1].key, elems[-2].key)) for i in range(0, self.size / 10) ] new_elems = map( IntElem, self.generate_elems(elems[1].key, elems[-2].key, self.size / 10)) min_ne = min(new_elems) max_ne = max(new_elems) vo = SkipListVO.range_query(sl, min_ne, max_ne, HashCoin()) try: ret_elems = vo.verify(min_ne, max_ne, sl.root.label) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' % e.msg) self.assertEqual(ret_elems, [x for x in elems if min_ne <= x <= max_ne]) self.assertEqual(vo.root.label, sl.root.label) with HashSocketSLClient.new(elems, IntElem(elems[0].key - 1), IntElem(elems[-1].key + 1), IntElem, self.host, self.port) as client: lower = random.choice(elems[1:self.size / 3]) upper = random.choice(elems[2 * self.size / 3:-1]) self.assertEqual(SkipListVO.range_query(sl, lower, upper), client.range_query(lower, upper)) self.assertEqual(SkipListVO.range_query(sl, min_ne, max_ne), client.range_query(min_ne, max_ne)) for elem in new_elems: vo.insert(elem) for elem in new_elems: sl.insert(elem) self.assertEqual(sl.root.label, vo.root.label) try: new_root = client.batch_insert(old_root, new_elems, min_ne, max_ne) except VerificationObjectException as e: self.assertTrue(False, 'Error: %s' % e.msg) self.assertEqual(new_root, vo.root.label) self.assertEqual(sl.root.label, new_root) self.assertEqual( len(sl.to_list_of_lists()[-1]) - 2, self.size + (self.size / 10)) for elem in new_elems: proof = client.query(elem) self.assertEqual(proof, sl.contains(elem)) self.assertEqual(AuthSkipList.verify(proof[1]), sl.root.label)