def test_stringify(self):
for i in range(0, self.num_iters):
elems = [IntElem(random.randint(0, 100000000))
for i in range(0, 1000)]
elems.sort()
sl = AuthSkipList.new(elems, IntElem(-1), IntElem(100000001))
lower = random.choice(elems[1:200])
upper = random.choice(elems[500:-1])
vo = SkipListVO.range_query(sl, lower, upper)
self.assertTrue(isinstance(vo, SkipListVO))
serialized = vo.serialize()
new_vo = SkipListVO.deserialize(serialized, IntElem)
print 'vo:'
for l in vo.to_list_of_lists():
print l
print
print new_vo.root
print 'new_vo:'
for l in new_vo.to_list_of_lists():
print l
self.assertEqual(vo, new_vo)
def test_range(self):
""" Ensure that range queries work correctly
"""
for i in range(0, self.num_iters):
elems = []
# hacked up to avoid repeated elements in elems
i = 0
while i < 5:
x = IntElem(random.randint(-50, 50))
if x not in elems:
i = i + 1
elems.append(x)
sl = AuthSkipList.new(elems, IntElem(-101), IntElem(101))
lower = random.choice(sorted(elems)[0:2])
upper = random.choice(sorted(elems)[3:5])
vo = SkipListVO.range_query(sl, lower, upper)
try:
elems = vo.verify(lower, upper, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' %e.msg)
self.assertEqual(elems,
[e for e in sl.to_list_of_lists()[-1]
if lower <= e <= upper])
def test_multiple_inserts(self):
for i in range(0, self.num_iters):
orig_seed = random.randint(0, 10000000)
elems = [IntElem(random.randint(0, 100000000))
for i in range(0, 100)]
elems.sort()
sl = AuthSkipList.new(elems, IntElem(elems[0].key-1),
IntElem(elems[-1].key+1),
SeededCoin(orig_seed))
new_elems = [IntElem(random.randint(elems[1].key, elems[-2].key))
for i in range(0, 10)]
min_ne = min(new_elems)
max_ne = max(new_elems)
vo = SkipListVO.range_query(sl, min_ne, max_ne)
try:
ret_elems = vo.verify(min_ne, max_ne, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' %e.msg)
seed = random.randint(0, 1000000)
vo.coin = SeededCoin(seed)
sl.coin = SeededCoin(seed)
self.assertEqual(vo.root.label, sl.root.label)
for elem in new_elems:
vo.insert(elem)
sl.insert(elem)
self.assertEqual(sl.root.label, vo.root.label, 'failed on index %d' %new_elems.index(elem))
self.assertEqual(sl.root.label, vo.root.label)
def test_different_single_insert(self):
for i in range(0, self.num_iters):
print
print i
orig_seed = random.randint(0, 10000000)
elems = [IntElem(random.randint(0, 100)) for i in range(0, 5)]
elems.sort()
sl = AuthSkipList.new(elems, IntElem(elems[0].key-1),
IntElem(elems[-1].key+1),
SeededCoin(orig_seed))
new_elem = IntElem(random.randint(elems[1].key, elems[-2].key))
min_ne = elems[1]
max_ne = elems[-2]
vo = SkipListVO.range_query(sl, min_ne, max_ne)
try:
ret_elems = vo.verify(min_ne, max_ne, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' %e.msg)
seed = random.randint(0, 1000000)
vo.coin = SeededCoin(seed)
sl.coin = SeededCoin(seed)
self.assertEqual(vo.root.label, sl.root.label)
vo.insert(new_elem)
sl.insert(new_elem)
self.assertEqual(sl.root.label, vo.root.label)
def test_empty_vo_fails(self):
elems = self.elems
sl = self.sl
lower = self.lower
upper = self.upper
vo = self.vo
# Make sure it won't verify an empty VO
bad_vo = SkipListVO(IntElem(0),
IntElem(10),
VONode(None, sl.root.down.label, sl.root.right.label, -1),
None)
passed = False
try:
bad_vo.verify(lower, upper, sl.root.label)
except VerificationObjectException as e:
passed = True
self.assertTrue(passed, 'Should not verify an empty VO')
def test_vo_insert(self):
seed = str(time.time())
random.seed(seed)
for i in range(0, self.num_iters):
elems = [IntElem(random.randint(0, 100000000))
for i in range(0, self.size)]
num_elems = self.size
elems.sort()
for i in range(len(elems)-1):
if elems[i] == elems[i+1]:
elems[i+1] = elems[i+1] + 1
orig_elems = elems
min_elem = IntElem(-1)
max_elem = IntElem(100000001)
sl = AuthSkipList.new(elems, min_elem, max_elem, RecordedCoin())
left = IntElem(random.randint(5, 6000))
right = IntElem(random.randint(left.key+100, 9000))
vo = SkipListVO.range_query(
sl, left, right, PrefixCoin([]))
try:
elems = vo.verify(left, right, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' %e.msg)
self.assertEqual(elems, [e for e in orig_elems if left<=e<=right])
elem = IntElem(random.randint(left.key+1, right.key-1))
while elem in orig_elems:
elem = IntElem(random.randint(left.key+1, right.key-1))
bisect.insort(orig_elems, elem)
sl.coin.read()
sl.insert(elem)
vo.coin.extend(sl.coin.read())
vo.insert(elem)
try:
elems = vo.verify(left, right, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' %e.msg)
self.assertEqual(sl.root.label, vo.root.label)
self.assertEqual(elems, [e for e in orig_elems if left<=e<=right])
def setUp(self):
random.seed(int(time.time()))
# Setup for small, hand-made tests
self.elems = map(IntElem, [0, 5, 6, 10])
self.sl = AuthSkipList.new(self.elems,
IntElem(self.elems[0].key-1),
IntElem(self.elems[-1].key+1))
self.lower = IntElem(1)
self.upper = IntElem(9)
self.vo = SkipListVO.range_query(self.sl, self.lower, self.upper)
def range_query(self, lower, upper):
""" Query a range of elements beteween lower and upper (inclusive)
Arguments:
self - the skiplist object
lower - the (inclusive) lower bound of the range
upper - the (inclusive) upper bound of the range
Returns:
vo - a verification object (as in vo.py) for the given range
"""
_send(self.wfile, 'RANGE_QUERY')
_sendi(self.wfile, [lower.serialize(), upper.serialize()])
vo = _recv(self.rfile, 'client')
return SkipListVO.deserialize(vo, self.elemClass)
def test_big_range_query(self):
""" Check that range queries work
"""
lower_bound = -100000
upper_bound = 100000
num_elems = 1000
for i in range(0, self.num_iters):
elems = [random.randint(lower_bound, upper_bound)
for i in range(num_elems)]
elems = map(IntElem, set(elems))
sl = AuthSkipList.new(elems, IntElem(lower_bound-1), IntElem(upper_bound+1))
for j in range(0, self.num_iters):
lower = IntElem(random.randint(lower_bound/5, upper_bound/100))
upper = IntElem(random.randint(lower.key + 1000,
upper_bound/5))
vo = SkipListVO.range_query(sl, lower, upper)
try:
elems = vo.verify(lower, upper, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' %e.msg)
expected = [e for e in sl.to_list_of_lists()[-1]
if lower <= e <= upper]
if len(elems) != len(expected):
print 'num returned elems:', len(elems)
print 'num expected elems:', len(expected)
for x, y in zip(elems, expected):
print x,
print '---',
print y
self.assertEqual(elems, expected)
def test_remote_range_query_2(self):
for i in range(0, self.num_iters):
orig_seed = random.randint(0, 10000000)
elems = map(IntElem, self.generate_elems())
elems.sort()
sl = AuthSkipList.new(elems, IntElem(elems[0].key - 1),
IntElem(elems[-1].key + 1),
SeededCoin(orig_seed))
with SeededSocketSLClient.new(elems, IntElem(elems[0].key - 1),
IntElem(elems[-1].key + 1),
orig_seed, IntElem, self.host,
self.port) as client:
lower = random.choice(elems[1:self.size / 3])
upper = random.choice(elems[2 * self.size / 3:-1])
localvo = SkipListVO.range_query(sl, lower, upper)
othervo = client.range_query(lower, upper)
try:
ret_elems = localvo.verify(lower, upper, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' % e.msg)
self.assertEqual(ret_elems,
[x for x in elems if lower <= x <= upper])
try:
ret_elems = othervo.verify(lower, upper, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' % e.msg)
self.assertEqual(ret_elems,
[x for x in elems if lower <= x <= upper])
self.assertEqual(localvo, othervo)
def test_remote_range_query(self):
total_time = 0
for i in range(0, self.num_iters):
elems = map(IntElem, self.generate_elems())
elems.sort()
seed = random.randint(0, 100000)
sl = AuthSkipList.new(elems, IntElem(-1), IntElem(1000000001),
SeededCoin(seed))
start = time.time()
with SeededSocketSLClient.new(elems, IntElem(-1),
IntElem(1000000001), seed, IntElem,
self.host, self.port) as client:
## Get valid lower & upper bounds with enough room between
## them (about a third of the list) for a substantial range
## query
lower = random.choice(elems[1:self.size / 3])
upper = random.choice(elems[2 * self.size / 3:-1])
self.assertEqual(SkipListVO.range_query(sl, lower, upper),
client.range_query(lower, upper))
end = time.time()
total_time = total_time + (end - start)
print 'Average time taken: %s' % str(total_time / self.num_iters)
def test_remote_batch_insert_hash_coin(self):
for i in range(0, self.num_iters):
elems = map(IntElem, self.generate_elems())
elems.sort()
sl = AuthSkipList.new(elems, IntElem(elems[0].key - 1),
IntElem(elems[-1].key + 1), HashCoin())
old_root = sl.root.label
self.assertEqual(len(sl.to_list_of_lists()[-1]) - 2, self.size)
## Generate a small number (in this case, n/10) of random elements
## between the minimum and maximum elements (not including the
## boundary elements) to insert.
new_elems = [
IntElem(random.randint(elems[1].key, elems[-2].key))
for i in range(0, self.size / 10)
]
new_elems = map(
IntElem,
self.generate_elems(elems[1].key, elems[-2].key,
self.size / 10))
min_ne = min(new_elems)
max_ne = max(new_elems)
vo = SkipListVO.range_query(sl, min_ne, max_ne, HashCoin())
try:
ret_elems = vo.verify(min_ne, max_ne, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' % e.msg)
self.assertEqual(ret_elems,
[x for x in elems if min_ne <= x <= max_ne])
self.assertEqual(vo.root.label, sl.root.label)
with HashSocketSLClient.new(elems, IntElem(elems[0].key - 1),
IntElem(elems[-1].key + 1), IntElem,
self.host, self.port) as client:
lower = random.choice(elems[1:self.size / 3])
upper = random.choice(elems[2 * self.size / 3:-1])
self.assertEqual(SkipListVO.range_query(sl, lower, upper),
client.range_query(lower, upper))
self.assertEqual(SkipListVO.range_query(sl, min_ne, max_ne),
client.range_query(min_ne, max_ne))
for elem in new_elems:
vo.insert(elem)
for elem in new_elems:
sl.insert(elem)
self.assertEqual(sl.root.label, vo.root.label)
try:
new_root = client.batch_insert(old_root, new_elems, min_ne,
max_ne)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' % e.msg)
self.assertEqual(new_root, vo.root.label)
self.assertEqual(sl.root.label, new_root)
self.assertEqual(
len(sl.to_list_of_lists()[-1]) - 2,
self.size + (self.size / 10))
for elem in new_elems:
proof = client.query(elem)
self.assertEqual(proof, sl.contains(elem))
self.assertEqual(AuthSkipList.verify(proof[1]),
sl.root.label)
def test_remote_batch_insert_helper(self):
for i in range(0, self.num_iters):
orig_seed = random.randint(0, 10000000)
elems = map(IntElem, self.generate_elems())
elems.sort()
sl = AuthSkipList.new(elems, IntElem(elems[0].key - 1),
IntElem(elems[-1].key + 1),
SeededCoin(orig_seed))
new_elems = [
IntElem(random.randint(elems[1].key, elems[-2].key))
for i in range(0, 2)
]
min_ne = min(new_elems)
max_ne = max(new_elems)
while min_ne == max_ne:
new_elems = [
IntElem(random.randint(elems[1].key, elems[-2].key))
for i in range(0, 2)
]
min_ne = min(new_elems)
max_ne = max(new_elems)
vo = SkipListVO.range_query(sl, min_ne, max_ne)
try:
ret_elems = vo.verify(min_ne, max_ne, sl.root.label)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' % e.msg)
seed = random.randint(0, 1000000)
vo.coin = SeededCoin(seed)
sl.coin = SeededCoin(seed)
self.assertEqual(vo.root.label, sl.root.label)
orig_label = sl.root.label
for elem in new_elems:
vo.insert(elem)
sl.insert(elem)
self.assertEqual(sl.root.label, vo.root.label,
'failed on index %d' % new_elems.index(elem))
self.assertEqual(sl.root.label, vo.root.label)
with SeededSocketSLClient.new(elems, IntElem(elems[0].key - 1),
IntElem(elems[-1].key + 1),
orig_seed, IntElem, self.host,
self.port) as client:
try:
new_root = client.batch_insert(orig_label, new_elems,
min_ne, max_ne, seed)
except VerificationObjectException as e:
self.assertTrue(False, 'Error: %s' % e.msg)
self.assertEqual(new_root, vo.root.label)
self.assertEqual(sl.root.label, new_root)
self.assertEqual(
len(sl.to_list_of_lists()[-1]) - 2,
len(elems) + len(new_elems))
for elem in new_elems:
proof = client.query(elem)
self.assertEqual(proof, sl.contains(elem))
self.assertEqual(AuthSkipList.verify(proof[1]),
sl.root.label)
def handle(self):
# Initialize the skiplist
# Can have a persistent skiplist once batch insertion has been
# implemented.
coin_type = _recv(self.rfile, 'handler')
if coin_type == 'SEED':
raw_seed = _recv(self.rfile, 'handler')
try:
seed = int(raw_seed)
except ValueError:
raise SkipListServerException(
'Non-number %s passed in for seed value' % raw_seed)
coin = SeededCoin(seed)
elif coin_type == 'PREFIX':
prefix = [p == 'True' for p in _recv_iterate(self.rfile, 'ELEMS')]
coin = PrefixCoin(prefix)
elif coin_type == 'HASH':
coin = HashCoin()
else:
raise SkipListServerException('Invalid coin type %s' % coin_type)
elems = [
self.server.elemClass.deserialize(elem)
for elem in _recv_iterate(self.rfile, 'END_ELEMS')
]
# Protocol: send the lower bound, then the upper bound, then the entire
# list of elements to insert at first
sl = AuthSkipList.new(elems[2:], elems[0], elems[1], coin)
while True:
task = self.rfile.readline().strip()
if task == 'QUERY':
raw_elem = _recv(self.rfile, 'handler')
elem = self.server.elemClass.deserialize(raw_elem)
proof = sl.contains(elem)
sproof = SLHandler.serialize_query_result(proof)
_send(self.wfile, sproof, 'handler')
elif task == 'RANGE_QUERY':
raw_lower, raw_upper = _recvi(self.rfile, 2, 'handler')
lower, upper = (self.server.elemClass.deserialize(raw_lower),
self.server.elemClass.deserialize(raw_upper))
vo = SkipListVO.range_query(sl, lower, upper)
_send(self.wfile, vo.serialize(), 'handler')
elif task == 'INSERT':
for elem in _recv_iterate(self.rfile, 'END_INSERT'):
sl.insert(self.server.elemClass.deserialize(elem))
elif task == 'REPREFIX':
prefix = [
int(y) for y in _recv_iterate(self.rfile, 'END_REPREFIX')
]
sl.coin.extend(prefix)
elif task == 'RESEED':
seed = int(_recv(self.rfile))
sl.coin.reseed(seed)
elif task == 'PING':
_send(self.wfile, 'PONG')
elif task == 'QUIT':
return
else:
raise Exception('Unknown task %s' % task)