def test_order_closest(self):
id0 = Id(BIN_ID0)
ordered_list = [
Id('\x00' * ID_SIZE_BYTES),
Id(BIN_ID0[:-1] + '\x06'),
Id(BIN_ID0[:9] + '\x01' * (ID_SIZE_BYTES - 9)),
Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)),
Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)),
Id('\x00' + '\xff' * (ID_SIZE_BYTES - 1)),
Id('\x53' * ID_SIZE_BYTES),
Id('\xff' * ID_SIZE_BYTES),
]
random_list = random.sample(ordered_list, len(ordered_list))
random_list_copy = random_list[:]
logger.debug('ordered list')
for e in ordered_list: logger.debug('%s' % e)
logger.debug('random order')
for e in random_list: logger.debug('%s' % e)
result_list = id0.order_closest(random_list)
logger.debug('order_closest result')
for e in result_list: logger.debug('%s' % e)
logger.debug('random order (it should not change)')
for e in random_list: logger.debug('%s' % e)
# make sure order_closest does not modify random_list
assert random_list == random_list_copy
for i, ordered_id in enumerate(ordered_list):
logger.debug('%d, %s, %s' % (i, ordered_id, result_list[i]))
assert ordered_id.bin_id == result_list[i].bin_id
def _get_id(self, k, kk=None):
try:
v = self._get_value(k, kk)
v = Id(v)
except (IdError):
raise MsgError, 'Value (%s:%s,%s) must be a valid Id' % (k, kk, v)
return v
def handle(self, data, addr):
#droid.log("New connection")
if data == "KILL_DHT":
return True # stop DHT
data_len = len(data)
i = 0
#print 'in: ',
remote_cid = data[:CHANNEL_SIZE]
i += CHANNEL_SIZE
#print '%r' % remote_cid,
channel = self.channel_m.get(remote_cid, addr)
if not channel:
#print 'Invalid channel id'
return
while i < data_len:
msg_type = ord(data[i])
i += 1
if msg_type == HANDSHAKE:
channel.remote_cid = data[i:i + CHANNEL_SIZE]
i += CHANNEL_SIZE
elif msg_type == DATA:
i = data_len # DATA always ends a datagram
elif msg_type == ACK:
i += TS_SIZE + BIN_SIZE
elif msg_type == HAVE:
i += BIN_SIZE
elif msg_type == HASH:
i += BIN_SIZE
channel.rhash = Id(data[i:i + HASH_SIZE])
i += HASH_SIZE
elif msg_type == PEX_RES:
i += 0 #no arguments
elif msg_type == PEX_REQ:
i += PEER_SIZE
elif msg_type == SIGNED_HASH:
print ` data `
raise NotImplemented
elif msg_type == HINT:
i += BIN_SIZE
elif msg_type == MSGTYPE_RCVD:
print ` data `
raise NotImplemented
elif msg_type == VERSION:
i += VERSION_SIZE
else:
print 'UNKNOWN: ', msg_type,
print ` data `
#raise NotImplemented
return
if remote_cid == CHANNEL_ZERO and channel.rhash:
#droid.log(">>>>>>> DHT: got HANDSHAKE from swift <<<<<<<")
self.pymdht.get_peers(channel, channel.rhash, self._on_peers_found,
channel.remote_addr[1])
# need to complete handshake
reply = ''.join((
channel.remote_cid,
chr(HANDSHAKE),
channel.local_cid,
))
self.socket.sendto(reply, channel.remote_addr)
def load_state(self):
self._my_id = None
self.loaded_nodes = []
try:
f = open(self.state_filename)
except(IOError):
return
# the first line contains this node's identifier
hex_id = f.readline().strip()
self._my_id = Id(hex_id)
# the rest of the lines contain routing table nodes
# FORMAT
# log_distance hex_id ip port rtt
for line in f:
_, hex_id, ip, port, _ = line.split()
addr = (ip, int(port))
node_ = Node(addr, Id(hex_id))
self.loaded_nodes.append(node_)
f.close
def load(filename):
my_id = None
nodes = []
try:
f = open(filename)
hex_id = f.readline().strip()
my_id = Id(hex_id)
for line in f:
_, hex_id, _, ip, port, _, _ = line.split()
addr = (ip, int(port))
node_ = Node(addr, Id(hex_id))
nodes.append(node_)
except (IOError):
logger.debug("No state saved, loading default.")
return None, []
except:
logger.exception("Error when loading state, loading default.")
#raise # debug only
return None, []
f.close
return my_id, nodes
def uncompact_nodes2(c_nodes):
nodes = []
for c_node in c_nodes:
node_id = Id(c_node[:ID_SIZE_BYTES])
try:
node_addr = uncompact_addr(c_node[ID_SIZE_BYTES:])
except (AddrError):
logger.warning('IPv6 addr in nodes2: %s' % c_node)
else:
node = Node(node_addr, node_id)
nodes.append(node)
return nodes
def load_state(self):
self._my_id = None
self.loaded_nodes = []
try:
f = open(self.state_filename)
except IOError:
return
try:
hex_id = f.readline().strip()
self._my_id = Id(hex_id)
for line in f:
_, hex_id, ip, port, _ = line.split()
addr = (ip, int(port))
node_ = Node(addr, Id(hex_id))
self.loaded_nodes.append(node_)
f.close()
except:
self._my_id = None
self.loaded_nodes = []
logger.error('state.dat is corrupted')
def test_create(self):
_ = Id(BIN_ID1)
_ = RandomId()
assert_raises(IdError, Id, 1)
assert_raises(IdError, Id, '1')
_ = Id('1' * 40) # Hexadecimal
assert_raises(IdError, Id, 'Z' * 40)
eq_(Id('\x00' * 20).bin_id, Id('0' * 40).bin_id)
eq_(Id('\xff' * 20), Id('f' * 40))
def _uncompact_nodes(c_nodes):
if len(c_nodes) % C_NODE_SIZE != 0:
raise MsgError, 'invalid size (%d) %s' % (len(c_nodes), c_nodes)
nodes = []
for begin in xrange(0, len(c_nodes), C_NODE_SIZE):
node_id = Id(c_nodes[begin:begin + ID_SIZE_BYTES])
try:
node_addr = uncompact_addr(c_nodes[begin + ID_SIZE_BYTES:begin +
C_NODE_SIZE])
except AddrError:
pass
else:
node = Node(node_addr, node_id)
nodes.append(node)
return nodes
def uncompact_nodes(c_nodes):
if len(c_nodes) % C_NODE_SIZE != 0:
logger.info('invalid size (%d) %s' % (len(c_nodes), c_nodes))
return []
nodes = []
for begin in xrange(0, len(c_nodes), C_NODE_SIZE):
node_id = Id(c_nodes[begin:begin + ID_SIZE_BYTES])
try:
node_addr = uncompact_addr(c_nodes[begin + ID_SIZE_BYTES:begin +
C_NODE_SIZE])
except AddrError:
pass
else:
node = Node(node_addr, node_id, version=None)
nodes.append(node)
return nodes
def test_log_distance(self):
id0 = Id(BIN_ID0)
id1 = Id(BIN_ID1)
id2 = Id(BIN_ID2)
eq_(id0.log_distance(id0), -1)
eq_(id0.log_distance(id1), ID_SIZE_BITS - 8)
eq_(id0.log_distance(id2), ID_SIZE_BITS - 7)
id_log = (
(Id('\x00' + '\xff' * (ID_SIZE_BYTES - 1)),
BITS_PER_BYTE * (ID_SIZE_BYTES - 1) - 1),
(Id('\x53' * ID_SIZE_BYTES), BITS_PER_BYTE * ID_SIZE_BYTES - 2),
(Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)),
(ID_SIZE_BYTES - 7) * BITS_PER_BYTE - 1),
(Id(BIN_ID0[:9] + '\x01' * (ID_SIZE_BYTES - 9)),
(ID_SIZE_BYTES - 10) * BITS_PER_BYTE),
(Id(BIN_ID0[:-1] + '\x06'), 2),
)
id2_log = (
(Id('\x41' * ID_SIZE_BYTES), Id('\x41' * ID_SIZE_BYTES), -1),
(Id('\x41' * ID_SIZE_BYTES), Id('\x01' * ID_SIZE_BYTES), 158),
(Id('\x41' * ID_SIZE_BYTES), Id('\x43' * ID_SIZE_BYTES), 153),
)
for (id_, log_) in id_log:
logger.debug('log_distance: %d' % id0.log_distance(id_))
logger.debug('expected: %d' % log_)
eq_(id0.log_distance(id_), log_)
for id1, id2, expected in id2_log:
eq_(id1.log_distance(id2), expected)
z = Id('\0' * 20)
eq_(z.log_distance(Id('\x00' * 19 + '\x00')), -1)
eq_(z.log_distance(Id('\x00' * 19 + '\x00')), -1)
eq_(z.log_distance(Id('\x00' * 19 + '\x00')), -1)
eq_(z.log_distance(Id('\x00' * 19 + '\x00')), -1)
eq_(z.log_distance(Id('\x00' * 19 + '\x00')), -1)
def test_bin_id_read_only(self):
id1 = Id(BIN_ID1)
id1.bin_id = BIN_ID2
def test_str(self):
id1 = Id(BIN_ID1)
assert BIN_ID1 == '%s' % id1
def test_distance(self):
id1 = Id(BIN_ID1)
id2 = Id(BIN_ID2)
dist1_2 = Id(DIST1_2)
assert id1.distance(id2).bin_id == dist1_2.bin_id
assert id2.distance(id1).bin_id == dist1_2.bin_id
def test_equal(self):
id1 = Id(BIN_ID0)
assert id1 == id1 # same instance
assert id1 == Id(BIN_ID0) #different instance, same value
assert id1 != Id(BIN_ID1)
def test_bin_id_read_only(self):
id1 = Id(BIN_ID1)
id1.bin_id = BIN_ID2
def test_is_hashable(self):
d = {Id(BIN_ID1): 1}
def handle(self, data, addr):
droid.log("New connection")
if data == "KILL_DHT":
return True # stop DHT
data_len = len(data)
i = 0
print 'in: ',
remote_cid = data[:CHANNEL_SIZE]
i += CHANNEL_SIZE
print '%r' % remote_cid,
channel = self.channel_m.get(remote_cid, addr)
if not channel:
print 'Invalid channel id'
return
while i < data_len:
msg_type = ord(data[i])
i += 1
if msg_type == HANDSHAKE:
print 'HAND',
channel.remote_cid = data[i:i + CHANNEL_SIZE]
i += CHANNEL_SIZE
elif msg_type == DATA:
print 'DATA',
i = data_len # DATA always ends a datagram
elif msg_type == ACK:
print 'ACK',
i += TS_SIZE + BIN_SIZE
elif msg_type == HAVE:
print 'HAVE',
i += BIN_SIZE
elif msg_type == HASH:
print 'HASH',
i += BIN_SIZE
channel.rhash = Id(data[i:i + HASH_SIZE])
i += HASH_SIZE
elif msg_type == PEX_RES:
print 'PRES',
i += 0 #no arguments
elif msg_type == PEX_REQ:
print 'PREQ',
i += PEER_SIZE
elif msg_type == SIGNED_HASH:
print 'SHASH',
print ` data `
raise NotImplemented
elif msg_type == HINT:
print 'HINT',
i += BIN_SIZE
elif msg_type == MSGTYPE_RCVD:
print 'MSGTYPE > not implemented',
print ` data `
raise NotImplemented
elif msg_type == VERSION:
print 'VERSION',
i += VERSION_SIZE
else:
print 'UNKNOWN: ', msg_type,
print ` data `
raise NotImplemented
print
if remote_cid == CHANNEL_ZERO and channel.rhash:
droid.log(">>>>>>> DHT: got HANDSHAKE from swift <<<<<<<")
self.dht.get_peers(channel, channel.rhash, self._on_peers_found, 0)
# need to complete handshake
reply = ''.join((
channel.remote_cid,
chr(HANDSHAKE),
channel.local_cid,
))
self.socket.sendto(reply, addr)
droid.log('>>>>>>>>>>>>> GETTING PEERS <<<<<<<<<<<<<<')
reply = ''.join((
channel.remote_cid,
chr(PEX_RES),
socket.inet_aton('130.161.211.194'), #Delft
chr(20050 >> 8),
chr(20050 % 256),
chr(PEX_RES),
socket.inet_aton('192.16.127.98'), #KTH
chr(20050 >> 8),
chr(20050 % 256),
))
self.socket.sendto(reply, addr)
def test_distance(self):
id1 = Id(BIN_ID1)
id2 = Id(BIN_ID2)
dist1_2 = Id(DIST1_2)
assert id1.distance(id2).bin_id == dist1_2.bin_id
assert id2.distance(id1).bin_id == dist1_2.bin_id
def test_has_repr(self):
eq_(repr(Id(BIN_ID1)), '01' * ID_SIZE_BYTES)
def test_order_closest(self):
id0 = Id(BIN_ID0)
ordered_list = [
Id('\x00' * ID_SIZE_BYTES),
Id(BIN_ID0[:-1] + '\x06'),
Id(BIN_ID0[:9] + '\x01' * (ID_SIZE_BYTES - 9)),
Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)),
Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)),
Id('\x00' + '\xff' * (ID_SIZE_BYTES - 1)),
Id('\x53' * ID_SIZE_BYTES),
Id('\xff' * ID_SIZE_BYTES),
]
random_list = random.sample(ordered_list, len(ordered_list))
random_list_copy = random_list[:]
logger.debug('ordered list')
for e in ordered_list:
logger.debug('%s' % e)
logger.debug('random order')
for e in random_list:
logger.debug('%s' % e)
result_list = id0.order_closest(random_list)
logger.debug('order_closest result')
for e in result_list:
logger.debug('%s' % e)
logger.debug('random order (it should not change)')
for e in random_list:
logger.debug('%s' % e)
# make sure order_closest does not modify random_list
assert random_list == random_list_copy
for i, ordered_id in enumerate(ordered_list):
logger.debug('%d, %s, %s' % (i, ordered_id, result_list[i]))
assert ordered_id.bin_id == result_list[i].bin_id
def test_log_distance(self):
id0 = Id(BIN_ID0)
id1 = Id(BIN_ID1)
id2 = Id(BIN_ID2)
eq_(id0.log_distance(id0), -1)
eq_(id0.log_distance(id1), ID_SIZE_BITS - 8)
eq_(id0.log_distance(id2), ID_SIZE_BITS - 7)
id_log = (
(Id('\x00' + '\xff' * (ID_SIZE_BYTES - 1)),
BITS_PER_BYTE * (ID_SIZE_BYTES - 1) - 1),
(Id('\x53' * ID_SIZE_BYTES),
BITS_PER_BYTE * ID_SIZE_BYTES - 2),
(Id(BIN_ID0[:7] + '\xff' * (ID_SIZE_BYTES - 7)),
(ID_SIZE_BYTES - 7) * BITS_PER_BYTE - 1),
(Id(BIN_ID0[:9] + '\x01' * (ID_SIZE_BYTES - 9)),
(ID_SIZE_BYTES - 10) * BITS_PER_BYTE),
(Id(BIN_ID0[:-1] + '\x06'),
2),
)
id2_log = (
(Id('\x41' * ID_SIZE_BYTES),
Id('\x41' * ID_SIZE_BYTES),
-1),
(Id('\x41' * ID_SIZE_BYTES),
Id('\x01' * ID_SIZE_BYTES),
158),
(Id('\x41' * ID_SIZE_BYTES),
Id('\x43' * ID_SIZE_BYTES),
153),
)
for (id_, log_) in id_log:
logger.debug('log_distance: %d' % id0.log_distance(id_))
logger.debug('expected: %d' % log_)
eq_(id0.log_distance(id_), log_)
for id1, id2, expected in id2_log:
eq_(id1.log_distance(id2), expected)
z = Id('\0'*20)
eq_(z.log_distance(Id('\x00'*19+'\x00')), -1)
eq_(z.log_distance(Id('\x00'*19+'\x00')), -1)
eq_(z.log_distance(Id('\x00'*19+'\x00')), -1)
eq_(z.log_distance(Id('\x00'*19+'\x00')), -1)
eq_(z.log_distance(Id('\x00'*19+'\x00')), -1)
from nose.tools import ok_, eq_, raises, assert_raises
import test_const as tc
import logging, logging_conf
import utils
from identifier import Id, ID_SIZE_BYTES
from node import Node, RoutingNode
import node
logging_conf.testing_setup(__name__)
logger = logging.getLogger('dht')
bin_id1 = '1' * ID_SIZE_BYTES
bin_id2 = '2' * ID_SIZE_BYTES
id1 = Id(bin_id1)
id2 = Id(bin_id2)
addr1 = ('127.0.0.1', 1111)
addr2 = ('127.0.0.1', 2222)
class TestNode:
def setup(self):
pass
def test_node(self):
node1 = Node(addr1, id1)
node2 = Node(addr2, id2)
node1b = Node(addr1, None)
node1ip = Node(('127.0.0.2', 1111), id1)
node1port = Node(addr2, id1)
for i in xrange(100, 100+NUM_NODES)]
NODES2 = [node.Node(addr, node_id) \
for addr, node_id in zip(ADDRS2, NODE2_IDS)]
PEERS2 = ADDRS2
IPS = ['1.2.3.' + str(i) for i in xrange(NUM_NODES)]
#TODO2: make this faster
num_nodes_per_ld = 20
NODES_LD_IH = [[]] * BITS_PER_BYTE
for ld in xrange(BITS_PER_BYTE, ID_SIZE_BITS):
NODES_LD_IH.append([])
common_id = INFO_HASH_ZERO.generate_close_id(ld)
#eq_(common_id.log_distance(INFO_HASH_ZERO), ld)
for i in xrange(num_nodes_per_ld):
this_id = Id(common_id.bin_id[:-1] + chr(i))
#eq_(this_id.log_distance(INFO_HASH_ZERO), ld)
NODES_LD_IH[ld].append(
node.Node(('128.0.0.' + str(i), ld), this_id))
num_nodes_per_ld = 50
NODES_LD_CL = [[]] * BITS_PER_BYTE
for ld in xrange(BITS_PER_BYTE, ID_SIZE_BITS):
NODES_LD_CL.append([])
common_id = CLIENT_ID.generate_close_id(ld)
#eq_(common_id.log_distance(INFO_HASH_ZERO), ld)
for i in xrange(num_nodes_per_ld):
this_id = Id(common_id.bin_id[:-1] + chr(i))
#eq_(this_id.log_distance(INFO_HASH_ZERO), ld)
NODES_LD_CL[ld].append(
node.Node(('128.0.0.' + str(i), ld), this_id))
def test_bin_id(self):
assert Id(BIN_ID1).bin_id == BIN_ID1
def _get_id(self, k, kk=None):
v = self._get_str(k, kk)
try:
return Id(v)
except (IdError):
raise MsgError('Value (%s:%s,%s) must be a valid Id' % (k, kk, v))
def test_has_repr(self):
eq_(repr(Id(BIN_ID1)), '<Id: ' + '01' * ID_SIZE_BYTES + '>')