def test_callFindNode(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("nodetofind"))
self.protocol.callFindNode(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_NODE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def rpc_find_node(self, sender, nodeid, key):
self.log.info("finding neighbors of %i in local table" %
long(nodeid.encode('hex'), 16))
source = Node(nodeid, sender[0], sender[1])
self.router.addContact(source)
node = Node(key)
return map(tuple, self.router.findNeighbors(node, exclude=source))
def test_create_proto(self):
rid = hashlib.sha1(str(random.getrandbits(255))).digest()
pubkey = digest("pubkey")
addr = objects.Node.IPAddress()
addr.ip = "127.0.0.1"
addr.port = 1234
relay_addr = objects.Node.IPAddress()
relay_addr.ip = "127.0.0.1"
relay_addr.port = 1234
n1 = objects.Node()
n1.guid = rid
n1.signedPublicKey = pubkey
n1.vendor = False
n1.nodeAddress.MergeFrom(addr)
n1.natType = objects.FULL_CONE
n2 = Node(rid, "127.0.0.1", 1234, digest("pubkey"), None, objects.FULL_CONE, False)
self.assertEqual(n1, n2.getProto())
n1.vendor = True
n1.relayAddress.MergeFrom(relay_addr)
n2 = Node(rid, "127.0.0.1", 1234, digest("pubkey"), ("127.0.0.1", 1234), objects.FULL_CONE, True)
self.assertEqual(n1, n2.getProto())
def test_rpc_find_without_value(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.arguments.append(digest("Keyword"))
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
m.arguments.extend([node2.getProto().SerializeToString(), node3.getProto().SerializeToString(), node1.getProto().SerializeToString()])
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
m = message.Message()
m.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_distanceCalculation(self):
ridone = hashlib.sha1(str(random.getrandbits(255)))
ridtwo = hashlib.sha1(str(random.getrandbits(255)))
shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16)
none = Node(ridone.digest())
ntwo = Node(ridtwo.digest())
self.assertEqual(none.distanceTo(ntwo), shouldbe)
def test_distanceCalculation(self):
ridone = hashlib.sha1(str(random.getrandbits(255)))
ridtwo = hashlib.sha1(str(random.getrandbits(255)))
shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16)
none = Node(ridone.digest())
ntwo = Node(ridtwo.digest())
self.assertEqual(none.distanceTo(ntwo), shouldbe)
def test_rpc_find_node(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_NODE")
m.arguments.append(digest("nodetofind"))
data = m.SerializeToString()
del m.arguments[-1]
m.arguments.extend([
node2.getProto().SerializeToString(),
node1.getProto().SerializeToString(),
node3.getProto().SerializeToString()
])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_refreshIDs(self):
node1 = Node(digest("id1"), "127.0.0.1", 12345, pubkey=digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, pubkey=digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, pubkey=digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
for b in self.protocol.router.buckets:
b.lastUpdated = (time.time() - 5000)
ids = self.protocol.getRefreshIDs()
self.assertTrue(len(ids) == 1)
def test_rpc_find_without_value(self):
self._connecting_to_connected()
node1 = Node(digest("id1"),
"127.0.0.1",
12345,
digest("key1"),
nat_type=objects.FULL_CONE)
node2 = Node(digest("id2"),
"127.0.0.1",
22222,
digest("key2"),
nat_type=objects.FULL_CONE)
node3 = Node(digest("id3"),
"127.0.0.1",
77777,
digest("key3"),
nat_type=objects.FULL_CONE)
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("Keyword"))
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
del m.arguments[-1]
m.arguments.extend([
node3.getProto().SerializeToString(),
node1.getProto().SerializeToString(),
node2.getProto().SerializeToString()
])
m.ClearField("signature")
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
a.ClearField("signature")
received_message = a.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_create_proto(self):
rid = hashlib.sha1(str(random.getrandbits(255))).digest()
pubkey = digest("pubkey")
vendor = True
n1 = objects.Node()
n1.guid = rid
n1.signedPublicKey = pubkey
n1.vendor = False
n2 = Node(rid, signed_pubkey=digest("pubkey"))
self.assertEqual(n1, n2.getProto())
n1.vendor = True
n2 = Node(rid, signed_pubkey=pubkey, vendor=vendor)
self.assertEqual(n1, n2.getProto())
def test_getNodeList(self):
node1 = Node(digest("id1"), "127.0.0.1", 1234, digest("key1"), None, FULL_CONE, True)
node2 = Node(digest("id2"), "248.130.11.21", 1111, digest("key2"), None, FULL_CONE, True)
node3 = Node(digest("id3"), "987.567.23.1", 23456, digest("key3"), None, FULL_CONE, False)
response = (True, (node1.getProto().SerializeToString(), node2.getProto().SerializeToString(),
node3.getProto().SerializeToString(),
"sdfasdfsd"))
r = RPCFindResponse(response)
nodes = r.getNodeList()
self.assertEqual(nodes[0].getProto(), node1.getProto())
self.assertEqual(nodes[1].getProto(), node2.getProto())
self.assertEqual(nodes[2].getProto(), node3.getProto())
def test_getNodeList(self):
node1 = Node(digest("id1"), "127.0.0.1", 12345, signed_pubkey=digest("key1"), vendor=True)
node2 = Node(digest("id2"), "127.0.0.1", 22222, signed_pubkey=digest("key2"), vendor=True)
node3 = Node(digest("id3"), "127.0.0.1", 77777, signed_pubkey=digest("key3"))
response = (True, (node1.getProto().SerializeToString(), node2.getProto().SerializeToString(),
node3.getProto().SerializeToString(),
"sdfasdfsd"))
r = RPCFindResponse(response)
nodes = r.getNodeList()
self.assertEqual(nodes[0].getProto(), node1.getProto())
self.assertEqual(nodes[1].getProto(), node2.getProto())
self.assertEqual(nodes[2].getProto(), node3.getProto())
def prepare():
"""
Prepare the application state.
"""
thisNodeIP = config.NODE_IP
thisNodePort = config.NODE_PORT
thisNodeHash = Hash(sha1(config.NODE_ID_NAME).digest())
AppState.thisNode = Node(thisNodeHash, (thisNodeIP, thisNodePort))
AppState.heartbeat = config.HEARTBEAT
AppState.tokenSecret = utils.randomBits(160)
AppState.maxPeersPerTorrent = config.MAX_PEERS_PER_TORRENT
AppState.k = config.K
AppState.maxNodesPerPucket = config.MAX_NODES_PER_BUCKET
AppState.routingTable = RoutingTable()
if config.PEER_STORAGE == 'file':
AppState.peerStorage = dht.peerstorage.FilePeerStorage(
config.PEER_STORAGE_DIR)
elif config.PEER_STORAGE == 'mysql':
AppState.peerStorage = dht.peerstorage.MySQLPeerStorage()
# {transactionID: {(RPCQuery, Node, timestamp)}}
AppState.outstandingQueries = {}
def loadState(cls, fname, ip_address, port, multiplexer, db, callback=None, storage=None):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
with open(fname, 'r') as f:
data = pickle.load(f)
if data['testnet'] != multiplexer.testnet:
raise Exception('Cache uses wrong network parameters')
n = Node(data['id'], ip_address, port, data['signed_pubkey'], data['vendor'])
s = Server(n, db, data['ksize'], data['alpha'], storage=storage)
s.protocol.connect_multiplexer(multiplexer)
if len(data['neighbors']) > 0:
if callback is not None:
s.bootstrap(data['neighbors']).addCallback(callback)
else:
s.bootstrap(data['neighbors'])
else:
# TODO: load seed from config file
if callback is not None:
s.bootstrap(s.querySeed("seed.openbazaar.org:8080",
"5b44be5c18ced1bc9400fe5e79c8ab90204f06bebacc04dd9c70a95eaca6e117"))\
.addCallback(callback)
else:
s.bootstrap(s.querySeed("seed.openbazaar.org:8080",
"5b44be5c18ced1bc9400fe5e79c8ab90204f06bebacc04dd9c70a95eaca6e117"))
return s
def test_callStore(self):
self._connecting_to_connected()
n = Node(digest("guid"), self.addr1[0], self.addr1[1],
digest("pubkey"), None, objects.FULL_CONE, False)
self.wire_protocol[self.addr1] = self.con
self.protocol.callStore(
n, digest("Keyword"), digest("Key"),
self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey,
m.sender.publicKey)
self.assertTrue(m.command == message.STORE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(
m.arguments[2],
self.protocol.sourceNode.getProto().SerializeToString())
def func(node, *args):
msgID = sha1(str(random.getrandbits(255))).digest()
m = Message()
m.messageID = msgID
m.sender.MergeFrom(self.sourceNode.getProto())
m.command = Command.Value(name.upper())
m.protoVer = PROTOCOL_VERSION
for arg in args:
m.arguments.append(str(arg))
m.testnet = self.multiplexer.testnet
data = m.SerializeToString()
address = (node.ip, node.port)
relay_addr = None
if node.nat_type == SYMMETRIC or \
(node.nat_type == RESTRICTED and self.sourceNode.nat_type == SYMMETRIC):
relay_addr = node.relay_node
d = defer.Deferred()
if m.command != HOLE_PUNCH:
timeout = reactor.callLater(self._waitTimeout, self.timeout, node)
self._outstanding[msgID] = [d, address, timeout]
self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID)))
self.multiplexer.send_message(data, address, relay_addr)
if self.multiplexer[address].state != State.CONNECTED and \
node.nat_type == RESTRICTED and \
self.sourceNode.nat_type != SYMMETRIC:
self.hole_punch(Node(digest("null"), node.relay_node[0], node.relay_node[1], nat_type=FULL_CONE),
address[0], address[1], "True")
self.log.debug("sending hole punch message to %s" % address[0] + ":" + str(address[1]))
return d
def setUp(self):
self.catcher = []
observer = self.catcher.append
log.addObserver(observer)
self.addCleanup(log.removeObserver, observer)
self.node = Node(digest("test"), "127.0.0.1", 1234)
self.router = RoutingTable(self, 20, self.node.id)
def test_callDelete(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callDelete(n, digest("Keyword"), digest("Key"),
digest("Signature"))
self.clock.advance(constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey,
m.sender.publicKey)
self.assertTrue(m.command == message.DELETE)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], digest("Signature"))
def resolve(self, guid):
"""
Given a guid return a `Node` object containing its ip and port or none if it's
not found.
Args:
guid: the 20 raw bytes representing the guid.
"""
node_to_find = Node(guid)
def check_for_node(nodes):
for node in nodes:
if node.id == node_to_find.id:
return node
return None
index = self.protocol.router.getBucketFor(node_to_find)
nodes = self.protocol.router.buckets[index].getNodes()
for node in nodes:
if node.id == node_to_find.id:
return defer.succeed(node)
nearest = self.protocol.router.findNeighbors(node_to_find)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
return defer.succeed(None)
spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
return spider.find().addCallback(check_for_node)
def test_addSameIP(self):
self.router.addContact(self.node)
self.router.addContact(Node(digest("asdf"), "127.0.0.1", 1234))
self.assertTrue(len(self.router.buckets), 1)
self.assertTrue(len(self.router.buckets[0].nodes), 1)
self.assertTrue(
self.router.buckets[0].getNodes()[0].id == digest("asdf"))
def delete(self, keyword, key, signature):
"""
Delete the given key/value pair from the keyword dictionary on the network.
To delete you must provide a signature covering the key that you wish to
delete. It will be verified against the public key stored in the value. We
use our ksize as alpha to make sure we reach as many nodes storing our value
as possible.
Args:
keyword: the `string` keyword where the data being deleted is stored.
key: the 20 byte hash of the data.
signature: a signature covering the key.
"""
self.log.info("deleting '%s':'%s' from the network" % (keyword, hexlify(key)))
dkey = digest(keyword)
def delete(nodes):
self.log.debug("deleting '%s' on %s" % (key, [str(i) for i in nodes]))
ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]
if self.storage.getSpecific(dkey, key) is not None:
self.storage.delete(dkey, key)
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to delete key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
return spider.find().addCallback(delete)
def receive_message(self, datagram):
if len(datagram) < 166:
self.log.warning(
"received datagram too small from %s, ignoring" %
self.addr)
return False
try:
m = Message()
m.ParseFromString(datagram)
self.node = Node(
m.sender.guid, m.sender.nodeAddress.ip,
m.sender.nodeAddress.port, m.sender.publicKey,
None if not m.sender.HasField("relayAddress") else
(m.sender.relayAddress.ip, m.sender.relayAddress.port),
m.sender.natType, m.sender.vendor)
self.remote_node_version = m.protoVer
if self.time_last_message == 0:
h = nacl.hash.sha512(m.sender.publicKey)
pow_hash = h[40:]
if int(pow_hash[:6],
16) >= 50 or m.sender.guid.encode("hex") != h[:40]:
raise Exception('Invalid GUID')
for processor in self.processors:
if m.command in processor or m.command == NOT_FOUND:
processor.receive_message(m, self.node,
self.connection,
self.ban_score)
if m.command != PING:
self.time_last_message = time.time()
except Exception:
# If message isn't formatted property then ignore
self.log.warning(
"received an invalid message from %s, ignoring" %
self.addr)
return False
def handle_shutdown(self):
try:
self.connection.unregister()
except Exception:
pass
if self.node is None:
self.node = Node(digest("null"), str(self.connection.dest_addr[0]),
int(self.connection.dest_addr[1]))
for processor in self.processors:
processor.timeout(self.node)
if self.addr:
self.log.info("connection with %s terminated" % self.addr)
try:
self.ban_score.scoring_loop.stop()
except Exception:
pass
try:
self.keep_alive_loop.stop()
except Exception:
pass
if self.relay_node == (self.connection.dest_addr[0], self.connection.dest_addr[1]):
self.log.info("Disconnected from relay node. Picking new one...")
self.change_relay_node()
def send(node_to_send):
n = node_to_send if node_to_send is not None else Node(
unhexlify(guid), "123.4.5.6", 1234)
self.factory.mserver.send_message(
n, recipient_encryption_key,
Plaintext_Message.Type.Value(message_type.upper()), message,
subject)
def test_acceptResponse(self):
self._connecting_to_connected()
def handle_response(resp):
self.assertTrue(resp[0])
self.assertEqual(resp[1][0], self.protocol.sourceNode.id)
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
d = self.protocol.callPing(n)
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
timeout = reactor.callLater(5, self.protocol._timeout, m.messageID)
self.protocol._outstanding[m.messageID] = (d, timeout)
m.arguments.append(self.protocol.sourceNode.id)
self.handler.receive_message(m.SerializeToString())
return d.addCallback(handle_response)
def test_transferKeyValues(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.protocol.addToRouter(mknode())
self.protocol.storage[digest("keyword")] = (
digest("key"),
self.protocol.sourceNode.getProto().SerializeToString())
self.protocol.transferKeyValues(
Node(digest("id"), self.addr1[0], self.addr1[1]))
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
x = message.Message()
x.ParseFromString(sent_message)
m = message.Message()
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.append(digest("keyword"))
m.arguments.append(digest("key"))
m.arguments.append(
self.protocol.sourceNode.getProto().SerializeToString())
self.assertEqual(x.sender, m.sender)
self.assertEqual(x.command, m.command)
self.assertEqual(x.arguments[0], m.arguments[0])
self.assertEqual(x.arguments[1], m.arguments[1])
self.assertEqual(x.arguments[2], m.arguments[2])
def test_handleFoundValues(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest,
dht.constants.KSIZE, dht.constants.ALPHA)
val = Value()
val.valueKey = digest("contractID")
val.serializedData = self.node1.getProto().SerializeToString()
val1 = val.SerializeToString()
value = spider._handleFoundValues([(val1, )])
self.assertEqual(value[0], val.SerializeToString())
# test handle multiple values
val.serializedData = self.node2.getProto().SerializeToString()
val2 = val.SerializeToString()
found_values = [(val1, ), (val1, ), (val2, )]
self.assertEqual(spider._handleFoundValues(found_values), (val1, ))
# test store value at nearest without value
spider.nearestWithoutValue.push(self.node1)
spider._handleFoundValues(found_values)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertTrue(len(self.proto_mock.send_datagram.call_args_list) > 1)
self.proto_mock.send_datagram.call_args_list = []
def parse_results(values):
if values is not None:
for v in values:
try:
val = Value()
val.ParseFromString(v)
n = objects.Node()
n.ParseFromString(val.serializedData)
node_to_ask = Node(
n.guid, n.nodeAddress.ip, n.nodeAddress.port,
n.publicKey,
None if not n.HasField("relayAddress") else
(n.relayAddress.ip, n.relayAddress.port),
n.natType, n.vendor)
if n.guid == KeyChain(self.factory.db).guid:
proto = self.factory.db.listings.get_proto()
l = Listings()
l.ParseFromString(proto)
for listing in l.listing:
if listing.contract_hash == val.valueKey:
respond(listing, node_to_ask)
else:
self.factory.mserver.get_contract_metadata(node_to_ask, val.valueKey)\
.addCallback(respond, node_to_ask)
except Exception:
pass
def rpc_find_value(self, sender, nodeid, key):
source = Node(nodeid, sender[0], sender[1])
self.router.addContact(source)
value = self.storage.get(key, None)
if value is None:
return self.rpc_find_node(sender, nodeid, key)
return {'value': value}
def change_relay_node(self):
potential_relay_nodes = []
for bucket in self.processors[0].router.buckets:
for node in bucket.nodes.values():
if node.nat_type == FULL_CONE:
potential_relay_nodes.append((node.ip, node.port))
if len(potential_relay_nodes) == 0:
for seed in SEEDS:
try:
potential_relay_nodes.append(
(socket.gethostbyname(seed[0].split(":")[0]),
28469 if self.processors[0].TESTNET else 18469))
except socket.gaierror:
pass
shuffle(potential_relay_nodes)
self.relay_node = potential_relay_nodes[0]
for processor in self.processors:
if PING in processor:
if (self.relay_node[0],
self.relay_node[1]) in processor.multiplexer:
processor.multiplexer[(self.relay_node[0],
self.relay_node[1])].shutdown()
processor.callPing(
Node(digest("null"),
self.relay_node[0],
self.relay_node[1],
relay_node=None,
nat_type=FULL_CONE))
def receive_message(self, datagram):
if len(datagram) < 166:
self.log.warning("received datagram too small from %s, ignoring" % self.addr)
return False
m = Message()
try:
m.ParseFromString(datagram)
self.node = Node(m.sender.guid,
m.sender.nodeAddress.ip,
m.sender.nodeAddress.port,
m.sender.publicKey,
None if not m.sender.HasField("relayAddress") else
(m.sender.relayAddress.ip, m.sender.relayAddress.port),
m.sender.natType,
m.sender.vendor)
pubkey = m.sender.publicKey
verify_key = nacl.signing.VerifyKey(pubkey)
signature = m.signature
m.ClearField("signature")
verify_key.verify(m.SerializeToString(), signature)
h = nacl.hash.sha512(m.sender.publicKey)
pow_hash = h[40:]
if int(pow_hash[:6], 16) >= 50 or m.sender.guid.encode("hex") != h[:40]:
raise Exception('Invalid GUID')
except Exception:
# If message isn't formatted property then ignore
self.log.warning("received an invalid message from %s, ignoring" % self.addr)
return False
for processor in self.processors:
if m.command in processor or m.command == NOT_FOUND:
processor.receive_message(m, self.node, self.connection, self.ban_score)
def loadState(cls,
fname,
ip_address,
port,
multiplexer,
db,
callback=None,
storage=None):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
with open(fname, 'r') as f:
data = pickle.load(f)
if data['testnet'] != multiplexer.testnet:
raise Exception('Cache uses wrong network parameters')
n = Node(data['id'], ip_address, port, data['signed_pubkey'],
data['vendor'])
s = Server(n, db, data['ksize'], data['alpha'], storage=storage)
s.protocol.connect_multiplexer(multiplexer)
if len(data['neighbors']) > 0:
if callback is not None:
s.bootstrap(data['neighbors']).addCallback(callback)
else:
s.bootstrap(data['neighbors'])
else:
if callback is not None:
s.bootstrap(s.querySeed(SEED))\
.addCallback(callback)
else:
s.bootstrap(s.querySeed(SEED))
return s
def initTable(results):
potential_relay_nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
h = nacl.hash.sha512(n.publicKey)
hash_pow = h[40:]
if int(hash_pow[:6], 16) >= 50 or hexlify(
n.guid) != h[:40]:
raise Exception('Invalid GUID')
node = Node(
n.guid, addr[0], addr[1], n.publicKey,
None if not n.HasField("relayAddress") else
(n.relayAddress.ip, n.relayAddress.port),
n.natType, n.vendor)
self.protocol.router.addContact(node)
if n.natType == objects.FULL_CONE:
potential_relay_nodes.append((addr[0], addr[1]))
except Exception:
self.log.warning(
"bootstrap node returned invalid GUID")
if len(potential_relay_nodes
) > 0 and self.node.nat_type != objects.FULL_CONE:
shuffle(potential_relay_nodes)
self.node.relay_node = potential_relay_nodes[0]
d.callback(True)
def transferKeyValues(self, node):
"""
Given a new node, send it all the keys/values it should be storing.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
ds = []
for keyword in self.storage.iterkeys():
keynode = Node(keyword)
neighbors = self.router.findNeighbors(keynode, exclude=node)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 \
or (newNodeClose and thisNodeClosest) \
or (thisNodeClosest and len(neighbors) < self.ksize):
for k, v in self.storage.iteritems(keyword):
ds.append(self.callStore(node, keyword, k, v))
return defer.gatherResults(ds)
def receive_message(self, message, connection, ban_score):
sender = Node(
message.sender.guid, message.sender.nodeAddress.ip,
message.sender.nodeAddress.port, message.sender.signedPublicKey,
None if not message.sender.HasField("relayAddress") else
(message.sender.relayAddress.ip, message.sender.relayAddress.port),
message.sender.natType, message.sender.vendor)
if message.testnet != self.multiplexer.testnet:
self.log.warning(
"received message from %s with incorrect network parameters." %
str(connection.dest_addr))
connection.shutdown()
return False
if message.protoVer < PROTOCOL_VERSION:
self.log.warning(
"received message from %s with incompatible protocol version."
% str(connection.dest_addr))
connection.shutdown()
return False
# Check that the GUID is valid. If not, ignore
if self.router.isNewNode(sender):
try:
pubkey = message.sender.signedPublicKey[len(message.sender.
signedPublicKey) -
32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(message.sender.signedPublicKey)
h = nacl.hash.sha512(message.sender.signedPublicKey)
pow_hash = h[64:128]
if int(pow_hash[:6], 16) >= 50 or hexlify(
message.sender.guid) != h[:40]:
raise Exception('Invalid GUID')
except Exception:
self.log.warning(
"received message from sender with invalid GUID, ignoring")
connection.shutdown()
return False
if message.sender.vendor:
self.db.VendorStore().save_vendor(
message.sender.guid.encode("hex"),
message.sender.SerializeToString())
msgID = message.messageID
if message.command == NOT_FOUND:
data = None
else:
data = tuple(message.arguments)
if msgID in self._outstanding:
self._acceptResponse(msgID, data, sender)
elif message.command != NOT_FOUND:
#ban_score.process_message(m)
self._acceptRequest(msgID,
str(Command.Name(message.command)).lower(),
data, sender, connection)
def bootstrap(self, addrs, deferred=None):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
self.log.info("bootstrapping with %s addresses, finding neighbors..." % len(addrs))
if deferred is None:
d = defer.Deferred()
else:
d = deferred
def initTable(results):
response = False
potential_relay_nodes = []
for addr, result in results.items():
if result[0]:
response = True
n = objects.Node()
try:
n.ParseFromString(result[1][0])
h = nacl.hash.sha512(n.publicKey)
hash_pow = h[40:]
if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
node = Node(n.guid, addr[0], addr[1], n.publicKey,
None if not n.HasField("relayAddress") else
(n.relayAddress.ip, n.relayAddress.port),
n.natType,
n.vendor)
self.protocol.router.addContact(node)
if n.natType == objects.FULL_CONE:
potential_relay_nodes.append((addr[0], addr[1]))
except Exception:
self.log.warning("bootstrap node returned invalid GUID")
if not response:
if self.protocol.multiplexer.testnet:
self.bootstrap(self.querySeed(SEEDS_TESTNET), d)
else:
self.bootstrap(self.querySeed(SEEDS), d)
return
if len(potential_relay_nodes) > 0 and self.node.nat_type != objects.FULL_CONE:
shuffle(potential_relay_nodes)
self.node.relay_node = potential_relay_nodes[0]
d.callback(True)
ds = {}
for addr in addrs:
if addr != (self.node.ip, self.node.port):
ds[addr] = self.protocol.ping(Node(digest("null"), addr[0], addr[1], nat_type=objects.FULL_CONE))
deferredDict(ds).addCallback(initTable)
return d
def __init__(self, private_key, public_key, alias=None):
self.private_key = private_key
self.public_key = public_key
self._node = Node()
if alias:
self.alias = alias
else:
self.alias = {}
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.addr3 = ("193.193.111.00", 99999)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "63d901c4d57cde34fc1f1e28b9af5d56ed342cae5c2fb470046d0130a4226b0c"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
h = nacl.hash.sha512(verify_key.encode())
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, verify_key.encode(), None, FULL_CONE, True)
self.db = Database(filepath="test.db")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db, self.signing_key)
self.wire_protocol = PulseShopProtocol(self.db, self.own_addr, FULL_CONE)
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol, None,
self.wire_protocol.ban_score)
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
self.node1 = Node(digest("id1"), self.addr1[0], self.addr1[1], digest("key1"), None, FULL_CONE, True)
self.node2 = Node(digest("id2"), self.addr2[0], self.addr2[1], digest("key2"), None, FULL_CONE, True)
self.node3 = Node(digest("id3"), self.addr3[0], self.addr3[1], digest("key3"), None, FULL_CONE, True)
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.addr3 = ("193.193.111.00", 99999)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
signed_pubkey = self.signing_key.sign(str(verify_key))
h = nacl.hash.sha512(signed_pubkey)
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
self.db = Database(filepath=":memory:")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db)
self.wire_protocol = OpenBazaarProtocol(self.own_addr)
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol)
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
self.node1 = Node(digest("id1"), self.addr1[0], self.addr1[1], digest("key1"), True)
self.node2 = Node(digest("id2"), self.addr2[0], self.addr2[1], digest("key2"), True)
self.node3 = Node(digest("id3"), self.addr3[0], self.addr3[1], digest("key3"), True)
def on_response_received(self, transaction):
# Update the id of the remote node.
transaction.response_node.id_20 = transaction.response["r"]["id"]
# Add the node that reponded to us. We most likely already have this one.
self._server.remote_nodes.add(transaction.response_node)
# A find_node or get_peers response may carry extra nodes.
if transaction.query["q"] == "find_node" or transaction.query["q"] == "get_peers":
nodes = transaction.response["r"].get("nodes", "")
for i in xrange(0, len(nodes), 26):
id_26 = nodes[i : i + 26]
if not self._server.remote_nodes.get_node(id_26):
self._server.remote_nodes.add(Node.from_id_26(id_26))
def _receiver(self):
"""
Run in a different thread, and have as only goal to receive messages,
parse them, and put them into a thread safe queue for further handling
"""
while True:
try:
# Receive message
raw_message, connect_info = self._sock.recvfrom(4096)
message = bdecode(raw_message)
# print "Received f " + str(connect_info) + ":\t" + str(message)
# This is a query from another peer.
if message['y'] == 'q':
# Retrieve or create source node.
id_26 = message['a']['id'] + socket.inet_aton(connect_info[0]) + struct.pack(r'!H', connect_info[1])
source_node = self.remote_nodes.get_node(id_26) or Node.from_id_26(id_26)
transaction = Transaction(query=message, query_node=source_node, response_node=self.node) # create transaction
self._running_transactions[transaction.id] = transaction # Add it to running transactions
for handler in self.on_query_received:
handler(transaction) # Notify listeners
elif message['y'] == 'e':
raise Exception
# This is a response to a transaction we started.
elif message['y'] == 'r':
transaction = self._running_transactions.pop(message["t"]) # pop transaction
transaction.response = message
for handler in self.on_response_received:
handler(transaction) # Notify listeners
else:
raise Exception
except socket.timeout:
pass
except BTFailure:
pass
except:
print traceback.format_exc()
def test_rpc_find_node(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"), nat_type=objects.FULL_CONE)
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"), nat_type=objects.FULL_CONE)
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"), nat_type=objects.FULL_CONE)
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_NODE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("nodetofind"))
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
del m.arguments[-1]
m.arguments.extend(
[
node2.getProto().SerializeToString(),
node1.getProto().SerializeToString(),
node3.getProto().SerializeToString(),
]
)
m.ClearField("signature")
expected_message = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
a.ClearField("signature")
received_message = a.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
class Drogulus(object):
"""
Represents a node in the Drogulus distributed hash table. This is the
class that generally should be instantiated.
"""
def __init__(self, private_key, public_key, alias=None):
self.private_key = private_key
self.public_key = public_key
self._node = Node()
if alias:
self.alias = alias
else:
self.alias = {}
def join(self):
"""
Causes the node to join the distributed hash table. Returns a deferred
that fires when the operation is complete.
"""
pass
def whois(self, public_key):
"""
Given the public key of an entity that uses the drogulus will return a
deferred that fires when information about them stored in the DHT is
retrieved.
"""
return self.get(public_key, None)
def get(self, public_key, key_name):
"""
Gets the value associated with a compound key made of the passed in
public key and meaningful key name. Returns a deferred that fires when
the value is retrieved.
"""
target = construct_key(public_key, key_name)
return self._node.retrieve(target)
def set(self, key_name, value, duplicate=DUPLICATION_COUNT, meta=None,
expires=EXPIRY_DURATION):
"""
Stores a value at a compound key made from the local node's public key
and the passed in meaningful key name. Returns a deferred that fires
when the value has been stored to duplicate number of nodes. An
optional meta dictionary and expires duration (to be added to the
current time) can also be specified.
"""
timestamp = time.time()
if meta is None:
meta = {}
if expires < 1:
expires = -1
else:
expires = timestamp + expires
signature = generate_signature(value, timestamp, expires, key_name,
meta, self.private_key)
return self._node.replicate(self.public_key, key_name, value,
timestamp, expires, meta, signature,
duplicate)
class ValueSpiderCrawlTest(unittest.TestCase):
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.addr3 = ("193.193.111.00", 99999)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "63d901c4d57cde34fc1f1e28b9af5d56ed342cae5c2fb470046d0130a4226b0c"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
h = nacl.hash.sha512(verify_key.encode())
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, verify_key.encode(), None, FULL_CONE, True)
self.db = Database(filepath="test.db")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db, self.signing_key)
self.wire_protocol = OpenBazaarProtocol(self.db, self.own_addr, FULL_CONE)
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol, None)
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
self.node1 = Node(digest("id1"), self.addr1[0], self.addr1[1], digest("key1"), None, FULL_CONE, True)
self.node2 = Node(digest("id2"), self.addr2[0], self.addr2[1], digest("key2"), None, FULL_CONE, True)
self.node3 = Node(digest("id3"), self.addr3[0], self.addr3[1], digest("key3"), None, FULL_CONE, True)
def tearDown(self):
self.con.shutdown()
self.wire_protocol.shutdown()
os.remove("test.db")
def test_find(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
spider.find()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 7)
def test_nodesFound(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
# test response with uncontacted nodes
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 7)
# test all been contacted
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
for peer in spider.nearest.getUncontacted():
spider.nearest.markContacted(peer)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node2.id: response}
resp = spider._nodesFound(responses)
self.assertTrue(resp is None)
# test didn't happen
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (False, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.assertTrue(len(spider.nearest) == 2)
# test got value
val = Value()
val.valueKey = digest("contractID")
val.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
val.ttl = 10
response = (True, ("value", val.SerializeToString()))
responses = {self.node3.id: response}
spider.nearestWithoutValue = NodeHeap(node, 1)
value = spider._nodesFound(responses)
self.assertEqual(value[0], val.SerializeToString())
def test_handleFoundValues(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
val = Value()
val.valueKey = digest("contractID")
val.serializedData = self.node1.getProto().SerializeToString()
val.ttl = 10
val1 = val.SerializeToString()
value = spider._handleFoundValues([val1])
self.assertEqual(value[0], val.SerializeToString())
# test handle multiple values
val.serializedData = self.node2.getProto().SerializeToString()
val2 = val.SerializeToString()
val.valueKey = digest("contractID2")
val3 = val.SerializeToString()
found_values = [val1, val2, val2, val3]
self.assertEqual(spider._handleFoundValues(found_values), [val3, val2])
# test store value at nearest without value
spider.nearestWithoutValue.push(self.node1)
spider._handleFoundValues(found_values)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertTrue(len(self.proto_mock.send_datagram.call_args_list) > 1)
self.proto_mock.send_datagram.call_args_list = []
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet, self.addr1)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1
class NodeSpiderCrawlTest(unittest.TestCase):
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.addr3 = ("193.193.111.00", 99999)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "63d901c4d57cde34fc1f1e28b9af5d56ed342cae5c2fb470046d0130a4226b0c"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
h = nacl.hash.sha512(verify_key.encode())
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, verify_key.encode(), None, FULL_CONE, True)
self.db = Database(filepath="test.db")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db, self.signing_key)
self.wire_protocol = PulseShopProtocol(self.db, self.own_addr, FULL_CONE)
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol, None,
self.wire_protocol.ban_score)
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
self.node1 = Node(digest("id1"), self.addr1[0], self.addr1[1], digest("key1"), None, FULL_CONE, True)
self.node2 = Node(digest("id2"), self.addr2[0], self.addr2[1], digest("key2"), None, FULL_CONE, True)
self.node3 = Node(digest("id3"), self.addr3[0], self.addr3[1], digest("key3"), None, FULL_CONE, True)
def tearDown(self):
self.con.shutdown()
self.wire_protocol.shutdown()
os.remove("test.db")
def test_find(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
spider.find()
self.clock.advance(constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
def test_nodesFound(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node1.getProto() in node_protos)
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
response = (False, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet, self.addr1)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1
class NodeSpiderCrawlTest(unittest.TestCase):
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.addr3 = ("193.193.111.00", 99999)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
signed_pubkey = self.signing_key.sign(str(verify_key))
h = nacl.hash.sha512(signed_pubkey)
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
self.db = Database(filepath=":memory:")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db)
self.wire_protocol = OpenBazaarProtocol(self.own_addr, "Full Cone")
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol)
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
self.node1 = Node(digest("id1"), self.addr1[0], self.addr1[1], digest("key1"), True)
self.node2 = Node(digest("id2"), self.addr2[0], self.addr2[1], digest("key2"), True)
self.node3 = Node(digest("id3"), self.addr3[0], self.addr3[1], digest("key3"), True)
def test_find(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
spider.find()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
def test_nodesFound(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node1.getProto() in node_protos)
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
response = (False, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1
def test_tuple(self):
n = Node('127.0.0.1', 0, 'testkey')
i = n.__iter__()
self.assertIn('127.0.0.1', i)
self.assertIn(0, i)
self.assertIn('testkey', i)
def test_tuple(self):
n = Node("127.0.0.1", 0, "testkey")
i = n.__iter__()
self.assertIn("127.0.0.1", i)
self.assertIn(0, i)
self.assertIn("testkey", i)
class ValueSpiderCrawlTest(unittest.TestCase):
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.addr3 = ("193.193.111.00", 99999)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
signed_pubkey = self.signing_key.sign(str(verify_key))
h = nacl.hash.sha512(signed_pubkey)
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
self.protocol = KademliaProtocol(self.node, self.storage, 20)
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.protocol.makeConnection(transport)
self.node1 = Node(digest("id1"), self.addr1[0], self.addr1[1], digest("key1"), True)
self.node2 = Node(digest("id2"), self.addr2[0], self.addr2[1], digest("key2"), True)
self.node3 = Node(digest("id3"), self.addr3[0], self.addr3[1], digest("key3"), True)
def tearDown(self):
self.con.shutdown()
self.protocol.shutdown()
def test_find(self):
self._connecting_to_connected()
self.protocol[self.addr1] = self.con
self.protocol[self.addr2] = self.con
self.protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
spider.find()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
def test_nodesFound(self):
self._connecting_to_connected()
self.protocol[self.addr1] = self.con
self.protocol[self.addr2] = self.con
self.protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
# test resonse with uncontacted nodes
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
# test all been contacted
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
for peer in spider.nearest.getUncontacted():
spider.nearest.markContacted(peer)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node2.id: response}
resp = spider._nodesFound(responses)
self.assertTrue(resp is None)
# test didn't happen
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (False, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.assertTrue(len(spider.nearest) == 2)
# test got value
val = Value()
val.valueKey = digest("contractID")
val.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
response = (True, ("value", val.SerializeToString()))
responses = {self.node3.id: response}
spider.nearestWithoutValue = NodeHeap(node, 1)
value = spider._nodesFound(responses)
self.assertEqual(value[0], val.SerializeToString())
def test_handleFoundValues(self):
self._connecting_to_connected()
self.protocol[self.addr1] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest, 20, 3)
val = Value()
val.valueKey = digest("contractID")
val.serializedData = self.node1.getProto().SerializeToString()
val1 = val.SerializeToString()
value = spider._handleFoundValues([(val1,)])
self.assertEqual(value[0], val.SerializeToString())
# test handle multiple values
val.serializedData = self.node2.getProto().SerializeToString()
val2 = val.SerializeToString()
found_values = [(val1,), (val1,), (val2,)]
self.assertEqual(spider._handleFoundValues(found_values), (val1,))
# test store value at nearest without value
spider.nearestWithoutValue.push(self.node1)
spider._handleFoundValues(found_values)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertTrue(len(self.proto_mock.send_datagram.call_args_list) > 1)
self.proto_mock.send_datagram.call_args_list = []
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1