def __init__(self, id, client_string='ssl:%s:%d'):
"""
Initialises the object representing the node with the given id.
"""
# The node's ID within the distributed hash table.
self.id = id
# The routing table stores information about other nodes on the DHT.
self._routing_table = RoutingTable(id)
# The local key/value store containing data held by this node.
self._data_store = DictDataStore()
# A dictionary of IDs for messages pending a response and associated
# deferreds to be fired when a response is completed.
self._pending = {}
# The template string to use when initiating a connection to another
# node on the network.
self._client_string = client_string
# The version of Drogulus that this node implements.
self.version = get_version()
log.msg('Initialised node with id: %r' % self.id)
class Node(object):
"""
This class represents a single local node in the DHT encapsulating its
presence in the network.
All interactions with the DHT network by a client application are
performed via this class (or a subclass).
"""
def __init__(self, id, client_string='ssl:%s:%d'):
"""
Initialises the object representing the node with the given id.
"""
# The node's ID within the distributed hash table.
self.id = id
# The routing table stores information about other nodes on the DHT.
self._routing_table = RoutingTable(id)
# The local key/value store containing data held by this node.
self._data_store = DictDataStore()
# A dictionary of IDs for messages pending a response and associated
# deferreds to be fired when a response is completed.
self._pending = {}
# The template string to use when initiating a connection to another
# node on the network.
self._client_string = client_string
# The version of Drogulus that this node implements.
self.version = get_version()
log.msg('Initialised node with id: %r' % self.id)
def join(self, seed_nodes=None):
"""
Causes the Node to join the DHT network. This should be called before
any other DHT operations. The seed_nodes argument must be a list of
already known contacts describing existing nodes on the network.
"""
if not seed_nodes:
raise ValueError('Seed nodes required for node to join network')
for contact in seed_nodes:
self._routing_table.add_contact(contact)
# Looking up the node's ID on the network will populate the routing
# table with fresh nodes as well as tell us who our nearest neighbours
# are.
# TODO: Add callback to kick off refresh of k-buckets in future..?
raise Exception('FIX ME!')
# Ensure the refresh of k-buckets is set up properly.
return NodeLookup(self.id, FindNode, self)
def message_received(self, message, protocol):
"""
Handles incoming messages.
"""
# Update the routing table.
peer = protocol.transport.getPeer()
other_node = Contact(message.node, peer.host, peer.port,
message.version, time.time())
log.msg('Message received from %s' % other_node)
log.msg(message)
self._routing_table.add_contact(other_node)
# Sort on message type and pass to handler method. Explicit > implicit.
if isinstance(message, Ping):
self.handle_ping(message, protocol)
elif isinstance(message, Pong):
self.handle_pong(message)
elif isinstance(message, Store):
self.handle_store(message, protocol, other_node)
elif isinstance(message, FindNode):
self.handle_find_node(message, protocol)
elif isinstance(message, FindValue):
self.handle_find_value(message, protocol)
elif isinstance(message, Error):
self.handle_error(message, protocol, other_node)
elif isinstance(message, Value):
self.handle_value(message, other_node)
elif isinstance(message, Nodes):
self.handle_nodes(message)
def send_message(self, contact, message):
"""
Sends a message to the specified contact, adds it to the _pending
dictionary and ensures it times-out after the correct period. If an
error occurs the deferred's errback is called.
"""
d = defer.Deferred()
# open network call.
client_string = self._client_string % (contact.address, contact.port)
client = clientFromString(reactor, client_string)
connection = client.connect(DHTFactory(self))
# Ensure the connection will potentially time out.
connection_timeout = reactor.callLater(constants.RPC_TIMEOUT,
connection.cancel)
def on_connect(protocol):
# Cancel pending connection_timeout if it's still active.
if connection_timeout.active():
connection_timeout.cancel()
# Send the message and add a timeout for the response.
protocol.sendMessage(message)
self._pending[message.uuid] = d
reactor.callLater(constants.RESPONSE_TIMEOUT, response_timeout,
message, protocol, self)
def on_error(error):
log.msg('***** ERROR ***** interacting with %s' % contact)
log.msg(error)
self._routing_table.remove_contact(message.node)
if message.uuid in self._pending:
del self._pending[message.uuid]
d.errback(error)
connection.addCallback(on_connect)
connection.addErrback(on_error)
return d
def trigger_deferred(self, message, error=False):
"""
Given a message, will attempt to retrieve the deferred and trigger it
with the appropriate callback or errback.
"""
if message.uuid in self._pending:
deferred = self._pending[message.uuid]
if error:
error.message = message
deferred.errback(error)
else:
deferred.callback(message)
# Remove the called deferred from the _pending dictionary.
del self._pending[message.uuid]
def handle_ping(self, message, protocol):
"""
Handles an incoming Ping message. Returns a Pong message using the
referenced protocol object.
"""
pong = Pong(message.uuid, self.id, self.version)
protocol.sendMessage(pong, True)
def handle_pong(self, message):
"""
Handles an incoming Pong message.
"""
self.trigger_deferred(message)
def handle_store(self, message, protocol, sender):
"""
Handles an incoming Store message. Checks the provenance and timeliness
of the message before storing locally. If there is a problem, removes
the untrustworthy peer from the routing table. Otherwise, at
REPLICATE_INTERVAL minutes in the future, the local node will attempt
to replicate the Store message elsewhere in the DHT if such time is
<= the message's expiry time.
Sends a Pong message if successful otherwise replies with an
appropriate Error.
"""
# Check provenance
is_valid, err_code = validate_message(message)
if is_valid:
# Ensure the node doesn't already have a more up-to-date version
# of the value.
current = self._data_store.get(message.key, False)
if current and (message.timestamp < current.timestamp):
# The node already has a later version of the value so
# return an error.
details = {
'new_timestamp': '%d' % current.timestamp
}
raise ValueError(8, constants.ERRORS[8], details,
message.uuid)
# Good to go, so store value.
self._data_store.set_item(message.key, message)
# Reply with a pong so the other end updates its routing table.
pong = Pong(message.uuid, self.id, self.version)
protocol.sendMessage(pong, True)
# At some future time attempt to replicate the Store message
# around the network IF it is within the message's expiry time.
raise Exception("FIX ME!")
# Need to check that callLater is called as part of the tests.
reactor.callLater(constants.REPLICATE_INTERVAL,
self.republish, message)
else:
# Remove from the routing table.
log.msg('Problem with Store command: %d - %s' %
(err_code, constants.ERRORS[err_code]))
self._routing_table.blacklist(sender)
# Return an error.
details = {
'message': 'You have been blacklisted.'
}
raise ValueError(err_code, constants.ERRORS[err_code], details,
message.uuid)
def handle_find_node(self, message, protocol):
"""
Handles an incoming FindNode message. Finds the details of up to K
other nodes closer to the target key that *this* node knows about.
Responds with a "Nodes" message containing the list of matching
nodes.
"""
target_key = message.key
# List containing tuples of information about the matching contacts.
other_nodes = [(n.id, n.address, n.port, n.version) for n in
self._routing_table.find_close_nodes(target_key)]
result = Nodes(message.uuid, self.id, other_nodes, self.version)
protocol.sendMessage(result, True)
def handle_find_value(self, message, protocol):
"""
Handles an incoming FindValue message. If the local node contains the
value associated with the requested key replies with an appropriate
"Value" message. Otherwise, responds with details of up to K other
nodes closer to the target key that the local node knows about. In
this case a "Nodes" message containing the list of matching nodes is
sent to the caller.
"""
match = self._data_store.get(message.key, False)
if match:
result = Value(message.uuid, self.id, match.key, match.value,
match.timestamp, match.expires, match.public_key,
match.name, match.meta, match.sig, match.version)
protocol.sendMessage(result, True)
else:
self.handle_find_node(message, protocol)
def handle_error(self, message, protocol, sender):
"""
Handles an incoming Error message. Currently, this simply logs the
error and closes the connection. In future this *may* remove the
sender from the routing table (depending on the error).
"""
# TODO: Handle error 8 (out of date data)
log.msg('***** ERROR ***** from %s' % sender)
log.msg(message)
def handle_value(self, message, sender):
"""
Handles an incoming Value message containing a value retrieved from
another node on the DHT. Ensures the message is valid and calls the
referenced deferred to signal the arrival of the value.
TODO: How to handle invalid messages and errback the deferred.
"""
# Check provenance
is_valid, err_code = validate_message(message)
if is_valid:
self.trigger_deferred(message)
else:
log.msg('Problem with incoming Value: %d - %s' %
(err_code, constants.ERRORS[err_code]))
log.msg(message)
# Remove the remote node from the routing table.
self._routing_table.remove_contact(sender.id, True)
error = ValueError(constants.ERRORS[err_code])
self.trigger_deferred(message, error)
def handle_nodes(self, message):
"""
Handles an incoming Nodes message containing information about other
nodes on the network that are close to a requested key.
"""
self.trigger_deferred(message)
def send_ping(self, contact):
"""
Sends a ping request to the given contact and returns a deferred
that is fired when the reply arrives or an error occurs.
"""
new_uuid = str(uuid4())
ping = Ping(new_uuid, self.id, self.version)
return self.send_message(contact, ping)
def send_store(self, contact, public_key, name, value, timestamp, expires,
meta, signature):
"""
Sends a Store message to the given contact. The value contained within
the message is stored against a key derived from the public_key and
name. Furthermore, the message is cryptographically signed using the
value, timestamp, expires, name and meta values.
"""
uuid = str(uuid4())
compound_key = construct_key(public_key, name)
store = Store(uuid, self.id, compound_key, value, timestamp, expires,
public_key, name, meta, signature, self.version)
return self.send_message(contact, store)
def send_find(self, contact, target, message_type):
"""
Sends a Find[Node|Value] message to the given contact with the
intention of obtaining information at the given target key. The type of
find message is specified by message_type.
"""
new_uuid = str(uuid4())
find_message = message_type(new_uuid, self.id, target, self.version)
deferred = self.send_message(contact, find_message)
return (new_uuid, deferred)
def _process_lookup_result(self, nearest_nodes, public_key, name, value,
timestamp, expires, meta, signature, length):
"""
Given a list of nearest nodes will return a list of send_store based
deferreds for the item to be stored in the DHT. The list will contain
up to "length" number of deferreds.
"""
list_of_deferreds = []
for contact in nearest_nodes[:length]:
deferred = self.send_store(contact, public_key, name, value,
timestamp, expires, meta, signature)
list_of_deferreds.append(deferred)
return list_of_deferreds
def replicate(self, public_key, name, value, timestamp, expires, meta,
signature, duplicate):
"""
Will replicate args to "duplicate" number of nodes in the distributed
hash table. Returns a deferred that will fire with a list of send_store
deferreds when "duplicate" number of closest nodes have been
identified.
Obviously, the list can be turned in to a deferred_list to fire when
the store commands have completed.
Even if "duplicate" is > K no more than K items will be contained
within the list result.
"""
if duplicate < 1:
# Guard to ensure meaningful duplication count.
raise ValueError('Duplication count may not be less than 1')
result = defer.Deferred()
compound_key = construct_key(public_key, name)
lookup = NodeLookup(compound_key, FindNode, self)
def on_success(nodes):
"""
A list of close nodes have been found so send store messages to
the "duplicate" closest number of them and fire the "result"
deferred with the resulting DeferredList of pending deferreds.
"""
deferreds = self._process_lookup_result(nodes, public_key, name,
value, timestamp, expires,
meta, signature, duplicate)
result.callback(deferreds)
def on_error(error):
"""
Catch all for errors during the lookup phase. Simply pass them on
via the "result" deferred.
"""
result.errback(error)
lookup.addCallback(on_success)
lookup.addErrback(on_error)
return result
def retrieve(self, key):
"""
Given a key, will try to retrieve associated value from the distributed
hash table. Returns a deferred that will fire when the operation is
complete or failed.
As the original Kademlia explains:
"For caching purposes, once a lookup succeeds, the requesting node
stores the <key, value> pair at the closest node it observed to the
key that did not return the value."
This method adds a callback to the NodeLookup to achieve this end.
"""
lookup = NodeLookup(key, FindValue, self)
def cache(result):
"""
Called once the lookup succeeds in order to store the item at the
node closest to the key that did not return the value.
"""
caching_contact = None
for candidate in lookup.shortlist:
if candidate in lookup.contacted:
caching_contact = candidate
break
if caching_contact:
log.msg("Caching to %r" % caching_contact)
self.send_store(caching_contact, result.public_key,
result.name, result.value, result.timestamp,
result.expires, result.meta, result.sig)
return result
lookup.addCallback(cache)
return lookup
def republish(self, message):
"""
Will check and republish a locally stored message to the wider network.
From the original Kademlia paper:
"To ensure the persistence of key-value pairs, nodes must periodically
republish keys. Otherwise, two phenomena may cause lookups for valid
keys to fail. First, some of the k nodes that initially get a key-value
pair when it is published may leave the network. Second, new nodes may
join the network with IDs closer to some published key than the nodes
on which the key-value pair was originally published. In both cases,
the nodes with a key-value pair must republish it so as once again to
ensure it is available on the k nodes closest to the key.
To compensate for nodes leaving the network, Kademlia republishes each
key-value pair once an hour. A naive implementation of this strategy
would require many messages - each of up to k nodes storing a key-value
pair would perform a node lookup followed by k - 1 STORE RPCs every
hour. Fortunately, the republish process can be heavily optimized.
First, when a node receives a STORE RPC for a given key-value pair, it
assumes the RPC was also issued to the other k - 1 closest nodes, and
thus the recipient will not republish the key-value pair in the next
hour. This ensures that as long as republication intervals are not
exactly synchronized, only one node will republish a given key-value
pair every hour.
A second optimization avoids performing node lookups before
republishing keys. As described in Section 2.4, to handle unbalanced
trees, nodes split k-buckets as required to ensure they have complete
knowledge of a surrounding subtree with at least k nodes. If, before
republishing key-value pairs, a node u refreshes all k-buckets in this
subtree of k nodes, it will automatically be able to figure out the
k closest nodes to a given key. These bucket refreshes can be amortized
over the republication of many keys.
To see why a node lookup is unnecessary after u refreshes buckets in
the sub-tree of size >= k, it is necessary to consider two cases. If
the key being republished falls in the ID range of the subtree, then
since the subtree is of size at least k and u has complete knowledge of
the subtree, clearly u must know the k closest nodes to the key. If,
on the other hand, the key lies outside the subtree, yet u was one of
the k closest nodes to the key, it must follow that u's k-buckets for
intervals closer to the key than the subtree all have fewer than k
entries. Hence, u will know all nodes in these k-buckets, which
together with knowledge of the subtree will include the k closest nodes
to the key.
When a new node joins the system, it must store any key-value pair to
which is is one of the k closest. Existing nodes, by similarly
exploiting complete knowledge of their surrounding subtrees, will know
which key-value pairs the new node should store. Any node learning of a
new node therefore issues STORE RPCs to transfer relevant key-value
pairs to the new node. To avoid redundant STORE RPCs, however, a node
only transfers a key-value pair if it's [sic] own ID is closer to the
key than are the IDs of other nodes."
Messages are only republished if the following requirements are met:
* They still exist in the local data store.
* They have not expired.
* They have not been updated for REPLICATE_INTERVAL seconds.
"""
pass
def __init__(self, num = -1): super(Compute,self).__init__() self.setNodeNum(num) self.rtables.append(RoutingTable())
def __init__(self, num = -1): super(Router,self).__init__() self.setNodeNum(num) self.num_ports = 5 self.rtables.extend([RoutingTable() for i in xrange(self.num_ports)]) self.prtables.extend([RoutingTable() for i in xrange(self.num_ports)])
class Node(object):
"""
This class represents a single local node in the DHT encapsulating its
presence in the network.
All interactions with the DHT network by a client application are
performed via this class (or a subclass).
"""
def __init__(self, id, client_string='ssl:%s:%d'):
"""
Initialises the object representing the node with the given id.
"""
# The node's ID within the distributed hash table.
self.id = id
# The routing table stores information about other nodes on the DHT.
self._routing_table = RoutingTable(id)
# The local key/value store containing data held by this node.
self._data_store = DictDataStore()
# A dictionary of IDs for messages pending a response and associated
# deferreds to be fired when a response is completed.
self._pending = {}
# The template string to use when initiating a connection to another
# node on the network.
self._client_string = client_string
# The version of Drogulus that this node implements.
self.version = get_version()
log.msg('Initialised node with id: %r' % self.id)
def join(self, seed_nodes=None):
"""
Causes the Node to join the DHT network. This should be called before
any other DHT operations. The seedNodes argument contains a list of
tuples describing existing nodes on the network in the form of their
IP address and port.
"""
pass
def message_received(self, message, protocol):
"""
Handles incoming messages.
"""
# Update the routing table.
peer = protocol.transport.getPeer()
other_node = Contact(message.node, peer.host, peer.port,
message.version, time.time())
log.msg('Message received from %s' % other_node)
log.msg(message)
self._routing_table.add_contact(other_node)
# Sort on message type and pass to handler method. Explicit > implicit.
if isinstance(message, Ping):
self.handle_ping(message, protocol)
elif isinstance(message, Pong):
self.handle_pong(message)
elif isinstance(message, Store):
self.handle_store(message, protocol, other_node)
elif isinstance(message, FindNode):
self.handle_find_node(message, protocol)
elif isinstance(message, FindValue):
self.handle_find_value(message, protocol)
elif isinstance(message, Error):
self.handle_error(message, protocol, other_node)
elif isinstance(message, Value):
self.handle_value(message, other_node)
elif isinstance(message, Nodes):
self.handle_nodes(message)
def send_message(self, contact, message):
"""
Sends a message to the specified contact, adds it to the _pending
dictionary and ensures it times-out after the correct period. If an
error occurs the deferred's errback is called.
"""
d = defer.Deferred()
# open network call.
client_string = self._client_string % (contact.address, contact.port)
client = clientFromString(reactor, client_string)
connection = client.connect(DHTFactory(self))
# Ensure the connection will potentially time out.
connection_timeout = reactor.callLater(constants.RPC_TIMEOUT,
connection.cancel)
def on_connect(protocol):
# Cancel pending connection_timeout if it's still active.
if connection_timeout.active():
connection_timeout.cancel()
# Send the message and add a timeout for the response.
protocol.sendMessage(message)
self._pending[message.uuid] = d
reactor.callLater(constants.RESPONSE_TIMEOUT, response_timeout,
message, protocol, self)
def on_error(error):
log.msg('***** ERROR ***** interacting with %s' % contact)
log.msg(error)
self._routing_table.remove_contact(message.node)
if message.uuid in self._pending:
del self._pending[message.uuid]
d.errback(error)
connection.addCallback(on_connect)
connection.addErrback(on_error)
return d
def trigger_deferred(self, message, error=False):
"""
Given a message, will attempt to retrieve the deferred and trigger it
with the appropriate callback or errback.
"""
if message.uuid in self._pending:
deferred = self._pending[message.uuid]
if error:
error.message = message
deferred.errback(error)
else:
deferred.callback(message)
# Remove the called deferred from the _pending dictionary.
del self._pending[message.uuid]
def handle_ping(self, message, protocol):
"""
Handles an incoming Ping message. Returns a Pong message using the
referenced protocol object.
"""
pong = Pong(message.uuid, self.id, self.version)
protocol.sendMessage(pong, True)
def handle_pong(self, message):
"""
Handles an incoming Pong message.
"""
self.trigger_deferred(message)
def handle_store(self, message, protocol, sender):
"""
Handles an incoming Store message. Checks the provenance and timeliness
of the message before storing locally. If there is a problem, removes
the untrustworthy peer from the routing table. Otherwise, at
REPLICATE_INTERVAL minutes in the future, the local node will attempt
to replicate the Store message elsewhere in the DHT if such time is
<= the message's expiry time.
Sends a Pong message if successful otherwise replies with an
appropriate Error.
"""
# Check provenance
is_valid, err_code = validate_message(message)
if is_valid:
# Ensure the node doesn't already have a more up-to-date version
# of the value.
current = self._data_store.get(message.key, False)
if current and (message.timestamp < current.timestamp):
# The node already has a later version of the value so
# return an error.
details = {
'new_timestamp': '%d' % current.timestamp
}
raise ValueError(8, constants.ERRORS[8], details,
message.uuid)
# Good to go, so store value.
self._data_store.set_item(message.key, message)
# Reply with a pong so the other end updates its routing table.
pong = Pong(message.uuid, self.id, self.version)
protocol.sendMessage(pong, True)
# At some future time attempt to replicate the Store message
# around the network IF it is within the message's expiry time.
reactor.callLater(constants.REPLICATE_INTERVAL,
self.replicate, message)
else:
# Remove from the routing table.
log.msg('Problem with Store command: %d - %s' %
(err_code, constants.ERRORS[err_code]))
self._routing_table.blacklist(sender)
# Return an error.
details = {
'message': 'You have been blacklisted.'
}
raise ValueError(err_code, constants.ERRORS[err_code], details,
message.uuid)
def handle_find_node(self, message, protocol):
"""
Handles an incoming FindNode message. Finds the details of up to K
other nodes closer to the target key that *this* node knows about.
Responds with a "Nodes" message containing the list of matching
nodes.
"""
target_key = message.key
# List containing tuples of information about the matching contacts.
other_nodes = [(n.id, n.address, n.port, n.version) for n in
self._routing_table.find_close_nodes(target_key)]
result = Nodes(message.uuid, self.id, other_nodes, self.version)
protocol.sendMessage(result, True)
def handle_find_value(self, message, protocol):
"""
Handles an incoming FindValue message. If the local node contains the
value associated with the requested key replies with an appropriate
"Value" message. Otherwise, responds with details of up to K other
nodes closer to the target key that the local node knows about. In
this case a "Nodes" message containing the list of matching nodes is
sent to the caller.
"""
match = self._data_store.get(message.key, False)
if match:
result = Value(message.uuid, self.id, match.key, match.value,
match.timestamp, match.expires, match.public_key,
match.name, match.meta, match.sig, match.version)
protocol.sendMessage(result, True)
else:
self.handle_find_node(message, protocol)
def handle_error(self, message, protocol, sender):
"""
Handles an incoming Error message. Currently, this simply logs the
error and closes the connection. In future this *may* remove the
sender from the routing table (depending on the error).
"""
# TODO: Handle error 8 (out of date data)
log.msg('***** ERROR ***** from %s' % sender)
log.msg(message)
def handle_value(self, message, sender):
"""
Handles an incoming Value message containing a value retrieved from
another node on the DHT. Ensures the message is valid and calls the
referenced deferred to signal the arrival of the value.
TODO: How to handle invalid messages and errback the deferred.
"""
# Check provenance
is_valid, err_code = validate_message(message)
if is_valid:
self.trigger_deferred(message)
else:
log.msg('Problem with incoming Value: %d - %s' %
(err_code, constants.ERRORS[err_code]))
log.msg(message)
# Remove the remote node from the routing table.
self._routing_table.remove_contact(sender.id, True)
error = ValueError(constants.ERRORS[err_code])
self.trigger_deferred(message, error)
def handle_nodes(self, message):
"""
Handles an incoming Nodes message containing information about other
nodes on the network that are close to a requested key.
"""
self.trigger_deferred(message)
def send_ping(self, contact):
"""
Sends a ping request to the given contact and returns a deferred
that is fired when the reply arrives or an error occurs.
"""
new_uuid = str(uuid4())
ping = Ping(new_uuid, self.id, self.version)
return self.send_message(contact, ping)
def send_store(self, contact, public_key, name, value, timestamp, expires,
meta, signature):
"""
Sends a Store message to the given contact. The value contained within
the message is stored against a key derived from the public_key and
name. Furthermore, the message is cryptographically signed using the
value, timestamp, expires, name and meta values.
"""
uuid = str(uuid4())
compound_key = construct_key(public_key, name)
store = Store(uuid, self.id, compound_key, value, timestamp, expires,
public_key, name, meta, signature, self.version)
return self.send_message(contact, store)
def send_find(self, contact, target, message_type):
"""
Sends a Find[Node|Value] message to the given contact with the
intention of obtaining information at the given target key. The type of
find message is specified by message_type.
"""
new_uuid = str(uuid4())
find_message = message_type(new_uuid, self.id, target, self.version)
deferred = self.send_message(contact, find_message)
return (new_uuid, deferred)
def _process_lookup_result(self, nearest_nodes, public_key, name, value,
timestamp, expires, meta, signature, length):
"""
Given a list of nearest nodes will return a list of send_store based
deferreds for the item to be stored in the DHT. The list will contain
up to "length" number of deferreds.
"""
list_of_deferreds = []
for contact in nearest_nodes[:length]:
deferred = self.send_store(contact, public_key, name, value,
timestamp, expires, meta, signature)
list_of_deferreds.append(deferred)
return list_of_deferreds
def replicate(self, public_key, name, value, timestamp, expires, meta,
signature, duplicate):
"""
Will replicate args to "duplicate" number of nodes in the distributed
hash table. Returns a deferred that will fire with a list of send_store
deferreds when "duplicate" number of closest nodes have been
identified.
Obviously, the list can be turned in to a deferred_list to fire when
the store commands have completed.
Even if "duplicate" is > K no more than K items will be contained
within the list result.
"""
if duplicate < 1:
# Guard to ensure meaningful duplication count.
raise ValueError('Duplication count may not be less than 1')
result = defer.Deferred()
compound_key = construct_key(public_key, name)
lookup = NodeLookup(compound_key, FindNode, self)
def on_success(nodes):
"""
A list of close nodes have been found so send store messages to
the "duplicate" closest number of them and fire the "result"
deferred with the resulting DeferredList of pending deferreds.
"""
deferreds = self._process_lookup_result(nodes, public_key, name,
value, timestamp, expires,
meta, signature, duplicate)
result.callback(deferreds)
def on_error(error):
"""
Catch all for errors during the lookup phase. Simply pass them on
via the "result" deferred.
"""
result.errback(error)
lookup.addCallback(on_success)
lookup.addErrback(on_error)
return result
def retrieve(self, key):
"""
Given a key, will try to retrieve associated value from the distributed
hash table. Returns a deferred that will fire when the operation is
complete or failed.
As the original Kademlia explains:
"For caching purposes, once a lookup succeeds, the requesting node
stores the <key, value> pair at the closest node it observed to the
key that did not return the value."
This method adds a callback to the NodeLookup to achieve this end.
"""
lookup = NodeLookup(key, FindValue, self)
def cache(result):
"""
Called once the lookup succeeds in order to store the item at the
node closest to the key that did not return the value.
"""
caching_contact = None
for candidate in lookup.shortlist:
if candidate in lookup.contacted:
caching_contact = candidate
break
if caching_contact:
log.msg("Caching to %r" % caching_contact)
self.send_store(caching_contact, result.public_key,
result.name, result.value, result.timestamp,
result.expires, result.meta, result.sig)
return result
lookup.addCallback(cache)
return lookup
class Node(object):
"""
This class represents a single local node in the DHT encapsulating its
presence in the network.
All interactions with the DHT network by a client application are
performed via this class (or a subclass).
"""
def __init__(self, id, client_string='ssl:%s:%d'):
"""
Initialises the object representing the node with the given id.
"""
# The node's ID within the distributed hash table.
self.id = id
# The routing table stores information about other nodes on the DHT.
self._routing_table = RoutingTable(id)
# The local key/value store containing data held by this node.
self._data_store = DictDataStore()
# A dictionary of IDs for messages pending a response and associated
# deferreds to be fired when a response is completed.
self._pending = {}
# The template string to use when initiating a connection to another
# node on the network.
self._client_string = client_string
# The version of Drogulus that this node implements.
self.version = get_version()
log.msg('Initialised node with id: %r' % self.id)
def join(self, seed_nodes=None):
"""
Causes the Node to join the DHT network. This should be called before
any other DHT operations. The seedNodes argument contains a list of
tuples describing existing nodes on the network in the form of their
IP address and port.
"""
pass
def message_received(self, message, protocol):
"""
Handles incoming messages.
"""
# Update the routing table.
peer = protocol.transport.getPeer()
other_node = Contact(message.node, peer.host, peer.port,
message.version, time.time())
log.msg('Message received from %s' % other_node)
log.msg(message)
self._routing_table.add_contact(other_node)
# Sort on message type and pass to handler method. Explicit > implicit.
if isinstance(message, Ping):
self.handle_ping(message, protocol)
elif isinstance(message, Pong):
self.handle_pong(message)
elif isinstance(message, Store):
self.handle_store(message, protocol, other_node)
elif isinstance(message, FindNode):
self.handle_find_node(message, protocol)
elif isinstance(message, FindValue):
self.handle_find_value(message, protocol)
elif isinstance(message, Error):
self.handle_error(message, protocol, other_node)
elif isinstance(message, Value):
self.handle_value(message, other_node)
elif isinstance(message, Nodes):
self.handle_nodes(message)
def send_message(self, contact, message):
"""
Sends a message to the specified contact, adds it to the _pending
dictionary and ensures it times-out after the correct period. If an
error occurs the deferred's errback is called.
"""
d = defer.Deferred()
# open network call.
client_string = self._client_string % (contact.address, contact.port)
client = clientFromString(reactor, client_string)
connection = client.connect(DHTFactory(self))
# Ensure the connection will potentially time out.
connection_timeout = reactor.callLater(constants.RPC_TIMEOUT,
connection.cancel)
def on_connect(protocol):
# Cancel pending connection_timeout if it's still active.
if connection_timeout.active():
connection_timeout.cancel()
# Send the message and add a timeout for the response.
protocol.sendMessage(message)
self._pending[message.uuid] = d
reactor.callLater(constants.RESPONSE_TIMEOUT, response_timeout,
message, protocol, self)
def on_error(error):
log.msg('***** ERROR ***** connecting to %s' % contact)
log.msg(error)
self._routing_table.remove_contact(message.node)
d.errback(error)
connection.addCallbacks(on_connect, on_error)
return d
def trigger_deferred(self, message, error=False):
"""
Given a message, will attempt to retrieve the deferred and trigger it
with the appropriate callback or errback.
"""
if message.uuid in self._pending:
deferred = self._pending[message.uuid]
if error:
error.message = message
deferred.errback(error)
else:
deferred.callback(message)
# Remove the called deferred from the _pending dictionary.
del self._pending[message.uuid]
def handle_ping(self, message, protocol):
"""
Handles an incoming Ping message. Returns a Pong message using the
referenced protocol object.
"""
pong = Pong(message.uuid, self.id, self.version)
protocol.sendMessage(pong, True)
def handle_pong(self, message):
"""
Handles an incoming Pong message.
"""
self.trigger_deferred(message)
def handle_store(self, message, protocol, sender):
"""
Handles an incoming Store message. Checks the provenance and timeliness
of the message before storing locally. If there is a problem, removes
the untrustworthy peer from the routing table. Otherwise, at
REPLICATE_INTERVAL minutes in the future, the local node will attempt
to replicate the Store message elsewhere in the DHT if such time is
<= the message's expiry time.
Sends a Pong message if successful otherwise replies with an
appropriate Error.
"""
# Check provenance
is_valid, err_code = validate_message(message)
if is_valid:
# Ensure the node doesn't already have a more up-to-date version
# of the value.
current = self._data_store.get(message.key, False)
if current and (message.timestamp < current.timestamp):
# The node already has a later version of the value so
# return an error.
details = {
'new_timestamp': '%d' % current.timestamp
}
raise ValueError(8, constants.ERRORS[8], details,
message.uuid)
# Good to go, so store value.
self._data_store.set_item(message.key, message)
# Reply with a pong so the other end updates its routing table.
pong = Pong(message.uuid, self.id, self.version)
protocol.sendMessage(pong, True)
# At some future time attempt to replicate the Store message
# around the network IF it is within the message's expiry time.
reactor.callLater(constants.REPLICATE_INTERVAL,
self.send_replicate, message)
else:
# Remove from the routing table.
log.msg('Problem with Store command: %d - %s' %
(err_code, constants.ERRORS[err_code]))
self._routing_table.remove_contact(sender.id, True)
# Return an error.
details = {
'message': 'You have been removed from remote routing table.'
}
raise ValueError(err_code, constants.ERRORS[err_code], details,
message.uuid)
def handle_find_node(self, message, protocol):
"""
Handles an incoming FindNode message. Finds the details of up to K
other nodes closer to the target key that *this* node knows about.
Responds with a "Nodes" message containing the list of matching
nodes.
"""
target_key = message.key
# List containing tuples of information about the matching contacts.
other_nodes = [(n.id, n.address, n.port, n.version) for n in
self._routing_table.find_close_nodes(target_key)]
result = Nodes(message.uuid, self.id, other_nodes, self.version)
protocol.sendMessage(result, True)
def handle_find_value(self, message, protocol):
"""
Handles an incoming FindValue message. If the local node contains the
value associated with the requested key replies with an appropriate
"Value" message. Otherwise, responds with details of up to K other
nodes closer to the target key that the local node knows about. In
this case a "Nodes" message containing the list of matching nodes is
sent to the caller.
"""
match = self._data_store.get(message.key, False)
if match:
result = Value(message.uuid, self.id, match.key, match.value,
match.timestamp, match.expires, match.public_key,
match.name, match.meta, match.sig, match.version)
protocol.sendMessage(result, True)
else:
self.handle_find_node(message, protocol)
def handle_error(self, message, protocol, sender):
"""
Handles an incoming Error message. Currently, this simply logs the
error and closes the connection. In future this *may* remove the
sender from the routing table (depending on the error).
"""
# TODO: Handle error 8 (out of date data)
log.msg('***** ERROR ***** from %s' % sender)
log.msg(message)
def handle_value(self, message, sender):
"""
Handles an incoming Value message containing a value retrieved from
another node on the DHT. Ensures the message is valid and calls the
referenced deferred to signal the arrival of the value.
TODO: How to handle invalid messages and errback the deferred.
"""
# Check provenance
is_valid, err_code = validate_message(message)
if is_valid:
self.trigger_deferred(message)
else:
log.msg('Problem with incoming Value: %d - %s' %
(err_code, constants.ERRORS[err_code]))
log.msg(message)
# Remove the remote node from the routing table.
self._routing_table.remove_contact(sender.id, True)
error = ValueError(constants.ERRORS[err_code])
self.trigger_deferred(message, error)
def handle_nodes(self, message):
"""
Handles an incoming Nodes message containing information about other
nodes on the network that are close to a requested key.
"""
self.trigger_deferred(message)
def iterative_lookup(self, key, message_class):
"""
A generic lookup function for finding nodes or values within the
distributed hash table. Takes a key that either references a value or
location in the hash-space. This function returns a deferred that will
fire wth the found value or a set of peers in the DHT that are close to
the key. The message class should be either FindNode or FindValue.
"""
pass
def send_ping(self, contact):
"""
Sends a ping request to the given contact and returns a deferred
that is fired when the reply arrives or an error occurs.
"""
new_uuid = str(uuid4())
ping = Ping(new_uuid, self.id, self.version)
return self.send_message(contact, ping)
def send_store(self, private_key, public_key, name, value,
timestamp, expires, meta):
"""
Sends a Store message to the given contact. The value contained within
the message is stored against a key derived from the public_key and
name. Furthermore, the message is cryptographically signed using the
value, timestamp, expires, name and meta values.
"""
new_uuid = str(uuid4())
signature = generate_signature(value, timestamp, expires, name, meta,
private_key)
compound_key = construct_key(public_key, name)
new_store = Store(new_uuid, self.id, compound_key, value, timestamp,
expires, public_key, name, meta, signature,
self.version)
return self.send_replicate(new_store)
def send_replicate(self, store_message):
"""
Sends an existing valid Store message (that will probably have
originated from a third party) to another peer on the network for the
purposes of replication / spreading popular values.
"""
# Check for expiry time..?
# Find closest node...
"""
new_uuid = str(uuid4())
store = Store(new_uuid, self.id, store_message.key,
store_message.value, store_message.timestamp,
store_message.expires, store_message.public_key,
store_message.name, store_message.meta,
store_message.sig, self.version)
return self.send_message(contact, store)"""
def send_find_node(self, contact, id):
"""
Sends a FindNode message to the given contact with the intention of
obtaining contact information about the node with the specified id.
"""
pass
def send_find_value(self, contact, key):
"""
Sends a FindValue message to the given contact with the intention of
obtaining the value associated with the specified key.
"""
pass