def test_validate_authorization(self):
Crypto.check_signature = Mock(return_value=True)
value = Mock()
value.data = 'data'
value.authorization.sign = 'sign'
value.authorization.pub_key.exp_time = None
value.authorization.pub_key.key = 'key'
value.persist_mode = PersistMode.SECURED
dkey = hashlib.sha1('key'.encode('utf8')).digest()
dval = digest(dkey.hex() + value.data +
str(value.authorization.pub_key.exp_time) +
str(value.persist_mode))
validate_authorization(dkey, value)
Crypto.check_signature.assert_called_with(dval, 'sign', 'key')
value.authorization.pub_key.exp_time = 6
dval = digest(dkey.hex() + value.data +
str(value.authorization.pub_key.exp_time) +
str(value.persist_mode))
validate_authorization(dkey, value)
Crypto.check_signature.assert_called_with(dval, 'sign', 'key')
value.authorization.pub_key.exp_time = 4
with self.assertRaises(AssertionError):
validate_authorization(
hashlib.sha1('key'.encode('utf8')).digest(), value)
value.authorization.pub_key.exp_time = 6
Crypto.check_signature = Mock(return_value=False)
with self.assertRaises(InvalidSignException):
validate_authorization(
hashlib.sha1('key'.encode('utf8')).digest(), value)
def __init__(self,
ksize=20,
alpha=3,
node_id=None,
storage=None,
discovery_mode='neighborhood',
loop=None,
max_peers=64,
dht=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.loop = loop if loop else asyncio.get_event_loop()
asyncio.set_event_loop(self.loop)
self.ksize = ksize
self.alpha = alpha
self.port = os.getenv('NETWORK_PORT', 5678)
self.storage = storage or ForgetfulStorage()
self.node = Node(digest(node_id) or digest(random.getrandbits(255)))
self.dht = dht
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
self.max_peers = max_peers
self.setup_stethoscope()
def turn_evil(self, evilPort):
old_ping = self.rpc_ping
old_find_node = self.rpc_find_node
old_find_value = self.rpc_find_value
self.router.node.port = evilPort;
if self.evilType == "poison":
self.rpc_find_node = self.poison_rpc_find_node
self.rpc_find_value = self.poison_rpc_find_value
self.false_neighbour_list = []
for i in range(0, 30):
fakeid = hashlib.sha1(str(random.getrandbits(255))).digest()
fake_neighbour = [fakeid,
'10.0.0.9',
self.router.node.port]
self.false_neighbour_list.append(fake_neighbour)
_log.debug("Node with port {} prepared to execute "
"poisoning attack".format(self.router.node.port))
elif self.evilType == "insert":
self.rpc_find_node = self.sybil_rpc_find_node
self.rpc_find_value = self.poison_rpc_find_value
ends = bytearray([0x01, 0x02, 0x03])
self.false_neighbour_list = []
for i in range(0, 9):
if i < 3:
key = digest("APA")
elif i > 5:
key = digest("KANIN")
else:
key = digest("KOALA")
key = key[:-1] + bytes(ends[i % 3])
self.false_neighbour_list.append((key,
'10.0.0.9',
self.router.node.port))
_log.debug("Node with port {} prepared to execute node "
"insertion attack".format(self.router.node.port))
elif self.evilType == "eclipse":
self.rpc_find_node = self.eclipse_rpc_find_node
self.rpc_find_value = self.eclipse_rpc_find_value
self.closest_neighbour = map(list,
self.router.findNeighbors((self.router.node)))
self.false_neighbour_list = []
for i in range(0, 10):
fakeid = hashlib.sha1(str(random.getrandbits(255))).digest()
self.false_neighbour_list.append((fakeid,
'10.0.0.9',
self.router.node.port))
_log.debug("Node with port {} prepared to execute eclipse "
"attack on {}".format(self.router.node.port,
self.closest_neighbour[0][2]))
elif self.evilType == "sybil":
self.rpc_find_node = self.sybil_rpc_find_node
self.rpc_find_value = self.poison_rpc_find_value
self.false_neighbour_list = []
for i in range(0, 30):
fakeid = [hashlib.sha1(str(random.getrandbits(255))).digest()]
fake_neighbour = [fakeid, '10.0.0.9', self.router.node.port]
self.false_neighbour_list.append(fake_neighbour)
_log.debug("Node with port {} prepared to execute "
"Sybil attack".format(self.router.node.port))
def test_check_signature(self):
"""
check_signature should validate signature
"""
public_key = '0224d2079e86e937224f08aa37a857ca6116546868edde549d0bd6b8536af9d554'
tcs_sig = '749625f8d70efae75ffd4a62e22c6534b2cbaa49212c454e6cfb7c5215e39ef01d0388999b2d38a24ad379245e1b4c69b9259b1c8c86bb011712999b4565192d'
value = digest('some_key').hex() + 'some_data' + str(None) + str(PersistMode.SECURED)
self.assertTrue(self.crypto.check_signature(digest(value), tcs_sig, public_key))
def turn_evil(self, evilPort):
old_ping = self.rpc_ping
old_find_node = self.rpc_find_node
old_find_value = self.rpc_find_value
self.router.node.port = evilPort
if self.evilType == "poison":
self.rpc_find_node = self.poison_rpc_find_node
self.rpc_find_value = self.poison_rpc_find_value
self.false_neighbour_list = []
for i in range(0, 30):
fakeid = hashlib.sha1(str(random.getrandbits(255))).digest()
fake_neighbour = [fakeid, '10.0.0.9', self.router.node.port]
self.false_neighbour_list.append(fake_neighbour)
_log.debug("Node with port {} prepared to execute "
"poisoning attack".format(self.router.node.port))
elif self.evilType == "insert":
self.rpc_find_node = self.sybil_rpc_find_node
self.rpc_find_value = self.poison_rpc_find_value
ends = bytearray([0x01, 0x02, 0x03])
self.false_neighbour_list = []
for i in range(0, 9):
if i < 3:
key = digest("APA")
elif i > 5:
key = digest("KANIN")
else:
key = digest("KOALA")
key = key[:-1] + bytes(ends[i % 3])
self.false_neighbour_list.append(
(key, '10.0.0.9', self.router.node.port))
_log.debug("Node with port {} prepared to execute node "
"insertion attack".format(self.router.node.port))
elif self.evilType == "eclipse":
self.rpc_find_node = self.eclipse_rpc_find_node
self.rpc_find_value = self.eclipse_rpc_find_value
self.closest_neighbour = map(
list, self.router.findNeighbors((self.router.node)))
self.false_neighbour_list = []
for i in range(0, 10):
fakeid = hashlib.sha1(str(random.getrandbits(255))).digest()
self.false_neighbour_list.append(
(fakeid, '10.0.0.9', self.router.node.port))
_log.debug("Node with port {} prepared to execute eclipse "
"attack on {}".format(self.router.node.port,
self.closest_neighbour[0][2]))
elif self.evilType == "sybil":
self.rpc_find_node = self.sybil_rpc_find_node
self.rpc_find_value = self.poison_rpc_find_value
self.false_neighbour_list = []
for i in range(0, 30):
fakeid = [hashlib.sha1(str(random.getrandbits(255))).digest()]
fake_neighbour = [fakeid, '10.0.0.9', self.router.node.port]
self.false_neighbour_list.append(fake_neighbour)
_log.debug("Node with port {} prepared to execute "
"Sybil attack".format(self.router.node.port))
def perform_stores():
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
ds.append(self.callStore(node, digest(key), value))
def test_bucket_basic(self):
ecd_key = generate_secret_key()
sourceNode = Node(digest(random.getrandbits(255)), ip="127.0.0.1", port=12345)
dummy_protocol = DummyProt(ecd_key, sourceNode, None, 4, talos_vc=None)
nodes = []
for i in range(1000):
nodes.append(Node(digest(random.getrandbits(255)), ip="127.0.0.1", port=i+10000))
for i in range(1000):
dummy_protocol.router.addContact(nodes[i])
for i in range(1000):
self.assertFalse(dummy_protocol.router.isNewNode(nodes[i]))
def poison_rpc_find_value(self, sender, nodeid, key, challenge, signature):
value = self.storage[digest(str(self.sourceNode.id.encode("hex").upper()) + "cert")]
if key == digest("APA") or \
key == digest("KANIN") or \
key == digest("KOALA"):
logger(self.sourceNode,
"Attacking node with port {} sent back "
"forged value".format(self.router.node.port))
value = "apelsin"
try:
signature = self.runtime_credentials.sign_data(challenge)
except:
_log.debug("signing poison find value failed")
return { 'value': value, 'signature': signature }
def poison_rpc_find_value(self, sender, nodeid, key, challenge, signature):
value = self.storage[digest(
str(self.sourceNode.id.encode("hex").upper()) + "cert")]
if key == digest("APA") or \
key == digest("KANIN") or \
key == digest("KOALA"):
logger(
self.sourceNode, "Attacking node with port {} sent back "
"forged value".format(self.router.node.port))
value = "apelsin"
try:
signature = self.runtime_credentials.sign_data(challenge)
except:
_log.debug("signing poison find value failed")
return {'value': value, 'signature': signature}
def __init__(self,
ksize=20,
alpha=3,
node_id=None,
storage=None,
custom_event_loop=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.storage = storage or ForgetfulStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
self.custom_event_loop = custom_event_loop
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
node = Node(dkey)
def store(nodes):
self.log.info("setting '%s' on %s" % (key, list(map(str, nodes))))
# if this node is close too, then store here as well
if self.node.distanceTo(node) < max([n.distanceTo(node) for n in nodes]):
self.storage[dkey] = value
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
d = defer.DeferredList(ds)
d.addCallback(self._anyRespondSuccess)
d.addErrback(self.onError)
return d
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
d = spider.find()
d.addCallback(store)
d.addErrback(self.onError)
return d
def welcomeIfNewNode(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if self.router.isNewNode(node):
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
ds.append(self.callStore(node, key, value))
self.router.addContact(node)
return defer.gatherResults(ds)
async def _persist_locally(self, key, new_value: Value):
"""
Validate and persist new value locally
:param key: plain value key
:param new_value: new value to persist on the network
"""
dkey = digest(key)
log.debug(f"Going to retrieve stored value for key: {dkey}")
value_response = await self.get(key)
if not self._get_dtl_record(dkey, new_value):
raise UnauthorizedOperationException()
if value_response.data:
stored_value = ValueFactory.create_from_string(
dkey, value_response.data)
if isinstance(stored_value, ControlledValue):
result = stored_value.add_value(new_value)
else:
validate_secure_value(dkey, new_value, stored_value)
result = new_value
else:
result = ValueFactory.create_from_value(new_value)
if not self._get_dtl_record(dkey, new_value):
raise UnauthorizedOperationException()
self.storage[dkey] = str(result)
async def get(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
log.info("Looking up key %s", key)
dkey = digest(key)
# if this node has it, return it
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to get key %s", key)
return NodeMessage.of_params(dkey, None)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
local_value = self.storage.get(dkey, None)
if local_value:
local_value = NodeMessage.of_params(dkey, local_value).to_json()
responses = await spider.find([local_value])
else:
responses = await spider.find()
most_common_response = select_most_common_response(dkey, responses)
return NodeMessage.of_params(dkey, most_common_response)
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
log.info("setting '%s' = '%s' on network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
def transferKeyValues(self, node):
"""
Given a new node, send it all the keys/values it should be storing.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
ds.append(self.callStore(node, key, value))
return defer.gatherResults(ds)
def append(self, key, value):
"""
Append the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
def append(nodes, mid):
self.log.info("setting '%s' on %s" % (key, map(str, nodes)))
# TODO: Must add transaction ID so we dont append multiple times.
print "org mid", mid
mid = uuid.uuid1().hex
print "new mid", mid
ds = [self.protocol.callAppend(node, mid, dkey, value) for node in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(append, "hej")
def append(self, key, value):
"""
For the given key append the given list values to the set in the network.
"""
dkey = digest(key)
node = Node(dkey)
def append_(nodes):
# if this node is close too, then store here as well
if not nodes or self.node.distanceTo(node) < max([n.distanceTo(node) for n in nodes]):
try:
pvalue = json.loads(value)
self.set_keys.add(dkey)
if dkey not in self.storage:
_log.debug("%s local append key: %s not in storage set value: %s" % (base64.b64encode(node.id), base64.b64encode(dkey), pvalue))
self.storage[dkey] = value
else:
old_value_ = self.storage[dkey]
old_value = json.loads(old_value_)
new_value = list(set(old_value + pvalue))
_log.debug("%s local append key: %s old: %s add: %s new: %s" % (base64.b64encode(node.id), base64.b64encode(dkey), old_value, pvalue, new_value))
self.storage[dkey] = json.dumps(new_value)
except:
_log.debug("Trying to append something not a JSON coded list %s" % value, exc_info=True)
ds = [self.protocol.callAppend(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
_log.debug("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(append_)
def remove(self, key, value):
"""
For the given key remove the given list values from the set in the network.
"""
dkey = digest(key)
node = Node(dkey)
_log.debug("Server:remove %s" % base64.b64encode(dkey))
def remove_(nodes):
# if this node is close too, then store here as well
max_distance = max([n.distanceTo(node) for n in nodes]) if nodes else sys.maxint
if self.node.distanceTo(node) < max_distance:
try:
pvalue = json.loads(value)
self.set_keys.add(dkey)
if dkey in self.storage:
old_value = json.loads(self.storage[dkey])
new_value = list(set(old_value) - set(pvalue))
self.storage[dkey] = json.dumps(new_value)
_log.debug("%s local remove key: %s old: %s remove: %s new: %s" % (base64.b64encode(node.id), base64.b64encode(dkey), old_value, pvalue, new_value))
except:
_log.debug("Trying to remove somthing not a JSON coded list %s" % value, exc_info=True)
ds = [self.protocol.callRemove(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(remove_)
def get_addr_chunk(self,
chunk_key,
policy_in=None,
time_keeper=TimeKeeper()):
# if this node has it, return it
if self.storage.has_value(chunk_key):
addr = self.protocol.get_address()
return defer.succeed("%s:%d" % (addr[0], addr[1]))
dkey = digest(chunk_key)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
self.log.debug("Crawling for key %s" % (binascii.hexlify(dkey), ))
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" %
binascii.hexlify(dkey))
return defer.succeed(None)
spider = TalosChunkSpiderCrawl(self.protocol,
self.httpprotocol_client,
node,
chunk_key,
nearest,
self.ksize,
self.alpha,
time_keeper=time_keeper)
return spider.find()
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
_log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
node = Node(dkey)
def store(nodes):
_log.debug("setting '%s' to %s on %s" %
(key, value, map(str, nodes)))
# if this node is close too, then store here as well
if (not nodes or self.node.distanceTo(node) < max(
[n.distanceTo(node) for n in nodes]) or dkey in self.storage):
_log.debug("setting '%s' to %s locally" % (key, value))
self.storage[dkey] = value
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
_log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
return spider.find().addCallback(store)
def welcome_if_new(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if not self.router.is_new_node(node):
return
log.info("never seen %s before, adding to router", node)
for key, value in self.storage:
keynode = Node(digest(key))
neighbors = self.router.find_neighbors(keynode)
if neighbors:
last = neighbors[-1].distance_to(keynode)
new_node_close = node.distance_to(keynode) < last
first = neighbors[0].distance_to(keynode)
this_closest = self.source_node.distance_to(keynode) < first
if not neighbors or (new_node_close and this_closest):
asyncio.ensure_future(self.call_store(node, key, value))
self.router.add_contact(node)
async def run_test():
server = Server()
def async_return(result):
f = asyncio.Future()
f.set_result(result)
return f
get_signed_value = get_signed_value_with_keys(priv_key_path='kademlia/tests/resources/key.der',
pub_key_path='kademlia/tests/resources/public.der')
get_signed_message = get_signed_message_with_keys(priv_key_path='kademlia/tests/resources/key.der',
pub_key_path='kademlia/tests/resources/public.der')
key_test = 'test key'
dkey_test = digest(key_test)
data = json.dumps(get_signed_value(dkey_test, 'data', PersistMode.SECURED).to_json())
value = get_signed_value(dkey_test, data, PersistMode.SECURED)
server._call_remote_persist = Mock(return_value=async_return(True))
server.get = Mock(return_value=async_return(get_signed_message(dkey_test, data)))
server.set_digest = Mock(return_value=async_return(True))
Server._get_dtl_record = Mock(return_value=True)
await server.set('test key', value)
server.get.assert_called_with('test key')
server.stop()
def rpc_store(self, sender, nodeid, key, value):
source = kademlia.node.Node(nodeid, sender[0], sender[1])
self.welcomeIfNewNode(source)
self.log.debug("got a store request from %s" % str(sender))
header, payload = default_constant_splitter(value, return_remainder=True)
if header == constants.BYTESTRING_IS_URSULA_IFACE_INFO:
from nucypher.characters import Ursula
stranger_ursula = Ursula.from_bytes(payload,
federated_only=self.sourceNode.federated_only) # TODO: Is federated_only the right thing here?
if stranger_ursula.interface_is_valid() and key == digest(stranger_ursula.canonical_public_address):
self.sourceNode._node_storage[stranger_ursula.checksum_public_address] = stranger_ursula # TODO: 340
return True
else:
self.log.warning("Got request to store invalid node: {} / {}".format(key, value))
self.illegal_keys_seen.append(key)
return False
elif header == constants.BYTESTRING_IS_TREASURE_MAP:
from nucypher.policy.models import TreasureMap
try:
treasure_map = TreasureMap.from_bytes(payload)
self.log.info("Storing TreasureMap: {} / {}".format(key, value))
self.sourceNode._treasure_maps[treasure_map.public_id()] = value
return True
except TreasureMap.InvalidSignature:
self.log.warning("Got request to store invalid TreasureMap: {} / {}".format(key, value))
self.illegal_keys_seen.append(key)
return False
else:
self.log.info(
"Got request to store bad k/v: {} / {}".format(key, value))
return False
def provide_treasure_map(self, treasure_map_id_as_hex):
# For now, grab the TreasureMap for the DHT storage. Soon, no do that. #TODO!
treasure_map_id = binascii.unhexlify(treasure_map_id_as_hex)
treasure_map_bytes = self.server.storage.get(digest(treasure_map_id))
headers = {'Content-Type': 'application/octet-stream'}
return Response(content=treasure_map_bytes, headers=headers)
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
key = digest(bytes(key))
return await self.set_digest(key, value)
def transferKeyValues(self, node):
"""
Given a new node, send it all the keys/values it should be storing.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
_log.debug("**** transfer key values %s ****" % node)
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
_log.debug("transfer? nbr neighbors=%d, key=%s, value=%s" %
(len(neighbors), base64.b64encode(key), str(value)))
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
if key in self.set_keys:
_log.debug("transfer append key value key=%s, value=%s" %
(base64.b64encode(key), str(value)))
ds.append(self.callAppend(node, key, value))
else:
_log.debug("transfer store key value key=%s, value=%s" %
(base64.b64encode(key), str(value)))
ds.append(self.callStore(node, key, value))
return defer.gatherResults(ds)
def store(nodes):
self.log.info("setting '%s' on %s" % (hkey, map(str, nodes)))
# if this node is close too, then store here as well
if self.node.distanceTo(node) < max([n.distanceTo(node) for n in nodes]):
chunk = CloudChunk.decode(value)
if not digest(chunk.key) == dkey:
return {'error': 'key missmatch'}
def handle_policy(policy):
time_keeper.stop_clock(ENTRY_FETCH_POLICY)
# Hack no chunk id given -> no key checks, key is in the encoded chunk
id = time_keeper.start_clock_unique()
self.storage.store_check_chunk(chunk, None, policy, time_keeper=time_keeper)
time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id)
id = time_keeper.start_clock_unique()
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(_anyRespondSuccess, time_keeper, id,
ENTRY_STORE_TO_ALL_NODES)
if not policy_in is None:
return handle_policy(policy_in)
time_keeper.start_clock()
return self.talos_vc.get_policy_with_txid(chunk.get_tag_hex()).addCallback(handle_policy)
id = time_keeper.start_clock_unique()
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(_anyRespondSuccess, time_keeper, id, ENTRY_STORE_TO_ALL_NODES)
def welcomeIfNewNode(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if not self.router.isNewNode(node):
return
self.log.info(
"never seen %s before, adding to router and setting nearby " %
node)
# TODO: 331 and 340 next two lines
ursulas = [(id, bytes(node))
for id, node in self.sourceNode._node_storage.items()]
for key, value in tuple(
self.sourceNode._treasure_maps.items()) + tuple(ursulas):
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
asyncio.ensure_future(self.callStore(node, key, value))
self.router.addContact(node)
def __init__(self, ksize=20, alpha=3, id=None, storage=None,
talos_vc=None, rebub_delay=3600, tls_port=-1):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
id: The id for this node on the network.
storage: An instance that implements :interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or TalosLevelDBDHTStorage("./leveldb")
self.node = Node(id or digest(random.getrandbits(255)))
def start_looping_call(num_seconds):
self.refreshLoop = LoopingCall(self.refreshTable).start(num_seconds)
self.delay = rebub_delay
task.deferLater(reactor, rebub_delay, start_looping_call, rebub_delay)
self.talos_vc = talos_vc or AsyncPolicyApiClient()
self.protocol = TalosKademliaProtocol(self.node, self.storage, ksize, talos_vc=self.talos_vc)
self.httpprotocol_client = None
self.tls_port = tls_port
def transferKeyValues(self, node):
"""
Given a new node, send it all the keys/values it should be storing.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
_log.debug("**** transfer key values %s ****" % node)
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
_log.debug("transfer? nbr neighbors=%d, key=%s, value=%s" % (len(neighbors), base64.b64encode(key), str(value)))
if len(neighbors) > 0:
newNodeClose = node.distanceTo(keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
if key in self.set_keys:
_log.debug("transfer append key value key=%s, value=%s" % (base64.b64encode(key), str(value)))
ds.append(self.callAppend(node, key, value))
else:
_log.debug("transfer store key value key=%s, value=%s" % (base64.b64encode(key), str(value)))
ds.append(self.callStore(node, key, value))
return defer.gatherResults(ds)
def welcome_if_new(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if not self.router.is_new_node(node):
return
log.info("never seen %s before, adding to router", node)
for key, value in self.storage:
keynode = Node(digest(key))
neighbors = self.router.find_neighbors(keynode)
if neighbors:
last = neighbors[-1].distance_to(keynode)
new_node_close = node.distance_to(keynode) < last
first = neighbors[0].distance_to(keynode)
this_closest = self.source_node.distance_to(keynode) < first
if not neighbors or (new_node_close and this_closest):
asyncio.ensure_future(self.call_store(node, key, value))
self.router.add_contact(node)
def receive_treasure_map(self, treasure_map_id, request: http.Request):
from nucypher.policy.models import TreasureMap
try:
treasure_map = TreasureMap.from_bytes(
bytes_representation=request.body, verify=True)
except TreasureMap.InvalidSignature:
do_store = False
else:
do_store = treasure_map.public_id() == treasure_map_id
if do_store:
self.log.info("{} storing TreasureMap {}".format(
self, treasure_map_id))
self.dht_server.set_now(
binascii.unhexlify(treasure_map_id),
constants.BYTESTRING_IS_TREASURE_MAP + bytes(treasure_map))
# TODO 341 - what if we already have this TreasureMap?
self.treasure_maps[digest(treasure_map_id)] = treasure_map
return Response(content=bytes(treasure_map), status_code=202)
else:
# TODO: Make this a proper 500 or whatever.
self.log.info(
"Bad TreasureMap ID; not storing {}".format(treasure_map_id))
assert False
def get_concat(self, key):
"""
Get a key if the network has it. Assuming it is a list that should be combined.
@return: C{None} if not found, the value otherwise.
"""
dkey = digest(key)
# Always try to do a find even if we have it, due to the concatenation of all results
exists, value = self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
_log.debug(
"Server:get_concat key=%s, value=%s, exists=%s, nbr nearest=%d" %
(base64.b64encode(dkey), value, exists, len(nearest)))
if len(nearest) == 0:
# No neighbors but we had it, return that value
if exists:
return defer.succeed(value)
self.log.warning("There are no known neighbors to get key %s" %
key)
return defer.succeed(None)
spider = ValueListSpiderCrawl(self.protocol,
node,
nearest,
self.ksize,
self.alpha,
local_value=value if exists else None)
return spider.find()
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
return self.digest_set(dkey, value)
def poison_rpc_find_value(self, sender, nodeid, key, challenge, signature):
value = self.storage[digest(
str(self.sourceNode.id.encode("hex").upper()) + "cert")]
if key == digest("APA") or \
key == digest("KANIN") or \
key == digest("KOALA"):
logger(
self.sourceNode, "Attacking node with port {} sent back "
"forged value".format(self.router.node.port))
value = "apelsin"
try:
private = OpenSSL.crypto.load_privatekey(
OpenSSL.crypto.FILETYPE_PEM, self.priv_key, '')
signature = OpenSSL.crypto.sign(private, challenge, "sha256")
except:
_log.debug("signing poison find value failed")
return {'value': value, 'signature': signature}
def attach_server(self, ksize=20, alpha=3, id=None,
storage=None, *args, **kwargs):
# TODO: Network-wide deterministic ID generation (ie, auction or
# whatever) See #136.
if not id:
id = digest(secure_random(32))
super().attach_server(ksize, alpha, id, storage)
self.attach_rest_server(db_name=self.db_name)
def test_alice_sets_treasure_map_on_network(enacted_policy, ursulas):
"""
Having enacted all the policies of a PolicyGroup, Alice creates a TreasureMap and sends it to Ursula via the DHT.
"""
_, packed_encrypted_treasure_map, _, _ = enacted_policy.publish_treasure_map()
treasure_map_as_set_on_network = ursulas[0].server.storage[
digest(enacted_policy.treasure_map_dht_key())]
assert treasure_map_as_set_on_network == b"trmap" + packed_encrypted_treasure_map
def poison_rpc_find_value(self, sender, nodeid, key, challenge, signature):
value = self.storage[digest(str(self.sourceNode.id.encode("hex").upper()) + "cert")]
if key == digest("APA") or \
key == digest("KANIN") or \
key == digest("KOALA"):
logger(self.sourceNode,
"Attacking node with port {} sent back "
"forged value".format(self.router.node.port))
value = "apelsin"
try:
private = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM,
self.priv_key,
'')
signature = OpenSSL.crypto.sign(private,
challenge,
"sha256")
except:
_log.debug("signing poison find value failed")
return { 'value': value, 'signature': signature }
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
if not check_dht_value_type(value):
raise TypeError(
"Value must be of type int, float, bool, str, or bytes"
)
log.info("setting '%s' = '%s' on network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
def get(self, key):
"""
Get a key if the network has it.
@return: C{None} if not found, the value otherwise.
"""
node = Node(digest(key))
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find()
def __init__(self, ksize=20, alpha=3, id=None):
"""
Create a server instance. This will start listening on the given port.
@param port: UDP port to listen on
@param k: The k parameter from the paper
@param alpha: The alpha parameter from the paper
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def get(self, key, default=None):
self.cull()
if key in self.data:
value = self[key]
hash = pybitcoin.hash.hex_hash160(value)
test_key = digest(hash)
if key != test_key:
self.log.info("hash(value) doesn't match, ignoring value")
return default
return self[key]
return default
def __init__(self, ksize=20, alpha=3, id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
id: The id for this node on the network.
storage: An instance that implements :interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def get(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return defer.succeed(self.storage.get(dkey))
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find()
def get(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return defer.succeed(self.storage.get(dkey))
node = Node(dkey)
#the quorum
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
return self.protocol.callFindKey(dkey, nearest)
def remove(self, key, value):
"""
For the given key remove the given list values from the set in the network.
"""
dkey = digest(key)
def remove(nodes):
ds = [self.protocol.callRemove(node, dkey, value) for node in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(remove)
def set(self, key, value):
"""
Set the given key to the given value in the network.
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(key)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
return self.protocol.callStore(dkey, value, nearest)
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.storage = storage or ForgetfulStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
def store(nodes):
self.log.info("setting '%s' on %s" % (key, map(str, nodes)))
ds = [self.protocol.callStore(node, dkey, value) for node in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def __setitem__(self, key, value):
try:
test_value = json.loads(value)
except:
self.log.info("value not JSON, not storing")
return
hash = pybitcoin.hash.hex_hash160(value)
test_key = digest(hash)
if key != test_key:
self.log.info("hash(value) doesn't match, not storing")
return
if key in self.data:
del self.data[key]
self.data[key] = (time.time(), value)
self.cull()
async def get(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
log.info("Looking up key %s", key)
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.find_neighbors(node)
if not nearest:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
return await spider.find()
def get_concat(self, key):
"""
Get a key if the network has it. Assuming it is a list that should be combined.
@return: C{None} if not found, the value otherwise.
"""
dkey = digest(key)
# Always try to do a find even if we have it, due to the concatenation of all results
exists, value = self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
_log.debug("Server:get_concat key=%s, value=%s, exists=%s, nbr nearest=%d" % (base64.b64encode(dkey), value,
exists, len(nearest)))
if len(nearest) == 0:
# No neighbors but we had it, return that value
if exists:
return defer.succeed(value)
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
spider = ValueListSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha,
local_value=value if exists else None)
return spider.find()
def test_digest(self):
dig = hashlib.sha1(b'1').digest()
self.assertEqual(dig, digest(1))
dig = hashlib.sha1(b'another').digest()
self.assertEqual(dig, digest('another'))
def test_digest(self):
d = hashlib.sha1('1').digest()
self.assertEqual(d, digest(1))
d = hashlib.sha1('another').digest()
self.assertEqual(d, digest('another'))