class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
self.config = config
self.log = Logger(system=self)
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def getPublicKey(self, keyId):
"""
Get the public key from the network, and return only if the key is
the one that matches the keyId based on hash.
"""
def verify(key):
if key is not None and PublicKey(key).getKeyId() == keyId:
return key
return None
self.log.debug("Getting key text for key id %s" % keyId)
return self.kserver.get(keyId).addCallback(verify)
def resolve(self, keyId):
def parse(locations):
self.log.debug("Locations for %s: %s" % (keyId, locations))
results = []
if locations is None or locations == "":
return results
for location in locations.split(','):
host, port = location.split(':')
results.append((host, int(port)))
return results
d = self.kserver.get("%s-location" % keyId)
return d.addCallback(parse)
def announceLocation(self, myKeyId, myPublicKey):
def announce(ips):
ips = self.localAddresses() + ips
ipports = map(lambda ip: "%s:%i" % (ip, self.config['s2s.port']), ips)
return self.kserver.set("%s-location" % myKeyId, ",".join(ipports))
d = self.kserver.set(myKeyId, str(myPublicKey))
d.addCallback(lambda _: self.kserver.inetVisibleIP())
return d.addCallback(announce)
def localAddresses(self):
result = []
for iface in netifaces.interfaces():
addys = netifaces.ifaddresses(iface).get(netifaces.AF_INET)
result += [ addy['addr'] for addy in (addys or []) if addy['addr'] != '127.0.0.1' ]
return result
def __init__(self, keyPath, authorizedKeysDir):
"""
Handle all key interactions.
@param keyPath: Path to the key to use as this server's id. If
it doesn't exist, it will be created.
@param authorizedKeysDir: Path to a directory to read/write
authorized keys to. They are stored in memory in the keystore
upon this object's construction.
"""
self.log = Logger(system=self)
# first, check our key
if not os.path.isfile(keyPath):
self.log.debug("%s doesn't exist; creating new keypair" % keyPath)
self.kpair = KeyPair.generate()
self.kpair.store(keyPath)
else:
self.log.debug("using %s as this node's keypair" % keyPath)
self.kpair = KeyPair.load(keyPath)
# load other keys
self.authorizedKeysDir = authorizedKeysDir
self.refreshAuthorizedKeys()
def __init__(self, resolver, contextFactory, keyStore, storage):
self.connections = {}
self.resolver = resolver
self.contextFactory = contextFactory
self.keyStore = keyStore
self.storage = storage
self.log = Logger(system=self)
class FriendsList(object):
def __init__(self, storage, keyStore, resolver):
self.storage = storage
self.keyStore = keyStore
self.resolver = resolver
self.log = Logger(system=self)
def addFriendById(self, name, keyId):
"""
Lookup a public key with the given keyId and save if found.
"""
d = self.resolver.getPublicKey(keyId)
return d.addCallback(self._addFriendById, name)
def _addFriendById(self, keyvalue, name):
if keyvalue is None:
raise KeyNotResolvedError("Could not find key for %s" % name)
return self.addFriend(keyvalue, name)
def addFriend(self, publicKey, name):
"""
Add a friend with the given public key.
"""
self.log.debug("Adding key belonging to %s: %s" % (name, publicKey))
pk = PublicKey(publicKey)
path = str(Path(pk.getKeyId()))
self.storage.grantAccess(pk.getKeyId(), path)
self.keyStore.setAuthorizedKey(pk, name)
f = Friend(pk.getKeyId(), name, publicKey)
return defer.succeed(f)
def removeFriend(self, name):
return self.keyStore.removeAuthorizedKey(name)
def getFriends(self):
friends = []
for key in self.keyStore.getAuthorizedKeysList():
friend = Friend(key.getKeyId(), key.name, str(key))
friends.append(friend)
return friends
def __iter__(self):
for friend in self.getFriends():
yield friend
def __len__(self):
return len(self.getFriends())
def __init__(self, keyStore, storage, resolver):
self.keyStore = keyStore
self.storage = storage
self.contextFactory = PFSContextFactory(self.keyStore)
self.pool = ConnectionPool(resolver, self.contextFactory, self.keyStore, self.storage)
self.protocolFactory = TintProtocolFactory(self.pool)
self.friends = FriendsList(self.storage, self.keyStore, resolver)
self.log = Logger(system=self)
class AnyDBMStorage(ObservableStorage):
implements(IStorage)
def __init__(self, filename):
super(ObservableStorage, self).__init__()
self.filename = filename
self.log = Logger(system=self)
self.db = anydbm.open(self.filename, 'c')
def get(self, key, default=None):
key = Path.normalize(key)
self.log.debug("Getting %s" % key)
value = self.db[key] if key in self.db else default
return defer.succeed(value)
def set(self, key, value):
key = Path.normalize(key)
self.log.debug("Setting %s = %s" % (key, value))
preexisting = key in self.db
self.db[key] = str(value)
return self.publishChange(key, value, preexisting)
def push(self, key, value):
path = Path(key)
mkey = path.join('_maxid').path
def dopush(current):
id = int(current) + 1
vkey = path.join(id).path
d = defer.gatherResults([self.set(mkey, id), self.set(vkey, value)])
return d.addCallback(lambda _: self.publishChange(vkey, value, False))
return self.get(mkey, '-1').addCallback(dopush)
def ls(self, key, offset=0, length=100):
length = min(length, 1000)
kids = set([])
path = Path(key)
for k in imap(Path, self.db.keys()):
if k in path:
kids.add(path.childFrom(k))
kids = [ str(kid) for kid in kids if kid != '_maxid' ]
kids.sort()
results = []
for k in kids[offset:][:length]:
results.append({ 'key': k, 'value': self.db[path.join(k).path] })
return defer.succeed(results)
def __init__(self, config, bootstrapNeighbors):
self.config = config
self.log = Logger(system=self)
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def __init__(self, connectionPool, timeout=10):
"""
We have access to the connection pool so we can store connections
there after they've been successfully made.
"""
MsgPackProtocol.__init__(self, timeout)
self.connectionPool = connectionPool
self.log = Logger(system=self)
self.storage = self.connectionPool.storage
self.peersKeyId = None
class ConnectionPool(object):
def __init__(self, resolver, contextFactory, keyStore, storage):
self.connections = {}
self.resolver = resolver
self.contextFactory = contextFactory
self.keyStore = keyStore
self.storage = storage
self.log = Logger(system=self)
def send(self, keyId, cmd, *args):
if keyId in self.connections:
return self.sendOnConnection(self.connections[keyId], cmd)
d = self.resolver.resolve(keyId)
d.addCallback(self.createConnection, keyId)
return d.addCallback(self.sendOnConnection, cmd, args)
def sendOnConnection(self, connection, cmd, args):
if connection is None:
return False
return connection.sendCommand(cmd, args)
def createConnection(self, addr, keyId):
if addr is None:
return False
host, port = addr
cc = ClientCreator(reactor, TintProtocol, self)
d = cc.connectSSL(host, port, self.contextFactory)
return d.addCallback(self.saveConnection, keyId)
def forgetConnection(self, keyId):
self.log.info("removing connection %s from pool" % keyId)
del self.connections[keyId]
def saveConnection(self, connection, keyId):
self.connections[keyId] = connection
self.log.info("saving connection %s in pool" % keyId)
return connection
class DefaultPermissions(object):
def __init__(self, storage):
self.storage = storage
self.log = Logger(system=self)
def accessAvailable(self, requestor, key):
def gather(results):
access = set()
for result in results:
if result is not None:
access.update(result.split(','))
return list(access)
self.log.info("Testing access for %s to %s" % (requestor, key))
ds = []
for path in Path(key).ancestors():
path = str(Path('a').join(requestor).join(path))
ds.append(self.storage.get(path, None))
return defer.gatherResults(ds).addCallback(gather)
def canAccess(self, requestor, key, optype='*'):
"""
@param key The path to the storage. Should always start with a '/'.
"""
def test(access):
return '*' in access or optype in access
d = self.accessAvailable(requestor, key)
return d.addCallback(test)
def grantAccess(self, requestor, key, optype="*"):
def test(access):
if not access:
path = Path('a').join(requestor).join(Path(key))
return self.storage.set(str(path), optype)
return defer.succeed(optype)
d = self.canAccess(requestor, key, optype)
return d.addCallback(test)
class TintProtocol(MsgPackProtocol):
def __init__(self, connectionPool, timeout=10):
"""
We have access to the connection pool so we can store connections
there after they've been successfully made.
"""
MsgPackProtocol.__init__(self, timeout)
self.connectionPool = connectionPool
self.log = Logger(system=self)
self.storage = self.connectionPool.storage
self.peersKeyId = None
def getPeersKeyId(self):
if self.peersKeyId is None:
cert = self.transport.getPeerCertificate()
if cert is not None:
issuerCommonName = cert.get_issuer().commonName
key = self.connectionPool.keyStore.getIssuerPublicKey(issuerCommonName)
self.peersKeyId = key.getKeyId()
return self.peersKeyId
def dataReceived(self, data):
self.connectionPool.saveConnection(self, self.getPeersKeyId())
self.log.debug("received data from %s: %s" % (self.getPeersKeyId(), data))
MsgPackProtocol.dataReceived(self, data)
def connectionLost(self, reason):
keyId = self.getPeersKeyId()
self.log.warning("Connection to %s lost: %s" % (keyId, str(reason)))
self.connectionPool.forgetConnection(keyId)
MsgPackProtocol.connectionLost(self, reason)
def connectionFailed(self, reason):
self.log.warning("Connection failed: %s" % str(reason))
def cmd_get(self, key):
return self.storage.get(self.getPeersKeyId(), key)
def cmd_set(self, key, value):
return self.storage.set(self.getPeersKeyId(), key, value)
def cmd_push(self, key, value):
return self.storage.push(self.getPeersKeyId(), key, value)
def cmd_ls(self, key, offset, length):
return self.storage.ls(self.getPeersKeyId(), key, offset, length)
class ConnectionPool(object):
def __init__(self, resolver, contextFactory, keyStore, storage):
self.connections = {}
self.resolver = resolver
self.contextFactory = contextFactory
self.keyStore = keyStore
self.storage = storage
self.log = Logger(system=self)
def send(self, keyId, cmd, *args):
if keyId in self.connections:
return self.sendOnConnection(self.connections[keyId], cmd, args)
d = self.resolver.resolve(keyId)
d.addCallback(self.createConnection, keyId)
return d.addCallback(self.sendOnConnection, cmd, args)
def sendOnConnection(self, connection, cmd, args):
if connection is None:
return False
return connection.sendCommand(cmd, args)
def createConnection(self, addrs, keyId):
if len(addrs) == 0:
raise HostUnreachableError("Cannot connect to %s" % keyId)
host, port = addrs.pop()
self.log.debug("Attempting to create connection to %s:%i" % (host, port))
cc = ClientCreator(reactor, TintProtocol, self)
d = cc.connectSSL(host, port, self.contextFactory, timeout=5)
d.addCallback(self.saveConnection, keyId)
if len(addrs) > 0:
d.addErrback(lambda _: self.createConnection(addrs, keyId))
return d
def forgetConnection(self, keyId):
self.log.info("removing connection %s from pool" % keyId)
if keyId in self.connections:
del self.connections[keyId]
def saveConnection(self, connection, keyId):
self.connections[keyId] = connection
self.log.info("saving connection %s in pool" % keyId)
return connection
def __init__(self, filename):
super(ObservableStorage, self).__init__()
self.filename = filename
self.log = Logger(system=self)
self.db = anydbm.open(self.filename, 'c')
class KeyStore(service.Service):
def __init__(self, keyPath, authorizedKeysDir):
"""
Handle all key interactions.
@param keyPath: Path to the key to use as this server's id. If
it doesn't exist, it will be created.
@param authorizedKeysDir: Path to a directory to read/write
authorized keys to. They are stored in memory in the keystore
upon this object's construction.
"""
self.log = Logger(system=self)
# first, check our key
if not os.path.isfile(keyPath):
self.log.debug("%s doesn't exist; creating new keypair" % keyPath)
self.kpair = KeyPair.generate()
self.kpair.store(keyPath)
else:
self.log.debug("using %s as this node's keypair" % keyPath)
self.kpair = KeyPair.load(keyPath)
# load other keys
self.authorizedKeysDir = authorizedKeysDir
self.refreshAuthorizedKeys()
def startService(self):
self.log.debug("Using keypair id: %s" % self.getKeyId())
service.Service.startService(self)
def getKeyId(self):
return self.kpair.getKeyId()
def getPublicKey(self):
return self.kpair.getPublicKey()
def generateSignedTmpKeyPair(self, expiresIn):
self.log.debug("Creating a new tmp keypair that expires in %i seconds" % expiresIn)
return self.kpair.generateSignedTmpKeyPair(expiresIn)
def refreshAuthorizedKeys(self):
self.authorizedKeys = {}
pemFilter = os.path.join(self.authorizedKeysDir, "*.pem")
for fname in glob.glob(pemFilter):
self.log.debug("Loading %s into authorized keys" % fname)
keyId = PublicKey.load(fname).getIssuer()
if keyId in self.authorizedKeys:
msg = "There are two keys with the same issuer - %s and %s."
msg += " Only one can be used - please delete the duplicate."
raise DuplicateIssuer(msg % (fname, self.authorizedKeys[keyId]))
self.authorizedKeys[keyId] = fname
def getIssuerPublicKey(self, issuerCommonName):
# this shouldn't happen - if a connection is successfully made,
# then that means the key *must* be in the local authorized keys.
# However, this is a double check, cause safety first and all that.
if issuerCommonName not in self.authorizedKeys:
msg = "Issuer %s could not be found in local authorized keys"
raise InvalidIssuer(msg % issuerCommonName)
return PublicKey.load(self.authorizedKeys[issuerCommonName])
def getAuthorizedKeysList(self):
return [PublicKey.load(fname) for fname in self]
def removeAuthorizedKey(self, fname):
path = os.path.join(self.authorizedKeysDir, fname + ".pem")
if not os.path.isfile(path):
return False
try:
os.remove(path)
self.refreshAuthorizedKeys()
return True
except:
return False
def setAuthorizedKey(self, publicKey, fname):
path = os.path.join(self.authorizedKeysDir, fname + ".pem")
if not fname.isalnum():
msg = "Filename %s is not alpha numeric"
raise InvalidAuthorizedKeyFilename(msg % fname)
if path in self:
msg = "There is already a key with the same filename - %s."
msg += " If you want to update - please delete first."
raise DuplicateKeyFilename(msg % fname)
if publicKey.getIssuer() in self.authorizedKeys:
msg = "There are two keys with the same issuer - %s and %s."
msg += " Only one can be used - please delete the duplicate."
raise DuplicateIssuer(msg % (fname, self.authorizedKeys[publicKey.getIssuer()]))
publicKey.store(path)
self.refreshAuthorizedKeys()
def __len__(self):
return len(self.authorizedKeys)
def __iter__(self):
return self.authorizedKeys.itervalues()
def __init__(self, storage):
self.storage = storage
self.log = Logger(system=self)
def __init__(self, storage, keyStore, resolver):
self.storage = storage
self.keyStore = keyStore
self.resolver = resolver
self.log = Logger(system=self)
class Peer(object):
def __init__(self, keyStore, storage, resolver):
self.keyStore = keyStore
self.storage = storage
self.contextFactory = PFSContextFactory(self.keyStore)
self.pool = ConnectionPool(resolver, self.contextFactory, self.keyStore, self.storage)
self.protocolFactory = TintProtocolFactory(self.pool)
self.friends = FriendsList(self.storage, self.keyStore, resolver)
self.log = Logger(system=self)
def getKeyId(self):
"""
Get the keyId used by this peer (this peer's identifier).
This is stored in the key store.
"""
return self.keyStore.getKeyId()
def getPublicKey(self):
"""
Get the keyId used by this peer (this peer's identifier).
This is stored in the key store.
"""
return self.keyStore.getPublicKey()
def set(self, hostKeyId, storagePath, storageValue):
"""
Set a value on a host.
@param hostKeyId: The key id for the destination host to set the
given key. This could be the local host, in which case the hostKey
will be the same as this C{Peer}'s keyStore keyId.
@param storagePath: The path to the key to set. For instance, this
could be something like /chat/<somekey>/inbox.
@param storageValue: The value to set.
"""
if hostKeyId == self.getKeyId():
return self.storage.set(hostKeyId, storagePath, storageValue)
return self.pool.send(hostKeyId, 'set', storagePath, storageValue)
def get(self, hostKeyId, storagePath):
"""
Get a value from a host.
@param hostKeyId: The key id for the destination host to get the
given key. This could be the local host, in which case the hostKey
will be the same as this C{Peer}'s keyStore keyId.
@param storagePath: The path to the key to get. For instance, this
could be something like /chat/<somekey>/inbox.
"""
if hostKeyId == self.getKeyId():
self.log.debug("getting storagePath %s on self" % storagePath)
return self.storage.get(hostKeyId, storagePath)
self.log.debug("getting storagePath %s on %s" % (storagePath, hostKeyId))
return self.pool.send(hostKeyId, 'get', storagePath)
def push(self, hostKeyId, storagePath, storageValue):
"""
Given key, create a new key at <key>/<id> with the given value, where <id>
is an auto-incrementing integer value starting at 0.
"""
if hostKeyId == self.getKeyId():
return self.storage.push(hostKeyId, storagePath, storageValue)
return self.pool.send(hostKeyId, 'push', storagePath, storageValue)
def ls(self, hostKeyId, storagePath, offset, length):
"""
Given key, get all children keys (with the given offset and length). Length cannot
be more than 1000.
"""
if hostKeyId == self.getKeyId():
return self.storage.ls(hostKeyId, storagePath, offset, length)
return self.pool.send(hostKeyId, 'ls', storagePath, offset, length)
class Peer(object):
def __init__(self, keyStore, storage, resolver):
self.keyStore = keyStore
self.storage = storage
self.resolver = resolver
self.contextFactory = PFSContextFactory(self.keyStore)
self.pool = ConnectionPool(self.resolver, self.contextFactory, self.keyStore, self.storage)
self.protocolFactory = TintProtocolFactory(self.pool)
self.log = Logger(system=self)
def getKeyId(self):
"""
Get the keyId used by this peer (this peer's identifier).
This is stored in the key store.
"""
return self.keyStore.getKeyId()
def getPublicKey(self):
"""
Get the keyId used by this peer (this peer's identifier).
This is stored in the key store.
"""
return self.keyStore.getPublicKey()
def set(self, hostKeyId, storagePath, storageValue):
"""
Set a value on a host.
@param hostKeyId: The key id for the destination host to set the
given key. This could be the local host, in which case the hostKey
will be the same as this C{Peer}'s keyStore keyId.
@param storagePath: The path to the key to set. For instance, this
could be something like /chat/<somekey>/inbox.
@param storageValue: The value to set.
"""
if hostKeyId == self.getKeyId():
return self.storage.set(hostKeyId, storagePath, storageValue)
return self.pool.send(hostKeyId, 'set', storagePath, storageValue)
def get(self, hostKeyId, storagePath):
"""
Get a value from a host.
@param hostKeyId: The key id for the destination host to get the
given key. This could be the local host, in which case the hostKey
will be the same as this C{Peer}'s keyStore keyId.
@param storagePath: The path to the key to get. For instance, this
could be something like /chat/<somekey>/inbox.
"""
if hostKeyId == self.getKeyId():
self.log.debug("getting storagePath %s on self" % storagePath)
return self.storage.get(hostKeyId, storagePath)
self.log.debug("getting storagePath %s on %s" % (storagePath, hostKeyId))
return self.pool.send(hostKeyId, 'get', storagePath)
def incr(self, hostKeyId, storagePath, amount=1, default=0):
"""
Increment a value on a host.
@param hostKeyId: The key id for the destination host to increment the
given key. This could be the local host, in which case the hostKey
will be the same as this C{Peer}'s keyStore keyId.
@param storagePath: The path to the key to increment. For instance, this
could be something like /chat/<somekey>/inbox.
@param amount: The amount to increment. This could be negative, which
would make this actually a decrement. By default this is 1.
@param default: The amount to use as the default if no value exists
at the given storagePath. By default this is 0.
"""
if hostKeyId == self.getKeyId():
return self.storage.incr(hostKeyId, storagePath, amount, default)
return self.pool.send(hostKeyId, 'incr', storagePath, amount, default)