def setUp(self):
self.kfiles()
self.l = []
for i in range(self.num):
self.l.append(
Khashmir('', self.startport + i,
'/tmp/kh%s.db' % (self.startport + i)))
reactor.iterate()
reactor.iterate()
for i in self.l:
i.addContact(self.localip, self.l[randrange(0, self.num)].port)
i.addContact(self.localip, self.l[randrange(0, self.num)].port)
i.addContact(self.localip, self.l[randrange(0, self.num)].port)
reactor.iterate()
reactor.iterate()
reactor.iterate()
for i in self.l:
self.done = 0
i.findCloseNodes(self._done)
while not self.done:
reactor.iterate()
for i in self.l:
self.done = 0
i.findCloseNodes(self._done)
while not self.done:
reactor.iterate()
def join(self, deferred=None):
"""See L{apt_p2p.interfaces.IDHT}.
@param deferred: the deferred to callback when the join is complete
(optional, defaults to creating a new deferred and returning it)
"""
# Check for multiple simultaneous joins
if self.joining:
if deferred:
deferred.errback(DHTError("a join is already in progress"))
return
else:
raise DHTError, "a join is already in progress"
if deferred:
self.joining = deferred
else:
self.joining = defer.Deferred()
if self.config is None:
self.joining.errback(DHTError("configuration not loaded"))
return self.joining
# Create the new khashmir instance
if not self.khashmir:
self.khashmir = Khashmir(self.config, self.cache_dir)
self.outstandingJoins = 0
for node in self.bootstrap:
host, port = node.rsplit(':', 1)
port = int(port)
self.outstandingJoins += 1
# Translate host names into IP addresses
if isIPAddress(host):
self._join_gotIP(host, port)
else:
reactor.resolve(host).addCallbacks(self._join_gotIP,
self._join_resolveFailed,
callbackArgs=(port, ),
errbackArgs=(host, port))
return self.joining
def join(self, deferred = None):
"""See L{apt_p2p.interfaces.IDHT}.
@param deferred: the deferred to callback when the join is complete
(optional, defaults to creating a new deferred and returning it)
"""
# Check for multiple simultaneous joins
if self.joining:
if deferred:
deferred.errback(DHTError("a join is already in progress"))
return
else:
raise DHTError, "a join is already in progress"
if deferred:
self.joining = deferred
else:
self.joining = defer.Deferred()
if self.config is None:
self.joining.errback(DHTError("configuration not loaded"))
return self.joining
# Create the new khashmir instance
if not self.khashmir:
self.khashmir = Khashmir(self.config, self.cache_dir)
self.outstandingJoins = 0
for node in self.bootstrap:
host, port = node.rsplit(':', 1)
port = int(port)
self.outstandingJoins += 1
# Translate host names into IP addresses
if isIPAddress(host):
self._join_gotIP(host, port)
else:
reactor.resolve(host).addCallbacks(self._join_gotIP,
self._join_resolveFailed,
callbackArgs = (port, ),
errbackArgs = (host, port))
return self.joining
class DHT:
"""The main interface instance to the Khashmir DHT.
@type config: C{dictionary}
@ivar config: the DHT configuration values
@type cache_dir: C{string}
@ivar cache_dir: the directory to use for storing files
@type bootstrap: C{list} of C{string}
@ivar bootstrap: the nodes to contact to bootstrap into the system
@type bootstrap_node: C{boolean}
@ivar bootstrap_node: whether this node is a bootstrap node
@type joining: L{twisted.internet.defer.Deferred}
@ivar joining: if a join is underway, the deferred that will signal it's end
@type joined: C{boolean}
@ivar joined: whether the DHT network has been successfully joined
@type outstandingJoins: C{int}
@ivar outstandingJoins: the number of bootstrap nodes that have yet to respond
@type next_rejoin: C{int}
@ivar next_rejoin: the number of seconds before retrying the next join
@type foundAddrs: C{list} of (C{string}, C{int})
@ivar foundAddrs: the IP address an port that were returned by bootstrap nodes
@type storing: C{dictionary}
@ivar storing: keys are keys for which store requests are active, values
are dictionaries with keys the values being stored and values the
deferred to call when complete
@type retrieving: C{dictionary}
@ivar retrieving: keys are the keys for which getValue requests are active,
values are lists of the deferreds waiting for the requests
@type retrieved: C{dictionary}
@ivar retrieved: keys are the keys for which getValue requests are active,
values are list of the values returned so far
@type factory: L{twisted.web2.channel.HTTPFactory}
@ivar factory: the factory to use to serve HTTP requests for statistics
@type config_parser: L{apt_p2p.apt_p2p_conf.AptP2PConfigParser}
@ivar config_parser: the configuration info for the main program
@type section: C{string}
@ivar section: the section of the configuration info that applies to the DHT
@type khashmir: L{khashmir.Khashmir}
@ivar khashmir: the khashmir DHT instance to use
"""
if _web2:
implements(IDHT, IDHTStats, IDHTStatsFactory)
else:
implements(IDHT, IDHTStats)
def __init__(self):
"""Initialize the DHT."""
self.config = None
self.cache_dir = ''
self.bootstrap = []
self.bootstrap_node = False
self.joining = None
self.khashmir = None
self.joined = False
self.outstandingJoins = 0
self.next_rejoin = 20
self.foundAddrs = []
self.storing = {}
self.retrieving = {}
self.retrieved = {}
self.factory = None
def loadConfig(self, config, section):
"""See L{apt_p2p.interfaces.IDHT}."""
self.config_parser = config
self.section = section
self.config = {}
# Get some initial values
self.cache_dir = os.path.join(self.config_parser.get(section, 'cache_dir'), khashmir_dir)
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
self.bootstrap = self.config_parser.getstringlist(section, 'BOOTSTRAP')
self.bootstrap_node = self.config_parser.getboolean(section, 'BOOTSTRAP_NODE')
for k in self.config_parser.options(section):
# The numbers in the config file
if k in ['CONCURRENT_REQS', 'STORE_REDUNDANCY',
'RETRIEVE_VALUES', 'MAX_FAILURES', 'PORT']:
self.config[k] = self.config_parser.getint(section, k)
# The times in the config file
elif k in ['CHECKPOINT_INTERVAL', 'MIN_PING_INTERVAL',
'BUCKET_STALENESS', 'KEY_EXPIRE',
'KRPC_TIMEOUT', 'KRPC_INITIAL_DELAY']:
self.config[k] = self.config_parser.gettime(section, k)
# The booleans in the config file
elif k in ['SPEW', 'LOCAL_OK']:
self.config[k] = self.config_parser.getboolean(section, k)
# Everything else is a string
else:
self.config[k] = self.config_parser.get(section, k)
def join(self, deferred = None):
"""See L{apt_p2p.interfaces.IDHT}.
@param deferred: the deferred to callback when the join is complete
(optional, defaults to creating a new deferred and returning it)
"""
# Check for multiple simultaneous joins
if self.joining:
if deferred:
deferred.errback(DHTError("a join is already in progress"))
return
else:
raise DHTError, "a join is already in progress"
if deferred:
self.joining = deferred
else:
self.joining = defer.Deferred()
if self.config is None:
self.joining.errback(DHTError("configuration not loaded"))
return self.joining
# Create the new khashmir instance
if not self.khashmir:
self.khashmir = Khashmir(self.config, self.cache_dir)
self.outstandingJoins = 0
for node in self.bootstrap:
host, port = node.rsplit(':', 1)
port = int(port)
self.outstandingJoins += 1
# Translate host names into IP addresses
if isIPAddress(host):
self._join_gotIP(host, port)
else:
reactor.resolve(host).addCallbacks(self._join_gotIP,
self._join_resolveFailed,
callbackArgs = (port, ),
errbackArgs = (host, port))
return self.joining
def _join_gotIP(self, ip, port):
"""Join the DHT using a single bootstrap nodes IP address."""
self.khashmir.addContact(ip, port, self._join_single, self._join_error)
def _join_resolveFailed(self, err, host, port):
"""Failed to lookup the IP address of the bootstrap node."""
log.msg('Failed to find an IP address for host: (%r, %r)' % (host, port))
log.err(err)
self.outstandingJoins -= 1
if self.outstandingJoins <= 0:
self.khashmir.findCloseNodes(self._join_complete)
def _join_single(self, addr):
"""Process the response from the bootstrap node.
Finish the join by contacting close nodes.
"""
self.outstandingJoins -= 1
if addr:
self.foundAddrs.append(addr)
if addr or self.outstandingJoins <= 0:
self.khashmir.findCloseNodes(self._join_complete)
log.msg('Got back from bootstrap node: %r' % (addr,))
def _join_error(self, failure = None):
"""Process an error in contacting the bootstrap node.
If no bootstrap nodes remain, finish the process by contacting
close nodes.
"""
self.outstandingJoins -= 1
log.msg("bootstrap node could not be reached")
if self.outstandingJoins <= 0:
self.khashmir.findCloseNodes(self._join_complete)
def _join_complete(self, result):
"""End the joining process and return the addresses found for this node."""
if not self.joined and isinstance(result, list) and len(result) > 1:
self.joined = True
if self.joining and self.outstandingJoins <= 0:
df = self.joining
self.joining = None
if self.joined or self.bootstrap_node:
self.joined = True
df.callback(self.foundAddrs)
else:
# Try to join later using exponential backoff delays
log.msg('Join failed, retrying in %d seconds' % self.next_rejoin)
reactor.callLater(self.next_rejoin, self.join, df)
self.next_rejoin *= 2
def getAddrs(self):
"""Get the list of addresses returned by bootstrap nodes for this node."""
return self.foundAddrs
def leave(self):
"""See L{apt_p2p.interfaces.IDHT}."""
if self.config is None:
raise DHTError, "configuration not loaded"
if self.joined or self.joining:
if self.joining:
self.joining.errback(DHTError('still joining when leave was called'))
self.joining = None
self.joined = False
self.khashmir.shutdown()
def _normKey(self, key):
"""Normalize the length of keys used in the DHT."""
# Extend short keys with null bytes
if len(key) < HASH_LENGTH:
key = key + '\000'*(HASH_LENGTH - len(key))
# Truncate long keys
elif len(key) > HASH_LENGTH:
key = key[:HASH_LENGTH]
return key
def getValue(self, key):
"""See L{apt_p2p.interfaces.IDHT}."""
if self.config is None:
return defer.fail(DHTError("configuration not loaded"))
if not self.joined:
return defer.fail(DHTError("have not joined a network yet"))
d = defer.Deferred()
key = self._normKey(key)
if key not in self.retrieving:
self.khashmir.valueForKey(key, self._getValue)
self.retrieving.setdefault(key, []).append(d)
return d
def _getValue(self, key, result):
"""Process a returned list of values from the DHT."""
# Save the list of values to return when it is complete
if result:
self.retrieved.setdefault(key, []).extend([bdecode(r) for r in result])
else:
# Empty list, the get is complete, return the result
final_result = []
if key in self.retrieved:
final_result = self.retrieved[key]
del self.retrieved[key]
for i in range(len(self.retrieving[key])):
d = self.retrieving[key].pop(0)
d.callback(final_result)
del self.retrieving[key]
def storeValue(self, key, value):
"""See L{apt_p2p.interfaces.IDHT}."""
if self.config is None:
return defer.fail(DHTError("configuration not loaded"))
if not self.joined:
return defer.fail(DHTError("have not joined a network yet"))
d = defer.Deferred()
key = self._normKey(key)
bvalue = bencode(value)
if key in self.storing and bvalue in self.storing[key]:
raise DHTError, "already storing that key with the same value"
self.khashmir.storeValueForKey(key, bvalue, self._storeValue)
self.storing.setdefault(key, {})[bvalue] = d
return d
def _storeValue(self, key, bvalue, result):
"""Process the response from the DHT."""
if key in self.storing and bvalue in self.storing[key]:
# Check if the store succeeded
if len(result) > 0:
self.storing[key][bvalue].callback(result)
else:
self.storing[key][bvalue].errback(DHTError('could not store value %s in key %s' % (bvalue, key)))
del self.storing[key][bvalue]
if len(self.storing[key].keys()) == 0:
del self.storing[key]
def getStats(self):
"""See L{apt_p2p.interfaces.IDHTStats}."""
return self.khashmir.getStats()
def getStatsFactory(self):
"""See L{apt_p2p.interfaces.IDHTStatsFactory}."""
assert _web2, "NOT IMPLEMENTED: twisted.web2 must be installed to use the stats factory."
if self.factory is None:
# Create a simple HTTP factory for stats
class StatsResource(resource.Resource):
def __init__(self, manager):
self.manager = manager
def render(self, ctx):
return http.Response(
200,
{'content-type': http_headers.MimeType('text', 'html')},
'<html><body>\n\n' + self.manager.getStats() + '\n</body></html>\n')
def locateChild(self, request, segments):
log.msg('Got HTTP stats request from %s' % (request.remoteAddr, ))
return self, ()
self.factory = channel.HTTPFactory(server.Site(StatsResource(self)))
return self.factory
class DHT:
"""The main interface instance to the Khashmir DHT.
@type config: C{dictionary}
@ivar config: the DHT configuration values
@type cache_dir: C{string}
@ivar cache_dir: the directory to use for storing files
@type bootstrap: C{list} of C{string}
@ivar bootstrap: the nodes to contact to bootstrap into the system
@type bootstrap_node: C{boolean}
@ivar bootstrap_node: whether this node is a bootstrap node
@type joining: L{twisted.internet.defer.Deferred}
@ivar joining: if a join is underway, the deferred that will signal it's end
@type joined: C{boolean}
@ivar joined: whether the DHT network has been successfully joined
@type outstandingJoins: C{int}
@ivar outstandingJoins: the number of bootstrap nodes that have yet to respond
@type next_rejoin: C{int}
@ivar next_rejoin: the number of seconds before retrying the next join
@type foundAddrs: C{list} of (C{string}, C{int})
@ivar foundAddrs: the IP address an port that were returned by bootstrap nodes
@type storing: C{dictionary}
@ivar storing: keys are keys for which store requests are active, values
are dictionaries with keys the values being stored and values the
deferred to call when complete
@type retrieving: C{dictionary}
@ivar retrieving: keys are the keys for which getValue requests are active,
values are lists of the deferreds waiting for the requests
@type retrieved: C{dictionary}
@ivar retrieved: keys are the keys for which getValue requests are active,
values are list of the values returned so far
@type factory: L{twisted.web2.channel.HTTPFactory}
@ivar factory: the factory to use to serve HTTP requests for statistics
@type config_parser: L{apt_p2p.apt_p2p_conf.AptP2PConfigParser}
@ivar config_parser: the configuration info for the main program
@type section: C{string}
@ivar section: the section of the configuration info that applies to the DHT
@type khashmir: L{khashmir.Khashmir}
@ivar khashmir: the khashmir DHT instance to use
"""
if _web2:
implements(IDHT, IDHTStats, IDHTStatsFactory)
else:
implements(IDHT, IDHTStats)
def __init__(self):
"""Initialize the DHT."""
self.config = None
self.cache_dir = ''
self.bootstrap = []
self.bootstrap_node = False
self.joining = None
self.khashmir = None
self.joined = False
self.outstandingJoins = 0
self.next_rejoin = 20
self.foundAddrs = []
self.storing = {}
self.retrieving = {}
self.retrieved = {}
self.factory = None
def loadConfig(self, config, section):
"""See L{apt_p2p.interfaces.IDHT}."""
self.config_parser = config
self.section = section
self.config = {}
# Get some initial values
self.cache_dir = os.path.join(
self.config_parser.get(section, 'cache_dir'), khashmir_dir)
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
self.bootstrap = self.config_parser.getstringlist(section, 'BOOTSTRAP')
self.bootstrap_node = self.config_parser.getboolean(
section, 'BOOTSTRAP_NODE')
for k in self.config_parser.options(section):
# The numbers in the config file
if k in [
'CONCURRENT_REQS', 'STORE_REDUNDANCY', 'RETRIEVE_VALUES',
'MAX_FAILURES', 'PORT'
]:
self.config[k] = self.config_parser.getint(section, k)
# The times in the config file
elif k in [
'CHECKPOINT_INTERVAL', 'MIN_PING_INTERVAL',
'BUCKET_STALENESS', 'KEY_EXPIRE', 'KRPC_TIMEOUT',
'KRPC_INITIAL_DELAY'
]:
self.config[k] = self.config_parser.gettime(section, k)
# The booleans in the config file
elif k in ['SPEW', 'LOCAL_OK']:
self.config[k] = self.config_parser.getboolean(section, k)
# Everything else is a string
else:
self.config[k] = self.config_parser.get(section, k)
def join(self, deferred=None):
"""See L{apt_p2p.interfaces.IDHT}.
@param deferred: the deferred to callback when the join is complete
(optional, defaults to creating a new deferred and returning it)
"""
# Check for multiple simultaneous joins
if self.joining:
if deferred:
deferred.errback(DHTError("a join is already in progress"))
return
else:
raise DHTError, "a join is already in progress"
if deferred:
self.joining = deferred
else:
self.joining = defer.Deferred()
if self.config is None:
self.joining.errback(DHTError("configuration not loaded"))
return self.joining
# Create the new khashmir instance
if not self.khashmir:
self.khashmir = Khashmir(self.config, self.cache_dir)
self.outstandingJoins = 0
for node in self.bootstrap:
host, port = node.rsplit(':', 1)
port = int(port)
self.outstandingJoins += 1
# Translate host names into IP addresses
if isIPAddress(host):
self._join_gotIP(host, port)
else:
reactor.resolve(host).addCallbacks(self._join_gotIP,
self._join_resolveFailed,
callbackArgs=(port, ),
errbackArgs=(host, port))
return self.joining
def _join_gotIP(self, ip, port):
"""Join the DHT using a single bootstrap nodes IP address."""
self.khashmir.addContact(ip, port, self._join_single, self._join_error)
def _join_resolveFailed(self, err, host, port):
"""Failed to lookup the IP address of the bootstrap node."""
log.msg('Failed to find an IP address for host: (%r, %r)' %
(host, port))
log.err(err)
self.outstandingJoins -= 1
if self.outstandingJoins <= 0:
self.khashmir.findCloseNodes(self._join_complete)
def _join_single(self, addr):
"""Process the response from the bootstrap node.
Finish the join by contacting close nodes.
"""
self.outstandingJoins -= 1
if addr:
self.foundAddrs.append(addr)
if addr or self.outstandingJoins <= 0:
self.khashmir.findCloseNodes(self._join_complete)
log.msg('Got back from bootstrap node: %r' % (addr, ))
def _join_error(self, failure=None):
"""Process an error in contacting the bootstrap node.
If no bootstrap nodes remain, finish the process by contacting
close nodes.
"""
self.outstandingJoins -= 1
log.msg("bootstrap node could not be reached")
if self.outstandingJoins <= 0:
self.khashmir.findCloseNodes(self._join_complete)
def _join_complete(self, result):
"""End the joining process and return the addresses found for this node."""
if not self.joined and isinstance(result, list) and len(result) > 1:
self.joined = True
if self.joining and self.outstandingJoins <= 0:
df = self.joining
self.joining = None
if self.joined or self.bootstrap_node:
self.joined = True
df.callback(self.foundAddrs)
else:
# Try to join later using exponential backoff delays
log.msg('Join failed, retrying in %d seconds' %
self.next_rejoin)
reactor.callLater(self.next_rejoin, self.join, df)
self.next_rejoin *= 2
def getAddrs(self):
"""Get the list of addresses returned by bootstrap nodes for this node."""
return self.foundAddrs
def leave(self):
"""See L{apt_p2p.interfaces.IDHT}."""
if self.config is None:
raise DHTError, "configuration not loaded"
if self.joined or self.joining:
if self.joining:
self.joining.errback(
DHTError('still joining when leave was called'))
self.joining = None
self.joined = False
self.khashmir.shutdown()
def _normKey(self, key):
"""Normalize the length of keys used in the DHT."""
# Extend short keys with null bytes
if len(key) < HASH_LENGTH:
key = key + '\000' * (HASH_LENGTH - len(key))
# Truncate long keys
elif len(key) > HASH_LENGTH:
key = key[:HASH_LENGTH]
return key
def getValue(self, key):
"""See L{apt_p2p.interfaces.IDHT}."""
if self.config is None:
return defer.fail(DHTError("configuration not loaded"))
if not self.joined:
return defer.fail(DHTError("have not joined a network yet"))
d = defer.Deferred()
key = self._normKey(key)
if key not in self.retrieving:
self.khashmir.valueForKey(key, self._getValue)
self.retrieving.setdefault(key, []).append(d)
return d
def _getValue(self, key, result):
"""Process a returned list of values from the DHT."""
# Save the list of values to return when it is complete
if result:
self.retrieved.setdefault(key,
[]).extend([bdecode(r) for r in result])
else:
# Empty list, the get is complete, return the result
final_result = []
if key in self.retrieved:
final_result = self.retrieved[key]
del self.retrieved[key]
for i in range(len(self.retrieving[key])):
d = self.retrieving[key].pop(0)
d.callback(final_result)
del self.retrieving[key]
def storeValue(self, key, value):
"""See L{apt_p2p.interfaces.IDHT}."""
if self.config is None:
return defer.fail(DHTError("configuration not loaded"))
if not self.joined:
return defer.fail(DHTError("have not joined a network yet"))
d = defer.Deferred()
key = self._normKey(key)
bvalue = bencode(value)
if key in self.storing and bvalue in self.storing[key]:
raise DHTError, "already storing that key with the same value"
self.khashmir.storeValueForKey(key, bvalue, self._storeValue)
self.storing.setdefault(key, {})[bvalue] = d
return d
def _storeValue(self, key, bvalue, result):
"""Process the response from the DHT."""
if key in self.storing and bvalue in self.storing[key]:
# Check if the store succeeded
if len(result) > 0:
self.storing[key][bvalue].callback(result)
else:
self.storing[key][bvalue].errback(
DHTError('could not store value %s in key %s' %
(bvalue, key)))
del self.storing[key][bvalue]
if len(self.storing[key].keys()) == 0:
del self.storing[key]
def getStats(self):
"""See L{apt_p2p.interfaces.IDHTStats}."""
return self.khashmir.getStats()
def getStatsFactory(self):
"""See L{apt_p2p.interfaces.IDHTStatsFactory}."""
assert _web2, "NOT IMPLEMENTED: twisted.web2 must be installed to use the stats factory."
if self.factory is None:
# Create a simple HTTP factory for stats
class StatsResource(resource.Resource):
def __init__(self, manager):
self.manager = manager
def render(self, ctx):
return http.Response(
200, {
'content-type': http_headers.MimeType(
'text', 'html')
}, '<html><body>\n\n' + self.manager.getStats() +
'\n</body></html>\n')
def locateChild(self, request, segments):
log.msg('Got HTTP stats request from %s' %
(request.remoteAddr, ))
return self, ()
self.factory = channel.HTTPFactory(server.Site(
StatsResource(self)))
return self.factory
