def data_came_in(self, amount):
if not self.got_anything:
self.got_anything = True
self.start = bttime() - 2
self.last = self.start
self.left -= amount
return
self.update(bttime(), amount)
def data_came_in(self, amount):
if not self.got_anything:
self.got_anything = True
self.start = bttime() - 2
self.last = self.start
self.left -= amount
return
self.update(bttime(), amount)
def get_time_left(self):
if not self.got_anything:
return None
t = bttime()
if t - self.last > 15:
self.update(t, 0)
return self.remaining
def _create(self, hostname='localhost'):
# Make the RSA key
self.rsakey = RSA.gen_key(2048, m2.RSA_F4)
if self.secure:
# Save the key, AES256-CBC encrypted
self.rsakey.save_key(self.keyfile, 'aes_256_cbc')
else:
# Save the key unencrypted.
self.rsakey.save_key(self.keyfile,
None,
callback=m2util.no_passphrase_callback)
# Make the public key
pkey = EVP.PKey()
pkey.assign_rsa(self.rsakey, 0)
# Generate the certificate
self.cert = X509.X509()
self.cert.set_serial_number(long(bttime()))
self.cert.set_version(0x2)
self.cert.set_pubkey(pkey)
# Set the name on the certificate
name = X509.X509_Name()
name.CN = hostname
self.cert.set_subject(name)
self.cert.set_issuer(name)
# Set the period of time the cert is valid for (5 years from issue)
notBefore = m2.x509_get_not_before(self.cert.x509)
notAfter = m2.x509_get_not_after(self.cert.x509)
m2.x509_gmtime_adj(notBefore, 0)
m2.x509_gmtime_adj(notAfter, 60 * 60 * 24 * 365 * 5)
# Sign the certificate
self.cert.sign(pkey, 'sha1')
# Save it
self.cert.save_pem(self.certfile)
def schedule(self, delay, func, context=None):
""" Insert a task into the queue in a threadsafe manner """
if threading.currentThread() == self.thread:
if self.contexts.get(context, False):
bisect.insort(self.tasks, (bttime() + delay, func, context))
else:
self._external_schedule(delay, func, context)
def request_status(self, infohash, want_spew, want_fileinfo):
torrent = self.torrents.get(infohash)
if torrent is None or torrent.state != RUNNING:
return
status = torrent.dl.get_status(want_spew, want_fileinfo)
if torrent.finishtime is not None:
now = bttime()
uptotal = status['upTotal'] + torrent.uptotal_old
downtotal = status['downTotal'] + torrent.downtotal_old
ulspeed = status['upRate']
if self.queue:
ratio = self.next_torrent_ratio / 100
else:
ratio = self.last_torrent_ratio / 100
if ratio <= 0 or ulspeed <= 0:
rem = 1e99
else:
rem = (downtotal * ratio - uptotal) / ulspeed
if self.queue:
rem = min(
rem, torrent.finishtime +
self.config['next_torrent_time'] * 60 - now)
rem = max(rem, torrent.finishtime + 120 - now)
if rem <= 0:
rem = 1
if rem == 1e99:
rem = None
status['timeEst'] = rem
self.run_ui_task(self.ui.update_status, infohash, status)
def _dump_state(self):
self.last_save_time = bttime()
r = []
def write_entry(infohash, t):
if t.dlpath is None:
assert t.state == ASKING_LOCATION
r.append(infohash.encode('hex') + '\n')
else:
r.append(
infohash.encode('hex') + ' ' + str(t.uptotal) + ' ' +
str(t.downtotal) + ' ' + t.dlpath.encode('string_escape') +
'\n')
r.append('Anomos UI state file, version 3\n')
r.append('Running torrents\n')
for infohash in self.running_torrents:
write_entry(infohash, self.torrents[infohash])
r.append('Queued torrents\n')
for infohash in self.queue:
write_entry(infohash, self.torrents[infohash])
r.append('Known torrents\n')
for infohash in self.other_torrents:
write_entry(infohash, self.torrents[infohash])
r.append('End\n')
f = None
try:
f = file(os.path.join(self.config['data_dir'], 'ui_state'), 'wb')
f.write(''.join(r))
f.close()
except Exception, e:
self.global_error("ERROR", 'Could not save UI state: ' + str(e))
if f is not None:
f.close()
def _queue_loop(self):
if self.doneflag.isSet():
return
self.multitorrent.schedule(20, self._queue_loop)
now = bttime()
if self.queue and self.starting_torrent is None:
if self.config['next_torrent_time'] == 0:
mintime = 0
else:
mintime = now - self.config['next_torrent_time'] * 60
minratio = self.next_torrent_ratio / 100
else:
mintime = 0
minratio = self.last_torrent_ratio / 100
if not minratio:
return
for infohash in self.running_torrents:
t = self.torrents[infohash]
if t.state == RUN_QUEUED:
continue
totals = t.dl.get_total_transfer()
# not updated for remaining torrents if one is stopped, who cares
t.uptotal = t.uptotal_old + totals[0]
t.downtotal = t.downtotal_old + totals[1]
if t.finishtime is None or t.finishtime > now - 120:
continue
if t.finishtime > mintime:
if t.uptotal < t.downtotal * minratio:
continue
self.change_torrent_state(infohash, RUNNING, KNOWN)
break
if self.running_torrents and self.last_save_time < now - 300:
self._dump_state()
def _dump_state(self):
self.last_save_time = bttime()
r = []
def write_entry(infohash, t):
if t.dlpath is None:
assert t.state == ASKING_LOCATION
r.append(infohash.encode('hex') + '\n')
else:
r.append(infohash.encode('hex') + ' ' + str(t.uptotal) + ' ' +
str(t.downtotal)+' '+t.dlpath.encode('string_escape')+'\n')
r.append('Anomos UI state file, version 3\n')
r.append('Running torrents\n')
for infohash in self.running_torrents:
write_entry(infohash, self.torrents[infohash])
r.append('Queued torrents\n')
for infohash in self.queue:
write_entry(infohash, self.torrents[infohash])
r.append('Known torrents\n')
for infohash in self.other_torrents:
write_entry(infohash, self.torrents[infohash])
r.append('End\n')
f = None
try:
f = file(os.path.join(self.config['data_dir'], 'ui_state'), 'wb')
f.write(''.join(r))
f.close()
except Exception, e:
self.global_error("ERROR", 'Could not save UI state: ' + str(e))
if f is not None:
f.close()
def _queue_loop(self):
if self.doneflag.isSet():
return
self.multitorrent.schedule(20, self._queue_loop)
now = bttime()
if self.queue and self.starting_torrent is None:
if self.config['next_torrent_time'] == 0:
mintime = 0
else:
mintime = now - self.config['next_torrent_time'] * 60
minratio = self.next_torrent_ratio / 100
else:
mintime = 0
minratio = self.last_torrent_ratio / 100
if not minratio:
return
for infohash in self.running_torrents:
t = self.torrents[infohash]
if t.state == RUN_QUEUED:
continue
totals = t.dl.get_total_transfer()
# not updated for remaining torrents if one is stopped, who cares
t.uptotal = t.uptotal_old + totals[0]
t.downtotal = t.downtotal_old + totals[1]
if t.finishtime is None or t.finishtime > now - 120:
continue
if t.finishtime > mintime:
if t.uptotal < t.downtotal * minratio:
continue
self.change_torrent_state(infohash, RUNNING, KNOWN)
break
if self.running_torrents and self.last_save_time < now - 300:
self._dump_state()
def schedule(self, delay, func, context=None):
""" Insert a task into the queue in a threadsafe manner """
if threading.currentThread() == self.thread:
if self.contexts.get(context, False):
bisect.insort(self.tasks, (bttime() + delay, func, context))
else:
self._external_schedule(delay, func, context)
def update(self, ip, params):
self.last_seen = bttime()
port = int(params.get('port'))
# IP or port changed so we should natcheck again
if (ip, port) != (self.ip, self.port):
self.nat = True # Assume peer is NAT'd
if is_valid_ip(ip):
self.num_natcheck = 0
else: # Don't natcheck invalid addresses
self.num_natcheck = INFINITY
# Mark any failed peers
for x in params.get('failed', []):
self.failed(x)
# Update stats
ihash = params.get('info_hash')
rl = params.get('relayed')
if rl is not None:
self.relayed_total = rl
# Remove any stopped torrents
if params.get('event') == 'stopped':
if self.infohashes.has_key(ihash):
del self.infohashes[ihash]
else:
# Update upload/download/left
if self.infohashes.has_key(ihash):
p_ul, p_dl, p_left = self.infohashes[ihash]
else:
p_ul, p_dl, p_left = (0,0,0)
ul = params.get('uploaded', p_ul)
dl = params.get('downloaded', p_dl)
left = params.get('left', p_left)
self.infohashes[ihash] = (int(ul), int(dl), int(left))
def request_status(self, infohash, want_spew, want_fileinfo):
torrent = self.torrents.get(infohash)
if torrent is None or torrent.state != RUNNING:
return
status = torrent.dl.get_status(want_spew, want_fileinfo)
if torrent.finishtime is not None:
now = bttime()
uptotal = status['upTotal'] + torrent.uptotal_old
downtotal = status['downTotal'] + torrent.downtotal_old
ulspeed = status['upRate']
if self.queue:
ratio = self.next_torrent_ratio / 100
else:
ratio = self.last_torrent_ratio / 100
if ratio <= 0 or ulspeed <= 0:
rem = 1e99
else:
rem = (downtotal * ratio - uptotal) / ulspeed
if self.queue:
rem = min(rem, torrent.finishtime +
self.config['next_torrent_time'] * 60 - now)
rem = max(rem, torrent.finishtime + 120 - now)
if rem <= 0:
rem = 1
if rem == 1e99:
rem = None
status['timeEst'] = rem
self.run_ui_task(self.ui.update_status, infohash, status)
def _create(self, hostname='localhost'):
# Make the RSA key
self.rsakey = RSA.gen_key(2048, m2.RSA_F4)
if self.secure:
# Save the key, AES256-CBC encrypted
self.rsakey.save_key(self.keyfile, 'aes_256_cbc')
else:
# Save the key unencrypted.
self.rsakey.save_key(self.keyfile, None, callback=m2util.no_passphrase_callback)
# Make the public key
pkey = EVP.PKey()
pkey.assign_rsa(self.rsakey, 0)
# Generate the certificate
self.cert = X509.X509()
self.cert.set_serial_number(long(bttime()))
self.cert.set_version(0x2)
self.cert.set_pubkey(pkey)
# Set the name on the certificate
name = X509.X509_Name()
name.CN = hostname
self.cert.set_subject(name)
self.cert.set_issuer(name)
# Set the period of time the cert is valid for (5 years from issue)
notBefore = m2.x509_get_not_before(self.cert.x509)
notAfter = m2.x509_get_not_after(self.cert.x509)
m2.x509_gmtime_adj(notBefore, 0)
m2.x509_gmtime_adj(notAfter, 60*60*24*365*5)
# Sign the certificate
self.cert.sign(pkey, 'sha1')
# Save it
self.cert.save_pem(self.certfile)
def get_time_left(self):
if not self.got_anything:
return None
t = bttime()
if t - self.last > 15:
self.update(t, 0)
return self.remaining
def stats(self):
self.multitorrent.schedule(self.config['display_interval'], self.stats)
data = []
for infohash in self.torrent_list:
cache = self.torrent_cache[infohash]
if self.config['display_path']:
name = cache['path']
else:
name = cache['name']
size = cache['length']
d = self.downloads[infohash]
progress = '0.0%'
peers = 0
seeds = 0
seedsmsg = "S"
dist = 0.0
uprate = 0.0
dnrate = 0.0
upamt = 0
dnamt = 0
t = 0
msg = ''
if d is None:
status = 'waiting for hash check'
else:
stats = d.get_status()
status = stats['activity']
progress = '%.1f%%' % (int(stats['fractionDone']*1000)/10.0)
if d.started and not d.closed:
s = stats
dist = s['numCopies']
if d.is_seed:
seeds = 0 # s['numOldSeeds']
seedsmsg = "s"
else:
if s['numSeeds'] + s['numPeers']:
t = stats['timeEst']
if t is None:
t = -1
if t == 0: # unlikely
t = 0.01
status = 'downloading'
else:
t = -1
status = 'connecting to peers'
seeds = s['numSeeds']
dnrate = stats['downRate']
peers = s['numPeers']
uprate = stats['upRate']
upamt = s['upTotal']
dnamt = s['downTotal']
if d.messages and (d.closed or d.messages[-1][0] + 300 > bttime()):
msg = d.messages[-1][2]
data.append(( name, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, size, t, msg ))
stop = self.output.display(data)
if stop:
self.doneflag.set()
def update_rate(self, amount):
self.total += amount
t = bttime()
self.rate = (self.rate * (self.last - self.ratesince) +
amount) / (t - self.ratesince)
self.last = t
if self.ratesince < t - self.max_rate_period:
self.ratesince = t - self.max_rate_period
def update_rate(self, amount):
self.total += amount
t = bttime()
self.rate = (self.rate * (self.last - self.ratesince) +
amount) / (t - self.ratesince)
self.last = t
if self.ratesince < t - self.max_rate_period:
self.ratesince = t - self.max_rate_period
def do_tasks(self):
""" Do all tasks with timestamps <= than current time """
context = None
try:
while len(self.tasks) > 0 and self.tasks[0][0] <= bttime():
_, f, context = self.tasks.pop(0)
apply(f)
except Exception, e:
if context is not None:
context.got_exception(e)
def do_tasks(self):
""" Do all tasks with timestamps <= than current time """
context = None
try:
while len(self.tasks) > 0 and self.tasks[0][0] <= bttime():
_, f, context = self.tasks.pop(0)
apply(f)
except Exception, e:
if context is not None:
context.got_exception(e)
def add_neighbor(self, peerid, nid, ip, port):
"""
Assign Neighbor ID to peer
@type peerid: string
@type nid: int
"""
self.neighbors.setdefault(peerid, {'nid':nid,'ip':ip, 'port':port})
self.id_map[nid] = peerid
self.last_modified = bttime()
if self.nbrs_needed > 0:
self.nbrs_needed -= 1
def rm_neighbor(self, peerid):
"""
Remove connection to neighbor
@type peerid: string
"""
edge = self.neighbors.get(peerid)
if edge:
del self.id_map[edge['nid']]
del self.neighbors[peerid]
self.last_modified = bttime()
self.nbrs_needed += 1
def finished(self, torrent):
infohash = torrent.infohash
if infohash == self.starting_torrent:
t = self.torrents[infohash]
if self.queue:
ratio = self.next_torrent_ratio / 100
else:
ratio = self.last_torrent_ratio / 100
if ratio and t.uptotal >= t.downtotal * ratio:
raise BTShutdown("Not starting torrent as it already meets "
"the current settings for when to stop seeding")
self.torrents[torrent.infohash].finishtime = bttime()
def _check(self):
if self.current_started is not None:
if self.current_started <= bttime() - 58:
log.warning("Tracker announce still not complete "
"%d seconds after starting it" %
int(bttime() - self.current_started))
## Announce has been hanging for too long, retry it.
if int(bttime() - self.current_started) >= 180:
self._announce(STARTED)
return
if self.basequery is None:
self.basequery = self._makequery()
self._announce(STARTED)
return
if self.changed_port:
self._announce(STOPPED)
self.changed_port = False
self.basequery = None
return
if self.finish:
self.finish = False
self._announce(COMPLETED)
return
if self.fail_wait is not None:
if self.last_time + self.fail_wait <= bttime():
self._announce(STARTED)
return
if self.last_time > bttime() - self.config['rerequest_interval']:
return
getmore = bool(self.neighbors.failed_connections())
#TODO: also reannounce when TCs have failed
if getmore or bttime() - self.last_time > self.announce_interval:
self._announce()
def finished(self, torrent):
infohash = torrent.infohash
if infohash == self.starting_torrent:
t = self.torrents[infohash]
if self.queue:
ratio = self.next_torrent_ratio / 100
else:
ratio = self.last_torrent_ratio / 100
if ratio and t.uptotal >= t.downtotal * ratio:
raise BTShutdown(
"Not starting torrent as it already meets "
"the current settings for when to stop seeding")
self.torrents[torrent.infohash].finishtime = bttime()
def got_piece(self, index, begin, piece):
try:
self.active_requests.remove((index, begin, len(piece)))
except ValueError:
self.downloader.discarded_bytes += len(piece)
return False
if self.downloader.storage.endgame:
self.downloader.all_requests.remove((index, begin, len(piece)))
self.last = bttime()
self.measure.update_rate(len(piece))
self.downloader.measurefunc(len(piece))
self.downloader.downmeasure.update_rate(len(piece))
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
if self.downloader.storage.endgame:
while self.downloader.storage.do_I_have_requests(index):
nb, nl = self.downloader.storage.new_request(index)
self.downloader.all_requests.append((index, nb, nl))
for d in self.downloader.downloads:
d.fix_download_endgame()
return False
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more([index])
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.storage.endgame:
for d in self.downloader.downloads:
if d is not self and d.interested:
if d.choked:
d.fix_download_endgame()
else:
try:
d.active_requests.remove(
(index, begin, len(piece)))
except ValueError:
continue
d.stream.send_cancel(index, begin, len(piece))
d.fix_download_endgame()
self._request_more()
if self.downloader.picker.am_I_complete():
for d in [
i for i in self.downloader.downloads
if i.have.numfalse == 0
]:
d.stream.close()
return self.downloader.storage.do_I_have(index)
def loop(self):
try:
while not self.doneflag.isSet():
self._pop_externally_added()
period = 1e9
if len(self.tasks) > 0:
# Poll until the next task is set to execute
period = max(0, self.tasks[0][0] - bttime())
asyncore.poll(period)
self.do_tasks()
except KeyboardInterrupt:
#TODO: cleanup?
pass
except:
log.critical('\n'+traceback.format_exc())
def loop(self):
try:
while not self.doneflag.isSet():
self._pop_externally_added()
period = 1e9
if len(self.tasks) > 0:
# Poll until the next task is set to execute
period = max(0, self.tasks[0][0] - bttime())
asyncore.poll(period)
self.do_tasks()
except KeyboardInterrupt:
#TODO: cleanup?
pass
except:
log.critical('\n' + traceback.format_exc())
def _postrequest(self, data=None, errormsg=None):
self.current_started = None
self.last_time = bttime()
if errormsg is not None:
log.warning(errormsg)
self._fail()
return
try:
# Here's where we receive/decrypt data from the tracker
r = bdecode(data)
check_peers(r)
except BTFailure, e:
if data != '':
log.error('bad data from tracker - ' + str(e))
self._fail()
return
def got_piece(self, index, begin, piece):
try:
self.active_requests.remove((index, begin, len(piece)))
except ValueError:
self.downloader.discarded_bytes += len(piece)
return False
if self.downloader.storage.endgame:
self.downloader.all_requests.remove((index, begin, len(piece)))
self.last = bttime()
self.measure.update_rate(len(piece))
self.downloader.measurefunc(len(piece))
self.downloader.downmeasure.update_rate(len(piece))
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
if self.downloader.storage.endgame:
while self.downloader.storage.do_I_have_requests(index):
nb, nl = self.downloader.storage.new_request(index)
self.downloader.all_requests.append((index, nb, nl))
for d in self.downloader.downloads:
d.fix_download_endgame()
return False
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more([index])
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.storage.endgame:
for d in self.downloader.downloads:
if d is not self and d.interested:
if d.choked:
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin, len(piece)))
except ValueError:
continue
d.stream.send_cancel(index, begin, len(piece))
d.fix_download_endgame()
self._request_more()
if self.downloader.picker.am_I_complete():
for d in [i for i in self.downloader.downloads if i.have.numfalse == 0]:
d.stream.close()
return self.downloader.storage.do_I_have(index)
def _announce(self, event=None):
self.current_started = bttime()
query = ('%s&uploaded=%d&downloaded=%d&relayed=%d&left=%d' %
(self.basequery, self.up(), self.down(), self.neighbors.relayed(),
self.amount_left()))
if self.neighbors.count() >= self.config['max_initiate']:
query += '&numwant=0'
else:
query += '&compact=1'
if event is not None:
query += '&event=' + ['started', 'completed', 'stopped'][event]
if event == STARTED:
query += '&sessionid='+b64encode(self.sessionid)
if self.config['ip']:
query += '&ip=' + gethostbyname(self.config['ip'])
self.failed_peers = self.neighbors.failed_connections()
if self.failed_peers:
query += '&failed=' + b64encode(''.join(self.failed_peers))
Thread(target=self._rerequest, args=[query]).start()
def make_meta_file(path, url, piece_len_exp, flag=Event(), progress=dummy,
comment=None, target=None, encoding='ascii'):
piece_length = 2 ** piece_len_exp
a, b = os.path.split(path)
if not target:
if b == '':
f = a + '.atorrent'
else:
f = os.path.join(a, b + '.atorrent')
else:
f = target
info = makeinfo(path, piece_length, flag, progress, encoding)
if flag.isSet():
return
check_info(info)
h = file(f, 'wb')
data = {'info': info, 'announce': url.strip(), 'creation date': int(bttime()), 'anon': '1'}
if comment:
data['comment'] = comment
h.write(bencode(data))
h.close()
def __init__(self, name, pubkey, ip, port, sid):
"""
@param name: Peer ID to be assigned to this SimPeer
@type name: string
@param pubkey: RSA Public Key to use when encrypting to this peer
@type pubkey: Anomos.Crypto.RSAPubKey
"""
self.name = name
self.ip = ip
self.port = port
self.pubkey = Anomos.Crypto.PeerCert(pubkey)
self.neighbors = {} # {PeerID: {nid:#, ip:"", port:#}}
self.id_map = {} # {NeighborID : PeerID}
self.infohashes = {} # {infohash: (uploaded, downloaded, left)}
self.relayed_total = 0
self.last_seen = 0 # Time of last client announce
self.last_modified = bttime() # Time when client was last modified
self.failed_nbrs = []
self.nbrs_needed = 0
self.sessionid = sid
self.num_natcheck = 0
self.nat = True # assume NAT
def __init__(self, max_rate_period, fudge=5):
self.max_rate_period = max_rate_period
self.ratesince = bttime() - fudge
self.last = self.ratesince
self.rate = 0.0
self.total = 0
def __init__(self, max_rate_period, fudge=5):
self.max_rate_period = max_rate_period
self.ratesince = bttime() - fudge
self.last = self.ratesince
self.rate = 0.0
self.total = 0
def stats(self):
self.multitorrent.schedule(self.config['display_interval'], self.stats)
data = []
for infohash in self.torrent_list:
cache = self.torrent_cache[infohash]
if self.config['display_path']:
name = cache['path']
else:
name = cache['name']
size = cache['length']
d = self.downloads[infohash]
progress = '0.0%'
peers = 0
seeds = 0
seedsmsg = "S"
dist = 0.0
uprate = 0.0
dnrate = 0.0
upamt = 0
dnamt = 0
t = 0
msg = ''
if d is None:
status = 'waiting for hash check'
else:
stats = d.get_status()
status = stats['activity']
progress = '%.1f%%' % (int(stats['fractionDone'] * 1000) /
10.0)
if d.started and not d.closed:
s = stats
dist = s['numCopies']
if d.is_seed:
seeds = 0 # s['numOldSeeds']
seedsmsg = "s"
else:
if s['numSeeds'] + s['numPeers']:
t = stats['timeEst']
if t is None:
t = -1
if t == 0: # unlikely
t = 0.01
status = 'downloading'
else:
t = -1
status = 'connecting to peers'
seeds = s['numSeeds']
dnrate = stats['downRate']
peers = s['numPeers']
uprate = stats['upRate']
upamt = s['upTotal']
dnamt = s['downTotal']
if d.messages and (d.closed
or d.messages[-1][0] + 300 > bttime()):
msg = d.messages[-1][2]
data.append((name, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, size, t, msg))
stop = self.output.display(data)
if stop:
self.doneflag.set()
def is_snubbed(self):
return bttime() - self.last > self.downloader.snub_time
def is_snubbed(self):
return bttime() - self.last > self.downloader.snub_time
