def __init__(self, downloader, stream):
self.downloader = downloader
self.stream = stream
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.config['max_rate_period'])
self.peermeasure = Measure(max(downloader.storage.piece_size / 10000,
20))
self.have = Bitfield(downloader.numpieces)
self.last = 0
self.example_interest = None
self.backlog = 2
self.guard = BadDataGuard(self)
def __init__(self, downloader, stream):
self.downloader = downloader
self.stream = stream
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.config['max_rate_period'])
self.peermeasure = Measure(
max(downloader.storage.piece_size / 10000, 20))
self.have = Bitfield(downloader.numpieces)
self.last = 0
self.example_interest = None
self.backlog = 2
self.guard = BadDataGuard(self)
def __init__(self, config, certificate, ssl_ctx, sessionid, schedule,
ratelimiter):
self.config = config
self.certificate = certificate
self.ssl_ctx = ssl_ctx
self.sessionid = sessionid
self.schedule = schedule
self.ratelimiter = ratelimiter
self.neighbors = {}
self.relay_measure = Measure(self.config['max_rate_period'])
self.relay_count = 0
self.incomplete = {}
self.torrents = {}
self.waiting_tcs = {}
self.failedPeers = []
def __init__(self, stream, ratelimiter, totalup, choker,
storage, max_slice_length, max_rate_period):
self.stream = stream
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.max_slice_length = max_slice_length
self.max_rate_period = max_rate_period
self.choked = True
self.unchoke_time = None
self.interested = False
self.buffer = []
self.measure = Measure(max_rate_period)
if storage.do_I_have_anything():
stream.send_bitfield(storage.get_have_list())
class SingleDownload(object):
def __init__(self, downloader, stream):
self.downloader = downloader
self.stream = stream
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.config['max_rate_period'])
self.peermeasure = Measure(max(downloader.storage.piece_size / 10000,
20))
self.have = Bitfield(downloader.numpieces)
self.last = 0
self.example_interest = None
self.backlog = 2
self.guard = BadDataGuard(self)
def _backlog(self):
backlog = 2 + int(4 * self.measure.get_rate() /
self.downloader.chunksize)
if backlog > 50:
backlog = max(50, int(.075 * backlog))
self.backlog = backlog
return backlog
def disconnected(self):
self.downloader.lost_peer(self)
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.lost_have(i)
self._letgo()
self.guard.download = None
def _letgo(self):
if not self.active_requests:
return
if self.downloader.storage.endgame:
self.active_requests = []
return
lost = []
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
if index not in lost:
lost.append(index)
self.active_requests = []
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more(lost)
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(l):
d.interested = True
d.stream.send_interested()
break
def got_choke(self):
if not self.choked:
self.choked = True
self._letgo()
def got_unchoke(self):
if self.choked:
self.choked = False
if self.interested:
self._request_more()
def got_piece(self, index, begin, piece):
try:
self.active_requests.remove((index, begin, len(piece)))
except ValueError:
self.downloader.discarded_bytes += len(piece)
return False
if self.downloader.storage.endgame:
self.downloader.all_requests.remove((index, begin, len(piece)))
self.last = bttime()
self.measure.update_rate(len(piece))
self.downloader.measurefunc(len(piece))
self.downloader.downmeasure.update_rate(len(piece))
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
if self.downloader.storage.endgame:
while self.downloader.storage.do_I_have_requests(index):
nb, nl = self.downloader.storage.new_request(index)
self.downloader.all_requests.append((index, nb, nl))
for d in self.downloader.downloads:
d.fix_download_endgame()
return False
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more([index])
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.storage.endgame:
for d in self.downloader.downloads:
if d is not self and d.interested:
if d.choked:
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin, len(piece)))
except ValueError:
continue
d.stream.send_cancel(index, begin, len(piece))
d.fix_download_endgame()
self._request_more()
if self.downloader.picker.am_I_complete():
for d in [i for i in self.downloader.downloads if i.have.numfalse == 0]:
d.stream.close()
return self.downloader.storage.do_I_have(index)
def _want(self, index):
return self.have[index] and self.downloader.storage.do_I_have_requests(index)
def _request_more(self, indices = None):
assert not self.choked
if len(self.active_requests) >= self._backlog():
return
if self.downloader.storage.endgame:
self.fix_download_endgame()
return
lost_interests = []
while len(self.active_requests) < self.backlog:
if indices is None:
interest = self.downloader.picker.next(self._want, self.have.numfalse == 0)
else:
interest = None
for i in indices:
if self.have[i] and self.downloader.storage.do_I_have_requests(i):
interest = i
break
if interest is None:
break
if not self.interested:
self.interested = True
self.stream.send_interested()
self.example_interest = interest
self.downloader.picker.requested(interest, self.have.numfalse == 0)
while len(self.active_requests) < (self.backlog-2) * 5 + 2:
begin, length = self.downloader.storage.new_request(interest)
self.active_requests.append((interest, begin, length))
self.stream.send_request(interest, begin, length)
if not self.downloader.storage.do_I_have_requests(interest):
lost_interests.append(interest)
break
if not self.active_requests and self.interested:
self.interested = False
self.stream.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(d._want, d.have.numfalse == 0)
if interest is None:
d.interested = False
d.stream.send_not_interested()
else:
d.example_interest = interest
if self.downloader.storage.endgame:
self.downloader.all_requests = []
for d in self.downloader.downloads:
self.downloader.all_requests.extend(d.active_requests)
for d in self.downloader.downloads:
d.fix_download_endgame()
def fix_download_endgame(self):
want = [a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests]
if self.interested and not self.active_requests and not want:
self.interested = False
self.stream.send_not_interested()
return
if not self.interested and want:
self.interested = True
self.stream.send_interested()
if self.choked or len(self.active_requests) >= self._backlog():
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
self.stream.send_request(piece, begin, length)
def got_have(self, index):
if self.have[index]:
return
if index == self.downloader.numpieces-1:
self.peermeasure.update_rate(self.downloader.storage.total_length-
(self.downloader.numpieces-1)*self.downloader.storage.piece_size)
else:
self.peermeasure.update_rate(self.downloader.storage.piece_size)
self.have[index] = True
self.downloader.picker.got_have(index)
if self.downloader.picker.am_I_complete() and self.have.numfalse == 0:
self.stream.close()
return
if self.downloader.storage.endgame:
self.fix_download_endgame()
elif self.downloader.storage.do_I_have_requests(index):
if not self.choked:
self._request_more([index])
else:
if not self.interested:
self.interested = True
self.stream.send_interested()
def got_have_bitfield(self, have):
if self.downloader.picker.am_I_complete() and have.numfalse == 0:
self.stream.close()
return
self.have = have
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.got_have(i)
if self.downloader.storage.endgame:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.interested = True
self.stream.send_interested()
return
for i in xrange(len(self.have)):
if self.have[i] and self.downloader.storage.do_I_have_requests(i):
self.interested = True
self.stream.send_interested()
return
def get_rate(self):
return self.measure.get_rate()
def is_snubbed(self):
return bttime() - self.last > self.downloader.snub_time
class SingleDownload(object):
def __init__(self, downloader, stream):
self.downloader = downloader
self.stream = stream
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.config['max_rate_period'])
self.peermeasure = Measure(
max(downloader.storage.piece_size / 10000, 20))
self.have = Bitfield(downloader.numpieces)
self.last = 0
self.example_interest = None
self.backlog = 2
self.guard = BadDataGuard(self)
def _backlog(self):
backlog = 2 + int(
4 * self.measure.get_rate() / self.downloader.chunksize)
if backlog > 50:
backlog = max(50, int(.075 * backlog))
self.backlog = backlog
return backlog
def disconnected(self):
self.downloader.lost_peer(self)
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.lost_have(i)
self._letgo()
self.guard.download = None
def _letgo(self):
if not self.active_requests:
return
if self.downloader.storage.endgame:
self.active_requests = []
return
lost = []
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
if index not in lost:
lost.append(index)
self.active_requests = []
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more(lost)
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(
l):
d.interested = True
d.stream.send_interested()
break
def got_choke(self):
if not self.choked:
self.choked = True
self._letgo()
def got_unchoke(self):
if self.choked:
self.choked = False
if self.interested:
self._request_more()
def got_piece(self, index, begin, piece):
try:
self.active_requests.remove((index, begin, len(piece)))
except ValueError:
self.downloader.discarded_bytes += len(piece)
return False
if self.downloader.storage.endgame:
self.downloader.all_requests.remove((index, begin, len(piece)))
self.last = bttime()
self.measure.update_rate(len(piece))
self.downloader.measurefunc(len(piece))
self.downloader.downmeasure.update_rate(len(piece))
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
if self.downloader.storage.endgame:
while self.downloader.storage.do_I_have_requests(index):
nb, nl = self.downloader.storage.new_request(index)
self.downloader.all_requests.append((index, nb, nl))
for d in self.downloader.downloads:
d.fix_download_endgame()
return False
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more([index])
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.storage.endgame:
for d in self.downloader.downloads:
if d is not self and d.interested:
if d.choked:
d.fix_download_endgame()
else:
try:
d.active_requests.remove(
(index, begin, len(piece)))
except ValueError:
continue
d.stream.send_cancel(index, begin, len(piece))
d.fix_download_endgame()
self._request_more()
if self.downloader.picker.am_I_complete():
for d in [
i for i in self.downloader.downloads
if i.have.numfalse == 0
]:
d.stream.close()
return self.downloader.storage.do_I_have(index)
def _want(self, index):
return self.have[index] and self.downloader.storage.do_I_have_requests(
index)
def _request_more(self, indices=None):
assert not self.choked
if len(self.active_requests) >= self._backlog():
return
if self.downloader.storage.endgame:
self.fix_download_endgame()
return
lost_interests = []
while len(self.active_requests) < self.backlog:
if indices is None:
interest = self.downloader.picker.next(self._want,
self.have.numfalse == 0)
else:
interest = None
for i in indices:
if self.have[
i] and self.downloader.storage.do_I_have_requests(
i):
interest = i
break
if interest is None:
break
if not self.interested:
self.interested = True
self.stream.send_interested()
self.example_interest = interest
self.downloader.picker.requested(interest, self.have.numfalse == 0)
while len(self.active_requests) < (self.backlog - 2) * 5 + 2:
begin, length = self.downloader.storage.new_request(interest)
self.active_requests.append((interest, begin, length))
self.stream.send_request(interest, begin, length)
if not self.downloader.storage.do_I_have_requests(interest):
lost_interests.append(interest)
break
if not self.active_requests and self.interested:
self.interested = False
self.stream.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(
d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(d._want,
d.have.numfalse == 0)
if interest is None:
d.interested = False
d.stream.send_not_interested()
else:
d.example_interest = interest
if self.downloader.storage.endgame:
self.downloader.all_requests = []
for d in self.downloader.downloads:
self.downloader.all_requests.extend(d.active_requests)
for d in self.downloader.downloads:
d.fix_download_endgame()
def fix_download_endgame(self):
want = [
a for a in self.downloader.all_requests
if self.have[a[0]] and a not in self.active_requests
]
if self.interested and not self.active_requests and not want:
self.interested = False
self.stream.send_not_interested()
return
if not self.interested and want:
self.interested = True
self.stream.send_interested()
if self.choked or len(self.active_requests) >= self._backlog():
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
self.stream.send_request(piece, begin, length)
def got_have(self, index):
if self.have[index]:
return
if index == self.downloader.numpieces - 1:
self.peermeasure.update_rate(self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_size)
else:
self.peermeasure.update_rate(self.downloader.storage.piece_size)
self.have[index] = True
self.downloader.picker.got_have(index)
if self.downloader.picker.am_I_complete() and self.have.numfalse == 0:
self.stream.close()
return
if self.downloader.storage.endgame:
self.fix_download_endgame()
elif self.downloader.storage.do_I_have_requests(index):
if not self.choked:
self._request_more([index])
else:
if not self.interested:
self.interested = True
self.stream.send_interested()
def got_have_bitfield(self, have):
if self.downloader.picker.am_I_complete() and have.numfalse == 0:
self.stream.close()
return
self.have = have
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.got_have(i)
if self.downloader.storage.endgame:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.interested = True
self.stream.send_interested()
return
for i in xrange(len(self.have)):
if self.have[i] and self.downloader.storage.do_I_have_requests(i):
self.interested = True
self.stream.send_interested()
return
def get_rate(self):
return self.measure.get_rate()
def is_snubbed(self):
return bttime() - self.last > self.downloader.snub_time
class Upload(object):
def __init__(self, stream, ratelimiter, totalup, choker,
storage, max_slice_length, max_rate_period):
self.stream = stream
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.max_slice_length = max_slice_length
self.max_rate_period = max_rate_period
self.choked = True
self.unchoke_time = None
self.interested = False
self.buffer = []
self.measure = Measure(max_rate_period)
if storage.do_I_have_anything():
stream.send_bitfield(storage.get_have_list())
def got_not_interested(self):
if self.interested:
self.interested = False
del self.buffer[:]
self.choker.not_interested(self.stream)
def got_interested(self):
if not self.interested:
self.interested = True
self.choker.interested(self.stream)
def get_upload_chunk(self):
if not self.buffer:
return None
index, begin, length = self.buffer.pop(0)
piece = self.storage.get_piece(index, begin, length)
if piece is None:
self.stream.close()
return None
self.measure.update_rate(len(piece))
self.totalup.update_rate(len(piece))
return (index, begin, piece)
def got_request(self, index, begin, length):
if not self.interested or length > self.max_slice_length:
self.stream.close()
return
if not self.stream.choke_sent:
self.buffer.append((index, begin, length))
if self.stream.next_upload is None and \
self.stream.is_flushed():
self.ratelimiter.queue(self.stream)
def got_cancel(self, index, begin, length):
try:
self.buffer.remove((index, begin, length))
except ValueError:
pass
def choke(self):
if not self.choked:
self.choked = True
self.stream.send_choke()
def sent_choke(self):
assert self.choked
del self.buffer[:]
def unchoke(self, time):
if self.choked:
self.choked = False
self.unchoke_time = time
self.stream.send_unchoke()
def has_queries(self):
return len(self.buffer) > 0
def get_rate(self):
return self.measure.get_rate()
class _SingleTorrent(object):
def __init__(self, event_handler, singleport_listener, ratelimiter, filepool,
config, neighbors, certificate, sessionid):
self.event_handler = event_handler
self._singleport_listener = singleport_listener
self._ratelimiter = ratelimiter
self._filepool = filepool
self.config = dict(config)
self._storage = None
self._storagewrapper = None
self._ratemeasure = None
self._upmeasure = None
self._downmeasure = None
self._torrent = None
self._rerequest = None
self._statuscollecter = None
self._announced = False
self._listening = False
self.reserved_ports = []
self.reported_port = None
self._myfiles = None
self.started = False
self.is_seed = False
self.closed = False
self.infohash = None
self.file_size = None
self._doneflag = threading.Event()
self.finflag = threading.Event()
self._hashcheck_thread = None
self._contfunc = None
self._activity = ('Initial startup', 0)
self.feedback = None
self.messages = []
self.neighbors = neighbors
self.certificate = certificate
self.sessionid = sessionid
def schedule(self, delay, func):
self.event_handler.schedule(delay, func, context=self)
def start_download(self, *args, **kwargs):
it = self._start_download(*args, **kwargs)
def cont():
try:
it.next()
except StopIteration:
self._contfunc = None
def contfunc():
self.schedule(0, cont)
self._contfunc = contfunc
contfunc()
def _start_download(self, metainfo, feedback, save_path):
# GTK Crash Hack
import time
time.sleep(.2)
self.feedback = feedback
self._set_auto_uploads()
self.infohash = metainfo.infohash
self.file_size = metainfo.file_size
if not metainfo.reported_errors:
metainfo.show_encoding_errors(log.error)
if metainfo.is_batch:
myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs]
else:
myfiles = [save_path]
self._filepool.add_files(myfiles, self)
self._myfiles = myfiles
self._storage = Storage(self.config, self._filepool, zip(myfiles,
metainfo.sizes))
resumefile = None
if self.config['data_dir']:
filename = os.path.join(self.config['data_dir'], 'resume',
self.infohash.encode('hex'))
if os.path.exists(filename):
try:
resumefile = file(filename, 'rb')
if self._storage.check_fastresume(resumefile) == 0:
resumefile.close()
resumefile = None
except Exception, e:
log.info("Could not load fastresume data: "+
str(e) + ". Will perform full hash check.")
if resumefile is not None:
resumefile.close()
resumefile = None
def data_flunked(amount, index):
self._ratemeasure.data_rejected(amount)
log.info('piece %d failed hash check, '
're-downloading it' % index)
backthread_exception = None
def hashcheck():
def statusfunc(activity = None, fractionDone = 0):
if activity is None:
activity = self._activity[0]
self._activity = (activity, fractionDone)
try:
self._storagewrapper = StorageWrapper(self._storage,
self.config, metainfo.hashes, metainfo.piece_length,
self._finished, statusfunc, self._doneflag, data_flunked,
self.infohash, resumefile)
except:
backthread_exception = sys.exc_info()
self._contfunc()
thread = threading.Thread(target = hashcheck)
thread.setDaemon(False)
self._hashcheck_thread = thread
thread.start()
yield None
self._hashcheck_thread = None
if resumefile is not None:
resumefile.close()
if backthread_exception:
a, b, c = backthread_exception
raise a, b, c
if self._storagewrapper.amount_left == 0:
self._finished()
choker = Choker(self.config, self.schedule, self.finflag.isSet)
upmeasure = Measure(self.config['max_rate_period'])
downmeasure = Measure(self.config['max_rate_period'])
self._upmeasure = upmeasure
self._downmeasure = downmeasure
self._ratemeasure = RateMeasure(self._storagewrapper.
amount_left_with_partials)
picker = PiecePicker(len(metainfo.hashes), self.config)
for i in xrange(len(metainfo.hashes)):
if self._storagewrapper.do_I_have(i):
picker.complete(i)
for i in self._storagewrapper.stat_dirty:
picker.requested(i)
def kickpeer(connection):
def kick():
connection.close()
self.schedule(0, kick)
downloader = Downloader(self.config, self._storagewrapper, picker,
len(metainfo.hashes), downmeasure,
self._ratemeasure.data_came_in, kickpeer)
def make_upload(connection):
return Upload(connection, self._ratelimiter, upmeasure, choker,
self._storagewrapper, self.config['max_slice_length'],
self.config['max_rate_period'])
self._torrent = Torrent(self.infohash, make_upload,
downloader, len(metainfo.hashes), self)
self.reported_port = self.config['forwarded_port']
if not self.reported_port:
self.reported_port = self._singleport_listener.get_port(self.neighbors)
self.reserved_ports.append(self.reported_port)
self.neighbors.add_torrent(self.infohash, self._torrent)
self._listening = True
self._rerequest = Rerequester(metainfo.announce, self.config,
self.schedule, self.neighbors, self._storagewrapper.get_amount_left,
upmeasure.get_total, downmeasure.get_total, self.reported_port,
self.infohash, self.finflag, self.internal_shutdown,
self._announce_done, self.certificate, self.sessionid)
self._statuscollecter = DownloaderFeedback(choker, upmeasure.get_rate,
downmeasure.get_rate, upmeasure.get_total, downmeasure.get_total,
self.neighbors.get_relay_stats, self._ratemeasure.get_time_left,
self._ratemeasure.get_size_left, self.file_size, self.finflag,
downloader, self._myfiles)
self._announced = True
self._rerequest.begin()
self.started = True
if not self.finflag.isSet():
self._activity = ('downloading', 0)
self.feedback.started(self)
