def __init__(self, downloader, stream): self.downloader = downloader self.stream = stream self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.config['max_rate_period']) self.peermeasure = Measure(max(downloader.storage.piece_size / 10000, 20)) self.have = Bitfield(downloader.numpieces) self.last = 0 self.example_interest = None self.backlog = 2 self.guard = BadDataGuard(self)
def __init__(self, downloader, stream): self.downloader = downloader self.stream = stream self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.config['max_rate_period']) self.peermeasure = Measure( max(downloader.storage.piece_size / 10000, 20)) self.have = Bitfield(downloader.numpieces) self.last = 0 self.example_interest = None self.backlog = 2 self.guard = BadDataGuard(self)
def __init__(self, config, certificate, ssl_ctx, sessionid, schedule, ratelimiter): self.config = config self.certificate = certificate self.ssl_ctx = ssl_ctx self.sessionid = sessionid self.schedule = schedule self.ratelimiter = ratelimiter self.neighbors = {} self.relay_measure = Measure(self.config['max_rate_period']) self.relay_count = 0 self.incomplete = {} self.torrents = {} self.waiting_tcs = {} self.failedPeers = []
def __init__(self, stream, ratelimiter, totalup, choker, storage, max_slice_length, max_rate_period): self.stream = stream self.ratelimiter = ratelimiter self.totalup = totalup self.choker = choker self.storage = storage self.max_slice_length = max_slice_length self.max_rate_period = max_rate_period self.choked = True self.unchoke_time = None self.interested = False self.buffer = [] self.measure = Measure(max_rate_period) if storage.do_I_have_anything(): stream.send_bitfield(storage.get_have_list())
class SingleDownload(object): def __init__(self, downloader, stream): self.downloader = downloader self.stream = stream self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.config['max_rate_period']) self.peermeasure = Measure(max(downloader.storage.piece_size / 10000, 20)) self.have = Bitfield(downloader.numpieces) self.last = 0 self.example_interest = None self.backlog = 2 self.guard = BadDataGuard(self) def _backlog(self): backlog = 2 + int(4 * self.measure.get_rate() / self.downloader.chunksize) if backlog > 50: backlog = max(50, int(.075 * backlog)) self.backlog = backlog return backlog def disconnected(self): self.downloader.lost_peer(self) for i in xrange(len(self.have)): if self.have[i]: self.downloader.picker.lost_have(i) self._letgo() self.guard.download = None def _letgo(self): if not self.active_requests: return if self.downloader.storage.endgame: self.active_requests = [] return lost = [] for index, begin, length in self.active_requests: self.downloader.storage.request_lost(index, begin, length) if index not in lost: lost.append(index) self.active_requests = [] ds = [d for d in self.downloader.downloads if not d.choked] shuffle(ds) for d in ds: d._request_more(lost) for d in self.downloader.downloads: if d.choked and not d.interested: for l in lost: if d.have[l] and self.downloader.storage.do_I_have_requests(l): d.interested = True d.stream.send_interested() break def got_choke(self): if not self.choked: self.choked = True self._letgo() def got_unchoke(self): if self.choked: self.choked = False if self.interested: self._request_more() def got_piece(self, index, begin, piece): try: self.active_requests.remove((index, begin, len(piece))) except ValueError: self.downloader.discarded_bytes += len(piece) return False if self.downloader.storage.endgame: self.downloader.all_requests.remove((index, begin, len(piece))) self.last = bttime() self.measure.update_rate(len(piece)) self.downloader.measurefunc(len(piece)) self.downloader.downmeasure.update_rate(len(piece)) if not self.downloader.storage.piece_came_in(index, begin, piece, self.guard): if self.downloader.storage.endgame: while self.downloader.storage.do_I_have_requests(index): nb, nl = self.downloader.storage.new_request(index) self.downloader.all_requests.append((index, nb, nl)) for d in self.downloader.downloads: d.fix_download_endgame() return False ds = [d for d in self.downloader.downloads if not d.choked] shuffle(ds) for d in ds: d._request_more([index]) return False if self.downloader.storage.do_I_have(index): self.downloader.picker.complete(index) if self.downloader.storage.endgame: for d in self.downloader.downloads: if d is not self and d.interested: if d.choked: d.fix_download_endgame() else: try: d.active_requests.remove((index, begin, len(piece))) except ValueError: continue d.stream.send_cancel(index, begin, len(piece)) d.fix_download_endgame() self._request_more() if self.downloader.picker.am_I_complete(): for d in [i for i in self.downloader.downloads if i.have.numfalse == 0]: d.stream.close() return self.downloader.storage.do_I_have(index) def _want(self, index): return self.have[index] and self.downloader.storage.do_I_have_requests(index) def _request_more(self, indices = None): assert not self.choked if len(self.active_requests) >= self._backlog(): return if self.downloader.storage.endgame: self.fix_download_endgame() return lost_interests = [] while len(self.active_requests) < self.backlog: if indices is None: interest = self.downloader.picker.next(self._want, self.have.numfalse == 0) else: interest = None for i in indices: if self.have[i] and self.downloader.storage.do_I_have_requests(i): interest = i break if interest is None: break if not self.interested: self.interested = True self.stream.send_interested() self.example_interest = interest self.downloader.picker.requested(interest, self.have.numfalse == 0) while len(self.active_requests) < (self.backlog-2) * 5 + 2: begin, length = self.downloader.storage.new_request(interest) self.active_requests.append((interest, begin, length)) self.stream.send_request(interest, begin, length) if not self.downloader.storage.do_I_have_requests(interest): lost_interests.append(interest) break if not self.active_requests and self.interested: self.interested = False self.stream.send_not_interested() if lost_interests: for d in self.downloader.downloads: if d.active_requests or not d.interested: continue if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest): continue for lost in lost_interests: if d.have[lost]: break else: continue interest = self.downloader.picker.next(d._want, d.have.numfalse == 0) if interest is None: d.interested = False d.stream.send_not_interested() else: d.example_interest = interest if self.downloader.storage.endgame: self.downloader.all_requests = [] for d in self.downloader.downloads: self.downloader.all_requests.extend(d.active_requests) for d in self.downloader.downloads: d.fix_download_endgame() def fix_download_endgame(self): want = [a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests] if self.interested and not self.active_requests and not want: self.interested = False self.stream.send_not_interested() return if not self.interested and want: self.interested = True self.stream.send_interested() if self.choked or len(self.active_requests) >= self._backlog(): return shuffle(want) del want[self.backlog - len(self.active_requests):] self.active_requests.extend(want) for piece, begin, length in want: self.stream.send_request(piece, begin, length) def got_have(self, index): if self.have[index]: return if index == self.downloader.numpieces-1: self.peermeasure.update_rate(self.downloader.storage.total_length- (self.downloader.numpieces-1)*self.downloader.storage.piece_size) else: self.peermeasure.update_rate(self.downloader.storage.piece_size) self.have[index] = True self.downloader.picker.got_have(index) if self.downloader.picker.am_I_complete() and self.have.numfalse == 0: self.stream.close() return if self.downloader.storage.endgame: self.fix_download_endgame() elif self.downloader.storage.do_I_have_requests(index): if not self.choked: self._request_more([index]) else: if not self.interested: self.interested = True self.stream.send_interested() def got_have_bitfield(self, have): if self.downloader.picker.am_I_complete() and have.numfalse == 0: self.stream.close() return self.have = have for i in xrange(len(self.have)): if self.have[i]: self.downloader.picker.got_have(i) if self.downloader.storage.endgame: for piece, begin, length in self.downloader.all_requests: if self.have[piece]: self.interested = True self.stream.send_interested() return for i in xrange(len(self.have)): if self.have[i] and self.downloader.storage.do_I_have_requests(i): self.interested = True self.stream.send_interested() return def get_rate(self): return self.measure.get_rate() def is_snubbed(self): return bttime() - self.last > self.downloader.snub_time
class SingleDownload(object): def __init__(self, downloader, stream): self.downloader = downloader self.stream = stream self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.config['max_rate_period']) self.peermeasure = Measure( max(downloader.storage.piece_size / 10000, 20)) self.have = Bitfield(downloader.numpieces) self.last = 0 self.example_interest = None self.backlog = 2 self.guard = BadDataGuard(self) def _backlog(self): backlog = 2 + int( 4 * self.measure.get_rate() / self.downloader.chunksize) if backlog > 50: backlog = max(50, int(.075 * backlog)) self.backlog = backlog return backlog def disconnected(self): self.downloader.lost_peer(self) for i in xrange(len(self.have)): if self.have[i]: self.downloader.picker.lost_have(i) self._letgo() self.guard.download = None def _letgo(self): if not self.active_requests: return if self.downloader.storage.endgame: self.active_requests = [] return lost = [] for index, begin, length in self.active_requests: self.downloader.storage.request_lost(index, begin, length) if index not in lost: lost.append(index) self.active_requests = [] ds = [d for d in self.downloader.downloads if not d.choked] shuffle(ds) for d in ds: d._request_more(lost) for d in self.downloader.downloads: if d.choked and not d.interested: for l in lost: if d.have[l] and self.downloader.storage.do_I_have_requests( l): d.interested = True d.stream.send_interested() break def got_choke(self): if not self.choked: self.choked = True self._letgo() def got_unchoke(self): if self.choked: self.choked = False if self.interested: self._request_more() def got_piece(self, index, begin, piece): try: self.active_requests.remove((index, begin, len(piece))) except ValueError: self.downloader.discarded_bytes += len(piece) return False if self.downloader.storage.endgame: self.downloader.all_requests.remove((index, begin, len(piece))) self.last = bttime() self.measure.update_rate(len(piece)) self.downloader.measurefunc(len(piece)) self.downloader.downmeasure.update_rate(len(piece)) if not self.downloader.storage.piece_came_in(index, begin, piece, self.guard): if self.downloader.storage.endgame: while self.downloader.storage.do_I_have_requests(index): nb, nl = self.downloader.storage.new_request(index) self.downloader.all_requests.append((index, nb, nl)) for d in self.downloader.downloads: d.fix_download_endgame() return False ds = [d for d in self.downloader.downloads if not d.choked] shuffle(ds) for d in ds: d._request_more([index]) return False if self.downloader.storage.do_I_have(index): self.downloader.picker.complete(index) if self.downloader.storage.endgame: for d in self.downloader.downloads: if d is not self and d.interested: if d.choked: d.fix_download_endgame() else: try: d.active_requests.remove( (index, begin, len(piece))) except ValueError: continue d.stream.send_cancel(index, begin, len(piece)) d.fix_download_endgame() self._request_more() if self.downloader.picker.am_I_complete(): for d in [ i for i in self.downloader.downloads if i.have.numfalse == 0 ]: d.stream.close() return self.downloader.storage.do_I_have(index) def _want(self, index): return self.have[index] and self.downloader.storage.do_I_have_requests( index) def _request_more(self, indices=None): assert not self.choked if len(self.active_requests) >= self._backlog(): return if self.downloader.storage.endgame: self.fix_download_endgame() return lost_interests = [] while len(self.active_requests) < self.backlog: if indices is None: interest = self.downloader.picker.next(self._want, self.have.numfalse == 0) else: interest = None for i in indices: if self.have[ i] and self.downloader.storage.do_I_have_requests( i): interest = i break if interest is None: break if not self.interested: self.interested = True self.stream.send_interested() self.example_interest = interest self.downloader.picker.requested(interest, self.have.numfalse == 0) while len(self.active_requests) < (self.backlog - 2) * 5 + 2: begin, length = self.downloader.storage.new_request(interest) self.active_requests.append((interest, begin, length)) self.stream.send_request(interest, begin, length) if not self.downloader.storage.do_I_have_requests(interest): lost_interests.append(interest) break if not self.active_requests and self.interested: self.interested = False self.stream.send_not_interested() if lost_interests: for d in self.downloader.downloads: if d.active_requests or not d.interested: continue if d.example_interest is not None and self.downloader.storage.do_I_have_requests( d.example_interest): continue for lost in lost_interests: if d.have[lost]: break else: continue interest = self.downloader.picker.next(d._want, d.have.numfalse == 0) if interest is None: d.interested = False d.stream.send_not_interested() else: d.example_interest = interest if self.downloader.storage.endgame: self.downloader.all_requests = [] for d in self.downloader.downloads: self.downloader.all_requests.extend(d.active_requests) for d in self.downloader.downloads: d.fix_download_endgame() def fix_download_endgame(self): want = [ a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests ] if self.interested and not self.active_requests and not want: self.interested = False self.stream.send_not_interested() return if not self.interested and want: self.interested = True self.stream.send_interested() if self.choked or len(self.active_requests) >= self._backlog(): return shuffle(want) del want[self.backlog - len(self.active_requests):] self.active_requests.extend(want) for piece, begin, length in want: self.stream.send_request(piece, begin, length) def got_have(self, index): if self.have[index]: return if index == self.downloader.numpieces - 1: self.peermeasure.update_rate(self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_size) else: self.peermeasure.update_rate(self.downloader.storage.piece_size) self.have[index] = True self.downloader.picker.got_have(index) if self.downloader.picker.am_I_complete() and self.have.numfalse == 0: self.stream.close() return if self.downloader.storage.endgame: self.fix_download_endgame() elif self.downloader.storage.do_I_have_requests(index): if not self.choked: self._request_more([index]) else: if not self.interested: self.interested = True self.stream.send_interested() def got_have_bitfield(self, have): if self.downloader.picker.am_I_complete() and have.numfalse == 0: self.stream.close() return self.have = have for i in xrange(len(self.have)): if self.have[i]: self.downloader.picker.got_have(i) if self.downloader.storage.endgame: for piece, begin, length in self.downloader.all_requests: if self.have[piece]: self.interested = True self.stream.send_interested() return for i in xrange(len(self.have)): if self.have[i] and self.downloader.storage.do_I_have_requests(i): self.interested = True self.stream.send_interested() return def get_rate(self): return self.measure.get_rate() def is_snubbed(self): return bttime() - self.last > self.downloader.snub_time
class Upload(object): def __init__(self, stream, ratelimiter, totalup, choker, storage, max_slice_length, max_rate_period): self.stream = stream self.ratelimiter = ratelimiter self.totalup = totalup self.choker = choker self.storage = storage self.max_slice_length = max_slice_length self.max_rate_period = max_rate_period self.choked = True self.unchoke_time = None self.interested = False self.buffer = [] self.measure = Measure(max_rate_period) if storage.do_I_have_anything(): stream.send_bitfield(storage.get_have_list()) def got_not_interested(self): if self.interested: self.interested = False del self.buffer[:] self.choker.not_interested(self.stream) def got_interested(self): if not self.interested: self.interested = True self.choker.interested(self.stream) def get_upload_chunk(self): if not self.buffer: return None index, begin, length = self.buffer.pop(0) piece = self.storage.get_piece(index, begin, length) if piece is None: self.stream.close() return None self.measure.update_rate(len(piece)) self.totalup.update_rate(len(piece)) return (index, begin, piece) def got_request(self, index, begin, length): if not self.interested or length > self.max_slice_length: self.stream.close() return if not self.stream.choke_sent: self.buffer.append((index, begin, length)) if self.stream.next_upload is None and \ self.stream.is_flushed(): self.ratelimiter.queue(self.stream) def got_cancel(self, index, begin, length): try: self.buffer.remove((index, begin, length)) except ValueError: pass def choke(self): if not self.choked: self.choked = True self.stream.send_choke() def sent_choke(self): assert self.choked del self.buffer[:] def unchoke(self, time): if self.choked: self.choked = False self.unchoke_time = time self.stream.send_unchoke() def has_queries(self): return len(self.buffer) > 0 def get_rate(self): return self.measure.get_rate()
class _SingleTorrent(object): def __init__(self, event_handler, singleport_listener, ratelimiter, filepool, config, neighbors, certificate, sessionid): self.event_handler = event_handler self._singleport_listener = singleport_listener self._ratelimiter = ratelimiter self._filepool = filepool self.config = dict(config) self._storage = None self._storagewrapper = None self._ratemeasure = None self._upmeasure = None self._downmeasure = None self._torrent = None self._rerequest = None self._statuscollecter = None self._announced = False self._listening = False self.reserved_ports = [] self.reported_port = None self._myfiles = None self.started = False self.is_seed = False self.closed = False self.infohash = None self.file_size = None self._doneflag = threading.Event() self.finflag = threading.Event() self._hashcheck_thread = None self._contfunc = None self._activity = ('Initial startup', 0) self.feedback = None self.messages = [] self.neighbors = neighbors self.certificate = certificate self.sessionid = sessionid def schedule(self, delay, func): self.event_handler.schedule(delay, func, context=self) def start_download(self, *args, **kwargs): it = self._start_download(*args, **kwargs) def cont(): try: it.next() except StopIteration: self._contfunc = None def contfunc(): self.schedule(0, cont) self._contfunc = contfunc contfunc() def _start_download(self, metainfo, feedback, save_path): # GTK Crash Hack import time time.sleep(.2) self.feedback = feedback self._set_auto_uploads() self.infohash = metainfo.infohash self.file_size = metainfo.file_size if not metainfo.reported_errors: metainfo.show_encoding_errors(log.error) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles self._storage = Storage(self.config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if self.config['data_dir']: filename = os.path.join(self.config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: log.info("Could not load fastresume data: "+ str(e) + ". Will perform full hash check.") if resumefile is not None: resumefile.close() resumefile = None def data_flunked(amount, index): self._ratemeasure.data_rejected(amount) log.info('piece %d failed hash check, ' 're-downloading it' % index) backthread_exception = None def hashcheck(): def statusfunc(activity = None, fractionDone = 0): if activity is None: activity = self._activity[0] self._activity = (activity, fractionDone) try: self._storagewrapper = StorageWrapper(self._storage, self.config, metainfo.hashes, metainfo.piece_length, self._finished, statusfunc, self._doneflag, data_flunked, self.infohash, resumefile) except: backthread_exception = sys.exc_info() self._contfunc() thread = threading.Thread(target = hashcheck) thread.setDaemon(False) self._hashcheck_thread = thread thread.start() yield None self._hashcheck_thread = None if resumefile is not None: resumefile.close() if backthread_exception: a, b, c = backthread_exception raise a, b, c if self._storagewrapper.amount_left == 0: self._finished() choker = Choker(self.config, self.schedule, self.finflag.isSet) upmeasure = Measure(self.config['max_rate_period']) downmeasure = Measure(self.config['max_rate_period']) self._upmeasure = upmeasure self._downmeasure = downmeasure self._ratemeasure = RateMeasure(self._storagewrapper. amount_left_with_partials) picker = PiecePicker(len(metainfo.hashes), self.config) for i in xrange(len(metainfo.hashes)): if self._storagewrapper.do_I_have(i): picker.complete(i) for i in self._storagewrapper.stat_dirty: picker.requested(i) def kickpeer(connection): def kick(): connection.close() self.schedule(0, kick) downloader = Downloader(self.config, self._storagewrapper, picker, len(metainfo.hashes), downmeasure, self._ratemeasure.data_came_in, kickpeer) def make_upload(connection): return Upload(connection, self._ratelimiter, upmeasure, choker, self._storagewrapper, self.config['max_slice_length'], self.config['max_rate_period']) self._torrent = Torrent(self.infohash, make_upload, downloader, len(metainfo.hashes), self) self.reported_port = self.config['forwarded_port'] if not self.reported_port: self.reported_port = self._singleport_listener.get_port(self.neighbors) self.reserved_ports.append(self.reported_port) self.neighbors.add_torrent(self.infohash, self._torrent) self._listening = True self._rerequest = Rerequester(metainfo.announce, self.config, self.schedule, self.neighbors, self._storagewrapper.get_amount_left, upmeasure.get_total, downmeasure.get_total, self.reported_port, self.infohash, self.finflag, self.internal_shutdown, self._announce_done, self.certificate, self.sessionid) self._statuscollecter = DownloaderFeedback(choker, upmeasure.get_rate, downmeasure.get_rate, upmeasure.get_total, downmeasure.get_total, self.neighbors.get_relay_stats, self._ratemeasure.get_time_left, self._ratemeasure.get_size_left, self.file_size, self.finflag, downloader, self._myfiles) self._announced = True self._rerequest.begin() self.started = True if not self.finflag.isSet(): self._activity = ('downloading', 0) self.feedback.started(self)