def next_have(self, connection, looser_upload): if self.seed_time is None: self.seed_time = clock() return None if clock() < self.seed_time+10: # wait 10 seconds after seeing the first peers return None # to give time to grab have lists if not connection.upload.super_seeding: return None if connection in self.seed_connections: if looser_upload: num = 1 # send a new have even if it hasn't spread that piece elsewhere else: num = 2 if self.seed_got_haves[self.seed_connections[connection]] < num: return None if not connection.upload.was_ever_interested: # it never downloaded it? connection.upload.skipped_count += 1 if connection.upload.skipped_count >= 3: # probably another stealthed seed return -1 # signal to close it for tier in self.interests: for piece in tier: if not connection.download.have[piece]: seedint = self.level_in_interests[piece] self.level_in_interests[piece] += 1 # tweak it up one, so you don't duplicate effort if seedint == len(self.interests) - 1: self.interests.append([]) self._shift_over(piece, self.interests[seedint], self.interests[seedint + 1]) self.seed_got_haves[piece] = 0 # reset this self.seed_connections[connection] = piece connection.upload.seed_have_list.append(piece) return piece return -1 # something screwy; terminate connection
def next_have(self, connection, looser_upload): if self.seed_time is None: self.seed_time = clock() return None if clock( ) < self.seed_time + 10: # wait 10 seconds after seeing the first peers return None # to give time to grab have lists if not connection.upload.super_seeding: return None if connection in self.seed_connections: if looser_upload: num = 1 # send a new have even if it hasn't spread that piece elsewhere else: num = 2 if self.seed_got_haves[self.seed_connections[connection]] < num: return None if not connection.upload.was_ever_interested: # it never downloaded it? connection.upload.skipped_count += 1 if connection.upload.skipped_count >= 3: # probably another stealthed seed return -1 # signal to close it for tier in self.interests: for piece in tier: if not connection.download.have[piece]: seedint = self.level_in_interests[piece] self.level_in_interests[ piece] += 1 # tweak it up one, so you don't duplicate effort if seedint == len(self.interests) - 1: self.interests.append([]) self._shift_over(piece, self.interests[seedint], self.interests[seedint + 1]) self.seed_got_haves[piece] = 0 # reset this self.seed_connections[connection] = piece connection.upload.seed_have_list.append(piece) return piece return -1 # something screwy; terminate connection
def _round_robin(self): self.schedule(self._round_robin, 5) if self.super_seed: cons = range(len(self.connections)) to_close = [] count = self.config['min_uploads']-self.last_preferred if count > 0: # optimization shuffle(cons) for c in cons: i = self.picker.next_have(self.connections[c], count > 0) if i is None: continue if i < 0: to_close.append(self.connections[c]) continue self.connections[c].send_have(i) count -= 1 for c in to_close: c.close() if self.last_round_robin + self.round_robin_period < clock(): self.last_round_robin = clock() for i in xrange(1, len(self.connections)): c = self.connections[i] u = c.get_upload() if u.is_choked() and u.is_interested(): self.connections = self.connections[i:] + self.connections[:i] break self._rechoke()
def is_snubbed(self): # 2fastbt_ if not self.choked and clock() - self.last2 > self.downloader.snub_time and \ not self.connection.connection.is_helper_con() and \ not self.connection.connection.is_coordinator_con(): # _2fastbt for index, begin, length in self.active_requests: self.connection.send_cancel(index, begin, length) self.got_choke() # treat it just like a choke return clock() - self.last > self.downloader.snub_time
def got_piece(self, index, begin, hashlist, piece): """ Returns True if the piece is complete. """ length = len(piece) #if DEBUG: # print >> sys.stderr, 'Downloader: got piece of length %d' % length try: self.active_requests.remove((index, begin, length)) except ValueError: self.downloader.discarded += length return False if self.downloader.endgamemode: self.downloader.all_requests.remove((index, begin, length)) self.last = clock() self.last2 = clock() self.measure.update_rate(length) self.downloader.measurefunc(length) if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): self.downloader.piece_flunked(index) return False if self.downloader.storage.do_I_have(index): self.downloader.picker.complete(index) if self.downloader.endgamemode: for d in self.downloader.downloads: if d is not self: if d.interested: if d.choked: assert not d.active_requests d.fix_download_endgame() else: try: d.active_requests.remove( (index, begin, length)) except ValueError: continue d.connection.send_cancel(index, begin, length) d.fix_download_endgame() else: assert not d.active_requests self._request_more() self.downloader.check_complete(index) # BarterCast counter self.connection.total_downloaded += length return self.downloader.storage.do_I_have(index)
def got_piece(self, index, begin, hashlist, piece): """ Returns True if the piece is complete. """ length = len(piece) #if DEBUG: # print >> sys.stderr, 'Downloader: got piece of length %d' % length try: self.active_requests.remove((index, begin, length)) except ValueError: self.downloader.discarded += length return False if self.downloader.endgamemode: self.downloader.all_requests.remove((index, begin, length)) self.last = clock() self.last2 = clock() self.measure.update_rate(length) self.downloader.measurefunc(length) if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): self.downloader.piece_flunked(index) return False if self.downloader.storage.do_I_have(index): self.downloader.picker.complete(index) if self.downloader.endgamemode: for d in self.downloader.downloads: if d is not self: if d.interested: if d.choked: assert not d.active_requests d.fix_download_endgame() else: try: d.active_requests.remove((index, begin, length)) except ValueError: continue d.connection.send_cancel(index, begin, length) d.fix_download_endgame() else: assert not d.active_requests self._request_more() self.downloader.check_complete(index) # BarterCast counter self.connection.total_downloaded += length return self.downloader.storage.do_I_have(index)
def __init__(self, storage, picker, backlog, max_rate_period, numpieces, chunksize, measurefunc, snub_time, kickbans_ok, kickfunc, banfunc, scheduler = None): self.storage = storage self.picker = picker self.backlog = backlog self.max_rate_period = max_rate_period self.measurefunc = measurefunc self.totalmeasure = Measure(max_rate_period*storage.piece_length/storage.request_size) self.numpieces = numpieces self.chunksize = chunksize self.snub_time = snub_time self.kickfunc = kickfunc self.banfunc = banfunc self.disconnectedseeds = {} self.downloads = [] self.perip = {} self.gotbaddata = {} self.kicked = {} self.banned = {} self.kickbans_ok = kickbans_ok self.kickbans_halted = False self.super_seeding = False self.endgamemode = False self.endgame_queued_pieces = [] self.all_requests = [] self.discarded = 0L # self.download_rate = 25000 # 25K/s test rate self.download_rate = 0 self.bytes_requested = 0 self.last_time = clock() self.queued_out = {} self.requeueing = False self.paused = False self.scheduler = scheduler
def num_disconnected_seeds(self): # first expire old ones expired = [] for id, t in self.disconnectedseeds.items(): if clock() - t > EXPIRE_TIME: #Expire old seeds after so long expired.append(id) for id in expired: # self.picker.seed_disappeared() del self.disconnectedseeds[id] return len(self.disconnectedseeds)
def __init__(self, infohash, storage, picker, backlog, max_rate_period, numpieces, chunksize, measurefunc, snub_time, kickbans_ok, kickfunc, banfunc, bt1dl, scheduler = None): self.infohash = infohash self.b64_infohash = b64encode(infohash) self.storage = storage self.picker = picker self.backlog = backlog self.max_rate_period = max_rate_period self.measurefunc = measurefunc self.totalmeasure = Measure(max_rate_period*storage.piece_length/storage.request_size) self.numpieces = numpieces self.chunksize = chunksize self.snub_time = snub_time self.kickfunc = kickfunc self.banfunc = banfunc self.disconnectedseeds = {} self.downloads = [] self.perip = {} self.gotbaddata = {} self.kicked = {} self.banned = {} self.kickbans_ok = kickbans_ok self.kickbans_halted = False self.super_seeding = False self.endgamemode = False self.endgame_queued_pieces = [] self.all_requests = [] self.discarded = 0L self.download_rate = 0 # self.download_rate = 25000 # 25K/s test rate self.bytes_requested = 0 self.last_time = clock() self.queued_out = {} self.requeueing = False self.paused = False self.scheduler = scheduler # ProxyService_ # self.bt1dl = bt1dl self.proxydownloader = None # # _ProxyService # hack: we should not import this since it is not part of the # core nor should we import here, but otherwise we will get # import errors # # _event_reporter stores events that are logged somewhere... # from Tribler.Core.Statistics.StatusReporter import get_reporter_instance # self._event_reporter = get_reporter_instance() self._event_reporter = get_status_holder("LivingLab") # check periodicaly self.scheduler(self.dlr_periodic_check, 1)
def update(self): self.raw_server.add_task(self.update, self.probing_period) current_value = self.rate_measure.get_rate() / 1000. current_time = clock() if self.value is None or current_time - self.timestamp > self.max_period: self.value = current_value else: self.value = self.alpha * current_value + (1 - self.alpha) * self.value if self.max_rate > 0 and self.value > self.max_rate: self.value = self.max_rate self.timestamp = current_time
def __init__(self, config, schedule, picker, done = lambda: False): self.config = config self.round_robin_period = config['round_robin_period'] self.schedule = schedule self.picker = picker self.connections = [] self.last_preferred = 0 self.last_round_robin = clock() self.done = done self.super_seed = False self.paused = False schedule(self._round_robin, 5)
def update(self): if DEBUG: print >>sys.stderr, time.asctime(),'-', "RatePredictor:update" self.raw_server.add_task(self.update, self.probing_period) current_value = self.rate_measure.get_rate() / 1000. current_time = clock() if self.value is None or current_time - self.timestamp > self.max_period: self.value = current_value else: self.value = self.alpha * current_value + (1 - self.alpha) * self.value if self.max_rate > 0 and self.value > self.max_rate: self.value = self.max_rate self.timestamp = current_time
def send_unchoke(self): if self.send_choke_queued: self.send_choke_queued = False if DEBUG_NORMAL_MSGS: print >>sys.stderr,'CHOKE SUPPRESSED' else: self._send_message(UNCHOKE) if (self.partial_message or self.just_unchoked is None or not self.upload.interested or self.download.active_requests): self.just_unchoked = 0 else: self.just_unchoked = clock()
def display(self, dpflag = Event(), fractionDone = None, timeEst = None, downRate = None, upRate = None, activity = None, statistics = None, **kws): if self.last_update_time + 0.1 > clock() and fractionDone not in (0.0, 1.0) and activity is not None: return self.last_update_time = clock() if fractionDone is not None: self.percentDone = str(float(int(fractionDone * 1000)) / 10) if timeEst is not None: self.timeEst = hours(timeEst) if activity is not None and not self.done: self.timeEst = activity if downRate is not None: self.downRate = '%.1f kB/s' % (float(downRate) / (1 << 10)) if upRate is not None: self.upRate = '%.1f kB/s' % (float(upRate) / (1 << 10)) if statistics is not None: if (statistics.shareRating < 0) or (statistics.shareRating > 100): self.shareRating = 'oo (%.1f MB up / %.1f MB down)' % (float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20)) else: self.shareRating = '%.3f (%.1f MB up / %.1f MB down)' % (statistics.shareRating, float(statistics.upTotal) / (1<<20), float(statistics.downTotal) / (1<<20)) if not self.done: self.seedStatus = '%d seen now, plus %.3f distributed copies' % (statistics.numSeeds,0.001*int(1000*statistics.numCopies)) else: self.seedStatus = '%d seen recently, plus %.3f distributed copies' % (statistics.numOldSeeds,0.001*int(1000*statistics.numCopies)) self.peerStatus = '%d seen now, %.1f%% done at %.1f kB/s' % (statistics.numPeers,statistics.percentDone,float(statistics.torrentRate) / (1 << 10)) print '\n\n\n\n' for err in self.errors: print 'ERROR:\n' + err + '\n' print 'saving: ', self.file print 'percent done: ', self.percentDone print 'time left: ', self.timeEst print 'download to: ', self.downloadTo print 'download rate: ', self.downRate print 'upload rate: ', self.upRate print 'share rating: ', self.shareRating print 'seed status: ', self.seedStatus print 'peer status: ', self.peerStatus stdout.flush() dpflag.set()
def send_unchoke(self): if self.send_choke_queued: self.send_choke_queued = False if DEBUG_NORMAL_MSGS: print >> sys.stderr, 'CHOKE SUPPRESSED' else: self._send_message(UNCHOKE) if (self.partial_message or self.just_unchoked is None or not self.upload.interested or self.download.active_requests): self.just_unchoked = 0 else: self.just_unchoked = clock()
def __init__(self, storage, picker, backlog, max_rate_period, numpieces, chunksize, measurefunc, snub_time, kickbans_ok, kickfunc, banfunc, scheduler=None): self.storage = storage self.picker = picker self.backlog = backlog self.max_rate_period = max_rate_period self.measurefunc = measurefunc self.totalmeasure = Measure(max_rate_period * storage.piece_length / storage.request_size) self.numpieces = numpieces self.chunksize = chunksize self.snub_time = snub_time self.kickfunc = kickfunc self.banfunc = banfunc self.disconnectedseeds = {} self.downloads = [] self.perip = {} self.gotbaddata = {} self.kicked = {} self.banned = {} self.kickbans_ok = kickbans_ok self.kickbans_halted = False self.super_seeding = False self.endgamemode = False self.endgame_queued_pieces = [] self.all_requests = [] self.discarded = 0L # self.download_rate = 25000 # 25K/s test rate self.download_rate = 0 self.bytes_requested = 0 self.last_time = clock() self.queued_out = {} self.requeueing = False self.paused = False self.scheduler = scheduler
def queue_limit(self): if not self.download_rate: return 10e10 # that's a big queue! t = clock() self.bytes_requested -= (t - self.last_time) * self.download_rate self.last_time = t if not self.requeueing and self.queued_out and self.bytes_requested < 0: self.requeueing = True q = self.queued_out.keys() shuffle(q) self.queued_out = {} for d in q: d._request_more() self.requeueing = False if -self.bytes_requested > 5*self.download_rate: self.bytes_requested = -5*self.download_rate ql = max(int(-self.bytes_requested/self.chunksize), 0) # if DEBUG: # print >> sys.stderr, 'Downloader: download_rate: %s, bytes_requested: %s, chunk: %s -> queue limit: %d' % \ # (self.download_rate, self.bytes_requested, self.chunksize, ql) return ql
def queue_limit(self): if not self.download_rate: return 10e10 # that's a big queue! t = clock() self.bytes_requested -= (t - self.last_time) * self.download_rate self.last_time = t if not self.requeueing and self.queued_out and self.bytes_requested < 0: self.requeueing = True q = self.queued_out.keys() shuffle(q) self.queued_out = {} for d in q: d._request_more() self.requeueing = False if -self.bytes_requested > 5 * self.download_rate: self.bytes_requested = -5 * self.download_rate ql = max(int(-self.bytes_requested / self.chunksize), 0) if DEBUG: print >> sys.stderr, 'Downloader: download_rate: %s, bytes_requested: %s, chunk: %s -> queue limit: %d' % \ (self.download_rate, self.bytes_requested, self.chunksize, ql) return ql
def __init__(self, infohash, storage, picker, backlog, max_rate_period, numpieces, chunksize, measurefunc, snub_time, kickbans_ok, kickfunc, banfunc, bt1dl, scheduler=None): self.infohash = infohash self.b64_infohash = b64encode(infohash) self.storage = storage self.picker = picker self.backlog = backlog self.max_rate_period = max_rate_period self.measurefunc = measurefunc self.totalmeasure = Measure(max_rate_period * storage.piece_length / storage.request_size) self.numpieces = numpieces self.chunksize = chunksize self.snub_time = snub_time self.kickfunc = kickfunc self.banfunc = banfunc self.disconnectedseeds = {} self.downloads = [] self.perip = {} self.gotbaddata = {} self.kicked = {} self.banned = {} self.kickbans_ok = kickbans_ok self.kickbans_halted = False self.super_seeding = False self.endgamemode = False self.endgame_queued_pieces = [] self.all_requests = [] self.discarded = 0L self.download_rate = 0 # self.download_rate = 25000 # 25K/s test rate self.bytes_requested = 0 self.last_time = clock() self.queued_out = {} self.requeueing = False self.paused = False self.scheduler = scheduler # ProxyService_ # self.bt1dl = bt1dl self.proxydownloader = None # # _ProxyService # hack: we should not import this since it is not part of the # core nor should we import here, but otherwise we will get # import errors # # _event_reporter stores events that are logged somewhere... # from Tribler.Core.Statistics.StatusReporter import get_reporter_instance # self._event_reporter = get_reporter_instance() self._event_reporter = get_status_holder("LivingLab") # check periodicaly self.scheduler(self.dlr_periodic_check, 1)
def got_piece(self, index, begin, hashlist, piece): """ Returns True if the piece is complete. Note that in this case a -piece- means a chunk! """ if self.bad_performance_counter: self.bad_performance_counter -= 1 if DEBUG: print >>sys.stderr, "decreased bad_performance_counter to", self.bad_performance_counter length = len(piece) #if DEBUG: # print >> sys.stderr, 'Downloader: got piece of length %d' % length try: self.active_requests.remove((index, begin, length)) except ValueError: self.downloader.discarded += length return False if self.downloader.endgamemode: self.downloader.all_requests.remove((index, begin, length)) if DEBUG: print >>sys.stderr, "Downloader: got_piece: removed one request from all_requests", len(self.downloader.all_requests), "remaining" self.last = clock() self.last2 = clock() self.measure.update_rate(length) # Update statistic gatherer status = get_status_holder("LivingLab") s_download = status.get_or_create_status_element("downloaded",0) s_download.inc(length) self.short_term_measure.update_rate(length) self.downloader.measurefunc(length) if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): self.downloader.piece_flunked(index) return False # boudewijn: we need more accurate (if possibly invalid) # measurements on current download speed self.downloader.picker.got_piece(index, begin, length) if self.downloader.storage.do_I_have(index): # The piece (actual piece, not chunk) is complete self.downloader.picker.complete(index) # ProxyService_ # if self.downloader.proxydownloader: if DEBUG: print >>sys.stderr, "downloader: got_piece. Searching if piece", index, "was requested by a doe node." if index in self.downloader.proxydownloader.proxy.currently_downloading_pieces: # get_piece(index, 0, -1) returns the complete piece data [piece_data, hash_list] = self.downloader.storage.get_piece(index, 0, -1) self.downloader.proxydownloader.proxy.retrieved_piece(index, piece_data) # # _ProxyService if self.downloader.endgamemode: for d in self.downloader.downloads: if d is not self: if d.interested: if d.choked: assert not d.active_requests d.fix_download_endgame() else: try: d.active_requests.remove((index, begin, length)) except ValueError: continue d.connection.send_cancel(index, begin, length) d.fix_download_endgame() else: assert not d.active_requests self._request_more() self.downloader.check_complete(index) # BarterCast counter self.connection.total_downloaded += length return self.downloader.storage.do_I_have(index)
def got_unchoke(self): if self.choked: self.choked = False if self.interested: self._request_more(new_unchoke = True) self.last2 = clock()
def got_request(self, i, p, l): self.upload.got_request(i, p, l) if self.just_unchoked: self.connecter.ratelimiter.ping(clock() - self.just_unchoked) self.just_unchoked = 0
def got_unchoke(self): if self.choked: self.choked = False if self.interested: self._request_more(new_unchoke=True) self.last2 = clock()
def add_disconnected_seed(self, id): # if not self.disconnectedseeds.has_key(id): # self.picker.seed_seen_recently() self.disconnectedseeds[id] = clock()
def is_snubbed(self): if not self.choked and clock() - self.last2 > self.downloader.snub_time: for index, begin, length in self.active_requests: self.connection.send_cancel(index, begin, length) self.got_choke() # treat it just like a choke return clock() - self.last > self.downloader.snub_time
def add_disconnected_seed(self, id): # if not self.disconnectedseeds.has_key(id): # self.picker.seed_seen_recently() self.disconnectedseeds[id]=clock()
def got_piece(self, index, begin, hashlist, piece): """ Returns True if the piece is complete. Note that in this case a -piece- means a chunk! """ if self.bad_performance_counter: self.bad_performance_counter -= 1 if DEBUG: print >> sys.stderr, "decreased bad_performance_counter to", self.bad_performance_counter length = len(piece) #if DEBUG: # print >> sys.stderr, 'Downloader: got piece of length %d' % length try: self.active_requests.remove((index, begin, length)) except ValueError: self.downloader.discarded += length return False if self.downloader.endgamemode: self.downloader.all_requests.remove((index, begin, length)) if DEBUG: print >> sys.stderr, "Downloader: got_piece: removed one request from all_requests", len( self.downloader.all_requests), "remaining" self.last = clock() self.last2 = clock() self.measure.update_rate(length) # Update statistic gatherer status = get_status_holder("LivingLab") s_download = status.get_or_create_status_element("downloaded", 0) s_download.inc(length) self.short_term_measure.update_rate(length) self.downloader.measurefunc(length) if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): self.downloader.piece_flunked(index) return False # boudewijn: we need more accurate (if possibly invalid) # measurements on current download speed self.downloader.picker.got_piece(index, begin, length) if self.downloader.storage.do_I_have(index): # The piece (actual piece, not chunk) is complete self.downloader.picker.complete(index) # ProxyService_ # if self.downloader.proxydownloader: if DEBUG: print >> sys.stderr, "downloader: got_piece. Searching if piece", index, "was requested by a doe node." if index in self.downloader.proxydownloader.proxy.currently_downloading_pieces: # get_piece(index, 0, -1) returns the complete piece data [piece_data, hash_list ] = self.downloader.storage.get_piece(index, 0, -1) self.downloader.proxydownloader.proxy.retrieved_piece( index, piece_data) # # _ProxyService if self.downloader.endgamemode: for d in self.downloader.downloads: if d is not self: if d.interested: if d.choked: assert not d.active_requests d.fix_download_endgame() else: try: d.active_requests.remove( (index, begin, length)) except ValueError: continue d.connection.send_cancel(index, begin, length) d.fix_download_endgame() else: assert not d.active_requests self._request_more() self.downloader.check_complete(index) # BarterCast counter self.connection.total_downloaded += length return self.downloader.storage.do_I_have(index)