def __init__(self, downloader, connection): SingleDownloadHelperInterface.__init__(self) self.downloader = downloader self.connection = connection self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.max_rate_period) self.peermeasure = Measure(downloader.max_rate_period) self.raw_have = Bitfield(downloader.numpieces) self.have = Bitfield(downloader.numpieces) self.last = -1000 self.last2 = -1000 self.example_interest = None self.backlog = 2 self.ip = connection.get_ip() self.guard = BadDataGuard(self) self.app_mode = globalConfig.get_mode() self.white_list = None self.black_list = None self.app_mode = globalConfig.get_mode() if self.app_mode == 'node': source_node = globalConfig.get_value('source_node') support_nodes = globalConfig.get_value('support_nodes') if not globalConfig.get_value('allow_peers_download'): self.white_list = set() if source_node is not None and globalConfig.get_value( 'allow_source_download'): self.white_list.add(source_node[0]) if len(support_nodes) and globalConfig.get_value( 'allow_support_download'): self.white_list.update([addr[0] for addr in support_nodes]) else: self.black_list = set() if source_node is not None and not globalConfig.get_value( 'allow_source_download'): self.black_list.add(source_node[0]) if len(support_nodes) and not globalConfig.get_value( 'allow_support_download'): self.black_list.update([addr[0] for addr in support_nodes]) if len(self.black_list) == 0: self.black_list = None if DEBUG: log('download::__init__: white_list', self.white_list, 'black_list', self.black_list) self.helper = downloader.picker.helper self.proxy_have = Bitfield(downloader.numpieces) self.short_term_measure = Measure(5) self.bad_performance_counter = 0
def live_invalidate_ranges(self, toinvalidateranges, toinvalidateset): if len(toinvalidateranges) == 1: s, e = toinvalidateranges[0] emptyrange = [False for piece in xrange(s, e + 1)] for d in self.downloads: newhave = d.have[0:s] + emptyrange + d.have[e + 1:] d.have = Bitfield(length=len(newhave), fromarray=newhave) else: s1, e1 = toinvalidateranges[0] s2, e2 = toinvalidateranges[1] emptyrange1 = [False for piece in xrange(s1, e1 + 1)] emptyrange2 = [False for piece in xrange(s2, e2 + 1)] for d in self.downloads: newhave = emptyrange1 + d.have[e1 + 1:s2] + emptyrange2 d.have = Bitfield(length=len(newhave), fromarray=newhave)
def reset_have(self): if DEBUG: print >> sys.stderr, 'Downloader::reset_have: before self.have:', self.have.toboollist( ) validhave = Bitfield(self.downloader.numpieces) for i in self.downloader.picker.get_valid_range_iterator(): if self.raw_have[i]: validhave[i] = True self.have = validhave if DEBUG: print >> sys.stderr, 'Downloader::reset_have: after self.have:', self.have.toboollist( )
def aggregate_and_send_haves(self): if self.picker.helper: if DEBUG: print >> sys.stderr, 'Downloader: aggregate_and_send_haves: helper None or helper conn' haves_vector = [None] * len(self.downloads) for i in range(0, len(self.downloads)): haves_vector[i] = self.downloads[i].have aggregated_haves = Bitfield(self.numpieces) for piece in range(0, self.numpieces): aggregated_value = False for d in range(0, len(self.downloads)): aggregated_value = aggregated_value or haves_vector[d][ piece] aggregated_haves[piece] = aggregated_value self.picker.helper.send_proxy_have(aggregated_haves)
def __init__(self, downloader, connection): SingleDownloadHelperInterface.__init__(self) self.downloader = downloader self.connection = connection self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.max_rate_period) self.peermeasure = Measure(downloader.max_rate_period) self.raw_have = Bitfield(downloader.numpieces) self.have = Bitfield(downloader.numpieces) self.last = -1000 self.last2 = -1000 self.example_interest = None self.backlog = 2 self.ip = connection.get_ip() self.guard = BadDataGuard(self) self.app_mode = globalConfig.get_mode() self.white_list = None self.black_list = None self.app_mode = globalConfig.get_mode() if self.app_mode == 'node': source_node = globalConfig.get_value('source_node') support_nodes = globalConfig.get_value('support_nodes') if not globalConfig.get_value('allow_peers_download'): self.white_list = set() if source_node is not None and globalConfig.get_value('allow_source_download'): self.white_list.add(source_node[0]) if len(support_nodes) and globalConfig.get_value('allow_support_download'): self.white_list.update([ addr[0] for addr in support_nodes ]) else: self.black_list = set() if source_node is not None and not globalConfig.get_value('allow_source_download'): self.black_list.add(source_node[0]) if len(support_nodes) and not globalConfig.get_value('allow_support_download'): self.black_list.update([ addr[0] for addr in support_nodes ]) if len(self.black_list) == 0: self.black_list = None if DEBUG: log('download::__init__: white_list', self.white_list, 'black_list', self.black_list) self.helper = downloader.picker.helper self.proxy_have = Bitfield(downloader.numpieces) self.short_term_measure = Measure(5) self.bad_performance_counter = 0
def got_have_bitfield(self, have): if self.downloader.picker.am_I_complete() and have.complete(): if self.downloader.super_seeding: self.connection.send_bitfield(have.tostring()) self.connection.try_send_pex() def auto_close(): self.connection.close() self.downloader.add_disconnected_seed( self.connection.get_readable_id()) self.downloader.scheduler(auto_close, REPEX_LISTEN_TIME) return if DEBUGBF: st = time.time() self.raw_have = have if have.complete(): self.downloader.picker.got_seed() else: activerangeiterators = [] if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and self.downloader.picker.videostatus.get_live_startpos( ) is None: activeranges = have.get_active_ranges() if len(activeranges) == 0: activerangeiterators = [ self.downloader.picker.get_valid_range_iterator() ] else: for s, e in activeranges: activerangeiterators.append(xrange(s, e + 1)) else: activerangeiterators = [ self.downloader.picker.get_valid_range_iterator( skip_filter=True) ] if DEBUGBF: print >> sys.stderr, 'Downloader: got_have_field: live: Filtering bitfield', activerangeiterators if DEBUGBF: print >> sys.stderr, 'Downloader: got_have_field: live or normal filter' validhave = Bitfield(self.downloader.numpieces) for iterator in activerangeiterators: for i in iterator: if have[i]: validhave[i] = True self.downloader.picker.got_have(i, self.connection) if DEBUG: print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip( ), 'have', debug_format_have(have) print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip( ), 'validhave', debug_format_have(validhave) self.downloader.aggregate_and_send_haves() have = validhave if DEBUGBF: et = time.time() diff = et - st print >> sys.stderr, 'Download: got_have_field: took', diff self.have = have if self.downloader.endgamemode and not self.downloader.paused: for piece, begin, length in self.downloader.all_requests: if self.have[piece]: self.send_interested() break return self._check_interests()
class SingleDownload(SingleDownloadHelperInterface): def __init__(self, downloader, connection): SingleDownloadHelperInterface.__init__(self) self.downloader = downloader self.connection = connection self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.max_rate_period) self.peermeasure = Measure(downloader.max_rate_period) self.raw_have = Bitfield(downloader.numpieces) self.have = Bitfield(downloader.numpieces) self.last = -1000 self.last2 = -1000 self.example_interest = None self.backlog = 2 self.ip = connection.get_ip() self.guard = BadDataGuard(self) self.app_mode = globalConfig.get_mode() self.white_list = None self.black_list = None self.app_mode = globalConfig.get_mode() if self.app_mode == 'node': source_node = globalConfig.get_value('source_node') support_nodes = globalConfig.get_value('support_nodes') if not globalConfig.get_value('allow_peers_download'): self.white_list = set() if source_node is not None and globalConfig.get_value( 'allow_source_download'): self.white_list.add(source_node[0]) if len(support_nodes) and globalConfig.get_value( 'allow_support_download'): self.white_list.update([addr[0] for addr in support_nodes]) else: self.black_list = set() if source_node is not None and not globalConfig.get_value( 'allow_source_download'): self.black_list.add(source_node[0]) if len(support_nodes) and not globalConfig.get_value( 'allow_support_download'): self.black_list.update([addr[0] for addr in support_nodes]) if len(self.black_list) == 0: self.black_list = None if DEBUG: log('download::__init__: white_list', self.white_list, 'black_list', self.black_list) self.helper = downloader.picker.helper self.proxy_have = Bitfield(downloader.numpieces) self.short_term_measure = Measure(5) self.bad_performance_counter = 0 def _backlog(self, just_unchoked): self.backlog = int( min( 2 + int(4 * self.measure.get_rate() / self.downloader.chunksize), 2 * just_unchoked + self.downloader.queue_limit())) if DEBUG: log('downloader::sd::_backlog: backlog', self.backlog, 'rate', self.measure.get_rate(), 'chunksize', self.downloader.chunksize, 'just_unchoked', just_unchoked, 'queue_limit', self.downloader.queue_limit()) if self.backlog > 50: self.backlog = int(max(50, self.backlog * 0.075)) if DEBUG: log('downloader::sd::_backlog: fix backlog', self.backlog) return self.backlog def disconnected(self): self.downloader.lost_peer(self) if self.have.complete() and self.downloader.storage.is_endgame(): self.downloader.add_disconnected_seed( self.connection.get_readable_id()) self._letgo() self.guard.download = None def _letgo(self): if self.downloader.queued_out.has_key(self): del self.downloader.queued_out[self] if not self.active_requests: return if self.downloader.endgamemode: self.active_requests = [] return lost = {} for index, begin, length in self.active_requests: self.downloader.storage.request_lost(index, begin, length) lost[index] = 1 lost = lost.keys() self.active_requests = [] if self.downloader.paused: return ds = [d for d in self.downloader.downloads if not d.choked] shuffle(ds) for d in ds: d._request_more() for d in self.downloader.downloads: if d.choked and not d.interested: for l in lost: if d.have[l] and self.downloader.storage.do_I_have_requests( l): d.send_interested() break def got_choke(self): if not self.choked: if DEBUG: log('downloader::got_choke: got choke: ip', self.connection.get_ip()) self.choked = True self._letgo() elif DEBUG: log('downloader::got_choke: already choked: ip', self.connection.get_ip()) def got_unchoke(self): if self.choked: if DEBUG: log('downloader::got_unchoke: got unchoke: ip', self.connection.get_ip(), 'interested', self.interested) self.choked = False if self.interested: self._request_more(new_unchoke=True) self.last2 = clock() elif DEBUG: log('downloader::got_unchoke: already unchoked: ip', self.connection.get_ip()) def is_choked(self): return self.choked def is_interested(self): return self.interested def send_interested(self): if not self.interested: if DEBUG: log('downloader::send_interested: send interested: ip', self.connection.get_ip()) self.interested = True self.connection.send_interested() elif DEBUG: log('downloader::send_interested: already interested: ip', self.connection.get_ip()) def send_not_interested(self): if self.interested: if DEBUG: log('downloader::send_not_interested: send not interested: ip', self.connection.get_ip()) self.interested = False self.connection.send_not_interested() elif DEBUG: log('downloader::send_not_interested: already not interested: ip', self.connection.get_ip()) def got_piece(self, index, begin, hashlist, piece): if self.bad_performance_counter: self.bad_performance_counter -= 1 if DEBUG: print >> sys.stderr, 'decreased bad_performance_counter to', self.bad_performance_counter length = len(piece) try: self.active_requests.remove((index, begin, length)) except ValueError: self.downloader.discarded += length return False if self.downloader.endgamemode: self.downloader.all_requests.remove((index, begin, length)) if DEBUG: print >> sys.stderr, 'Downloader: got_piece: removed one request from all_requests', len( self.downloader.all_requests), 'remaining' self.last = clock() self.last2 = clock() self.measure.update_rate(length) self.short_term_measure.update_rate(length) self.downloader.measurefunc(length) if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): self.downloader.piece_flunked(index) return False self.downloader.picker.got_piece(index, begin, length) if self.downloader.storage.do_I_have(index): self.downloader.picker.complete(index) if self.downloader.endgamemode: for d in self.downloader.downloads: if d is not self: if d.interested: if d.choked: d.fix_download_endgame() else: try: d.active_requests.remove( (index, begin, length)) except ValueError: continue d.connection.send_cancel(index, begin, length) d.fix_download_endgame() self._request_more() self.downloader.check_complete(index) self.connection.total_downloaded += length return self.downloader.storage.do_I_have(index) def helper_forces_unchoke(self): self.choked = False def _request_more(self, new_unchoke=False, slowpieces=[]): if self.helper is not None and self.is_frozen_by_helper(): if DEBUG: print >> sys.stderr, 'Downloader: _request_more: blocked, returning' return if self.app_mode == 'node': ip = self.connection.get_ip() if DEBUG: log('download::_request_more: check ip', ip) if self.white_list is not None and ip not in self.white_list: if DEBUG: log( 'download::_request_more: peer is not in the white list: ip', ip) return if self.black_list is not None and ip in self.black_list: if DEBUG: log( 'download::_request_more: peer is in the black list: ip', ip) return if self.choked: if DEBUG: print >> sys.stderr, 'Downloader: _request_more: choked, returning' return if self.connection.connection.is_coordinator_con(): if DEBUG: print >> sys.stderr, 'Downloader: _request_more: coordinator conn' return if self.downloader.endgamemode: self.fix_download_endgame(new_unchoke) if DEBUG: print >> sys.stderr, 'Downloader: _request_more: endgame mode, returning' return if self.downloader.paused: if DEBUG: print >> sys.stderr, 'Downloader: _request_more: paused, returning' return if len(self.active_requests) >= self._backlog(new_unchoke): if DEBUG: log('downloader::_request_more: more req than unchoke (active req: %d >= backlog: %d), download_rate=%d' % (len(self.active_requests), self._backlog(new_unchoke), self.downloader.download_rate)) if self.downloader.download_rate: wait_period = self.downloader.chunksize / self.downloader.download_rate / 2.0 if wait_period > 1.0: if DEBUG: print >> sys.stderr, 'Downloader: waiting for %f s to call _request_more again' % wait_period self.downloader.scheduler(self._request_more, wait_period) if not (self.active_requests or self.backlog): if DEBUG: print >> sys.stderr, 'Downloader::_request_more: queue out download' self.downloader.queued_out[self] = 1 return lost_interests = [] while len(self.active_requests) < self.backlog: interest = self.downloader.picker.next( self.have, self.downloader.storage.do_I_have_requests, self, self.downloader.too_many_partials(), self.connection.connection.is_helper_con(), slowpieces=slowpieces, connection=self.connection, proxyhave=self.proxy_have) diff = -1 if DEBUG: print >> sys.stderr, 'Downloader: _request_more: next() returned', interest, 'took %.5f' % diff if interest is None: break if self.helper and self.downloader.storage.inactive_requests[ interest] is None: self.connection.send_have(interest) break if self.helper and self.downloader.storage.inactive_requests[ interest] == []: break self.example_interest = interest self.send_interested() loop = True while len(self.active_requests) < self.backlog and loop: request = self.downloader.storage.new_request(interest) if request is None: log( 'downloader::_request_more: new_request returned none: index', interest) lost_interests.append(interest) break begin, length = request if DEBUG: log('downloader::_request_more: new_request', interest, begin, length, 'to', self.connection.connection.get_ip(), self.connection.connection.get_port()) self.downloader.picker.requested(interest, begin, length) self.active_requests.append((interest, begin, length)) self.connection.send_request(interest, begin, length) self.downloader.chunk_requested(length) if not self.downloader.storage.do_I_have_requests(interest): loop = False lost_interests.append(interest) if not self.active_requests: self.send_not_interested() if lost_interests: for d in self.downloader.downloads: if d.active_requests or not d.interested: continue if d.example_interest is not None and self.downloader.storage.do_I_have_requests( d.example_interest): continue for lost in lost_interests: if d.have[lost]: break else: continue interest = self.downloader.picker.next( d.have, self.downloader.storage.do_I_have_requests, self, self.downloader.too_many_partials(), self.connection.connection.is_helper_con(), willrequest=False, connection=self.connection, proxyhave=self.proxy_have) diff = -1 if DEBUG: print >> sys.stderr, 'Downloader: _request_more: next()2 returned', interest, 'took %.5f' % diff if interest is not None: if self.helper and self.downloader.storage.inactive_requests[ interest] is None: self.connection.send_have(interest) break if self.helper and self.downloader.storage.inactive_requests[ interest] == []: break if interest is None: d.send_not_interested() else: d.example_interest = interest if not self.downloader.endgamemode and self.downloader.storage.is_endgame( ) and not (self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming): self.downloader.start_endgame() def fix_download_endgame(self, new_unchoke=False): if self.downloader.paused or self.connection.connection.is_coordinator_con( ): if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: paused', self.downloader.paused, 'or is_coordinator_con', self.connection.connection.is_coordinator_con( ) return if len(self.active_requests) >= self._backlog(new_unchoke): if not (self.active_requests or self.backlog) and not self.choked: self.downloader.queued_out[self] = 1 if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: returned' return want = [ a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests and (self.helper is None or self.connection.connection.is_helper_con() or not self.helper.is_ignored(a[0])) ] if not (self.active_requests or want): self.send_not_interested() if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: not interested' return if want: self.send_interested() if self.choked: if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: choked' return shuffle(want) del want[self.backlog - len(self.active_requests):] self.active_requests.extend(want) for piece, begin, length in want: if self.helper is None or self.connection.connection.is_helper_con( ) or self.helper.reserve_piece(piece, self): self.connection.send_request(piece, begin, length) self.downloader.chunk_requested(length) def got_invalidate(self, index): if DEBUG: log('downloader::got_invalidate: index', index) if not self.have[index]: return self.have[index] = False self.downloader.picker.lost_have(index) def got_have(self, index): if index == self.downloader.numpieces - 1: self.downloader.totalmeasure.update_rate( self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_length) self.peermeasure.update_rate(self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_length) else: self.downloader.totalmeasure.update_rate( self.downloader.storage.piece_length) self.peermeasure.update_rate(self.downloader.storage.piece_length) self.raw_have[index] = True if not self.downloader.picker.is_valid_piece(index): if DEBUG: print >> sys.stderr, 'Downloader::got_have: invalid piece: index', index, 'ip', self.connection.get_ip( ) if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and not self.connection.supports_piece_invalidate( ): i = self.downloader.picker.videostatus.live_piece_to_invalidate( index) if DEBUG: log('downloader::got_have: invalidate old piece: i', i, 'ip', self.connection.get_ip()) self.got_invalidate(i) if self.have[index]: return self.have[index] = True self.downloader.picker.got_have(index, self.connection) if DEBUG: print >> sys.stderr, '>>>debug: got have:', self.connection.get_ip( ), 'piece', index, 'have', debug_format_have( self.have ), 'choked', self.choked, 'interested', self.interested self.downloader.aggregate_and_send_haves() if self.have.complete(): self.downloader.picker.became_seed() if self.downloader.picker.am_I_complete(): self.downloader.add_disconnected_seed( self.connection.get_readable_id()) self.connection.close() return if self.downloader.endgamemode: self.fix_download_endgame() elif not self.downloader.paused and not self.downloader.picker.is_blocked( index) and self.downloader.storage.do_I_have_requests(index): if not self.choked: if DEBUG: log('downloader::got_have: not choked, request more') self._request_more() else: if DEBUG: log('downloader::got_have: choked, send interested') self.send_interested() elif DEBUG: print >> sys.stderr, 'downloader::got_have: do not request more: paused', self.downloader.paused, 'is_blocked', self.downloader.picker.is_blocked( index ), 'have_requests', self.downloader.storage.do_I_have_requests( index) def _check_interests(self): if self.interested or self.downloader.paused: return for i in xrange(len(self.have)): if self.have[i] and not self.downloader.picker.is_blocked(i) and ( self.downloader.endgamemode or self.downloader.storage.do_I_have_requests(i)): self.send_interested() return def got_have_bitfield(self, have): if self.downloader.picker.am_I_complete() and have.complete(): if self.downloader.super_seeding: self.connection.send_bitfield(have.tostring()) self.connection.try_send_pex() def auto_close(): self.connection.close() self.downloader.add_disconnected_seed( self.connection.get_readable_id()) self.downloader.scheduler(auto_close, REPEX_LISTEN_TIME) return if DEBUGBF: st = time.time() self.raw_have = have if have.complete(): self.downloader.picker.got_seed() else: activerangeiterators = [] if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and self.downloader.picker.videostatus.get_live_startpos( ) is None: activeranges = have.get_active_ranges() if len(activeranges) == 0: activerangeiterators = [ self.downloader.picker.get_valid_range_iterator() ] else: for s, e in activeranges: activerangeiterators.append(xrange(s, e + 1)) else: activerangeiterators = [ self.downloader.picker.get_valid_range_iterator( skip_filter=True) ] if DEBUGBF: print >> sys.stderr, 'Downloader: got_have_field: live: Filtering bitfield', activerangeiterators if DEBUGBF: print >> sys.stderr, 'Downloader: got_have_field: live or normal filter' validhave = Bitfield(self.downloader.numpieces) for iterator in activerangeiterators: for i in iterator: if have[i]: validhave[i] = True self.downloader.picker.got_have(i, self.connection) if DEBUG: print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip( ), 'have', debug_format_have(have) print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip( ), 'validhave', debug_format_have(validhave) self.downloader.aggregate_and_send_haves() have = validhave if DEBUGBF: et = time.time() diff = et - st print >> sys.stderr, 'Download: got_have_field: took', diff self.have = have if self.downloader.endgamemode and not self.downloader.paused: for piece, begin, length in self.downloader.all_requests: if self.have[piece]: self.send_interested() break return self._check_interests() def reset_have(self): if DEBUG: print >> sys.stderr, 'Downloader::reset_have: before self.have:', self.have.toboollist( ) validhave = Bitfield(self.downloader.numpieces) for i in self.downloader.picker.get_valid_range_iterator(): if self.raw_have[i]: validhave[i] = True self.have = validhave if DEBUG: print >> sys.stderr, 'Downloader::reset_have: after self.have:', self.have.toboollist( ) def get_rate(self): return self.measure.get_rate() def get_short_term_rate(self): return self.short_term_measure.get_rate() def is_snubbed(self, just_check=False): if not self.choked and not just_check and self.app_mode != 'node' and clock( ) - self.last2 > self.downloader.snub_time and not self.connection.connection.is_helper_con( ) and not self.connection.connection.is_coordinator_con(): for index, begin, length in self.active_requests: self.connection.send_cancel(index, begin, length) self.got_choke() return clock() - self.last > self.downloader.snub_time def peer_is_complete(self): return self.have.complete()
class SingleDownload(SingleDownloadHelperInterface): def __init__(self, downloader, connection): SingleDownloadHelperInterface.__init__(self) self.downloader = downloader self.connection = connection self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.max_rate_period) self.peermeasure = Measure(downloader.max_rate_period) self.raw_have = Bitfield(downloader.numpieces) self.have = Bitfield(downloader.numpieces) self.last = -1000 self.last2 = -1000 self.example_interest = None self.backlog = 2 self.ip = connection.get_ip() self.guard = BadDataGuard(self) self.app_mode = globalConfig.get_mode() self.white_list = None self.black_list = None self.app_mode = globalConfig.get_mode() if self.app_mode == 'node': source_node = globalConfig.get_value('source_node') support_nodes = globalConfig.get_value('support_nodes') if not globalConfig.get_value('allow_peers_download'): self.white_list = set() if source_node is not None and globalConfig.get_value('allow_source_download'): self.white_list.add(source_node[0]) if len(support_nodes) and globalConfig.get_value('allow_support_download'): self.white_list.update([ addr[0] for addr in support_nodes ]) else: self.black_list = set() if source_node is not None and not globalConfig.get_value('allow_source_download'): self.black_list.add(source_node[0]) if len(support_nodes) and not globalConfig.get_value('allow_support_download'): self.black_list.update([ addr[0] for addr in support_nodes ]) if len(self.black_list) == 0: self.black_list = None if DEBUG: log('download::__init__: white_list', self.white_list, 'black_list', self.black_list) self.helper = downloader.picker.helper self.proxy_have = Bitfield(downloader.numpieces) self.short_term_measure = Measure(5) self.bad_performance_counter = 0 def _backlog(self, just_unchoked): self.backlog = int(min(2 + int(4 * self.measure.get_rate() / self.downloader.chunksize), 2 * just_unchoked + self.downloader.queue_limit())) if DEBUG: log('downloader::sd::_backlog: backlog', self.backlog, 'rate', self.measure.get_rate(), 'chunksize', self.downloader.chunksize, 'just_unchoked', just_unchoked, 'queue_limit', self.downloader.queue_limit()) if self.backlog > 50: self.backlog = int(max(50, self.backlog * 0.075)) if DEBUG: log('downloader::sd::_backlog: fix backlog', self.backlog) return self.backlog def disconnected(self): self.downloader.lost_peer(self) if self.have.complete() and self.downloader.storage.is_endgame(): self.downloader.add_disconnected_seed(self.connection.get_readable_id()) self._letgo() self.guard.download = None def _letgo(self): if self.downloader.queued_out.has_key(self): del self.downloader.queued_out[self] if not self.active_requests: return if self.downloader.endgamemode: self.active_requests = [] return lost = {} for index, begin, length in self.active_requests: self.downloader.storage.request_lost(index, begin, length) lost[index] = 1 lost = lost.keys() self.active_requests = [] if self.downloader.paused: return ds = [ d for d in self.downloader.downloads if not d.choked ] shuffle(ds) for d in ds: d._request_more() for d in self.downloader.downloads: if d.choked and not d.interested: for l in lost: if d.have[l] and self.downloader.storage.do_I_have_requests(l): d.send_interested() break def got_choke(self): if not self.choked: if DEBUG: log('downloader::got_choke: got choke: ip', self.connection.get_ip()) self.choked = True self._letgo() elif DEBUG: log('downloader::got_choke: already choked: ip', self.connection.get_ip()) def got_unchoke(self): if self.choked: if DEBUG: log('downloader::got_unchoke: got unchoke: ip', self.connection.get_ip(), 'interested', self.interested) self.choked = False if self.interested: self._request_more(new_unchoke=True) self.last2 = clock() elif DEBUG: log('downloader::got_unchoke: already unchoked: ip', self.connection.get_ip()) def is_choked(self): return self.choked def is_interested(self): return self.interested def send_interested(self): if not self.interested: if DEBUG: log('downloader::send_interested: send interested: ip', self.connection.get_ip()) self.interested = True self.connection.send_interested() elif DEBUG: log('downloader::send_interested: already interested: ip', self.connection.get_ip()) def send_not_interested(self): if self.interested: if DEBUG: log('downloader::send_not_interested: send not interested: ip', self.connection.get_ip()) self.interested = False self.connection.send_not_interested() elif DEBUG: log('downloader::send_not_interested: already not interested: ip', self.connection.get_ip()) def got_piece(self, index, begin, hashlist, piece): if self.bad_performance_counter: self.bad_performance_counter -= 1 if DEBUG: print >> sys.stderr, 'decreased bad_performance_counter to', self.bad_performance_counter length = len(piece) try: self.active_requests.remove((index, begin, length)) except ValueError: self.downloader.discarded += length return False if self.downloader.endgamemode: self.downloader.all_requests.remove((index, begin, length)) if DEBUG: print >> sys.stderr, 'Downloader: got_piece: removed one request from all_requests', len(self.downloader.all_requests), 'remaining' self.last = clock() self.last2 = clock() self.measure.update_rate(length) self.short_term_measure.update_rate(length) self.downloader.measurefunc(length) if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard): self.downloader.piece_flunked(index) return False self.downloader.picker.got_piece(index, begin, length) if self.downloader.storage.do_I_have(index): self.downloader.picker.complete(index) if self.downloader.endgamemode: for d in self.downloader.downloads: if d is not self: if d.interested: if d.choked: d.fix_download_endgame() else: try: d.active_requests.remove((index, begin, length)) except ValueError: continue d.connection.send_cancel(index, begin, length) d.fix_download_endgame() self._request_more() self.downloader.check_complete(index) self.connection.total_downloaded += length return self.downloader.storage.do_I_have(index) def helper_forces_unchoke(self): self.choked = False def _request_more(self, new_unchoke = False, slowpieces = []): if self.helper is not None and self.is_frozen_by_helper(): if DEBUG: print >> sys.stderr, 'Downloader: _request_more: blocked, returning' return if self.app_mode == 'node': ip = self.connection.get_ip() if DEBUG: log('download::_request_more: check ip', ip) if self.white_list is not None and ip not in self.white_list: if DEBUG: log('download::_request_more: peer is not in the white list: ip', ip) return if self.black_list is not None and ip in self.black_list: if DEBUG: log('download::_request_more: peer is in the black list: ip', ip) return if self.choked: if DEBUG: print >> sys.stderr, 'Downloader: _request_more: choked, returning' return if self.connection.connection.is_coordinator_con(): if DEBUG: print >> sys.stderr, 'Downloader: _request_more: coordinator conn' return if self.downloader.endgamemode: self.fix_download_endgame(new_unchoke) if DEBUG: print >> sys.stderr, 'Downloader: _request_more: endgame mode, returning' return if self.downloader.paused: if DEBUG: print >> sys.stderr, 'Downloader: _request_more: paused, returning' return if len(self.active_requests) >= self._backlog(new_unchoke): if DEBUG: log('downloader::_request_more: more req than unchoke (active req: %d >= backlog: %d), download_rate=%d' % (len(self.active_requests), self._backlog(new_unchoke), self.downloader.download_rate)) if self.downloader.download_rate: wait_period = self.downloader.chunksize / self.downloader.download_rate / 2.0 if wait_period > 1.0: if DEBUG: print >> sys.stderr, 'Downloader: waiting for %f s to call _request_more again' % wait_period self.downloader.scheduler(self._request_more, wait_period) if not (self.active_requests or self.backlog): if DEBUG: print >> sys.stderr, 'Downloader::_request_more: queue out download' self.downloader.queued_out[self] = 1 return lost_interests = [] while len(self.active_requests) < self.backlog: interest = self.downloader.picker.next(self.have, self.downloader.storage.do_I_have_requests, self, self.downloader.too_many_partials(), self.connection.connection.is_helper_con(), slowpieces=slowpieces, connection=self.connection, proxyhave=self.proxy_have) diff = -1 if DEBUG: print >> sys.stderr, 'Downloader: _request_more: next() returned', interest, 'took %.5f' % diff if interest is None: break if self.helper and self.downloader.storage.inactive_requests[interest] is None: self.connection.send_have(interest) break if self.helper and self.downloader.storage.inactive_requests[interest] == []: break self.example_interest = interest self.send_interested() loop = True while len(self.active_requests) < self.backlog and loop: request = self.downloader.storage.new_request(interest) if request is None: log('downloader::_request_more: new_request returned none: index', interest) lost_interests.append(interest) break begin, length = request if DEBUG: log('downloader::_request_more: new_request', interest, begin, length, 'to', self.connection.connection.get_ip(), self.connection.connection.get_port()) self.downloader.picker.requested(interest, begin, length) self.active_requests.append((interest, begin, length)) self.connection.send_request(interest, begin, length) self.downloader.chunk_requested(length) if not self.downloader.storage.do_I_have_requests(interest): loop = False lost_interests.append(interest) if not self.active_requests: self.send_not_interested() if lost_interests: for d in self.downloader.downloads: if d.active_requests or not d.interested: continue if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest): continue for lost in lost_interests: if d.have[lost]: break else: continue interest = self.downloader.picker.next(d.have, self.downloader.storage.do_I_have_requests, self, self.downloader.too_many_partials(), self.connection.connection.is_helper_con(), willrequest=False, connection=self.connection, proxyhave=self.proxy_have) diff = -1 if DEBUG: print >> sys.stderr, 'Downloader: _request_more: next()2 returned', interest, 'took %.5f' % diff if interest is not None: if self.helper and self.downloader.storage.inactive_requests[interest] is None: self.connection.send_have(interest) break if self.helper and self.downloader.storage.inactive_requests[interest] == []: break if interest is None: d.send_not_interested() else: d.example_interest = interest if not self.downloader.endgamemode and self.downloader.storage.is_endgame() and not (self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming): self.downloader.start_endgame() def fix_download_endgame(self, new_unchoke = False): if self.downloader.paused or self.connection.connection.is_coordinator_con(): if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: paused', self.downloader.paused, 'or is_coordinator_con', self.connection.connection.is_coordinator_con() return if len(self.active_requests) >= self._backlog(new_unchoke): if not (self.active_requests or self.backlog) and not self.choked: self.downloader.queued_out[self] = 1 if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: returned' return want = [ a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests and (self.helper is None or self.connection.connection.is_helper_con() or not self.helper.is_ignored(a[0])) ] if not (self.active_requests or want): self.send_not_interested() if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: not interested' return if want: self.send_interested() if self.choked: if DEBUG: print >> sys.stderr, 'Downloader: fix_download_endgame: choked' return shuffle(want) del want[self.backlog - len(self.active_requests):] self.active_requests.extend(want) for piece, begin, length in want: if self.helper is None or self.connection.connection.is_helper_con() or self.helper.reserve_piece(piece, self): self.connection.send_request(piece, begin, length) self.downloader.chunk_requested(length) def got_invalidate(self, index): if DEBUG: log('downloader::got_invalidate: index', index) if not self.have[index]: return self.have[index] = False self.downloader.picker.lost_have(index) def got_have(self, index): if index == self.downloader.numpieces - 1: self.downloader.totalmeasure.update_rate(self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_length) self.peermeasure.update_rate(self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_length) else: self.downloader.totalmeasure.update_rate(self.downloader.storage.piece_length) self.peermeasure.update_rate(self.downloader.storage.piece_length) self.raw_have[index] = True if not self.downloader.picker.is_valid_piece(index): if DEBUG: print >> sys.stderr, 'Downloader::got_have: invalid piece: index', index, 'ip', self.connection.get_ip() if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and not self.connection.supports_piece_invalidate(): i = self.downloader.picker.videostatus.live_piece_to_invalidate(index) if DEBUG: log('downloader::got_have: invalidate old piece: i', i, 'ip', self.connection.get_ip()) self.got_invalidate(i) if self.have[index]: return self.have[index] = True self.downloader.picker.got_have(index, self.connection) if DEBUG: print >> sys.stderr, '>>>debug: got have:', self.connection.get_ip(), 'piece', index, 'have', debug_format_have(self.have), 'choked', self.choked, 'interested', self.interested self.downloader.aggregate_and_send_haves() if self.have.complete(): self.downloader.picker.became_seed() if self.downloader.picker.am_I_complete(): self.downloader.add_disconnected_seed(self.connection.get_readable_id()) self.connection.close() return if self.downloader.endgamemode: self.fix_download_endgame() elif not self.downloader.paused and not self.downloader.picker.is_blocked(index) and self.downloader.storage.do_I_have_requests(index): if not self.choked: if DEBUG: log('downloader::got_have: not choked, request more') self._request_more() else: if DEBUG: log('downloader::got_have: choked, send interested') self.send_interested() elif DEBUG: print >> sys.stderr, 'downloader::got_have: do not request more: paused', self.downloader.paused, 'is_blocked', self.downloader.picker.is_blocked(index), 'have_requests', self.downloader.storage.do_I_have_requests(index) def _check_interests(self): if self.interested or self.downloader.paused: return for i in xrange(len(self.have)): if self.have[i] and not self.downloader.picker.is_blocked(i) and (self.downloader.endgamemode or self.downloader.storage.do_I_have_requests(i)): self.send_interested() return def got_have_bitfield(self, have): if self.downloader.picker.am_I_complete() and have.complete(): if self.downloader.super_seeding: self.connection.send_bitfield(have.tostring()) self.connection.try_send_pex() def auto_close(): self.connection.close() self.downloader.add_disconnected_seed(self.connection.get_readable_id()) self.downloader.scheduler(auto_close, REPEX_LISTEN_TIME) return if DEBUGBF: st = time.time() self.raw_have = have if have.complete(): self.downloader.picker.got_seed() else: activerangeiterators = [] if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and self.downloader.picker.videostatus.get_live_startpos() is None: activeranges = have.get_active_ranges() if len(activeranges) == 0: activerangeiterators = [self.downloader.picker.get_valid_range_iterator()] else: for s, e in activeranges: activerangeiterators.append(xrange(s, e + 1)) else: activerangeiterators = [self.downloader.picker.get_valid_range_iterator(skip_filter=True)] if DEBUGBF: print >> sys.stderr, 'Downloader: got_have_field: live: Filtering bitfield', activerangeiterators if DEBUGBF: print >> sys.stderr, 'Downloader: got_have_field: live or normal filter' validhave = Bitfield(self.downloader.numpieces) for iterator in activerangeiterators: for i in iterator: if have[i]: validhave[i] = True self.downloader.picker.got_have(i, self.connection) if DEBUG: print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip(), 'have', debug_format_have(have) print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip(), 'validhave', debug_format_have(validhave) self.downloader.aggregate_and_send_haves() have = validhave if DEBUGBF: et = time.time() diff = et - st print >> sys.stderr, 'Download: got_have_field: took', diff self.have = have if self.downloader.endgamemode and not self.downloader.paused: for piece, begin, length in self.downloader.all_requests: if self.have[piece]: self.send_interested() break return self._check_interests() def reset_have(self): if DEBUG: print >> sys.stderr, 'Downloader::reset_have: before self.have:', self.have.toboollist() validhave = Bitfield(self.downloader.numpieces) for i in self.downloader.picker.get_valid_range_iterator(): if self.raw_have[i]: validhave[i] = True self.have = validhave if DEBUG: print >> sys.stderr, 'Downloader::reset_have: after self.have:', self.have.toboollist() def get_rate(self): return self.measure.get_rate() def get_short_term_rate(self): return self.short_term_measure.get_rate() def is_snubbed(self, just_check = False): if not self.choked and not just_check and self.app_mode != 'node' and clock() - self.last2 > self.downloader.snub_time and not self.connection.connection.is_helper_con() and not self.connection.connection.is_coordinator_con(): for index, begin, length in self.active_requests: self.connection.send_cancel(index, begin, length) self.got_choke() return clock() - self.last > self.downloader.snub_time def peer_is_complete(self): return self.have.complete()