class SingleDownload(SingleDownloadHelperInterface):
def __init__(self, downloader, connection):
SingleDownloadHelperInterface.__init__(self)
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.raw_have = Bitfield(downloader.numpieces)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
self.app_mode = globalConfig.get_mode()
self.white_list = None
self.black_list = None
self.app_mode = globalConfig.get_mode()
if self.app_mode == 'node':
source_node = globalConfig.get_value('source_node')
support_nodes = globalConfig.get_value('support_nodes')
if not globalConfig.get_value('allow_peers_download'):
self.white_list = set()
if source_node is not None and globalConfig.get_value(
'allow_source_download'):
self.white_list.add(source_node[0])
if len(support_nodes) and globalConfig.get_value(
'allow_support_download'):
self.white_list.update([addr[0] for addr in support_nodes])
else:
self.black_list = set()
if source_node is not None and not globalConfig.get_value(
'allow_source_download'):
self.black_list.add(source_node[0])
if len(support_nodes) and not globalConfig.get_value(
'allow_support_download'):
self.black_list.update([addr[0] for addr in support_nodes])
if len(self.black_list) == 0:
self.black_list = None
if DEBUG:
log('download::__init__: white_list', self.white_list,
'black_list', self.black_list)
self.helper = downloader.picker.helper
self.proxy_have = Bitfield(downloader.numpieces)
self.short_term_measure = Measure(5)
self.bad_performance_counter = 0
def _backlog(self, just_unchoked):
self.backlog = int(
min(
2 +
int(4 * self.measure.get_rate() / self.downloader.chunksize),
2 * just_unchoked + self.downloader.queue_limit()))
if DEBUG:
log('downloader::sd::_backlog: backlog', self.backlog, 'rate',
self.measure.get_rate(), 'chunksize',
self.downloader.chunksize, 'just_unchoked', just_unchoked,
'queue_limit', self.downloader.queue_limit())
if self.backlog > 50:
self.backlog = int(max(50, self.backlog * 0.075))
if DEBUG:
log('downloader::sd::_backlog: fix backlog', self.backlog)
return self.backlog
def disconnected(self):
self.downloader.lost_peer(self)
if self.have.complete() and self.downloader.storage.is_endgame():
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self._letgo()
self.guard.download = None
def _letgo(self):
if self.downloader.queued_out.has_key(self):
del self.downloader.queued_out[self]
if not self.active_requests:
return
if self.downloader.endgamemode:
self.active_requests = []
return
lost = {}
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
lost[index] = 1
lost = lost.keys()
self.active_requests = []
if self.downloader.paused:
return
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more()
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(
l):
d.send_interested()
break
def got_choke(self):
if not self.choked:
if DEBUG:
log('downloader::got_choke: got choke: ip',
self.connection.get_ip())
self.choked = True
self._letgo()
elif DEBUG:
log('downloader::got_choke: already choked: ip',
self.connection.get_ip())
def got_unchoke(self):
if self.choked:
if DEBUG:
log('downloader::got_unchoke: got unchoke: ip',
self.connection.get_ip(), 'interested', self.interested)
self.choked = False
if self.interested:
self._request_more(new_unchoke=True)
self.last2 = clock()
elif DEBUG:
log('downloader::got_unchoke: already unchoked: ip',
self.connection.get_ip())
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def send_interested(self):
if not self.interested:
if DEBUG:
log('downloader::send_interested: send interested: ip',
self.connection.get_ip())
self.interested = True
self.connection.send_interested()
elif DEBUG:
log('downloader::send_interested: already interested: ip',
self.connection.get_ip())
def send_not_interested(self):
if self.interested:
if DEBUG:
log('downloader::send_not_interested: send not interested: ip',
self.connection.get_ip())
self.interested = False
self.connection.send_not_interested()
elif DEBUG:
log('downloader::send_not_interested: already not interested: ip',
self.connection.get_ip())
def got_piece(self, index, begin, hashlist, piece):
if self.bad_performance_counter:
self.bad_performance_counter -= 1
if DEBUG:
print >> sys.stderr, 'decreased bad_performance_counter to', self.bad_performance_counter
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
if DEBUG:
print >> sys.stderr, 'Downloader: got_piece: removed one request from all_requests', len(
self.downloader.all_requests), 'remaining'
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.short_term_measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, hashlist,
piece, self.guard):
self.downloader.piece_flunked(index)
return False
self.downloader.picker.got_piece(index, begin, length)
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
d.fix_download_endgame()
else:
try:
d.active_requests.remove(
(index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
self._request_more()
self.downloader.check_complete(index)
self.connection.total_downloaded += length
return self.downloader.storage.do_I_have(index)
def helper_forces_unchoke(self):
self.choked = False
def _request_more(self, new_unchoke=False, slowpieces=[]):
if self.helper is not None and self.is_frozen_by_helper():
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: blocked, returning'
return
if self.app_mode == 'node':
ip = self.connection.get_ip()
if DEBUG:
log('download::_request_more: check ip', ip)
if self.white_list is not None and ip not in self.white_list:
if DEBUG:
log(
'download::_request_more: peer is not in the white list: ip',
ip)
return
if self.black_list is not None and ip in self.black_list:
if DEBUG:
log(
'download::_request_more: peer is in the black list: ip',
ip)
return
if self.choked:
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: choked, returning'
return
if self.connection.connection.is_coordinator_con():
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: coordinator conn'
return
if self.downloader.endgamemode:
self.fix_download_endgame(new_unchoke)
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: endgame mode, returning'
return
if self.downloader.paused:
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: paused, returning'
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if DEBUG:
log('downloader::_request_more: more req than unchoke (active req: %d >= backlog: %d), download_rate=%d'
% (len(self.active_requests), self._backlog(new_unchoke),
self.downloader.download_rate))
if self.downloader.download_rate:
wait_period = self.downloader.chunksize / self.downloader.download_rate / 2.0
if wait_period > 1.0:
if DEBUG:
print >> sys.stderr, 'Downloader: waiting for %f s to call _request_more again' % wait_period
self.downloader.scheduler(self._request_more, wait_period)
if not (self.active_requests or self.backlog):
if DEBUG:
print >> sys.stderr, 'Downloader::_request_more: queue out download'
self.downloader.queued_out[self] = 1
return
lost_interests = []
while len(self.active_requests) < self.backlog:
interest = self.downloader.picker.next(
self.have,
self.downloader.storage.do_I_have_requests,
self,
self.downloader.too_many_partials(),
self.connection.connection.is_helper_con(),
slowpieces=slowpieces,
connection=self.connection,
proxyhave=self.proxy_have)
diff = -1
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: next() returned', interest, 'took %.5f' % diff
if interest is None:
break
if self.helper and self.downloader.storage.inactive_requests[
interest] is None:
self.connection.send_have(interest)
break
if self.helper and self.downloader.storage.inactive_requests[
interest] == []:
break
self.example_interest = interest
self.send_interested()
loop = True
while len(self.active_requests) < self.backlog and loop:
request = self.downloader.storage.new_request(interest)
if request is None:
log(
'downloader::_request_more: new_request returned none: index',
interest)
lost_interests.append(interest)
break
begin, length = request
if DEBUG:
log('downloader::_request_more: new_request',
interest, begin, length, 'to',
self.connection.connection.get_ip(),
self.connection.connection.get_port())
self.downloader.picker.requested(interest, begin, length)
self.active_requests.append((interest, begin, length))
self.connection.send_request(interest, begin, length)
self.downloader.chunk_requested(length)
if not self.downloader.storage.do_I_have_requests(interest):
loop = False
lost_interests.append(interest)
if not self.active_requests:
self.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(
d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(
d.have,
self.downloader.storage.do_I_have_requests,
self,
self.downloader.too_many_partials(),
self.connection.connection.is_helper_con(),
willrequest=False,
connection=self.connection,
proxyhave=self.proxy_have)
diff = -1
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: next()2 returned', interest, 'took %.5f' % diff
if interest is not None:
if self.helper and self.downloader.storage.inactive_requests[
interest] is None:
self.connection.send_have(interest)
break
if self.helper and self.downloader.storage.inactive_requests[
interest] == []:
break
if interest is None:
d.send_not_interested()
else:
d.example_interest = interest
if not self.downloader.endgamemode and self.downloader.storage.is_endgame(
) and not (self.downloader.picker.videostatus
and self.downloader.picker.videostatus.live_streaming):
self.downloader.start_endgame()
def fix_download_endgame(self, new_unchoke=False):
if self.downloader.paused or self.connection.connection.is_coordinator_con(
):
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: paused', self.downloader.paused, 'or is_coordinator_con', self.connection.connection.is_coordinator_con(
)
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog) and not self.choked:
self.downloader.queued_out[self] = 1
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: returned'
return
want = [
a for a in self.downloader.all_requests
if self.have[a[0]] and a not in self.active_requests and
(self.helper is None or self.connection.connection.is_helper_con()
or not self.helper.is_ignored(a[0]))
]
if not (self.active_requests or want):
self.send_not_interested()
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: not interested'
return
if want:
self.send_interested()
if self.choked:
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: choked'
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
if self.helper is None or self.connection.connection.is_helper_con(
) or self.helper.reserve_piece(piece, self):
self.connection.send_request(piece, begin, length)
self.downloader.chunk_requested(length)
def got_invalidate(self, index):
if DEBUG:
log('downloader::got_invalidate: index', index)
if not self.have[index]:
return
self.have[index] = False
self.downloader.picker.lost_have(index)
def got_have(self, index):
if index == self.downloader.numpieces - 1:
self.downloader.totalmeasure.update_rate(
self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_length)
else:
self.downloader.totalmeasure.update_rate(
self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.piece_length)
self.raw_have[index] = True
if not self.downloader.picker.is_valid_piece(index):
if DEBUG:
print >> sys.stderr, 'Downloader::got_have: invalid piece: index', index, 'ip', self.connection.get_ip(
)
if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and not self.connection.supports_piece_invalidate(
):
i = self.downloader.picker.videostatus.live_piece_to_invalidate(
index)
if DEBUG:
log('downloader::got_have: invalidate old piece: i', i, 'ip',
self.connection.get_ip())
self.got_invalidate(i)
if self.have[index]:
return
self.have[index] = True
self.downloader.picker.got_have(index, self.connection)
if DEBUG:
print >> sys.stderr, '>>>debug: got have:', self.connection.get_ip(
), 'piece', index, 'have', debug_format_have(
self.have
), 'choked', self.choked, 'interested', self.interested
self.downloader.aggregate_and_send_haves()
if self.have.complete():
self.downloader.picker.became_seed()
if self.downloader.picker.am_I_complete():
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self.connection.close()
return
if self.downloader.endgamemode:
self.fix_download_endgame()
elif not self.downloader.paused and not self.downloader.picker.is_blocked(
index) and self.downloader.storage.do_I_have_requests(index):
if not self.choked:
if DEBUG:
log('downloader::got_have: not choked, request more')
self._request_more()
else:
if DEBUG:
log('downloader::got_have: choked, send interested')
self.send_interested()
elif DEBUG:
print >> sys.stderr, 'downloader::got_have: do not request more: paused', self.downloader.paused, 'is_blocked', self.downloader.picker.is_blocked(
index
), 'have_requests', self.downloader.storage.do_I_have_requests(
index)
def _check_interests(self):
if self.interested or self.downloader.paused:
return
for i in xrange(len(self.have)):
if self.have[i] and not self.downloader.picker.is_blocked(i) and (
self.downloader.endgamemode
or self.downloader.storage.do_I_have_requests(i)):
self.send_interested()
return
def got_have_bitfield(self, have):
if self.downloader.picker.am_I_complete() and have.complete():
if self.downloader.super_seeding:
self.connection.send_bitfield(have.tostring())
self.connection.try_send_pex()
def auto_close():
self.connection.close()
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self.downloader.scheduler(auto_close, REPEX_LISTEN_TIME)
return
if DEBUGBF:
st = time.time()
self.raw_have = have
if have.complete():
self.downloader.picker.got_seed()
else:
activerangeiterators = []
if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and self.downloader.picker.videostatus.get_live_startpos(
) is None:
activeranges = have.get_active_ranges()
if len(activeranges) == 0:
activerangeiterators = [
self.downloader.picker.get_valid_range_iterator()
]
else:
for s, e in activeranges:
activerangeiterators.append(xrange(s, e + 1))
else:
activerangeiterators = [
self.downloader.picker.get_valid_range_iterator(
skip_filter=True)
]
if DEBUGBF:
print >> sys.stderr, 'Downloader: got_have_field: live: Filtering bitfield', activerangeiterators
if DEBUGBF:
print >> sys.stderr, 'Downloader: got_have_field: live or normal filter'
validhave = Bitfield(self.downloader.numpieces)
for iterator in activerangeiterators:
for i in iterator:
if have[i]:
validhave[i] = True
self.downloader.picker.got_have(i, self.connection)
if DEBUG:
print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip(
), 'have', debug_format_have(have)
print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip(
), 'validhave', debug_format_have(validhave)
self.downloader.aggregate_and_send_haves()
have = validhave
if DEBUGBF:
et = time.time()
diff = et - st
print >> sys.stderr, 'Download: got_have_field: took', diff
self.have = have
if self.downloader.endgamemode and not self.downloader.paused:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.send_interested()
break
return
self._check_interests()
def reset_have(self):
if DEBUG:
print >> sys.stderr, 'Downloader::reset_have: before self.have:', self.have.toboollist(
)
validhave = Bitfield(self.downloader.numpieces)
for i in self.downloader.picker.get_valid_range_iterator():
if self.raw_have[i]:
validhave[i] = True
self.have = validhave
if DEBUG:
print >> sys.stderr, 'Downloader::reset_have: after self.have:', self.have.toboollist(
)
def get_rate(self):
return self.measure.get_rate()
def get_short_term_rate(self):
return self.short_term_measure.get_rate()
def is_snubbed(self, just_check=False):
if not self.choked and not just_check and self.app_mode != 'node' and clock(
) - self.last2 > self.downloader.snub_time and not self.connection.connection.is_helper_con(
) and not self.connection.connection.is_coordinator_con():
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke()
return clock() - self.last > self.downloader.snub_time
def peer_is_complete(self):
return self.have.complete()
class Upload:
def __init__(self, connection, ratelimiter, totalup, choker, storage, picker, config):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.config = config
self.max_slice_length = config['max_slice_length']
self.choked = True
self.cleared = True
self.interested = False
self.super_seeding = False
self.buffer = []
self.measure = Measure(config['max_rate_period'], config['upload_rate_fudge'])
self.was_ever_interested = False
if storage.get_amount_left() == 0:
if choker.super_seed:
self.super_seeding = True
self.seed_have_list = []
self.skipped_count = 0
elif config['breakup_seed_bitfield']:
bitfield, msgs = storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(storage.get_have_list())
elif storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
self.hashlist = []
def send_haves(self, connection):
have_list = self.storage.get_have_list()
print >> sys.stderr, 'Have list:', have_list
def send_bitfield(self, connection):
if self.storage.get_amount_left() == 0:
if not self.super_seeding:
if self.config['breakup_seed_bitfield']:
bitfield, msgs = self.storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(self.storage.get_have_list())
elif self.storage.do_I_have_anything():
connection.send_bitfield(self.storage.get_have_list())
def got_not_interested(self):
if self.interested:
self.interested = False
del self.buffer[:]
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
self.choker.not_interested(self.connection)
def got_interested(self):
if not self.interested:
self.interested = True
self.was_ever_interested = True
self.choker.interested(self.connection)
def get_upload_chunk(self):
if self.choked or not self.buffer:
return
index, begin, length = self.buffer.pop(0)
if False and self.config['buffer_reads']:
if index != self.piecedl:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = index
self.piecebuf, self.hashlist = self.storage.get_piece(index, 0, -1)
try:
piece = self.piecebuf[begin:begin + length]
except:
self.connection.close()
return
if begin == 0:
hashlist = self.hashlist
else:
hashlist = []
else:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = None
piece, hashlist = self.storage.get_piece(index, begin, length)
if piece is None:
self.connection.close()
return
self.measure.update_rate(len(piece))
self.totalup.update_rate(len(piece))
self.connection.total_uploaded += length
return (index,
begin,
hashlist,
piece)
def got_request(self, index, begin, length):
if self.super_seeding and index not in self.seed_have_list or not self.connection.connection.is_coordinator_con() and not self.interested or length > self.max_slice_length:
self.connection.close()
return
if not self.cleared:
self.buffer.append((index, begin, length))
if not self.choked and self.connection.next_upload is None:
self.ratelimiter.queue(self.connection)
def got_cancel(self, index, begin, length):
try:
self.buffer.remove((index, begin, length))
except ValueError:
pass
def choke(self):
if not self.choked:
if DEBUG:
log('uploader::choke: ip', self.connection.get_ip(), 'port', self.connection.get_port())
self.choked = True
self.connection.send_choke()
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def choke_sent(self):
del self.buffer[:]
self.cleared = True
def unchoke(self):
if self.choked:
try:
if DEBUG:
log('uploader::unchoke: ip', self.connection.get_ip(), 'port', self.connection.get_port())
if self.connection.send_unchoke():
self.choked = False
self.cleared = False
except:
pass
def disconnected(self):
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def has_queries(self):
return not self.choked and self.buffer
def get_rate(self):
return self.measure.get_rate()
class Upload:
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, config):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.config = config
self.max_slice_length = config['max_slice_length']
self.choked = True
self.cleared = True
self.interested = False
self.super_seeding = False
self.buffer = []
self.measure = Measure(config['max_rate_period'],
config['upload_rate_fudge'])
self.was_ever_interested = False
if storage.get_amount_left() == 0:
if choker.super_seed:
self.super_seeding = True
self.seed_have_list = []
self.skipped_count = 0
elif config['breakup_seed_bitfield']:
bitfield, msgs = storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(storage.get_have_list())
elif storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
self.hashlist = []
def send_haves(self, connection):
have_list = self.storage.get_have_list()
print >> sys.stderr, 'Have list:', have_list
def send_bitfield(self, connection):
if self.storage.get_amount_left() == 0:
if not self.super_seeding:
if self.config['breakup_seed_bitfield']:
bitfield, msgs = self.storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(self.storage.get_have_list())
elif self.storage.do_I_have_anything():
connection.send_bitfield(self.storage.get_have_list())
def got_not_interested(self):
if self.interested:
self.interested = False
del self.buffer[:]
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
self.choker.not_interested(self.connection)
def got_interested(self):
if not self.interested:
self.interested = True
self.was_ever_interested = True
self.choker.interested(self.connection)
def get_upload_chunk(self):
if self.choked or not self.buffer:
return
index, begin, length = self.buffer.pop(0)
if False and self.config['buffer_reads']:
if index != self.piecedl:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = index
self.piecebuf, self.hashlist = self.storage.get_piece(
index, 0, -1)
try:
piece = self.piecebuf[begin:begin + length]
except:
self.connection.close()
return
if begin == 0:
hashlist = self.hashlist
else:
hashlist = []
else:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = None
piece, hashlist = self.storage.get_piece(index, begin, length)
if piece is None:
self.connection.close()
return
self.measure.update_rate(len(piece))
self.totalup.update_rate(len(piece))
self.connection.total_uploaded += length
return (index, begin, hashlist, piece)
def got_request(self, index, begin, length):
if self.super_seeding and index not in self.seed_have_list or not self.connection.connection.is_coordinator_con(
) and not self.interested or length > self.max_slice_length:
self.connection.close()
return
if not self.cleared:
self.buffer.append((index, begin, length))
if not self.choked and self.connection.next_upload is None:
self.ratelimiter.queue(self.connection)
def got_cancel(self, index, begin, length):
try:
self.buffer.remove((index, begin, length))
except ValueError:
pass
def choke(self):
if not self.choked:
if DEBUG:
log('uploader::choke: ip', self.connection.get_ip(), 'port',
self.connection.get_port())
self.choked = True
self.connection.send_choke()
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def choke_sent(self):
del self.buffer[:]
self.cleared = True
def unchoke(self):
if self.choked:
try:
if DEBUG:
log('uploader::unchoke: ip', self.connection.get_ip(),
'port', self.connection.get_port())
if self.connection.send_unchoke():
self.choked = False
self.cleared = False
except:
pass
def disconnected(self):
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def has_queries(self):
return not self.choked and self.buffer
def get_rate(self):
return self.measure.get_rate()
class SingleDownload(SingleDownloadHelperInterface):
def __init__(self, downloader, connection):
SingleDownloadHelperInterface.__init__(self)
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.raw_have = Bitfield(downloader.numpieces)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
self.app_mode = globalConfig.get_mode()
self.white_list = None
self.black_list = None
self.app_mode = globalConfig.get_mode()
if self.app_mode == 'node':
source_node = globalConfig.get_value('source_node')
support_nodes = globalConfig.get_value('support_nodes')
if not globalConfig.get_value('allow_peers_download'):
self.white_list = set()
if source_node is not None and globalConfig.get_value('allow_source_download'):
self.white_list.add(source_node[0])
if len(support_nodes) and globalConfig.get_value('allow_support_download'):
self.white_list.update([ addr[0] for addr in support_nodes ])
else:
self.black_list = set()
if source_node is not None and not globalConfig.get_value('allow_source_download'):
self.black_list.add(source_node[0])
if len(support_nodes) and not globalConfig.get_value('allow_support_download'):
self.black_list.update([ addr[0] for addr in support_nodes ])
if len(self.black_list) == 0:
self.black_list = None
if DEBUG:
log('download::__init__: white_list', self.white_list, 'black_list', self.black_list)
self.helper = downloader.picker.helper
self.proxy_have = Bitfield(downloader.numpieces)
self.short_term_measure = Measure(5)
self.bad_performance_counter = 0
def _backlog(self, just_unchoked):
self.backlog = int(min(2 + int(4 * self.measure.get_rate() / self.downloader.chunksize), 2 * just_unchoked + self.downloader.queue_limit()))
if DEBUG:
log('downloader::sd::_backlog: backlog', self.backlog, 'rate', self.measure.get_rate(), 'chunksize', self.downloader.chunksize, 'just_unchoked', just_unchoked, 'queue_limit', self.downloader.queue_limit())
if self.backlog > 50:
self.backlog = int(max(50, self.backlog * 0.075))
if DEBUG:
log('downloader::sd::_backlog: fix backlog', self.backlog)
return self.backlog
def disconnected(self):
self.downloader.lost_peer(self)
if self.have.complete() and self.downloader.storage.is_endgame():
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self._letgo()
self.guard.download = None
def _letgo(self):
if self.downloader.queued_out.has_key(self):
del self.downloader.queued_out[self]
if not self.active_requests:
return
if self.downloader.endgamemode:
self.active_requests = []
return
lost = {}
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
lost[index] = 1
lost = lost.keys()
self.active_requests = []
if self.downloader.paused:
return
ds = [ d for d in self.downloader.downloads if not d.choked ]
shuffle(ds)
for d in ds:
d._request_more()
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(l):
d.send_interested()
break
def got_choke(self):
if not self.choked:
if DEBUG:
log('downloader::got_choke: got choke: ip', self.connection.get_ip())
self.choked = True
self._letgo()
elif DEBUG:
log('downloader::got_choke: already choked: ip', self.connection.get_ip())
def got_unchoke(self):
if self.choked:
if DEBUG:
log('downloader::got_unchoke: got unchoke: ip', self.connection.get_ip(), 'interested', self.interested)
self.choked = False
if self.interested:
self._request_more(new_unchoke=True)
self.last2 = clock()
elif DEBUG:
log('downloader::got_unchoke: already unchoked: ip', self.connection.get_ip())
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def send_interested(self):
if not self.interested:
if DEBUG:
log('downloader::send_interested: send interested: ip', self.connection.get_ip())
self.interested = True
self.connection.send_interested()
elif DEBUG:
log('downloader::send_interested: already interested: ip', self.connection.get_ip())
def send_not_interested(self):
if self.interested:
if DEBUG:
log('downloader::send_not_interested: send not interested: ip', self.connection.get_ip())
self.interested = False
self.connection.send_not_interested()
elif DEBUG:
log('downloader::send_not_interested: already not interested: ip', self.connection.get_ip())
def got_piece(self, index, begin, hashlist, piece):
if self.bad_performance_counter:
self.bad_performance_counter -= 1
if DEBUG:
print >> sys.stderr, 'decreased bad_performance_counter to', self.bad_performance_counter
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
if DEBUG:
print >> sys.stderr, 'Downloader: got_piece: removed one request from all_requests', len(self.downloader.all_requests), 'remaining'
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.short_term_measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, hashlist, piece, self.guard):
self.downloader.piece_flunked(index)
return False
self.downloader.picker.got_piece(index, begin, length)
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
self._request_more()
self.downloader.check_complete(index)
self.connection.total_downloaded += length
return self.downloader.storage.do_I_have(index)
def helper_forces_unchoke(self):
self.choked = False
def _request_more(self, new_unchoke = False, slowpieces = []):
if self.helper is not None and self.is_frozen_by_helper():
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: blocked, returning'
return
if self.app_mode == 'node':
ip = self.connection.get_ip()
if DEBUG:
log('download::_request_more: check ip', ip)
if self.white_list is not None and ip not in self.white_list:
if DEBUG:
log('download::_request_more: peer is not in the white list: ip', ip)
return
if self.black_list is not None and ip in self.black_list:
if DEBUG:
log('download::_request_more: peer is in the black list: ip', ip)
return
if self.choked:
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: choked, returning'
return
if self.connection.connection.is_coordinator_con():
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: coordinator conn'
return
if self.downloader.endgamemode:
self.fix_download_endgame(new_unchoke)
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: endgame mode, returning'
return
if self.downloader.paused:
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: paused, returning'
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if DEBUG:
log('downloader::_request_more: more req than unchoke (active req: %d >= backlog: %d), download_rate=%d' % (len(self.active_requests), self._backlog(new_unchoke), self.downloader.download_rate))
if self.downloader.download_rate:
wait_period = self.downloader.chunksize / self.downloader.download_rate / 2.0
if wait_period > 1.0:
if DEBUG:
print >> sys.stderr, 'Downloader: waiting for %f s to call _request_more again' % wait_period
self.downloader.scheduler(self._request_more, wait_period)
if not (self.active_requests or self.backlog):
if DEBUG:
print >> sys.stderr, 'Downloader::_request_more: queue out download'
self.downloader.queued_out[self] = 1
return
lost_interests = []
while len(self.active_requests) < self.backlog:
interest = self.downloader.picker.next(self.have, self.downloader.storage.do_I_have_requests, self, self.downloader.too_many_partials(), self.connection.connection.is_helper_con(), slowpieces=slowpieces, connection=self.connection, proxyhave=self.proxy_have)
diff = -1
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: next() returned', interest, 'took %.5f' % diff
if interest is None:
break
if self.helper and self.downloader.storage.inactive_requests[interest] is None:
self.connection.send_have(interest)
break
if self.helper and self.downloader.storage.inactive_requests[interest] == []:
break
self.example_interest = interest
self.send_interested()
loop = True
while len(self.active_requests) < self.backlog and loop:
request = self.downloader.storage.new_request(interest)
if request is None:
log('downloader::_request_more: new_request returned none: index', interest)
lost_interests.append(interest)
break
begin, length = request
if DEBUG:
log('downloader::_request_more: new_request', interest, begin, length, 'to', self.connection.connection.get_ip(), self.connection.connection.get_port())
self.downloader.picker.requested(interest, begin, length)
self.active_requests.append((interest, begin, length))
self.connection.send_request(interest, begin, length)
self.downloader.chunk_requested(length)
if not self.downloader.storage.do_I_have_requests(interest):
loop = False
lost_interests.append(interest)
if not self.active_requests:
self.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(d.have, self.downloader.storage.do_I_have_requests, self, self.downloader.too_many_partials(), self.connection.connection.is_helper_con(), willrequest=False, connection=self.connection, proxyhave=self.proxy_have)
diff = -1
if DEBUG:
print >> sys.stderr, 'Downloader: _request_more: next()2 returned', interest, 'took %.5f' % diff
if interest is not None:
if self.helper and self.downloader.storage.inactive_requests[interest] is None:
self.connection.send_have(interest)
break
if self.helper and self.downloader.storage.inactive_requests[interest] == []:
break
if interest is None:
d.send_not_interested()
else:
d.example_interest = interest
if not self.downloader.endgamemode and self.downloader.storage.is_endgame() and not (self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming):
self.downloader.start_endgame()
def fix_download_endgame(self, new_unchoke = False):
if self.downloader.paused or self.connection.connection.is_coordinator_con():
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: paused', self.downloader.paused, 'or is_coordinator_con', self.connection.connection.is_coordinator_con()
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog) and not self.choked:
self.downloader.queued_out[self] = 1
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: returned'
return
want = [ a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests and (self.helper is None or self.connection.connection.is_helper_con() or not self.helper.is_ignored(a[0])) ]
if not (self.active_requests or want):
self.send_not_interested()
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: not interested'
return
if want:
self.send_interested()
if self.choked:
if DEBUG:
print >> sys.stderr, 'Downloader: fix_download_endgame: choked'
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
if self.helper is None or self.connection.connection.is_helper_con() or self.helper.reserve_piece(piece, self):
self.connection.send_request(piece, begin, length)
self.downloader.chunk_requested(length)
def got_invalidate(self, index):
if DEBUG:
log('downloader::got_invalidate: index', index)
if not self.have[index]:
return
self.have[index] = False
self.downloader.picker.lost_have(index)
def got_have(self, index):
if index == self.downloader.numpieces - 1:
self.downloader.totalmeasure.update_rate(self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.total_length - (self.downloader.numpieces - 1) * self.downloader.storage.piece_length)
else:
self.downloader.totalmeasure.update_rate(self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.piece_length)
self.raw_have[index] = True
if not self.downloader.picker.is_valid_piece(index):
if DEBUG:
print >> sys.stderr, 'Downloader::got_have: invalid piece: index', index, 'ip', self.connection.get_ip()
if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and not self.connection.supports_piece_invalidate():
i = self.downloader.picker.videostatus.live_piece_to_invalidate(index)
if DEBUG:
log('downloader::got_have: invalidate old piece: i', i, 'ip', self.connection.get_ip())
self.got_invalidate(i)
if self.have[index]:
return
self.have[index] = True
self.downloader.picker.got_have(index, self.connection)
if DEBUG:
print >> sys.stderr, '>>>debug: got have:', self.connection.get_ip(), 'piece', index, 'have', debug_format_have(self.have), 'choked', self.choked, 'interested', self.interested
self.downloader.aggregate_and_send_haves()
if self.have.complete():
self.downloader.picker.became_seed()
if self.downloader.picker.am_I_complete():
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self.connection.close()
return
if self.downloader.endgamemode:
self.fix_download_endgame()
elif not self.downloader.paused and not self.downloader.picker.is_blocked(index) and self.downloader.storage.do_I_have_requests(index):
if not self.choked:
if DEBUG:
log('downloader::got_have: not choked, request more')
self._request_more()
else:
if DEBUG:
log('downloader::got_have: choked, send interested')
self.send_interested()
elif DEBUG:
print >> sys.stderr, 'downloader::got_have: do not request more: paused', self.downloader.paused, 'is_blocked', self.downloader.picker.is_blocked(index), 'have_requests', self.downloader.storage.do_I_have_requests(index)
def _check_interests(self):
if self.interested or self.downloader.paused:
return
for i in xrange(len(self.have)):
if self.have[i] and not self.downloader.picker.is_blocked(i) and (self.downloader.endgamemode or self.downloader.storage.do_I_have_requests(i)):
self.send_interested()
return
def got_have_bitfield(self, have):
if self.downloader.picker.am_I_complete() and have.complete():
if self.downloader.super_seeding:
self.connection.send_bitfield(have.tostring())
self.connection.try_send_pex()
def auto_close():
self.connection.close()
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self.downloader.scheduler(auto_close, REPEX_LISTEN_TIME)
return
if DEBUGBF:
st = time.time()
self.raw_have = have
if have.complete():
self.downloader.picker.got_seed()
else:
activerangeiterators = []
if self.downloader.picker.videostatus and self.downloader.picker.videostatus.live_streaming and self.downloader.picker.videostatus.get_live_startpos() is None:
activeranges = have.get_active_ranges()
if len(activeranges) == 0:
activerangeiterators = [self.downloader.picker.get_valid_range_iterator()]
else:
for s, e in activeranges:
activerangeiterators.append(xrange(s, e + 1))
else:
activerangeiterators = [self.downloader.picker.get_valid_range_iterator(skip_filter=True)]
if DEBUGBF:
print >> sys.stderr, 'Downloader: got_have_field: live: Filtering bitfield', activerangeiterators
if DEBUGBF:
print >> sys.stderr, 'Downloader: got_have_field: live or normal filter'
validhave = Bitfield(self.downloader.numpieces)
for iterator in activerangeiterators:
for i in iterator:
if have[i]:
validhave[i] = True
self.downloader.picker.got_have(i, self.connection)
if DEBUG:
print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip(), 'have', debug_format_have(have)
print >> sys.stderr, '>>>debug: got bitfield:', self.connection.get_ip(), 'validhave', debug_format_have(validhave)
self.downloader.aggregate_and_send_haves()
have = validhave
if DEBUGBF:
et = time.time()
diff = et - st
print >> sys.stderr, 'Download: got_have_field: took', diff
self.have = have
if self.downloader.endgamemode and not self.downloader.paused:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.send_interested()
break
return
self._check_interests()
def reset_have(self):
if DEBUG:
print >> sys.stderr, 'Downloader::reset_have: before self.have:', self.have.toboollist()
validhave = Bitfield(self.downloader.numpieces)
for i in self.downloader.picker.get_valid_range_iterator():
if self.raw_have[i]:
validhave[i] = True
self.have = validhave
if DEBUG:
print >> sys.stderr, 'Downloader::reset_have: after self.have:', self.have.toboollist()
def get_rate(self):
return self.measure.get_rate()
def get_short_term_rate(self):
return self.short_term_measure.get_rate()
def is_snubbed(self, just_check = False):
if not self.choked and not just_check and self.app_mode != 'node' and clock() - self.last2 > self.downloader.snub_time and not self.connection.connection.is_helper_con() and not self.connection.connection.is_coordinator_con():
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke()
return clock() - self.last > self.downloader.snub_time
def peer_is_complete(self):
return self.have.complete()