def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, max_slice_length, max_rate_period, fudge,
buffer_reads):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.max_slice_length = max_slice_length
self.max_rate_period = max_rate_period
self.buffer_reads = buffer_reads
self.choked = True
self.cleared = True
self.interested = False
self.buffer = []
self.measure = Measure(max_rate_period, fudge)
self.was_ever_interested = False
if storage.get_amount_left() == 0 and choker.super_seed:
self.super_seeding = True # flag, and don't send bitfield
self.seed_have_list = [] # set from piecepicker
self.skipped_count = 0
else:
self.super_seeding = False
if storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def __init__(self, storage, picker, backlog, max_rate_period, numpieces,
chunksize, measurefunc, snub_time, kickbans_ok, kickfunc,
banfunc):
self.storage = storage
self.picker = picker
self.backlog = backlog
self.max_rate_period = max_rate_period
self.measurefunc = measurefunc
self.totalmeasure = Measure(max_rate_period * storage.piece_length /
storage.request_size)
self.numpieces = numpieces
self.chunksize = chunksize
self.snub_time = snub_time
self.kickfunc = kickfunc
self.banfunc = banfunc
self.disconnectedseeds = {}
self.downloads = []
self.perip = {}
self.gotbaddata = {}
self.kicked = {}
self.banned = {}
self.kickbans_ok = kickbans_ok
self.kickbans_halted = False
self.super_seeding = False
self.endgamemode = False
self.endgame_queued_pieces = []
self.all_requests = []
self.discarded = 0L
# self.download_rate = 25000 # 25K/s test rate
self.download_rate = 0
self.bytes_requested = 0
self.last_time = clock()
self.queued_out = {}
self.requeueing = False
self.paused = False
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, max_slice_length, max_rate_period, fudge, buffer_reads):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.max_slice_length = max_slice_length
self.max_rate_period = max_rate_period
self.buffer_reads = buffer_reads
self.choked = True
self.cleared = True
self.interested = False
self.buffer = []
self.measure = Measure(max_rate_period, fudge)
self.was_ever_interested = False
if storage.get_amount_left() == 0 and choker.super_seed:
self.super_seeding = True # flag, and don't send bitfield
self.seed_have_list = [] # set from piecepicker
self.skipped_count = 0
else:
self.super_seeding = False
if storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, config):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.config = config
self.max_slice_length = config['max_slice_length']
self.choked = True
self.cleared = True
self.interested = False
self.super_seeding = False
self.buffer = []
self.measure = Measure(config['max_rate_period'], config['upload_rate_fudge'])
self.was_ever_interested = False
if storage.get_amount_left() == 0:
if choker.super_seed:
self.super_seeding = True # flag, and don't send bitfield
self.seed_have_list = [] # set from piecepicker
self.skipped_count = 0
else:
if config['breakup_seed_bitfield']:
bitfield, msgs = storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(storage.get_have_list())
else:
if storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, config):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.config = config
self.max_slice_length = config['max_slice_length']
self.choked = True
self.cleared = True
self.interested = False
self.super_seeding = False
# ASSAF: I think this is the list of requested pieces
# Every entry is a tuple in the form of (index, begin, length)
self.buffer = []
self.measure = Measure(config['max_rate_period'],
config['upload_rate_fudge'])
self.was_ever_interested = False
if storage.get_amount_left() == 0:
if choker.super_seed:
self.super_seeding = True # flag, and don't send bitfield
self.seed_have_list = [] # set from piecepicker
self.skipped_count = 0
else:
if config['breakup_seed_bitfield']:
bitfield, msgs = storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(storage.get_have_list())
else:
if storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
def __init__(self, downloader, url):
self.downloader = downloader
self.baseurl = url
try:
(scheme, self.netloc, path, pars, query, fragment) = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
try:
self.connection = HTTPConnection(self.netloc)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
def __init__(self, downloader, url):
self.downloader = downloader
self.baseurl = url
try:
(scheme, self.netloc, path, pars, query, fragment) = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
try:
self.connection = HTTPConnection(self.netloc)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
class Upload:
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, config):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.config = config
self.max_slice_length = config['max_slice_length']
self.choked = True
self.cleared = True
self.interested = False
self.super_seeding = False
self.buffer = []
self.measure = Measure(config['max_rate_period'],
config['upload_rate_fudge'])
self.was_ever_interested = False
if storage.get_amount_left() == 0:
if choker.super_seed:
self.super_seeding = True # flag, and don't send bitfield
self.seed_have_list = [] # set from piecepicker
self.skipped_count = 0
else:
if config['breakup_seed_bitfield']:
bitfield, msgs = storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(storage.get_have_list())
else:
if storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
def got_not_interested(self):
if self.interested:
self.interested = False
del self.buffer[:]
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
self.choker.not_interested(self.connection)
def got_interested(self):
if not self.interested:
self.interested = True
self.was_ever_interested = True
self.choker.interested(self.connection)
def get_upload_chunk(self):
if self.choked or not self.buffer:
return None
index, begin, length = self.buffer.pop(0)
if self.config['buffer_reads']:
if index != self.piecedl:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = index
self.piecebuf = self.storage.get_piece(index, 0, -1)
try:
piece = self.piecebuf[begin:begin + length]
assert len(piece) == length
except:
# fails if storage.get_piece returns None or if out of range
self.connection.close()
return None
else:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = None
piece = self.storage.get_piece(index, begin, length)
if piece is None:
self.connection.close()
return None
self.measure.update_rate(len(piece))
self.totalup.update_rate(len(piece))
return (index, begin, piece)
def got_request(self, index, begin, length):
if self.super_seeding and index not in self.seed_have_list or \
not self.interested or length > self.max_slice_length:
self.connection.close()
return
if not self.cleared:
self.buffer.append((index, begin, length))
if not self.choked and self.connection.next_upload is None:
self.ratelimiter.queue(self.connection)
def got_cancel(self, index, begin, length):
try:
self.buffer.remove((index, begin, length))
except ValueError:
pass
def choke(self):
if not self.choked:
self.choked = True
self.connection.send_choke()
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def choke_sent(self):
del self.buffer[:]
self.cleared = True
def unchoke(self):
if self.choked:
self.choked = False
self.cleared = False
self.connection.send_unchoke()
def disconnected(self):
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def has_queries(self):
return not self.choked and len(self.buffer) > 0
def get_rate(self):
return self.measure.get_rate()
class SingleDownload:
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
self.choked = True
self.interested = False
self.active_requests = []
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.have = FAKEHAVELIST
def _backlog(self, just_unchoked):
self.backlog = min(
2+int(4*self.measure.get_rate()/self.downloader.chunksize),
(2*just_unchoked)+self.downloader.queue_limit() )
if self.backlog > 50:
self.backlog = max(50, self.backlog * 0.075)
return self.backlog
def disconnected(self):
self.downloader.lost_peer(self)
if self.have.complete():
self.downloader.picker.lost_seed()
else:
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.lost_have(i)
if self.have.complete() and self.downloader.storage.is_endgame():
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self._letgo()
self.guard.download = None
def _letgo(self):
if self.downloader.queued_out.has_key(self):
del self.downloader.queued_out[self]
if not self.active_requests:
return
if self.downloader.endgamemode:
self.active_requests = []
return
lost = {}
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
lost[index] = 1
lost = lost.keys()
self.active_requests = []
if self.downloader.paused:
return
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more()
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(l):
d.send_interested()
break
def got_choke(self):
if not self.choked:
self.choked = True
self._letgo()
def got_unchoke(self):
if self.choked:
self.choked = False
if self.interested:
self._request_more(new_unchoke = True)
self.last2 = clock()
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def send_interested(self):
if not self.interested:
self.interested = True
self.connection.send_interested()
if not self.choked:
self.last2 = clock()
def send_not_interested(self):
if self.interested:
self.interested = False
self.connection.send_not_interested()
def got_piece(self, index, begin, piece):
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, piece, self.guard):
self.downloader.piece_flunked(index)
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
return self.downloader.storage.do_I_have(index)
def _request_more(self, new_unchoke = False):
assert not self.choked
if self.downloader.endgamemode:
self.fix_download_endgame(new_unchoke)
return
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog):
self.downloader.queued_out[self] = 1
return
lost_interests = []
while len(self.active_requests) < self.backlog:
interest = self.downloader.picker.next(self.have,
self.downloader.storage.do_I_have_requests,
self.downloader.too_many_partials())
if interest is None:
break
self.example_interest = interest
self.send_interested()
loop = True
while len(self.active_requests) < self.backlog and loop:
begin, length = self.downloader.storage.new_request(interest)
self.downloader.picker.requested(interest)
self.active_requests.append((interest, begin, length))
self.connection.send_request(interest, begin, length)
self.downloader.chunk_requested(length)
if not self.downloader.storage.do_I_have_requests(interest):
loop = False
lost_interests.append(interest)
if not self.active_requests:
self.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(d.have,
self.downloader.storage.do_I_have_requests,
self.downloader.too_many_partials())
if interest is None:
d.send_not_interested()
else:
d.example_interest = interest
if self.downloader.storage.is_endgame():
self.downloader.start_endgame()
def fix_download_endgame(self, new_unchoke = False):
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog) and not self.choked:
self.downloader.queued_out[self] = 1
return
want = [a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests]
if not (self.active_requests or want):
self.send_not_interested()
return
if want:
self.send_interested()
if self.choked:
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
self.connection.send_request(piece, begin, length)
self.downloader.chunk_requested(length)
def got_have(self, index):
if index == self.downloader.numpieces-1:
self.downloader.totalmeasure.update_rate(self.downloader.storage.total_length-(self.downloader.numpieces-1)*self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.total_length-(self.downloader.numpieces-1)*self.downloader.storage.piece_length)
else:
self.downloader.totalmeasure.update_rate(self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.piece_length)
if not self.have[index]:
self.have[index] = True
self.downloader.picker.got_have(index)
if self.have.complete():
self.downloader.picker.became_seed()
if self.downloader.storage.am_I_complete():
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self.connection.close()
else:
if not self.choked:
self._request_more()
else:
self.send_interested()
elif self.downloader.endgamemode:
self.fix_download_endgame()
elif ( not self.downloader.paused
and not self.downloader.picker.is_blocked(index)
and self.downloader.storage.do_I_have_requests(index) ):
if not self.choked:
self._request_more()
else:
self.send_interested()
return self.have.complete()
def _check_interests(self):
if self.interested or self.downloader.paused:
return
for i in xrange(len(self.have)):
if ( self.have[i] and not self.downloader.picker.is_blocked(i)
and ( self.downloader.endgamemode
or self.downloader.storage.do_I_have_requests(i) ) ):
self.send_interested()
return
def got_have_bitfield(self, have):
if self.downloader.storage.am_I_complete() and have.complete():
if self.downloader.super_seeding:
self.connection.send_bitfield(have) # be nice, show you're a seed too
self.connection.close()
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
return False
self.have = have
if have.complete():
self.downloader.picker.got_seed()
else:
for i in xrange(len(have)):
if have[i]:
self.downloader.picker.got_have(i)
if self.downloader.endgamemode and not self.downloader.paused:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.send_interested()
break
else:
self._check_interests()
return have.complete()
def get_rate(self):
return self.measure.get_rate()
def is_snubbed(self):
if ( self.interested and not self.choked
and clock() - self.last2 > self.downloader.snub_time ):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
class SingleDownload:
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def _backlog(self, just_unchoked):
self.backlog = min(
2 + int(4 * self.measure.get_rate() / self.downloader.chunksize),
(2 * just_unchoked) + self.downloader.queue_limit())
if self.backlog > 50:
self.backlog = max(50, self.backlog * 0.075)
return self.backlog
def disconnected(self):
self.downloader.lost_peer(self)
if self.have.complete():
self.downloader.picker.lost_seed(self.connection.isVODPeer())
else:
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.lost_have(i)
if self.have.complete() and self.downloader.storage.is_endgame():
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self._letgo()
self.guard.download = None
def _letgo(self):
if self.downloader.queued_out.has_key(self):
del self.downloader.queued_out[self]
if not self.active_requests:
return
if self.downloader.endgamemode:
self.active_requests = []
return
lost = {}
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
lost[index] = 1
lost = lost.keys()
self.active_requests = []
if self.downloader.paused:
return
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more()
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(
l):
d.send_interested()
break
def got_choke(self):
if not self.choked:
self.choked = True
self._letgo()
def got_unchoke(self):
if self.choked:
self.choked = False
if self.interested:
self._request_more(new_unchoke=True)
self.last2 = clock()
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def send_interested(self):
if not self.interested:
self.interested = True
self.connection.send_interested()
if not self.choked:
self.last2 = clock()
def send_not_interested(self):
if self.interested:
self.interested = False
self.connection.send_not_interested()
def got_piece(self, index, begin, piece):
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
self.downloader.piece_flunked(index)
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove(
(index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
return self.downloader.storage.do_I_have(index)
def _request_more(self, new_unchoke=False):
assert not self.choked
if self.downloader.endgamemode:
self.fix_download_endgame(new_unchoke)
return
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog):
self.downloader.queued_out[self] = 1
return
lost_interests = []
while len(self.active_requests) < self.backlog:
interest = self.downloader.picker.next(
self.have,
self.downloader.storage.do_I_have_requests,
complete_first=self.downloader.too_many_partials(),
rate=self.measure.get_rate())
if interest is None:
break
self.example_interest = interest
self.send_interested()
loop = True
while len(self.active_requests) < self.backlog and loop:
begin, length = self.downloader.storage.new_request(interest)
self.downloader.picker.requested(interest)
self.active_requests.append((interest, begin, length))
self.connection.send_request(interest, begin, length)
self.downloader.chunk_requested(length)
if not self.downloader.storage.do_I_have_requests(interest):
loop = False
lost_interests.append(interest)
if not self.active_requests:
self.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(
d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(
d.have,
self.downloader.storage.do_I_have_requests,
complete_first=self.downloader.too_many_partials(),
rate=d.measure.get_rate())
if interest is None:
d.send_not_interested()
else:
d.example_interest = interest
if self.downloader.storage.is_endgame():
self.downloader.start_endgame()
def fix_download_endgame(self, new_unchoke=False):
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog) and not self.choked:
self.downloader.queued_out[self] = 1
return
want = [
a for a in self.downloader.all_requests
if self.have[a[0]] and a not in self.active_requests
]
if not (self.active_requests or want):
self.send_not_interested()
return
if want:
self.send_interested()
if self.choked:
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
self.connection.send_request(piece, begin, length)
self.downloader.chunk_requested(length)
def got_have(self, index):
if index == self.downloader.numpieces - 1:
self.downloader.totalmeasure.update_rate(
self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_length)
else:
self.downloader.totalmeasure.update_rate(
self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.piece_length)
if not self.have[index]:
self.have[index] = True
self.downloader.picker.got_have(index)
if self.have.complete():
self.downloader.picker.became_seed(self.connection.isVODPeer())
if self.downloader.storage.am_I_complete():
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self.connection.close()
elif self.downloader.endgamemode:
self.fix_download_endgame()
elif (not self.downloader.paused
and not self.downloader.picker.is_blocked(index)
and self.downloader.storage.do_I_have_requests(index)):
if not self.choked:
self._request_more()
else:
self.send_interested()
return self.have.complete()
def _check_interests(self):
if self.interested or self.downloader.paused:
return
for i in xrange(len(self.have)):
if (self.have[i] and not self.downloader.picker.is_blocked(i)
and (self.downloader.endgamemode
or self.downloader.storage.do_I_have_requests(i))):
self.send_interested()
return
def got_have_bitfield(self, have):
if self.downloader.storage.am_I_complete() and have.complete():
if self.downloader.super_seeding:
self.connection.send_bitfield(
have.tostring()) # be nice, show you're a seed too
self.connection.close()
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
return False
self.have = have
if have.complete():
self.downloader.picker.got_seed(self.connection.isVODPeer())
else:
for i in xrange(len(have)):
if have[i]:
self.downloader.picker.got_have(i)
if self.downloader.endgamemode and not self.downloader.paused:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.send_interested()
break
else:
self._check_interests()
return have.complete()
def get_rate(self):
return self.measure.get_rate()
def is_snubbed(self):
if (self.interested and not self.choked
and clock() - self.last2 > self.downloader.snub_time):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
class Upload:
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, config):
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
self.choker = choker
self.storage = storage
self.picker = picker
self.config = config
self.max_slice_length = config['max_slice_length']
self.choked = True
self.cleared = True
self.interested = False
self.super_seeding = False
self.buffer = []
self.measure = Measure(config['max_rate_period'],
config['upload_rate_fudge'])
self.was_ever_interested = False
if storage.get_amount_left() == 0:
if choker.super_seed:
self.super_seeding = True # flag, and don't send bitfield
self.seed_have_list = [] # set from piecepicker
self.skipped_count = 0
else:
if config['breakup_seed_bitfield']:
bitfield, msgs = storage.get_have_list_cloaked()
connection.send_bitfield(bitfield)
for have in msgs:
connection.send_have(have)
else:
connection.send_bitfield(storage.get_have_list())
else:
if storage.do_I_have_anything():
connection.send_bitfield(storage.get_have_list())
self.piecedl = None
self.piecebuf = None
def got_not_interested(self):
if self.interested:
self.interested = False
del self.buffer[:]
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
self.choker.not_interested(self.connection)
def got_interested(self):
if not self.interested:
self.interested = True
self.was_ever_interested = True
self.choker.interested(self.connection)
def get_upload_chunk(self):
if self.choked or not self.buffer:
return None
index, begin, length = self.buffer.pop(0)
if self.config['buffer_reads']:
if index != self.piecedl:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = index
self.piecebuf = self.storage.get_piece(index, 0, -1)
try:
piece = self.piecebuf[begin:begin + length]
assert len(piece) == length
except: # fails if storage.get_piece returns None or if out of range
self.connection.close()
return None
else:
if self.piecebuf:
self.piecebuf.release()
self.piecedl = None
piece = self.storage.get_piece(index, begin, length)
if piece is None:
self.connection.close()
return None
self.measure.update_rate(len(piece))
self.totalup.update_rate(len(piece))
return (index, begin, piece)
def got_request(self, index, begin, length):
if ((self.super_seeding and not index in self.seed_have_list)
or not self.interested or length > self.max_slice_length):
self.connection.close()
return
if not self.cleared:
self.buffer.append((index, begin, length))
if not self.choked and self.connection.next_upload is None:
self.ratelimiter.queue(self.connection)
def got_cancel(self, index, begin, length):
try:
self.buffer.remove((index, begin, length))
except ValueError:
pass
def choke(self):
if not self.choked:
self.choked = True
self.connection.send_choke()
self.piecedl = None
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def choke_sent(self):
del self.buffer[:]
self.cleared = True
def unchoke(self):
if self.choked:
self.choked = False
self.cleared = False
self.connection.send_unchoke()
def disconnected(self):
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def has_queries(self):
return not self.choked and len(self.buffer) > 0
def get_rate(self):
return self.measure.get_rate()
class SingleDownload:
def __init__(self, downloader, url):
self.downloader = downloader
self.baseurl = url
try:
(scheme, self.netloc, path, pars, query, fragment) = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
try:
self.connection = HTTPConnection(self.netloc)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
def resched(self, len=None):
if len is None:
len = self.retry_period
if self.errorcount > 3:
len = len * (self.errorcount - 2)
self.downloader.rawserver.add_task(self.download, len)
def _want(self, index):
if self.endflag:
return self.downloader.storage.do_I_have_requests(index)
else:
return self.downloader.storage.is_unstarted(index)
def download(self):
self.cancelled = False
if self.downloader.picker.am_I_complete():
self.downloader.downloads.remove(self)
return
self.index = self.downloader.picker.next(haveall, self._want)
if self.index is None and not self.endflag and \
not self.downloader.peerdownloader.has_downloaders():
self.endflag = True
self.index = self.downloader.picker.next(haveall, self._want)
if self.index is None:
self.endflag = True
self.resched()
else:
self.url = (self.seedurl + '&piece=' + str(self.index))
self._get_requests()
if self.request_size < \
self.downloader.storage._piecelen(self.index):
self.url += '&ranges=' + self._request_ranges()
rq = Thread(target=self._request)
rq.setDaemon(False)
rq.start()
self.active = True
def _request(self):
import encodings.ascii
import encodings.punycode
import encodings.idna
self.error = None
self.received_data = None
try:
self.connection.request('GET', self.url, None,
{'User-Agent': VERSION})
r = self.connection.getresponse()
self.connection_status = r.status
self.received_data = r.read()
except Exception as e:
self.error = 'error accessing http seed: ' + str(e)
try:
self.connection.close()
except:
pass
try:
self.connection = HTTPConnection(self.netloc)
except:
# will cause an exception and retry next cycle
self.connection = None
self.downloader.rawserver.add_task(self.request_finished)
def request_finished(self):
self.active = False
if self.error is not None:
if self.goodseed:
self.downloader.errorfunc(self.error)
self.errorcount += 1
if self.received_data:
self.errorcount = 0
if not self._got_data():
self.received_data = None
if not self.received_data:
self._release_requests()
self.downloader.peerdownloader.piece_flunked(self.index)
if self._retry_period:
self.resched(self._retry_period)
self._retry_period = None
return
self.resched()
def _got_data(self):
if self.connection_status == 503: # seed is busy
try:
self.retry_period = max(int(self.received_data), 5)
except:
pass
return False
if self.connection_status != 200:
self.errorcount += 1
return False
self._retry_period = 1
if len(self.received_data) != self.request_size:
if self.goodseed:
self.downloader.errorfunc('corrupt data from http seed - '
'redownloading')
return False
self.measure.update_rate(len(self.received_data))
self.downloader.measurefunc(len(self.received_data))
if self.cancelled:
return False
if not self._fulfill_requests():
return False
if not self.goodseed:
self.goodseed = True
self.downloader.seedsfound += 1
if self.downloader.storage.do_I_have(self.index):
self.downloader.picker.complete(self.index)
self.downloader.peerdownloader.check_complete(self.index)
self.downloader.gotpiecefunc(self.index)
return True
def _get_requests(self):
self.requests = []
self.request_size = 0L
while self.downloader.storage.do_I_have_requests(self.index):
r = self.downloader.storage.new_request(self.index)
self.requests.append(r)
self.request_size += r[1]
self.requests.sort()
def _fulfill_requests(self):
start = 0L
success = True
while self.requests:
begin, length = self.requests.pop(0)
if not self.downloader.storage.piece_came_in(
self.index, begin,
self.received_data[start:start + length]):
success = False
break
start += length
return success
def _release_requests(self):
for begin, length in self.requests:
self.downloader.storage.request_lost(self.index, begin, length)
self.requests = []
def _request_ranges(self):
s = ''
begin, length = self.requests[0]
for begin1, length1 in self.requests[1:]:
if begin + length == begin1:
length += length1
continue
else:
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
begin, length = begin1, length1
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
return s
class SingleDownload:
def __init__(self, downloader, url):
self.downloader = downloader
self.baseurl = url
try:
(scheme, self.netloc, path, pars, query, fragment) = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
try:
self.connection = HTTPConnection(self.netloc)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
def resched(self, len=None):
if len is None:
len = self.retry_period
if self.errorcount > 3:
len = len * (self.errorcount - 2)
self.downloader.rawserver.add_task(self.download, len)
def _want(self, index):
if self.endflag:
return self.downloader.storage.do_I_have_requests(index)
else:
return self.downloader.storage.is_unstarted(index)
def download(self):
self.cancelled = False
if self.downloader.picker.am_I_complete():
self.downloader.downloads.remove(self)
return
self.index = self.downloader.picker.next(haveall, self._want)
if self.index is None and not self.endflag and \
not self.downloader.peerdownloader.has_downloaders():
self.endflag = True
self.index = self.downloader.picker.next(haveall, self._want)
if self.index is None:
self.endflag = True
self.resched()
else:
self.url = (self.seedurl + '&piece=' + str(self.index))
self._get_requests()
if self.request_size < \
self.downloader.storage._piecelen(self.index):
self.url += '&ranges=' + self._request_ranges()
rq = Thread(target=self._request)
rq.setDaemon(False)
rq.start()
self.active = True
def _request(self):
import encodings.ascii
import encodings.punycode
import encodings.idna
self.error = None
self.received_data = None
try:
self.connection.request('GET', self.url, None,
{'User-Agent': VERSION})
r = self.connection.getresponse()
self.connection_status = r.status
self.received_data = r.read()
except Exception as e:
self.error = 'error accessing http seed: ' + str(e)
try:
self.connection.close()
except:
pass
try:
self.connection = HTTPConnection(self.netloc)
except:
# will cause an exception and retry next cycle
self.connection = None
self.downloader.rawserver.add_task(self.request_finished)
def request_finished(self):
self.active = False
if self.error is not None:
if self.goodseed:
self.downloader.errorfunc(self.error)
self.errorcount += 1
if self.received_data:
self.errorcount = 0
if not self._got_data():
self.received_data = None
if not self.received_data:
self._release_requests()
self.downloader.peerdownloader.piece_flunked(self.index)
if self._retry_period:
self.resched(self._retry_period)
self._retry_period = None
return
self.resched()
def _got_data(self):
if self.connection_status == 503: # seed is busy
try:
self.retry_period = max(int(self.received_data), 5)
except:
pass
return False
if self.connection_status != 200:
self.errorcount += 1
return False
self._retry_period = 1
if len(self.received_data) != self.request_size:
if self.goodseed:
self.downloader.errorfunc('corrupt data from http seed - '
'redownloading')
return False
self.measure.update_rate(len(self.received_data))
self.downloader.measurefunc(len(self.received_data))
if self.cancelled:
return False
if not self._fulfill_requests():
return False
if not self.goodseed:
self.goodseed = True
self.downloader.seedsfound += 1
if self.downloader.storage.do_I_have(self.index):
self.downloader.picker.complete(self.index)
self.downloader.peerdownloader.check_complete(self.index)
self.downloader.gotpiecefunc(self.index)
return True
def _get_requests(self):
self.requests = []
self.request_size = 0L
while self.downloader.storage.do_I_have_requests(self.index):
r = self.downloader.storage.new_request(self.index)
self.requests.append(r)
self.request_size += r[1]
self.requests.sort()
def _fulfill_requests(self):
start = 0L
success = True
while self.requests:
begin, length = self.requests.pop(0)
if not self.downloader.storage.piece_came_in(
self.index, begin,
self.received_data[start:start + length]):
success = False
break
start += length
return success
def _release_requests(self):
for begin, length in self.requests:
self.downloader.storage.request_lost(self.index, begin, length)
self.requests = []
def _request_ranges(self):
s = ''
begin, length = self.requests[0]
for begin1, length1 in self.requests[1:]:
if begin + length == begin1:
length += length1
continue
else:
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
begin, length = begin1, length1
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
return s
