def pickle(self):
if self.have.complete():
return {'pieces': 1}
pieces = Bitfield(len(self.hashes))
places = []
partials = []
for p in range(len(self.hashes)):
if self.blocked[p] or p not in self.places:
continue
h = self.have[p]
pieces[p] = h
pp = self.dirty.get(p)
if not h and not pp: # no data
places.extend([self.places[p], self.places[p]])
elif self.places[p] != p:
places.extend([p, self.places[p]])
if h or not pp:
continue
pp.sort()
r = []
while len(pp) > 1:
if pp[0][0] + pp[0][1] == pp[1][0]:
pp[0] = list(pp[0])
pp[0][1] += pp[1][1]
del pp[1]
else:
r.extend(pp[0])
del pp[0]
r.extend(pp[0])
partials.extend([p, r])
return {
'pieces': pieces.tostring(),
'places': places,
'partials': partials
}
def get_have_list_cloaked(self):
if self.have_cloaked_data is None:
newhave = Bitfield(copyfrom=self.have)
unhaves = []
n = min(randrange(2, 5),
len(self.hashes)) # between 2-4 unless torrent is small
while len(unhaves) < n:
unhave = randrange(min(32, len(
self.hashes))) # all in first 4 bytes
if not unhave in unhaves:
unhaves.append(unhave)
newhave[unhave] = False
self.have_cloaked_data = (newhave.tostring(), unhaves)
return self.have_cloaked_data
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def got_message(self, connection, message):
c = self.connections[connection]
t = message[0]
if t == BITFIELD and c.got_anything:
connection.close()
return
c.got_anything = True
if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED]
and len(message) != 1):
connection.close()
return
if t == CHOKE:
c.download.got_choke()
elif t == UNCHOKE:
c.download.got_unchoke()
elif t == INTERESTED:
if not c.download.have.complete():
c.upload.got_interested()
elif t == NOT_INTERESTED:
c.upload.got_not_interested()
elif t == HAVE:
if len(message) != 5:
connection.close()
return
i = toint(message[1:])
if i >= self.numpieces:
connection.close()
return
if c.download.got_have(i):
c.upload.got_not_interested()
elif t == BITFIELD:
try:
b = Bitfield(self.numpieces, message[1:])
except ValueError:
connection.close()
return
if c.download.got_have_bitfield(b):
c.upload.got_not_interested()
elif t == REQUEST:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.got_request(i, toint(message[5:9]), toint(message[9:]))
elif t == CANCEL:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_cancel(i, toint(message[5:9]), toint(message[9:]))
elif t == PIECE:
if len(message) <= 9:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
if c.download.got_piece(i, toint(message[5:9]), message[9:]):
self.got_piece(i)
else:
connection.close()
class SingleDownload:
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def _backlog(self, just_unchoked):
self.backlog = min(
2 + int(4 * self.measure.get_rate() / self.downloader.chunksize),
(2 * just_unchoked) + self.downloader.queue_limit())
if self.backlog > 50:
self.backlog = max(50, self.backlog * 0.075)
return self.backlog
def disconnected(self):
self.downloader.lost_peer(self)
if self.have.complete():
self.downloader.picker.lost_seed()
else:
for i in range(len(self.have)):
if self.have[i]:
self.downloader.picker.lost_have(i)
if self.have.complete() and self.downloader.storage.is_endgame():
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self._letgo()
self.guard.download = None
def _letgo(self):
if self in self.downloader.queued_out:
del self.downloader.queued_out[self]
if not self.active_requests:
return
if self.downloader.endgamemode:
self.active_requests = []
return
lost = {}
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
lost[index] = 1
lost = list(lost.keys())
self.active_requests = []
if self.downloader.paused:
return
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more()
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(
l):
d.send_interested()
break
def got_choke(self):
if not self.choked:
self.choked = True
self._letgo()
def got_unchoke(self):
if self.choked:
self.choked = False
if self.interested:
self._request_more(new_unchoke=True)
self.last2 = clock()
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def send_interested(self):
if not self.interested:
self.interested = True
self.connection.send_interested()
if not self.choked:
self.last2 = clock()
def send_not_interested(self):
if self.interested:
self.interested = False
self.connection.send_not_interested()
def got_piece(self, index, begin, piece):
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, piece,
self.guard):
self.downloader.piece_flunked(index)
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove(
(index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
return self.downloader.storage.do_I_have(index)
def _request_more(self, new_unchoke=False):
assert not self.choked
if self.downloader.endgamemode:
self.fix_download_endgame(new_unchoke)
return
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog):
self.downloader.queued_out[self] = 1
return
lost_interests = []
while len(self.active_requests) < self.backlog:
interest = self.downloader.picker.next(
self.have, self.downloader.storage.do_I_have_requests,
self.downloader.too_many_partials())
if interest is None:
break
self.example_interest = interest
self.send_interested()
loop = True
while len(self.active_requests) < self.backlog and loop:
begin, length = self.downloader.storage.new_request(interest)
self.downloader.picker.requested(interest)
self.active_requests.append((interest, begin, length))
self.connection.send_request(interest, begin, length)
self.downloader.chunk_requested(length)
if not self.downloader.storage.do_I_have_requests(interest):
loop = False
lost_interests.append(interest)
if not self.active_requests:
self.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(
d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(
d.have, self.downloader.storage.do_I_have_requests,
self.downloader.too_many_partials())
if interest is None:
d.send_not_interested()
else:
d.example_interest = interest
if self.downloader.storage.is_endgame():
self.downloader.start_endgame()
def fix_download_endgame(self, new_unchoke=False):
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog) and not self.choked:
self.downloader.queued_out[self] = 1
return
want = [
a for a in self.downloader.all_requests
if self.have[a[0]] and a not in self.active_requests
]
if not (self.active_requests or want):
self.send_not_interested()
return
if want:
self.send_interested()
if self.choked:
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
self.connection.send_request(piece, begin, length)
self.downloader.chunk_requested(length)
def got_have(self, index):
if index == self.downloader.numpieces - 1:
self.downloader.totalmeasure.update_rate(
self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.total_length -
(self.downloader.numpieces - 1) *
self.downloader.storage.piece_length)
else:
self.downloader.totalmeasure.update_rate(
self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.piece_length)
if not self.have[index]:
self.have[index] = True
self.downloader.picker.got_have(index)
if self.have.complete():
self.downloader.picker.became_seed()
if self.downloader.storage.am_I_complete():
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
self.connection.close()
elif self.downloader.endgamemode:
self.fix_download_endgame()
elif (not self.downloader.paused
and not self.downloader.picker.is_blocked(index)
and self.downloader.storage.do_I_have_requests(index)):
if not self.choked:
self._request_more()
else:
self.send_interested()
return self.have.complete()
def _check_interests(self):
if self.interested or self.downloader.paused:
return
for i in range(len(self.have)):
if (self.have[i] and not self.downloader.picker.is_blocked(i)
and (self.downloader.endgamemode
or self.downloader.storage.do_I_have_requests(i))):
self.send_interested()
return
def got_have_bitfield(self, have):
if self.downloader.storage.am_I_complete() and have.complete():
if self.downloader.super_seeding:
self.connection.send_bitfield(
have.tostring()) # be nice, show you're a seed too
self.connection.close()
self.downloader.add_disconnected_seed(
self.connection.get_readable_id())
return False
self.have = have
if have.complete():
self.downloader.picker.got_seed()
else:
for i in range(len(have)):
if have[i]:
self.downloader.picker.got_have(i)
if self.downloader.endgamemode and not self.downloader.paused:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.send_interested()
break
else:
self._check_interests()
return have.complete()
def get_rate(self):
return self.measure.get_rate()
def is_snubbed(self):
if (self.interested and not self.choked
and clock() - self.last2 > self.downloader.snub_time):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
class SingleDownload:
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def _backlog(self, just_unchoked):
self.backlog = min(
2+int(4*self.measure.get_rate()/self.downloader.chunksize),
(2*just_unchoked)+self.downloader.queue_limit() )
if self.backlog > 50:
self.backlog = max(50, self.backlog * 0.075)
return self.backlog
def disconnected(self):
self.downloader.lost_peer(self)
if self.have.complete():
self.downloader.picker.lost_seed()
else:
for i in xrange(len(self.have)):
if self.have[i]:
self.downloader.picker.lost_have(i)
if self.have.complete() and self.downloader.storage.is_endgame():
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self._letgo()
self.guard.download = None
def _letgo(self):
if self.downloader.queued_out.has_key(self):
del self.downloader.queued_out[self]
if not self.active_requests:
return
if self.downloader.endgamemode:
self.active_requests = []
return
lost = {}
for index, begin, length in self.active_requests:
self.downloader.storage.request_lost(index, begin, length)
lost[index] = 1
lost = lost.keys()
self.active_requests = []
if self.downloader.paused:
return
ds = [d for d in self.downloader.downloads if not d.choked]
shuffle(ds)
for d in ds:
d._request_more()
for d in self.downloader.downloads:
if d.choked and not d.interested:
for l in lost:
if d.have[l] and self.downloader.storage.do_I_have_requests(l):
d.send_interested()
break
def got_choke(self):
if not self.choked:
self.choked = True
self._letgo()
def got_unchoke(self):
if self.choked:
self.choked = False
if self.interested:
self._request_more(new_unchoke = True)
self.last2 = clock()
def is_choked(self):
return self.choked
def is_interested(self):
return self.interested
def send_interested(self):
if not self.interested:
self.interested = True
self.connection.send_interested()
if not self.choked:
self.last2 = clock()
def send_not_interested(self):
if self.interested:
self.interested = False
self.connection.send_not_interested()
def got_piece(self, index, begin, piece):
length = len(piece)
try:
self.active_requests.remove((index, begin, length))
except ValueError:
self.downloader.discarded += length
return False
if self.downloader.endgamemode:
self.downloader.all_requests.remove((index, begin, length))
self.last = clock()
self.last2 = clock()
self.measure.update_rate(length)
self.downloader.measurefunc(length)
if not self.downloader.storage.piece_came_in(index, begin, piece, self.guard):
self.downloader.piece_flunked(index)
return False
if self.downloader.storage.do_I_have(index):
self.downloader.picker.complete(index)
if self.downloader.endgamemode:
for d in self.downloader.downloads:
if d is not self:
if d.interested:
if d.choked:
assert not d.active_requests
d.fix_download_endgame()
else:
try:
d.active_requests.remove((index, begin, length))
except ValueError:
continue
d.connection.send_cancel(index, begin, length)
d.fix_download_endgame()
else:
assert not d.active_requests
self._request_more()
self.downloader.check_complete(index)
return self.downloader.storage.do_I_have(index)
def _request_more(self, new_unchoke = False):
assert not self.choked
if self.downloader.endgamemode:
self.fix_download_endgame(new_unchoke)
return
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog):
self.downloader.queued_out[self] = 1
return
lost_interests = []
while len(self.active_requests) < self.backlog:
interest = self.downloader.picker.next(self.have,
self.downloader.storage.do_I_have_requests,
self.downloader.too_many_partials())
if interest is None:
break
self.example_interest = interest
self.send_interested()
loop = True
while len(self.active_requests) < self.backlog and loop:
begin, length = self.downloader.storage.new_request(interest)
self.downloader.picker.requested(interest)
self.active_requests.append((interest, begin, length))
self.connection.send_request(interest, begin, length)
self.downloader.chunk_requested(length)
if not self.downloader.storage.do_I_have_requests(interest):
loop = False
lost_interests.append(interest)
if not self.active_requests:
self.send_not_interested()
if lost_interests:
for d in self.downloader.downloads:
if d.active_requests or not d.interested:
continue
if d.example_interest is not None and self.downloader.storage.do_I_have_requests(d.example_interest):
continue
for lost in lost_interests:
if d.have[lost]:
break
else:
continue
interest = self.downloader.picker.next(d.have,
self.downloader.storage.do_I_have_requests,
self.downloader.too_many_partials())
if interest is None:
d.send_not_interested()
else:
d.example_interest = interest
if self.downloader.storage.is_endgame():
self.downloader.start_endgame()
def fix_download_endgame(self, new_unchoke = False):
if self.downloader.paused:
return
if len(self.active_requests) >= self._backlog(new_unchoke):
if not (self.active_requests or self.backlog) and not self.choked:
self.downloader.queued_out[self] = 1
return
want = [a for a in self.downloader.all_requests if self.have[a[0]] and a not in self.active_requests]
if not (self.active_requests or want):
self.send_not_interested()
return
if want:
self.send_interested()
if self.choked:
return
shuffle(want)
del want[self.backlog - len(self.active_requests):]
self.active_requests.extend(want)
for piece, begin, length in want:
self.connection.send_request(piece, begin, length)
self.downloader.chunk_requested(length)
def got_have(self, index):
if index == self.downloader.numpieces-1:
self.downloader.totalmeasure.update_rate(self.downloader.storage.total_length-(self.downloader.numpieces-1)*self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.total_length-(self.downloader.numpieces-1)*self.downloader.storage.piece_length)
else:
self.downloader.totalmeasure.update_rate(self.downloader.storage.piece_length)
self.peermeasure.update_rate(self.downloader.storage.piece_length)
if not self.have[index]:
self.have[index] = True
self.downloader.picker.got_have(index)
if self.have.complete():
self.downloader.picker.became_seed()
if self.downloader.storage.am_I_complete():
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
self.connection.close()
elif self.downloader.endgamemode:
self.fix_download_endgame()
elif ( not self.downloader.paused
and not self.downloader.picker.is_blocked(index)
and self.downloader.storage.do_I_have_requests(index) ):
if not self.choked:
self._request_more()
else:
self.send_interested()
return self.have.complete()
def _check_interests(self):
if self.interested or self.downloader.paused:
return
for i in xrange(len(self.have)):
if ( self.have[i] and not self.downloader.picker.is_blocked(i)
and ( self.downloader.endgamemode
or self.downloader.storage.do_I_have_requests(i) ) ):
self.send_interested()
return
def got_have_bitfield(self, have):
if self.downloader.storage.am_I_complete() and have.complete():
if self.downloader.super_seeding:
self.connection.send_bitfield(have.tostring()) # be nice, show you're a seed too
self.connection.close()
self.downloader.add_disconnected_seed(self.connection.get_readable_id())
return False
self.have = have
if have.complete():
self.downloader.picker.got_seed()
else:
for i in xrange(len(have)):
if have[i]:
self.downloader.picker.got_have(i)
if self.downloader.endgamemode and not self.downloader.paused:
for piece, begin, length in self.downloader.all_requests:
if self.have[piece]:
self.send_interested()
break
else:
self._check_interests()
return have.complete()
def get_rate(self):
return self.measure.get_rate()
def is_snubbed(self):
if ( self.interested and not self.choked
and clock() - self.last2 > self.downloader.snub_time ):
for index, begin, length in self.active_requests:
self.connection.send_cancel(index, begin, length)
self.got_choke() # treat it just like a choke
return clock() - self.last > self.downloader.snub_time
def unpickle(self, data, valid_places):
got = {}
places = {}
dirty = {}
download_history = {}
stat_active = {}
stat_numfound = self.stat_numfound
amount_obtained = self.amount_obtained
amount_inactive = self.amount_inactive
amount_left = self.amount_left
inactive_requests = [x for x in self.inactive_requests]
restored_partials = []
try:
if data['pieces'] == 1: # a seed
assert not data.get('places', None)
assert not data.get('partials', None)
have = Bitfield(len(self.hashes))
for i in range(len(self.hashes)):
have[i] = True
assert have.complete()
_places = []
_partials = []
else:
have = Bitfield(len(self.hashes), data['pieces'])
_places = data['places']
assert len(_places) % 2 == 0
_places = [_places[x:x + 2] for x in range(0, len(_places), 2)]
_partials = data['partials']
assert len(_partials) % 2 == 0
_partials = [
_partials[x:x + 2] for x in range(0, len(_partials), 2)
]
for index, place in _places:
if place not in valid_places:
continue
assert index not in got
assert place not in got
places[index] = place
got[index] = 1
got[place] = 1
for index in range(len(self.hashes)):
if have[index]:
if index not in places:
if index not in valid_places:
have[index] = False
continue
assert index not in got
places[index] = index
got[index] = 1
length = self._piecelen(index)
amount_obtained += length
stat_numfound += 1
amount_inactive -= length
amount_left -= length
inactive_requests[index] = None
for index, plist in _partials:
assert index not in dirty
assert not have[index]
if index not in places:
if index not in valid_places:
continue
assert index not in got
places[index] = index
got[index] = 1
assert len(plist) % 2 == 0
plist = [plist[x:x + 2] for x in range(0, len(plist), 2)]
dirty[index] = plist
stat_active[index] = 1
download_history[index] = {}
# invert given partials
length = self._piecelen(index)
l = []
if plist[0][0] > 0:
l.append((0, plist[0][0]))
for i in range(len(plist) - 1):
end = plist[i][0] + plist[i][1]
assert not end > plist[i + 1][0]
l.append((end, plist[i + 1][0] - end))
end = plist[-1][0] + plist[-1][1]
assert not end > length
if end < length:
l.append((end, length - end))
# split them to request_size
ll = []
amount_obtained += length
amount_inactive -= length
for nb, nl in l:
while nl > 0:
r = min(nl, self.request_size)
ll.append((nb, r))
amount_inactive += r
amount_obtained -= r
nb += self.request_size
nl -= self.request_size
inactive_requests[index] = ll
restored_partials.append(index)
assert amount_obtained + amount_inactive == self.amount_desired
except:
# print_exc()
return [] # invalid data, discard everything
self.have = have
self.places = places
self.dirty = dirty
self.download_history = download_history
self.stat_active = stat_active
self.stat_numfound = stat_numfound
self.amount_obtained = amount_obtained
self.amount_inactive = amount_inactive
self.amount_left = amount_left
self.inactive_requests = inactive_requests
return restored_partials
def __init__(self,
storage,
request_size,
hashes,
piece_size,
finished,
failed,
statusfunc=dummy_status,
flag=fakeflag(),
check_hashes=True,
data_flunked=lambda x: None,
backfunc=None,
config={},
unpauseflag=fakeflag(True)):
self.storage = storage
self.request_size = int(request_size)
self.hashes = hashes
self.piece_size = int(piece_size)
self.piece_length = int(piece_size)
self.finished = finished
self.failed = failed
self.statusfunc = statusfunc
self.flag = flag
self.check_hashes = check_hashes
self.data_flunked = data_flunked
self.backfunc = backfunc
self.config = config
self.unpauseflag = unpauseflag
self.alloc_type = config.get('alloc_type', 'normal')
self.double_check = config.get('double_check', 0)
self.triple_check = config.get('triple_check', 0)
if self.triple_check:
self.double_check = True
self.bgalloc_enabled = False
self.bgalloc_active = False
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= self.piece_size * (len(hashes) - 1):
raise ValueError('bad data in responsefile - total too small')
if self.total_length > self.piece_size * len(hashes):
raise ValueError('bad data in responsefile - total too big')
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.amount_obtained = 0
self.amount_desired = self.total_length
self.have = Bitfield(len(hashes))
self.have_cloaked_data = None
self.blocked = [False] * len(hashes)
self.blocked_holes = []
self.blocked_movein = Olist()
self.blocked_moveout = Olist()
self.waschecked = [False] * len(hashes)
self.places = {}
self.holes = []
self.stat_active = {}
self.stat_new = {}
self.dirty = {}
self.stat_numflunked = 0
self.stat_numdownloaded = 0
self.stat_numfound = 0
self.download_history = {}
self.failed_pieces = {}
self.out_of_place = 0
self.write_buf_max = config['write_buffer_size'] * 1048576
self.write_buf_size = 0
self.write_buf = {} # structure: piece: [(start, data), ...]
self.write_buf_list = []
self.initialize_tasks = [[
'checking existing data', 0, self.init_hashcheck,
self.hashcheckfunc
], ['moving data', 1, self.init_movedata, self.movedatafunc
], ['allocating disk space', 1, self.init_alloc, self.allocfunc]]
self.backfunc(self._bgalloc, 0.1)
self.backfunc(self._bgsync, max(self.config['auto_flush'] * 60, 60))
class StorageWrapper:
def __init__(self,
storage,
request_size,
hashes,
piece_size,
finished,
failed,
statusfunc=dummy_status,
flag=fakeflag(),
check_hashes=True,
data_flunked=lambda x: None,
backfunc=None,
config={},
unpauseflag=fakeflag(True)):
self.storage = storage
self.request_size = int(request_size)
self.hashes = hashes
self.piece_size = int(piece_size)
self.piece_length = int(piece_size)
self.finished = finished
self.failed = failed
self.statusfunc = statusfunc
self.flag = flag
self.check_hashes = check_hashes
self.data_flunked = data_flunked
self.backfunc = backfunc
self.config = config
self.unpauseflag = unpauseflag
self.alloc_type = config.get('alloc_type', 'normal')
self.double_check = config.get('double_check', 0)
self.triple_check = config.get('triple_check', 0)
if self.triple_check:
self.double_check = True
self.bgalloc_enabled = False
self.bgalloc_active = False
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= self.piece_size * (len(hashes) - 1):
raise ValueError('bad data in responsefile - total too small')
if self.total_length > self.piece_size * len(hashes):
raise ValueError('bad data in responsefile - total too big')
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.amount_obtained = 0
self.amount_desired = self.total_length
self.have = Bitfield(len(hashes))
self.have_cloaked_data = None
self.blocked = [False] * len(hashes)
self.blocked_holes = []
self.blocked_movein = Olist()
self.blocked_moveout = Olist()
self.waschecked = [False] * len(hashes)
self.places = {}
self.holes = []
self.stat_active = {}
self.stat_new = {}
self.dirty = {}
self.stat_numflunked = 0
self.stat_numdownloaded = 0
self.stat_numfound = 0
self.download_history = {}
self.failed_pieces = {}
self.out_of_place = 0
self.write_buf_max = config['write_buffer_size'] * 1048576
self.write_buf_size = 0
self.write_buf = {} # structure: piece: [(start, data), ...]
self.write_buf_list = []
self.initialize_tasks = [[
'checking existing data', 0, self.init_hashcheck,
self.hashcheckfunc
], ['moving data', 1, self.init_movedata, self.movedatafunc
], ['allocating disk space', 1, self.init_alloc, self.allocfunc]]
self.backfunc(self._bgalloc, 0.1)
self.backfunc(self._bgsync, max(self.config['auto_flush'] * 60, 60))
def _bgsync(self):
if self.config['auto_flush']:
self.sync()
self.backfunc(self._bgsync, max(self.config['auto_flush'] * 60, 60))
def old_style_init(self):
while self.initialize_tasks:
msg, done, init, next = self.initialize_tasks.pop(0)
if init():
self.statusfunc(activity=msg, fractionDone=done)
t = clock() + STATS_INTERVAL
x = 0
while x is not None:
if t < clock():
t = clock() + STATS_INTERVAL
self.statusfunc(fractionDone=x)
self.unpauseflag.wait()
if self.flag.isSet():
return False
x = next()
self.statusfunc(fractionDone=0)
return True
def initialize(self, donefunc, statusfunc=None):
self.initialize_done = donefunc
if statusfunc is None:
statusfunc = self.statusfunc
self.initialize_status = statusfunc
self.initialize_next = None
self.backfunc(self._initialize)
def _initialize(self):
if not self.unpauseflag.isSet():
self.backfunc(self._initialize, 1)
return
if self.initialize_next:
x = self.initialize_next()
if x is None:
self.initialize_next = None
else:
self.initialize_status(fractionDone=x)
else:
if not self.initialize_tasks:
self.initialize_done()
return
msg, done, init, next = self.initialize_tasks.pop(0)
if init():
self.initialize_status(activity=msg, fractionDone=done)
self.initialize_next = next
self.backfunc(self._initialize)
def init_hashcheck(self):
if self.flag.isSet():
return False
self.check_list = []
if len(self.hashes) == 0 or self.amount_left == 0:
self.check_total = 0
self.finished()
return False
self.check_targets = {}
got = {}
for p, v in list(self.places.items()):
assert v not in got
got[v] = 1
for i in range(len(self.hashes)):
if i in self.places: # restored from pickled
self.check_targets[self.hashes[i]] = []
if self.places[i] == i:
continue
else:
assert i not in got
self.out_of_place += 1
if i in got:
continue
if self._waspre(i):
if self.blocked[i]:
self.places[i] = i
else:
self.check_list.append(i)
continue
if not self.check_hashes:
self.failed(
'told file complete on start-up, but data is missing')
return False
self.holes.append(i)
if self.blocked[i] or self.hashes[i] in self.check_targets:
self.check_targets[self.hashes[i]] = [
] # in case of a hash collision, discard
else:
self.check_targets[self.hashes[i]] = [i]
self.check_total = len(self.check_list)
self.check_numchecked = 0.0
self.lastlen = self._piecelen(len(self.hashes) - 1)
self.numchecked = 0.0
return self.check_total > 0
def _markgot(self, piece, pos):
if DEBUG:
print(str(piece) + ' at ' + str(pos))
self.places[piece] = pos
self.have[piece] = True
len = self._piecelen(piece)
self.amount_obtained += len
self.amount_left -= len
self.amount_inactive -= len
self.inactive_requests[piece] = None
self.waschecked[piece] = self.check_hashes
self.stat_numfound += 1
def hashcheckfunc(self):
if self.flag.isSet():
return None
if not self.check_list:
return None
i = self.check_list.pop(0)
if not self.check_hashes:
self._markgot(i, i)
else:
d1 = self.read_raw(i, 0, self.lastlen)
if d1 is None:
return None
sh = sha(d1[:])
d1.release()
sp = sh.digest()
d2 = self.read_raw(i, self.lastlen,
self._piecelen(i) - self.lastlen)
if d2 is None:
return None
sh.update(d2[:])
d2.release()
s = sh.digest()
if s == self.hashes[i]:
self._markgot(i, i)
elif (self.check_targets.get(s) and self._piecelen(i)
== self._piecelen(self.check_targets[s][-1])):
self._markgot(self.check_targets[s].pop(), i)
self.out_of_place += 1
elif (not self.have[-1] and sp == self.hashes[-1]
and (i == len(self.hashes) - 1
or not self._waspre(len(self.hashes) - 1))):
self._markgot(len(self.hashes) - 1, i)
self.out_of_place += 1
else:
self.places[i] = i
self.numchecked += 1
if self.amount_left == 0:
self.finished()
return (self.numchecked / self.check_total)
def init_movedata(self):
if self.flag.isSet():
return False
if self.alloc_type != 'sparse':
return False
self.storage.top_off() # sets file lengths to their final size
self.movelist = []
if self.out_of_place == 0:
for i in self.holes:
self.places[i] = i
self.holes = []
return False
self.tomove = float(self.out_of_place)
for i in range(len(self.hashes)):
if i not in self.places:
self.places[i] = i
elif self.places[i] != i:
self.movelist.append(i)
self.holes = []
return True
def movedatafunc(self):
if self.flag.isSet():
return None
if not self.movelist:
return None
i = self.movelist.pop(0)
old = self.read_raw(self.places[i], 0, self._piecelen(i))
if old is None:
return None
if not self.write_raw(i, 0, old):
return None
if self.double_check and self.have[i]:
if self.triple_check:
old.release()
old = self.read_raw(i, 0, self._piecelen(i), flush_first=True)
if old is None:
return None
if sha(old[:]).digest() != self.hashes[i]:
self.failed('download corrupted; please restart and resume')
return None
old.release()
self.places[i] = i
self.tomove -= 1
return (self.tomove / self.out_of_place)
def init_alloc(self):
if self.flag.isSet():
return False
if not self.holes:
return False
self.numholes = float(len(self.holes))
self.alloc_buf = chr(0xFF) * self.piece_size
if self.alloc_type == 'pre-allocate':
self.bgalloc_enabled = True
return True
if self.alloc_type == 'background':
self.bgalloc_enabled = True
if self.blocked_moveout:
return True
return False
def _allocfunc(self):
while self.holes:
n = self.holes.pop(0)
if self.blocked[n]: # assume not self.blocked[index]
if not self.blocked_movein:
self.blocked_holes.append(n)
continue
if n not in self.places:
b = self.blocked_movein.pop(0)
oldpos = self._move_piece(b, n)
self.places[oldpos] = oldpos
return None
if n in self.places:
oldpos = self._move_piece(n, n)
self.places[oldpos] = oldpos
return None
return n
return None
def allocfunc(self):
if self.flag.isSet():
return None
if self.blocked_moveout:
self.bgalloc_active = True
n = self._allocfunc()
if n is not None:
if self.blocked_moveout.includes(n):
self.blocked_moveout.remove(n)
b = n
else:
b = self.blocked_moveout.pop(0)
oldpos = self._move_piece(b, n)
self.places[oldpos] = oldpos
return len(self.holes) / self.numholes
if self.holes and self.bgalloc_enabled:
self.bgalloc_active = True
n = self._allocfunc()
if n is not None:
self.write_raw(n, 0, self.alloc_buf[:self._piecelen(n)])
self.places[n] = n
return len(self.holes) / self.numholes
self.bgalloc_active = False
return None
def bgalloc(self):
if self.bgalloc_enabled:
if not self.holes and not self.blocked_moveout and self.backfunc:
self.backfunc(self.storage.flush)
# force a flush whenever the "finish allocation" button is hit
self.bgalloc_enabled = True
return False
def _bgalloc(self):
self.allocfunc()
if self.config.get('alloc_rate', 0) < 0.1:
self.config['alloc_rate'] = 0.1
self.backfunc(
self._bgalloc,
float(self.piece_size) / (self.config['alloc_rate'] * 1048576))
def _waspre(self, piece):
return self.storage.was_preallocated(piece * self.piece_size,
self._piecelen(piece))
def _piecelen(self, piece):
if piece < len(self.hashes) - 1:
return self.piece_size
else:
return self.total_length - (piece * self.piece_size)
def get_amount_left(self):
return self.amount_left
def do_I_have_anything(self):
return self.amount_left < self.total_length
def _make_inactive(self, index):
length = self._piecelen(index)
l = []
x = 0
while x + self.request_size < length:
l.append((x, self.request_size))
x += self.request_size
l.append((x, length - x))
self.inactive_requests[index] = l
def is_endgame(self):
return not self.amount_inactive
def am_I_complete(self):
return self.amount_obtained == self.amount_desired
def reset_endgame(self, requestlist):
for index, begin, length in requestlist:
self.request_lost(index, begin, length)
def get_have_list(self):
return self.have.tostring()
def get_have_list_cloaked(self):
if self.have_cloaked_data is None:
newhave = Bitfield(copyfrom=self.have)
unhaves = []
n = min(randrange(2, 5),
len(self.hashes)) # between 2-4 unless torrent is small
while len(unhaves) < n:
unhave = randrange(min(32, len(
self.hashes))) # all in first 4 bytes
if not unhave in unhaves:
unhaves.append(unhave)
newhave[unhave] = False
self.have_cloaked_data = (newhave.tostring(), unhaves)
return self.have_cloaked_data
def do_I_have(self, index):
return self.have[index]
def do_I_have_requests(self, index):
return not not self.inactive_requests[index]
def is_unstarted(self, index):
return (not self.have[index] and not self.numactive[index]
and index not in self.dirty)
def get_hash(self, index):
return self.hashes[index]
def get_stats(self):
return self.amount_obtained, self.amount_desired
def new_request(self, index):
# returns (begin, length)
if self.inactive_requests[index] == 1:
self._make_inactive(index)
self.numactive[index] += 1
self.stat_active[index] = 1
if index not in self.dirty:
self.stat_new[index] = 1
rs = self.inactive_requests[index]
# r = min(rs)
# rs.remove(r)
r = rs.pop(0)
self.amount_inactive -= r[1]
return r
def write_raw(self, index, begin, data):
try:
self.storage.write(self.piece_size * index + begin, data)
return True
except IOError as e:
self.failed('IO Error: ' + str(e))
return False
def _write_to_buffer(self, piece, start, data):
if not self.write_buf_max:
return self.write_raw(self.places[piece], start, data)
self.write_buf_size += len(data)
while self.write_buf_size > self.write_buf_max:
old = self.write_buf_list.pop(0)
if not self._flush_buffer(old, True):
return False
if piece in self.write_buf:
self.write_buf_list.remove(piece)
else:
self.write_buf[piece] = []
self.write_buf_list.append(piece)
self.write_buf[piece].append((start, data))
return True
def _flush_buffer(self, piece, popped=False):
if piece not in self.write_buf:
return True
if not popped:
self.write_buf_list.remove(piece)
l = self.write_buf[piece]
del self.write_buf[piece]
l.sort()
for start, data in l:
self.write_buf_size -= len(data)
if not self.write_raw(self.places[piece], start, data):
return False
return True
def sync(self):
spots = {}
for p in self.write_buf_list:
spots[self.places[p]] = p
l = list(spots.keys())
l.sort()
for i in l:
try:
self._flush_buffer(spots[i])
except:
pass
try:
self.storage.sync()
except IOError as e:
self.failed('IO Error: ' + str(e))
except OSError as e:
self.failed('OS Error: ' + str(e))
def _move_piece(self, index, newpos):
oldpos = self.places[index]
if DEBUG:
print('moving ' + str(index) + ' from ' + str(oldpos) + ' to ' +
str(newpos))
assert oldpos != index
assert oldpos != newpos
assert index == newpos or newpos not in self.places
old = self.read_raw(oldpos, 0, self._piecelen(index))
if old is None:
return -1
if not self.write_raw(newpos, 0, old):
return -1
self.places[index] = newpos
if self.have[index] and (self.triple_check or
(self.double_check and index == newpos)):
if self.triple_check:
old.release()
old = self.read_raw(newpos,
0,
self._piecelen(index),
flush_first=True)
if old is None:
return -1
if sha(old[:]).digest() != self.hashes[index]:
self.failed('download corrupted; please restart and resume')
return -1
old.release()
if self.blocked[index]:
self.blocked_moveout.remove(index)
if self.blocked[newpos]:
self.blocked_movein.remove(index)
else:
self.blocked_movein.add(index)
else:
self.blocked_movein.remove(index)
if self.blocked[newpos]:
self.blocked_moveout.add(index)
else:
self.blocked_moveout.remove(index)
return oldpos
def _clear_space(self, index):
h = self.holes.pop(0)
n = h
if self.blocked[n]: # assume not self.blocked[index]
if not self.blocked_movein:
self.blocked_holes.append(n)
return True # repeat
if n not in self.places:
b = self.blocked_movein.pop(0)
oldpos = self._move_piece(b, n)
if oldpos < 0:
return False
n = oldpos
if n in self.places:
oldpos = self._move_piece(n, n)
if oldpos < 0:
return False
n = oldpos
if index == n or index in self.holes:
if n == h:
self.write_raw(n, 0, self.alloc_buf[:self._piecelen(n)])
self.places[index] = n
if self.blocked[n]:
# because n may be a spot cleared 10 lines above, it's possible
# for it to be blocked. While that spot could be left cleared
# and a new spot allocated, this condition might occur several
# times in a row, resulting in a significant amount of disk I/O,
# delaying the operation of the engine. Rather than do this,
# queue the piece to be moved out again, which will be performed
# by the background allocator, with which data movement is
# automatically limited.
self.blocked_moveout.add(index)
return False
for p, v in list(self.places.items()):
if v == index:
break
else:
self.failed('download corrupted; please restart and resume')
return False
self._move_piece(p, n)
self.places[index] = index
return False
def piece_came_in(self, index, begin, piece, source=None):
assert not self.have[index]
if index not in self.places:
while self._clear_space(index):
pass
if DEBUG:
print('new place for ' + str(index) + ' at ' +
str(self.places[index]))
if self.flag.isSet():
return
if index in self.failed_pieces:
old = self.read_raw(self.places[index], begin, len(piece))
if old is None:
return True
if old[:].tostring() != piece:
try:
self.failed_pieces[index][self.download_history[index]
[begin]] = 1
except:
self.failed_pieces[index][None] = 1
old.release()
self.download_history.setdefault(index, {})[begin] = source
if not self._write_to_buffer(index, begin, piece):
return True
self.amount_obtained += len(piece)
self.dirty.setdefault(index, []).append((begin, len(piece)))
self.numactive[index] -= 1
assert self.numactive[index] >= 0
if not self.numactive[index]:
del self.stat_active[index]
if index in self.stat_new:
del self.stat_new[index]
if self.inactive_requests[index] or self.numactive[index]:
return True
del self.dirty[index]
if not self._flush_buffer(index):
return True
length = self._piecelen(index)
data = self.read_raw(self.places[index],
0,
length,
flush_first=self.triple_check)
if data is None:
return True
hash = sha(data[:]).digest()
data.release()
if hash != self.hashes[index]:
self.amount_obtained -= length
self.data_flunked(length, index)
self.inactive_requests[index] = 1
self.amount_inactive += length
self.stat_numflunked += 1
self.failed_pieces[index] = {}
allsenders = {}
for d in list(self.download_history[index].values()):
allsenders[d] = 1
if len(allsenders) == 1:
culprit = list(allsenders.keys())[0]
if culprit is not None:
culprit.failed(index, bump=True)
del self.failed_pieces[index] # found the culprit already
return False
self.have[index] = True
self.inactive_requests[index] = None
self.waschecked[index] = True
self.amount_left -= length
self.stat_numdownloaded += 1
for d in list(self.download_history[index].values()):
if d is not None:
d.good(index)
del self.download_history[index]
if index in self.failed_pieces:
for d in list(self.failed_pieces[index].keys()):
if d is not None:
d.failed(index)
del self.failed_pieces[index]
if self.amount_left == 0:
self.finished()
return True
def request_lost(self, index, begin, length):
assert not (begin, length) in self.inactive_requests[index]
insort(self.inactive_requests[index], (begin, length))
self.amount_inactive += length
self.numactive[index] -= 1
if not self.numactive[index]:
del self.stat_active[index]
if index in self.stat_new:
del self.stat_new[index]
def get_piece(self, index, begin, length):
if not self.have[index]:
return None
data = None
if not self.waschecked[index]:
data = self.read_raw(self.places[index], 0, self._piecelen(index))
if data is None:
return None
if sha(data[:]).digest() != self.hashes[index]:
self.failed(
'told file complete on start-up, but piece failed hash check'
)
return None
self.waschecked[index] = True
if length == -1 and begin == 0:
return data # optimization
if length == -1:
if begin > self._piecelen(index):
return None
length = self._piecelen(index) - begin
if begin == 0:
return self.read_raw(self.places[index], 0, length)
elif begin + length > self._piecelen(index):
return None
if data is not None:
s = data[begin:begin + length]
data.release()
return s
data = self.read_raw(self.places[index], begin, length)
if data is None:
return None
s = data.getarray()
data.release()
return s
def read_raw(self, piece, begin, length, flush_first=False):
try:
return self.storage.read(self.piece_size * piece + begin, length,
flush_first)
except IOError as e:
self.failed('IO Error: ' + str(e))
return None
def set_file_readonly(self, n):
try:
self.storage.set_readonly(n)
except IOError as e:
self.failed('IO Error: ' + str(e))
except OSError as e:
self.failed('OS Error: ' + str(e))
def has_data(self, index):
return index not in self.holes and index not in self.blocked_holes
def doublecheck_data(self, pieces_to_check):
if not self.double_check:
return
sources = []
for p, v in list(self.places.items()):
if v in pieces_to_check:
sources.append(p)
assert len(sources) == len(pieces_to_check)
sources.sort()
for index in sources:
if self.have[index]:
piece = self.read_raw(self.places[index],
0,
self._piecelen(index),
flush_first=True)
if piece is None:
return False
if sha(piece[:]).digest() != self.hashes[index]:
self.failed(
'download corrupted; please restart and resume')
return False
piece.release()
return True
def reblock(self, new_blocked):
# assume downloads have already been canceled and chunks made inactive
for i in range(len(new_blocked)):
if new_blocked[i] and not self.blocked[i]:
length = self._piecelen(i)
self.amount_desired -= length
if self.have[i]:
self.amount_obtained -= length
continue
if self.inactive_requests[i] == 1:
self.amount_inactive -= length
continue
inactive = 0
for nb, nl in self.inactive_requests[i]:
inactive += nl
self.amount_inactive -= inactive
self.amount_obtained -= length - inactive
if self.blocked[i] and not new_blocked[i]:
length = self._piecelen(i)
self.amount_desired += length
if self.have[i]:
self.amount_obtained += length
continue
if self.inactive_requests[i] == 1:
self.amount_inactive += length
continue
inactive = 0
for nb, nl in self.inactive_requests[i]:
inactive += nl
self.amount_inactive += inactive
self.amount_obtained += length - inactive
self.blocked = new_blocked
self.blocked_movein = Olist()
self.blocked_moveout = Olist()
for p, v in list(self.places.items()):
if p != v:
if self.blocked[p] and not self.blocked[v]:
self.blocked_movein.add(p)
elif self.blocked[v] and not self.blocked[p]:
self.blocked_moveout.add(p)
self.holes.extend(self.blocked_holes) # reset holes list
self.holes.sort()
self.blocked_holes = []
'''
Pickled data format:
d['pieces'] = either a string containing a bitfield of complete pieces,
or the numeric value "1" signifying a seed. If it is
a seed, d['places'] and d['partials'] should be empty
and needn't even exist.
d['partials'] = [ piece, [ offset, length... ]... ]
a list of partial data that had been previously
downloaded, plus the given offsets. Adjacent partials
are merged so as to save space, and so that if the
request size changes then new requests can be
calculated more efficiently.
d['places'] = [ piece, place, {,piece, place ...} ]
the piece index, and the place it's stored.
If d['pieces'] specifies a complete piece or d['partials']
specifies a set of partials for a piece which has no
entry in d['places'], it can be assumed that
place[index] = index. A place specified with no
corresponding data in d['pieces'] or d['partials']
indicates allocated space with no valid data, and is
reserved so it doesn't need to be hash-checked.
'''
def pickle(self):
if self.have.complete():
return {'pieces': 1}
pieces = Bitfield(len(self.hashes))
places = []
partials = []
for p in range(len(self.hashes)):
if self.blocked[p] or p not in self.places:
continue
h = self.have[p]
pieces[p] = h
pp = self.dirty.get(p)
if not h and not pp: # no data
places.extend([self.places[p], self.places[p]])
elif self.places[p] != p:
places.extend([p, self.places[p]])
if h or not pp:
continue
pp.sort()
r = []
while len(pp) > 1:
if pp[0][0] + pp[0][1] == pp[1][0]:
pp[0] = list(pp[0])
pp[0][1] += pp[1][1]
del pp[1]
else:
r.extend(pp[0])
del pp[0]
r.extend(pp[0])
partials.extend([p, r])
return {
'pieces': pieces.tostring(),
'places': places,
'partials': partials
}
def unpickle(self, data, valid_places):
got = {}
places = {}
dirty = {}
download_history = {}
stat_active = {}
stat_numfound = self.stat_numfound
amount_obtained = self.amount_obtained
amount_inactive = self.amount_inactive
amount_left = self.amount_left
inactive_requests = [x for x in self.inactive_requests]
restored_partials = []
try:
if data['pieces'] == 1: # a seed
assert not data.get('places', None)
assert not data.get('partials', None)
have = Bitfield(len(self.hashes))
for i in range(len(self.hashes)):
have[i] = True
assert have.complete()
_places = []
_partials = []
else:
have = Bitfield(len(self.hashes), data['pieces'])
_places = data['places']
assert len(_places) % 2 == 0
_places = [_places[x:x + 2] for x in range(0, len(_places), 2)]
_partials = data['partials']
assert len(_partials) % 2 == 0
_partials = [
_partials[x:x + 2] for x in range(0, len(_partials), 2)
]
for index, place in _places:
if place not in valid_places:
continue
assert index not in got
assert place not in got
places[index] = place
got[index] = 1
got[place] = 1
for index in range(len(self.hashes)):
if have[index]:
if index not in places:
if index not in valid_places:
have[index] = False
continue
assert index not in got
places[index] = index
got[index] = 1
length = self._piecelen(index)
amount_obtained += length
stat_numfound += 1
amount_inactive -= length
amount_left -= length
inactive_requests[index] = None
for index, plist in _partials:
assert index not in dirty
assert not have[index]
if index not in places:
if index not in valid_places:
continue
assert index not in got
places[index] = index
got[index] = 1
assert len(plist) % 2 == 0
plist = [plist[x:x + 2] for x in range(0, len(plist), 2)]
dirty[index] = plist
stat_active[index] = 1
download_history[index] = {}
# invert given partials
length = self._piecelen(index)
l = []
if plist[0][0] > 0:
l.append((0, plist[0][0]))
for i in range(len(plist) - 1):
end = plist[i][0] + plist[i][1]
assert not end > plist[i + 1][0]
l.append((end, plist[i + 1][0] - end))
end = plist[-1][0] + plist[-1][1]
assert not end > length
if end < length:
l.append((end, length - end))
# split them to request_size
ll = []
amount_obtained += length
amount_inactive -= length
for nb, nl in l:
while nl > 0:
r = min(nl, self.request_size)
ll.append((nb, r))
amount_inactive += r
amount_obtained -= r
nb += self.request_size
nl -= self.request_size
inactive_requests[index] = ll
restored_partials.append(index)
assert amount_obtained + amount_inactive == self.amount_desired
except:
# print_exc()
return [] # invalid data, discard everything
self.have = have
self.places = places
self.dirty = dirty
self.download_history = download_history
self.stat_active = stat_active
self.stat_numfound = stat_numfound
self.amount_obtained = amount_obtained
self.amount_inactive = amount_inactive
self.amount_left = amount_left
self.inactive_requests = inactive_requests
return restored_partials