def pickle(self):
if self.have.complete():
return {'pieces': 1}
pieces = Bitfield(len(self.hashes))
places = []
partials = []
for p in range(len(self.hashes)):
if self.blocked[p] or p not in self.places:
continue
h = self.have[p]
pieces[p] = h
pp = self.dirty.get(p)
if not h and not pp: # no data
places.extend([self.places[p], self.places[p]])
elif self.places[p] != p:
places.extend([p, self.places[p]])
if h or not pp:
continue
pp.sort()
r = []
while len(pp) > 1:
if pp[0][0] + pp[0][1] == pp[1][0]:
pp[0] = list(pp[0])
pp[0][1] += pp[1][1]
del pp[1]
else:
r.extend(pp[0])
del pp[0]
r.extend(pp[0])
partials.extend([p, r])
return {
'pieces': pieces.tostring(),
'places': places,
'partials': partials
}
def get_have_list_cloaked(self):
if self.have_cloaked_data is None:
newhave = Bitfield(copyfrom=self.have)
unhaves = []
n = min(randrange(2, 5),
len(self.hashes)) # between 2-4 unless torrent is small
while len(unhaves) < n:
unhave = randrange(min(32, len(
self.hashes))) # all in first 4 bytes
if not unhave in unhaves:
unhaves.append(unhave)
newhave[unhave] = False
self.have_cloaked_data = (newhave.tostring(), unhaves)
return self.have_cloaked_data
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def got_message(self, connection, message):
c = self.connections[connection]
t = message[0]
if t == BITFIELD and c.got_anything:
connection.close()
return
c.got_anything = True
if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED]
and len(message) != 1):
connection.close()
return
if t == CHOKE:
c.download.got_choke()
elif t == UNCHOKE:
c.download.got_unchoke()
elif t == INTERESTED:
if not c.download.have.complete():
c.upload.got_interested()
elif t == NOT_INTERESTED:
c.upload.got_not_interested()
elif t == HAVE:
if len(message) != 5:
connection.close()
return
i = toint(message[1:])
if i >= self.numpieces:
connection.close()
return
if c.download.got_have(i):
c.upload.got_not_interested()
elif t == BITFIELD:
try:
b = Bitfield(self.numpieces, message[1:])
except ValueError:
connection.close()
return
if c.download.got_have_bitfield(b):
c.upload.got_not_interested()
elif t == REQUEST:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.got_request(i, toint(message[5:9]), toint(message[9:]))
elif t == CANCEL:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_cancel(i, toint(message[5:9]), toint(message[9:]))
elif t == PIECE:
if len(message) <= 9:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
if c.download.got_piece(i, toint(message[5:9]), message[9:]):
self.got_piece(i)
else:
connection.close()
def unpickle(self, data, valid_places):
got = {}
places = {}
dirty = {}
download_history = {}
stat_active = {}
stat_numfound = self.stat_numfound
amount_obtained = self.amount_obtained
amount_inactive = self.amount_inactive
amount_left = self.amount_left
inactive_requests = [x for x in self.inactive_requests]
restored_partials = []
try:
if data['pieces'] == 1: # a seed
assert not data.get('places', None)
assert not data.get('partials', None)
have = Bitfield(len(self.hashes))
for i in range(len(self.hashes)):
have[i] = True
assert have.complete()
_places = []
_partials = []
else:
have = Bitfield(len(self.hashes), data['pieces'])
_places = data['places']
assert len(_places) % 2 == 0
_places = [_places[x:x + 2] for x in range(0, len(_places), 2)]
_partials = data['partials']
assert len(_partials) % 2 == 0
_partials = [
_partials[x:x + 2] for x in range(0, len(_partials), 2)
]
for index, place in _places:
if place not in valid_places:
continue
assert index not in got
assert place not in got
places[index] = place
got[index] = 1
got[place] = 1
for index in range(len(self.hashes)):
if have[index]:
if index not in places:
if index not in valid_places:
have[index] = False
continue
assert index not in got
places[index] = index
got[index] = 1
length = self._piecelen(index)
amount_obtained += length
stat_numfound += 1
amount_inactive -= length
amount_left -= length
inactive_requests[index] = None
for index, plist in _partials:
assert index not in dirty
assert not have[index]
if index not in places:
if index not in valid_places:
continue
assert index not in got
places[index] = index
got[index] = 1
assert len(plist) % 2 == 0
plist = [plist[x:x + 2] for x in range(0, len(plist), 2)]
dirty[index] = plist
stat_active[index] = 1
download_history[index] = {}
# invert given partials
length = self._piecelen(index)
l = []
if plist[0][0] > 0:
l.append((0, plist[0][0]))
for i in range(len(plist) - 1):
end = plist[i][0] + plist[i][1]
assert not end > plist[i + 1][0]
l.append((end, plist[i + 1][0] - end))
end = plist[-1][0] + plist[-1][1]
assert not end > length
if end < length:
l.append((end, length - end))
# split them to request_size
ll = []
amount_obtained += length
amount_inactive -= length
for nb, nl in l:
while nl > 0:
r = min(nl, self.request_size)
ll.append((nb, r))
amount_inactive += r
amount_obtained -= r
nb += self.request_size
nl -= self.request_size
inactive_requests[index] = ll
restored_partials.append(index)
assert amount_obtained + amount_inactive == self.amount_desired
except:
# print_exc()
return [] # invalid data, discard everything
self.have = have
self.places = places
self.dirty = dirty
self.download_history = download_history
self.stat_active = stat_active
self.stat_numfound = stat_numfound
self.amount_obtained = amount_obtained
self.amount_inactive = amount_inactive
self.amount_left = amount_left
self.inactive_requests = inactive_requests
return restored_partials
def __init__(self,
storage,
request_size,
hashes,
piece_size,
finished,
failed,
statusfunc=dummy_status,
flag=fakeflag(),
check_hashes=True,
data_flunked=lambda x: None,
backfunc=None,
config={},
unpauseflag=fakeflag(True)):
self.storage = storage
self.request_size = int(request_size)
self.hashes = hashes
self.piece_size = int(piece_size)
self.piece_length = int(piece_size)
self.finished = finished
self.failed = failed
self.statusfunc = statusfunc
self.flag = flag
self.check_hashes = check_hashes
self.data_flunked = data_flunked
self.backfunc = backfunc
self.config = config
self.unpauseflag = unpauseflag
self.alloc_type = config.get('alloc_type', 'normal')
self.double_check = config.get('double_check', 0)
self.triple_check = config.get('triple_check', 0)
if self.triple_check:
self.double_check = True
self.bgalloc_enabled = False
self.bgalloc_active = False
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= self.piece_size * (len(hashes) - 1):
raise ValueError('bad data in responsefile - total too small')
if self.total_length > self.piece_size * len(hashes):
raise ValueError('bad data in responsefile - total too big')
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.amount_obtained = 0
self.amount_desired = self.total_length
self.have = Bitfield(len(hashes))
self.have_cloaked_data = None
self.blocked = [False] * len(hashes)
self.blocked_holes = []
self.blocked_movein = Olist()
self.blocked_moveout = Olist()
self.waschecked = [False] * len(hashes)
self.places = {}
self.holes = []
self.stat_active = {}
self.stat_new = {}
self.dirty = {}
self.stat_numflunked = 0
self.stat_numdownloaded = 0
self.stat_numfound = 0
self.download_history = {}
self.failed_pieces = {}
self.out_of_place = 0
self.write_buf_max = config['write_buffer_size'] * 1048576
self.write_buf_size = 0
self.write_buf = {} # structure: piece: [(start, data), ...]
self.write_buf_list = []
self.initialize_tasks = [[
'checking existing data', 0, self.init_hashcheck,
self.hashcheckfunc
], ['moving data', 1, self.init_movedata, self.movedatafunc
], ['allocating disk space', 1, self.init_alloc, self.allocfunc]]
self.backfunc(self._bgalloc, 0.1)
self.backfunc(self._bgsync, max(self.config['auto_flush'] * 60, 60))
