def pickle(self):
if self.have.complete:
return {'pieces': 1}
pieces = Bitfield(len(self.hashes))
places = []
partials = []
for p in xrange(len(self.hashes)):
if self.blocked[p] or p not in self.places:
continue
h = self.have[p]
pieces[p] = h
pp = self.dirty.get(p)
if not h and not pp: # no data
places.extend([self.places[p], self.places[p]])
elif self.places[p] != p:
places.extend([p, self.places[p]])
if h or not pp:
continue
pp.sort()
r = []
while len(pp) > 1:
if pp[0][0] + pp[0][1] == pp[1][0]:
pp[0] = list(pp[0])
pp[0][1] += pp[1][1]
del pp[1]
else:
r.extend(pp[0])
del pp[0]
r.extend(pp[0])
partials.extend([p, r])
return {'pieces': str(pieces), 'places': places, 'partials': partials}
def test_bitfield(self):
"""Unit test Bitfield"""
self.assertRaises(ValueError, Bitfield, 7, b'ab')
self.assertRaises(ValueError, Bitfield, 7, b'ab')
self.assertRaises(ValueError, Bitfield, 9, b'abc')
self.assertRaises(ValueError, Bitfield, 0, b'a')
self.assertRaises(ValueError, Bitfield, 1, b'')
self.assertRaises(ValueError, Bitfield, 7, b'')
self.assertRaises(ValueError, Bitfield, 8, b'')
self.assertRaises(ValueError, Bitfield, 9, b'a')
self.assertRaises(ValueError, Bitfield, 7, b'\x01')
self.assertRaises(ValueError, Bitfield, 9, b'\x00\x40')
self.assertEqual(bytes(Bitfield(0, b'')), b'')
self.assertEqual(bytes(Bitfield(1, b'\x80')), b'\x80')
self.assertEqual(bytes(Bitfield(7, b'\x02')), b'\x02')
self.assertEqual(bytes(Bitfield(8, b'\xFF')), b'\xFF')
self.assertEqual(bytes(Bitfield(9, b'\x00\x80')), b'\x00\x80')
testx = Bitfield(1)
self.assertEqual(testx.numfalse, 1)
testx[0] = 1
self.assertEqual(testx.numfalse, 0)
testx[0] = 1
self.assertEqual(testx.numfalse, 0)
self.assertEqual(bytes(testx), b'\x80')
testx = Bitfield(7)
self.assertEqual(len(testx), 7)
testx[6] = 1
self.assertEqual(testx.numfalse, 6)
self.assertEqual(bytes(testx), b'\x02')
testx = Bitfield(8)
testx[7] = 1
self.assertEqual(bytes(testx), b'\x01')
testx = Bitfield(9)
testx[8] = 1
self.assertEqual(testx.numfalse, 8)
self.assertEqual(bytes(testx), b'\x00\x80')
testx = Bitfield(8, b'\xc4')
self.assertEqual(len(testx), 8)
self.assertEqual(testx.numfalse, 5)
self.assertEqual(bytes(testx), b'\xc4')
def get_have_list_cloaked(self):
if self.have_cloaked_data is None:
newhave = Bitfield(copyfrom=self.have)
unhaves = []
# between 2-4 unless torrent is small
n = min(random.randrange(2, 5), len(self.hashes))
while len(unhaves) < n:
# all in first 4 bytes
unhave = random.randrange(min(32, len(self.hashes)))
if not unhave in unhaves:
unhaves.append(unhave)
newhave[unhave] = False
self.have_cloaked_data = (str(newhave), unhaves)
return self.have_cloaked_data
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.peermeasure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = -1000
self.last2 = -1000
self.example_interest = None
self.backlog = 2
self.ip = connection.get_ip()
self.guard = BadDataGuard(self)
def unpickle(self, data, valid_places):
got = set()
places = {}
dirty = {}
download_history = {}
stat_active = set()
stat_numfound = self.stat_numfound
amount_obtained = self.amount_obtained
amount_inactive = self.amount_inactive
amount_left = self.amount_left
inactive_requests = [x for x in self.inactive_requests]
restored_partials = []
try:
if data['pieces'] == 1: # a seed
assert not data.get('places', None)
assert not data.get('partials', None)
have = Bitfield(len(self.hashes), val=True)
assert have.complete
_places = []
_partials = []
else:
have = Bitfield(len(self.hashes), data['pieces'])
_places = data['places']
assert len(_places) % 2 == 0
_places = [
_places[x:x + 2] for x in xrange(0, len(_places), 2)
]
_partials = data['partials']
assert len(_partials) % 2 == 0
_partials = [
_partials[x:x + 2] for x in xrange(0, len(_partials), 2)
]
for index, place in _places:
if place not in valid_places:
continue
assert index not in got
assert place not in got
places[index] = place
got.add(index)
got.add(place)
for index in xrange(len(self.hashes)):
if have[index]:
if index not in places:
if index not in valid_places:
have[index] = False
continue
assert index not in got
places[index] = index
got.add(index)
length = self._piecelen(index)
amount_obtained += length
stat_numfound += 1
amount_inactive -= length
amount_left -= length
inactive_requests[index] = None
for index, plist in _partials:
assert index not in dirty
assert not have[index]
if index not in places:
if index not in valid_places:
continue
assert index not in got
places[index] = index
got.add(index)
assert len(plist) % 2 == 0
plist = [plist[x:x + 2] for x in xrange(0, len(plist), 2)]
dirty[index] = plist
stat_active.add(index)
download_history[index] = {}
# invert given partials
length = self._piecelen(index)
l = []
if plist[0][0] > 0:
l.append((0, plist[0][0]))
for pieceA, pieceB in zip(plist[:-1], plist[1:]):
end = pieceA[0] + pieceA[1]
assert not end > pieceB[0]
l.append((end, pieceB[0] - end))
end = pieceB[0] + pieceB[1]
assert not end > length
if end < length:
l.append((end, length - end))
# split them to request_size
ll = []
amount_obtained += length
amount_inactive -= length
for nb, nl in l:
while nl > 0:
r = min(nl, self.request_size)
ll.append((nb, r))
amount_inactive += r
amount_obtained -= r
nb += self.request_size
nl -= self.request_size
inactive_requests[index] = ll
restored_partials.append(index)
assert amount_obtained + amount_inactive == self.amount_desired
except Exception:
# print_exc()
return [] # invalid data, discard everything
self.have = have
self.places = places
self.dirty = dirty
self.download_history = download_history
self.stat_active = stat_active
self.stat_numfound = stat_numfound
self.amount_obtained = amount_obtained
self.amount_inactive = amount_inactive
self.amount_left = amount_left
self.inactive_requests = inactive_requests
return restored_partials
def __init__(self,
storage,
request_size,
hashes,
piece_size,
finished,
failed,
statusfunc=dummy_status,
flag=fakeflag(),
check_hashes=True,
data_flunked=lambda x: None,
backfunc=None,
config={},
unpauseflag=fakeflag(True)):
self.storage = storage
self.request_size = long(request_size)
self.hashes = hashes
self.piece_size = long(piece_size)
self.piece_length = long(piece_size)
self.finished = finished
self.failed = failed
self.statusfunc = statusfunc
self.flag = flag
self.check_hashes = check_hashes
self.data_flunked = data_flunked
self.backfunc = backfunc
self.config = config
self.unpauseflag = unpauseflag
self.alloc_type = config.get('alloc_type', 'normal')
self.double_check = config.get('double_check', 0)
self.triple_check = config.get('triple_check', 0)
if self.triple_check:
self.double_check = True
self.bgalloc_enabled = False
self.bgalloc_active = False
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= self.piece_size * (len(hashes) - 1):
raise ValueError('bad data in responsefile - total too small')
if self.total_length > self.piece_size * len(hashes):
raise ValueError('bad data in responsefile - total too big')
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.amount_obtained = 0
self.amount_desired = self.total_length
self.have = Bitfield(len(hashes))
self.have_cloaked_data = None
self.blocked = [False] * len(hashes)
self.blocked_holes = []
self.blocked_movein = OrderedSet()
self.blocked_moveout = OrderedSet()
self.waschecked = [False] * len(hashes)
self.places = {}
self.holes = []
self.stat_active = set()
self.stat_new = set()
self.dirty = {}
self.stat_numflunked = 0
self.stat_numdownloaded = 0
self.stat_numfound = 0
self.download_history = {}
self.failed_pieces = {}
self.out_of_place = 0
self.write_buf_max = config['write_buffer_size'] * 1048576L
self.write_buf_size = 0L
self.write_buf = {} # structure: piece: [(start, data), ...]
self.write_buf_list = []
self.initialize_tasks = [[
'checking existing data', 0, self.init_hashcheck,
self.hashcheckfunc
], ['moving data', 1, self.init_movedata, self.movedatafunc
], ['allocating disk space', 1, self.init_alloc, self.allocfunc]]
self.backfunc(self._bgalloc, 0.1)
self.backfunc(self._bgsync, max(self.config['auto_flush'] * 60, 60))
def got_message(self, connection, message):
c = self.connections[connection]
t = message[0]
if t == BITFIELD and c.got_anything:
connection.close()
return
c.got_anything = True
if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED]
and len(message) != 1):
connection.close()
return
if t == CHOKE:
c.download.got_choke()
elif t == UNCHOKE:
c.download.got_unchoke()
elif t == INTERESTED:
if not c.download.have.complete():
c.upload.got_interested()
elif t == NOT_INTERESTED:
c.upload.got_not_interested()
elif t == HAVE:
if len(message) != 5:
connection.close()
return
i = toint(message[1:])
if i >= self.numpieces:
connection.close()
return
if c.download.got_have(i):
c.upload.got_not_interested()
elif t == BITFIELD:
try:
b = Bitfield(self.numpieces, message[1:])
except ValueError:
connection.close()
return
if c.download.got_have_bitfield(b):
c.upload.got_not_interested()
elif t == REQUEST:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.got_request(i, toint(message[5:9]), toint(message[9:]))
elif t == CANCEL:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_cancel(i, toint(message[5:9]), toint(message[9:]))
elif t == PIECE:
if len(message) <= 9:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
if c.download.got_piece(i, toint(message[5:9]), message[9:]):
self.got_piece(i)
else:
connection.close()
def got_message(self, connection, message):
c = self.connections[connection]
t = message[:1]
if DEBUG2:
print(c.ccount, 'message received', ord(t))
if t == BITFIELD and c.got_anything:
if DEBUG2:
print(c.ccount, 'misplaced bitfield')
connection.close()
return
c.got_anything = True
if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED]
and len(message) != 1):
if DEBUG2:
print(c.ccount, 'bad message length')
connection.close()
return
if t == CHOKE:
c.download.got_choke()
elif t == UNCHOKE:
c.download.got_unchoke()
elif t == INTERESTED:
if not c.download.have.complete:
c.upload.got_interested()
elif t == NOT_INTERESTED:
c.upload.got_not_interested()
elif t == HAVE:
if len(message) != 5:
if DEBUG2:
print(c.ccount, 'bad message length')
connection.close()
return
i = int.from_bytes(message[1:], 'big')
if i >= self.numpieces:
if DEBUG2:
print(c.ccount, 'bad piece number')
connection.close()
return
if c.download.got_have(i):
c.upload.got_not_interested()
elif t == BITFIELD:
try:
b = Bitfield(self.numpieces, message[1:])
except ValueError:
if DEBUG2:
print(c.ccount, 'bad bitfield')
connection.close()
return
if c.download.got_have_bitfield(b):
c.upload.got_not_interested()
elif t == REQUEST:
if len(message) != 13:
if DEBUG2:
print(c.ccount, 'bad message length')
connection.close()
return
piece_num = int.from_bytes(message[1:5], 'big')
if piece_num >= self.numpieces:
if DEBUG2:
print(c.ccount, 'bad piece number')
connection.close()
return
c.got_request(piece_num, int.from_bytes(message[5:9], 'big'),
int.from_bytes(message[9:], 'big'))
elif t == CANCEL:
if len(message) != 13:
if DEBUG2:
print(c.ccount, 'bad message length')
connection.close()
return
i = int.from_bytes(message[1:5], 'big')
if i >= self.numpieces:
if DEBUG2:
print(c.ccount, 'bad piece number')
connection.close()
return
c.upload.got_cancel(i, int.from_bytes(message[5:9], 'big'),
int.from_bytes(message[9:], 'big'))
elif t == PIECE:
if len(message) <= 9:
if DEBUG2:
print(c.ccount, 'bad message length')
connection.close()
return
i = int.from_bytes(message[1:5], 'big')
if i >= self.numpieces:
if DEBUG2:
print(c.ccount, 'bad piece number')
connection.close()
return
if c.download.got_piece(i, int.from_bytes(message[5:9], 'big'),
message[9:]):
self.got_piece(i)
else:
connection.close()
