def pickle(self): if self.have.complete: return {'pieces': 1} pieces = Bitfield(len(self.hashes)) places = [] partials = [] for p in xrange(len(self.hashes)): if self.blocked[p] or p not in self.places: continue h = self.have[p] pieces[p] = h pp = self.dirty.get(p) if not h and not pp: # no data places.extend([self.places[p], self.places[p]]) elif self.places[p] != p: places.extend([p, self.places[p]]) if h or not pp: continue pp.sort() r = [] while len(pp) > 1: if pp[0][0] + pp[0][1] == pp[1][0]: pp[0] = list(pp[0]) pp[0][1] += pp[1][1] del pp[1] else: r.extend(pp[0]) del pp[0] r.extend(pp[0]) partials.extend([p, r]) return {'pieces': str(pieces), 'places': places, 'partials': partials}
def test_bitfield(self): """Unit test Bitfield""" self.assertRaises(ValueError, Bitfield, 7, b'ab') self.assertRaises(ValueError, Bitfield, 7, b'ab') self.assertRaises(ValueError, Bitfield, 9, b'abc') self.assertRaises(ValueError, Bitfield, 0, b'a') self.assertRaises(ValueError, Bitfield, 1, b'') self.assertRaises(ValueError, Bitfield, 7, b'') self.assertRaises(ValueError, Bitfield, 8, b'') self.assertRaises(ValueError, Bitfield, 9, b'a') self.assertRaises(ValueError, Bitfield, 7, b'\x01') self.assertRaises(ValueError, Bitfield, 9, b'\x00\x40') self.assertEqual(bytes(Bitfield(0, b'')), b'') self.assertEqual(bytes(Bitfield(1, b'\x80')), b'\x80') self.assertEqual(bytes(Bitfield(7, b'\x02')), b'\x02') self.assertEqual(bytes(Bitfield(8, b'\xFF')), b'\xFF') self.assertEqual(bytes(Bitfield(9, b'\x00\x80')), b'\x00\x80') testx = Bitfield(1) self.assertEqual(testx.numfalse, 1) testx[0] = 1 self.assertEqual(testx.numfalse, 0) testx[0] = 1 self.assertEqual(testx.numfalse, 0) self.assertEqual(bytes(testx), b'\x80') testx = Bitfield(7) self.assertEqual(len(testx), 7) testx[6] = 1 self.assertEqual(testx.numfalse, 6) self.assertEqual(bytes(testx), b'\x02') testx = Bitfield(8) testx[7] = 1 self.assertEqual(bytes(testx), b'\x01') testx = Bitfield(9) testx[8] = 1 self.assertEqual(testx.numfalse, 8) self.assertEqual(bytes(testx), b'\x00\x80') testx = Bitfield(8, b'\xc4') self.assertEqual(len(testx), 8) self.assertEqual(testx.numfalse, 5) self.assertEqual(bytes(testx), b'\xc4')
def get_have_list_cloaked(self): if self.have_cloaked_data is None: newhave = Bitfield(copyfrom=self.have) unhaves = [] # between 2-4 unless torrent is small n = min(random.randrange(2, 5), len(self.hashes)) while len(unhaves) < n: # all in first 4 bytes unhave = random.randrange(min(32, len(self.hashes))) if not unhave in unhaves: unhaves.append(unhave) newhave[unhave] = False self.have_cloaked_data = (str(newhave), unhaves) return self.have_cloaked_data
def __init__(self, downloader, connection): self.downloader = downloader self.connection = connection self.choked = True self.interested = False self.active_requests = [] self.measure = Measure(downloader.max_rate_period) self.peermeasure = Measure(downloader.max_rate_period) self.have = Bitfield(downloader.numpieces) self.last = -1000 self.last2 = -1000 self.example_interest = None self.backlog = 2 self.ip = connection.get_ip() self.guard = BadDataGuard(self)
def unpickle(self, data, valid_places): got = set() places = {} dirty = {} download_history = {} stat_active = set() stat_numfound = self.stat_numfound amount_obtained = self.amount_obtained amount_inactive = self.amount_inactive amount_left = self.amount_left inactive_requests = [x for x in self.inactive_requests] restored_partials = [] try: if data['pieces'] == 1: # a seed assert not data.get('places', None) assert not data.get('partials', None) have = Bitfield(len(self.hashes), val=True) assert have.complete _places = [] _partials = [] else: have = Bitfield(len(self.hashes), data['pieces']) _places = data['places'] assert len(_places) % 2 == 0 _places = [ _places[x:x + 2] for x in xrange(0, len(_places), 2) ] _partials = data['partials'] assert len(_partials) % 2 == 0 _partials = [ _partials[x:x + 2] for x in xrange(0, len(_partials), 2) ] for index, place in _places: if place not in valid_places: continue assert index not in got assert place not in got places[index] = place got.add(index) got.add(place) for index in xrange(len(self.hashes)): if have[index]: if index not in places: if index not in valid_places: have[index] = False continue assert index not in got places[index] = index got.add(index) length = self._piecelen(index) amount_obtained += length stat_numfound += 1 amount_inactive -= length amount_left -= length inactive_requests[index] = None for index, plist in _partials: assert index not in dirty assert not have[index] if index not in places: if index not in valid_places: continue assert index not in got places[index] = index got.add(index) assert len(plist) % 2 == 0 plist = [plist[x:x + 2] for x in xrange(0, len(plist), 2)] dirty[index] = plist stat_active.add(index) download_history[index] = {} # invert given partials length = self._piecelen(index) l = [] if plist[0][0] > 0: l.append((0, plist[0][0])) for pieceA, pieceB in zip(plist[:-1], plist[1:]): end = pieceA[0] + pieceA[1] assert not end > pieceB[0] l.append((end, pieceB[0] - end)) end = pieceB[0] + pieceB[1] assert not end > length if end < length: l.append((end, length - end)) # split them to request_size ll = [] amount_obtained += length amount_inactive -= length for nb, nl in l: while nl > 0: r = min(nl, self.request_size) ll.append((nb, r)) amount_inactive += r amount_obtained -= r nb += self.request_size nl -= self.request_size inactive_requests[index] = ll restored_partials.append(index) assert amount_obtained + amount_inactive == self.amount_desired except Exception: # print_exc() return [] # invalid data, discard everything self.have = have self.places = places self.dirty = dirty self.download_history = download_history self.stat_active = stat_active self.stat_numfound = stat_numfound self.amount_obtained = amount_obtained self.amount_inactive = amount_inactive self.amount_left = amount_left self.inactive_requests = inactive_requests return restored_partials
def __init__(self, storage, request_size, hashes, piece_size, finished, failed, statusfunc=dummy_status, flag=fakeflag(), check_hashes=True, data_flunked=lambda x: None, backfunc=None, config={}, unpauseflag=fakeflag(True)): self.storage = storage self.request_size = long(request_size) self.hashes = hashes self.piece_size = long(piece_size) self.piece_length = long(piece_size) self.finished = finished self.failed = failed self.statusfunc = statusfunc self.flag = flag self.check_hashes = check_hashes self.data_flunked = data_flunked self.backfunc = backfunc self.config = config self.unpauseflag = unpauseflag self.alloc_type = config.get('alloc_type', 'normal') self.double_check = config.get('double_check', 0) self.triple_check = config.get('triple_check', 0) if self.triple_check: self.double_check = True self.bgalloc_enabled = False self.bgalloc_active = False self.total_length = storage.get_total_length() self.amount_left = self.total_length if self.total_length <= self.piece_size * (len(hashes) - 1): raise ValueError('bad data in responsefile - total too small') if self.total_length > self.piece_size * len(hashes): raise ValueError('bad data in responsefile - total too big') self.numactive = [0] * len(hashes) self.inactive_requests = [1] * len(hashes) self.amount_inactive = self.total_length self.amount_obtained = 0 self.amount_desired = self.total_length self.have = Bitfield(len(hashes)) self.have_cloaked_data = None self.blocked = [False] * len(hashes) self.blocked_holes = [] self.blocked_movein = OrderedSet() self.blocked_moveout = OrderedSet() self.waschecked = [False] * len(hashes) self.places = {} self.holes = [] self.stat_active = set() self.stat_new = set() self.dirty = {} self.stat_numflunked = 0 self.stat_numdownloaded = 0 self.stat_numfound = 0 self.download_history = {} self.failed_pieces = {} self.out_of_place = 0 self.write_buf_max = config['write_buffer_size'] * 1048576L self.write_buf_size = 0L self.write_buf = {} # structure: piece: [(start, data), ...] self.write_buf_list = [] self.initialize_tasks = [[ 'checking existing data', 0, self.init_hashcheck, self.hashcheckfunc ], ['moving data', 1, self.init_movedata, self.movedatafunc ], ['allocating disk space', 1, self.init_alloc, self.allocfunc]] self.backfunc(self._bgalloc, 0.1) self.backfunc(self._bgsync, max(self.config['auto_flush'] * 60, 60))
def got_message(self, connection, message): c = self.connections[connection] t = message[0] if t == BITFIELD and c.got_anything: connection.close() return c.got_anything = True if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED] and len(message) != 1): connection.close() return if t == CHOKE: c.download.got_choke() elif t == UNCHOKE: c.download.got_unchoke() elif t == INTERESTED: if not c.download.have.complete(): c.upload.got_interested() elif t == NOT_INTERESTED: c.upload.got_not_interested() elif t == HAVE: if len(message) != 5: connection.close() return i = toint(message[1:]) if i >= self.numpieces: connection.close() return if c.download.got_have(i): c.upload.got_not_interested() elif t == BITFIELD: try: b = Bitfield(self.numpieces, message[1:]) except ValueError: connection.close() return if c.download.got_have_bitfield(b): c.upload.got_not_interested() elif t == REQUEST: if len(message) != 13: connection.close() return i = toint(message[1:5]) if i >= self.numpieces: connection.close() return c.got_request(i, toint(message[5:9]), toint(message[9:])) elif t == CANCEL: if len(message) != 13: connection.close() return i = toint(message[1:5]) if i >= self.numpieces: connection.close() return c.upload.got_cancel(i, toint(message[5:9]), toint(message[9:])) elif t == PIECE: if len(message) <= 9: connection.close() return i = toint(message[1:5]) if i >= self.numpieces: connection.close() return if c.download.got_piece(i, toint(message[5:9]), message[9:]): self.got_piece(i) else: connection.close()
def got_message(self, connection, message): c = self.connections[connection] t = message[:1] if DEBUG2: print(c.ccount, 'message received', ord(t)) if t == BITFIELD and c.got_anything: if DEBUG2: print(c.ccount, 'misplaced bitfield') connection.close() return c.got_anything = True if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED] and len(message) != 1): if DEBUG2: print(c.ccount, 'bad message length') connection.close() return if t == CHOKE: c.download.got_choke() elif t == UNCHOKE: c.download.got_unchoke() elif t == INTERESTED: if not c.download.have.complete: c.upload.got_interested() elif t == NOT_INTERESTED: c.upload.got_not_interested() elif t == HAVE: if len(message) != 5: if DEBUG2: print(c.ccount, 'bad message length') connection.close() return i = int.from_bytes(message[1:], 'big') if i >= self.numpieces: if DEBUG2: print(c.ccount, 'bad piece number') connection.close() return if c.download.got_have(i): c.upload.got_not_interested() elif t == BITFIELD: try: b = Bitfield(self.numpieces, message[1:]) except ValueError: if DEBUG2: print(c.ccount, 'bad bitfield') connection.close() return if c.download.got_have_bitfield(b): c.upload.got_not_interested() elif t == REQUEST: if len(message) != 13: if DEBUG2: print(c.ccount, 'bad message length') connection.close() return piece_num = int.from_bytes(message[1:5], 'big') if piece_num >= self.numpieces: if DEBUG2: print(c.ccount, 'bad piece number') connection.close() return c.got_request(piece_num, int.from_bytes(message[5:9], 'big'), int.from_bytes(message[9:], 'big')) elif t == CANCEL: if len(message) != 13: if DEBUG2: print(c.ccount, 'bad message length') connection.close() return i = int.from_bytes(message[1:5], 'big') if i >= self.numpieces: if DEBUG2: print(c.ccount, 'bad piece number') connection.close() return c.upload.got_cancel(i, int.from_bytes(message[5:9], 'big'), int.from_bytes(message[9:], 'big')) elif t == PIECE: if len(message) <= 9: if DEBUG2: print(c.ccount, 'bad message length') connection.close() return i = int.from_bytes(message[1:5], 'big') if i >= self.numpieces: if DEBUG2: print(c.ccount, 'bad piece number') connection.close() return if c.download.got_piece(i, int.from_bytes(message[5:9], 'big'), message[9:]): self.got_piece(i) else: connection.close()