class StorageWrapper: def __init__(self, storage, request_size, hashes, piece_size, finished, failed, statusfunc=dummy_status, flag=Event(), check_hashes=True, data_flunked=dummy_data_flunked): self.storage = storage self.request_size = request_size self.hashes = hashes self.piece_size = piece_size self.data_flunked = data_flunked self.total_length = storage.get_total_length() self.amount_left = self.total_length if self.total_length <= piece_size * (len(hashes) - 1): raise ValueError, 'bad data from tracker - total too small' if self.total_length > piece_size * len(hashes): raise ValueError, 'bad data from tracker - total too big' self.finished = finished self.failed = failed self.numactive = [0] * len(hashes) self.inactive_requests = [1] * len(hashes) self.amount_inactive = self.total_length self.endgame = False self.have = Bitfield(len(hashes)) self.waschecked = [check_hashes] * len(hashes) self.places = {} self.holes = [] if len(hashes) == 0: finished() return targets = {} total = len(hashes) for i in xrange(len(hashes)): if not self._waspre(i): targets.setdefault(hashes[i], []).append(i) total -= 1 numchecked = 0.0 if total and check_hashes: statusfunc({ "activity": 'checking existing file', "fractionDone": 0 }) def markgot(piece, pos, self=self, check_hashes=check_hashes): self.places[piece] = pos self.have[piece] = True self.amount_left -= self._piecelen(piece) self.amount_inactive -= self._piecelen(piece) self.inactive_requests[piece] = None self.waschecked[piece] = check_hashes lastlen = self._piecelen(len(hashes) - 1) for i in xrange(len(hashes)): if not self._waspre(i): self.holes.append(i) elif not check_hashes: markgot(i, i) else: sh = sha1(self.storage.read(piece_size * i, lastlen)) sp = sh.digest() sh.update( self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen)) s = sh.digest() if s == hashes[i]: markgot(i, i) elif targets.get(s) and self._piecelen(i) == self._piecelen( targets[s][-1]): markgot(targets[s].pop(), i) elif not self.have[len(hashes) - 1] and sp == hashes[-1] and ( i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)): markgot(len(hashes) - 1, i) else: self.places[i] = i if flag.isSet(): return numchecked += 1 statusfunc({ 'fractionDone': 1 - float(self.amount_left) / self.total_length }) if self.amount_left == 0: finished() def _waspre(self, piece): return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece)) def _piecelen(self, piece): if piece < len(self.hashes) - 1: return self.piece_size else: return self.total_length - piece * self.piece_size def get_amount_left(self): return self.amount_left def do_I_have_anything(self): return self.amount_left < self.total_length def _make_inactive(self, index): length = min(self.piece_size, self.total_length - self.piece_size * index) l = [] x = 0 while x + self.request_size < length: l.append((x, self.request_size)) x += self.request_size l.append((x, length - x)) self.inactive_requests[index] = l def is_endgame(self): return self.endgame def get_have_list(self): return self.have.tostring() def do_I_have(self, index): return self.have[index] def do_I_have_requests(self, index): return not not self.inactive_requests[index] def new_request(self, index): # returns (begin, length) if self.inactive_requests[index] == 1: self._make_inactive(index) self.numactive[index] += 1 rs = self.inactive_requests[index] r = min(rs) rs.remove(r) self.amount_inactive -= r[1] if self.amount_inactive == 0: self.endgame = True return r def piece_came_in(self, index, begin, piece): try: return self._piece_came_in(index, begin, piece) except IOError, e: self.failed('IO Error ' + str(e)) return True
class StorageWrapper: def __init__(self, storage, request_size, hashes, piece_size, finished, failed, statusfunc = dummy_status, flag = Event(), check_hashes = True, data_flunked = dummy_data_flunked): # The Storage instance. self.storage = storage # The size of blocks to request. self.request_size = request_size # An array of SHA-1 hashes for all pieces. self.hashes = hashes # The piece size, which should be a multiple of request_size. self.piece_size = piece_size # Method to call if a piece fails SHA-1 validation. self.data_flunked = data_flunked # The total bytes to download and save. self.total_length = storage.get_total_length() # The number of bytes left to download and validate. self.amount_left = self.total_length if self.total_length <= piece_size * (len(hashes) - 1): raise ValueError, 'bad data from tracker - total too small' if self.total_length > piece_size * len(hashes): raise ValueError, 'bad data from tracker - total too big' # Callback invoked once all pieces have been downloaded and validated. self.finished = finished # Callback invoked with a string describing any error. self.failed = failed # The number of outstanding requests for blocks belonging to a piece. self.numactive = [0] * len(hashes) # If an element is 1, then the piece has not been requested from peers. # If an elemtent is an array, it contains blocks that have not been requested from peers. self.inactive_requests = [1] * len(hashes) # The number of bytes that have not been downloaded or requested. # When this reaches 0, then we enter endgame mode. self.amount_inactive = self.total_length # Whether we are in endgame mode, which # "sends requests for all of its missing blocks to all of its peers." self.endgame = False # The bitfield of pieces we have downloaded, but not necessarily validated. self.have = Bitfield(len(hashes)) # Whether each piece has been validated by computing its SHA-1 hash. # If check_hashes is False, then validating preallocated segments is deferred. self.waschecked = [check_hashes] * len(hashes) # Maps each piece to what piece, or segment, it occupies on disk. # It may not be the right segment for the piece, and the piece may be incomplete. self.places = {} # Missing segments on disk. self.holes = [] if len(hashes) == 0: # If no hashes, then no data to download, so trivially finished. finished() return # Maps each SHA-1 hash to all pieces that have that hash. targets = {} # The total number of preallocated pieces. total = len(hashes) for i in xrange(len(hashes)): if not self._waspre(i): # This piece is not preallocated. targets.setdefault(hashes[i], []).append(i) total -= 1 numchecked = 0.0 if total and check_hashes: # There is at least one preallocated piece. Must determine what pieces they are. statusfunc({"activity" : 'checking existing file', "fractionDone" : 0}) def markgot(piece, pos, self = self, check_hashes = check_hashes): # Record the position of this piece and that we have it. self.places[piece] = pos self.have[piece] = True # This piece has been downloaded and validated if that option is enabled. self.amount_left -= self._piecelen(piece) self.amount_inactive -= self._piecelen(piece) # We won't be requesting this piece from peers. self.inactive_requests[piece] = None self.waschecked[piece] = check_hashes # Get the length of the last piece. lastlen = self._piecelen(len(hashes) - 1) for i in xrange(len(hashes)): if not self._waspre(i): # This piece is not preallocated, i.e. it has no segment of bytes on disk. self.holes.append(i) elif not check_hashes: # The corresponding segment on disk is full of bytes. # Assume that it belongs to this piece, meaning places[i] = i. markgot(i, i) else: # Only get here if check_hashes = True, so we called statusfunc earlier. # We're trying to figure out what piece is at segment i on disk. # The bytes in this segment on disk could belong to any piece. # Compute a hash of its first lastlen bytes in case it has the last piece. sh = sha(self.storage.read(piece_size * i, lastlen)) sp = sh.digest() # Compute a hash of all its bytes in the case that it has any other piece. sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen)) s = sh.digest() if s == hashes[i]: # This is piece i, occupying its correct segment on disk. markgot(i, i) elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]): # This is not the last piece, temporarily occupying the wrong segment. markgot(targets[s].pop(), i) elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)): # This is the last piece, temporarily occupying the wrong segment. markgot(len(hashes) - 1, i) else: # This segment has been allocated but it doesn't belong to any piece. # When this piece comes in, we can write to this segment directly. self.places[i] = i if flag.isSet(): return numchecked += 1 statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length}) if self.amount_left == 0: # All data has been downloaded and validated. finished() def _waspre(self, piece): # Returns whether all files containing this piece were preallocated. return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece)) def _piecelen(self, piece): # Return the length of the given piece. if piece < len(self.hashes) - 1: return self.piece_size else: return self.total_length - piece * self.piece_size def get_amount_left(self): return self.amount_left def do_I_have_anything(self): # Returns whether we've downloaded at least one piece. return self.amount_left < self.total_length def _make_inactive(self, index): # This assigns to length what _piecelen would return. length = min(self.piece_size, self.total_length - self.piece_size * index) # Will contain all blocks for this piece as (start byte, end byte) pairs. l = [] x = 0 while x + self.request_size < length: l.append((x, self.request_size)) x += self.request_size l.append((x, length - x)) # Replace the value of 1 with the array of blocks. self.inactive_requests[index] = l def is_endgame(self): return self.endgame def get_have_list(self): # Used when sending our bitfield to another peer. return self.have.tostring() def do_I_have(self, index): return self.have[index] def do_I_have_requests(self, index): # Similar to how you would convert to boolean in JavaScript, # this returns True if the element is 1 or a non-empty array, and # this returns False if the element is None or an empty array. return not not self.inactive_requests[index] def new_request(self, index): # returns (begin, length) if self.inactive_requests[index] == 1: # Create the blocks to request for this piece. self._make_inactive(index) # Increment count of blocks requested for this piece. self.numactive[index] += 1 # Get the block with the earliest start byte. rs = self.inactive_requests[index] r = min(rs) rs.remove(r) # Deduct block length from total bytes neither downloaded nor requested. self.amount_inactive -= r[1] if self.amount_inactive == 0: # All bytes either downloaded or requested, so enter endgame. self.endgame = True return r def piece_came_in(self, index, begin, piece): try: return self._piece_came_in(index, begin, piece) except IOError, e: self.failed('IO Error ' + str(e)) return True
from bitfield import Bitfield bitfield = Bitfield(8, '\xee') print 'value={0:b} complete={1}'.format(ord(bitfield.tostring()), bitfield.complete()) bitfield[3] = True bitfield[7] = True print 'value={0:b} complete={1}'.format(ord(bitfield.tostring()), bitfield.complete())
class StorageWrapper: def __init__(self, storage, request_size, hashes, piece_size, finished, failed, statusfunc = dummy_status, flag = Event(), check_hashes = True, data_flunked = dummy_data_flunked): self.storage = storage self.request_size = request_size self.hashes = hashes self.piece_size = piece_size self.data_flunked = data_flunked self.total_length = storage.get_total_length() self.amount_left = self.total_length if self.total_length <= piece_size * (len(hashes) - 1): raise ValueError, 'bad data from tracker - total too small' if self.total_length > piece_size * len(hashes): raise ValueError, 'bad data from tracker - total too big' self.finished = finished self.failed = failed self.numactive = [0] * len(hashes) self.inactive_requests = [1] * len(hashes) self.amount_inactive = self.total_length self.endgame = False self.have = Bitfield(len(hashes)) self.waschecked = [check_hashes] * len(hashes) self.places = {} self.holes = [] if len(hashes) == 0: finished() return targets = {} total = len(hashes) for i in xrange(len(hashes)): if not self._waspre(i): targets.setdefault(hashes[i], []).append(i) total -= 1 numchecked = 0.0 if total and check_hashes: statusfunc({"activity" : 'checking existing file', "fractionDone" : 0}) def markgot(piece, pos, self = self, check_hashes = check_hashes): self.places[piece] = pos self.have[piece] = True self.amount_left -= self._piecelen(piece) self.amount_inactive -= self._piecelen(piece) self.inactive_requests[piece] = None self.waschecked[piece] = check_hashes lastlen = self._piecelen(len(hashes) - 1) for i in xrange(len(hashes)): if not self._waspre(i): self.holes.append(i) elif not check_hashes: markgot(i, i) else: sh = sha(self.storage.read(piece_size * i, lastlen)) sp = sh.digest() sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen)) s = sh.digest() if s == hashes[i]: markgot(i, i) elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]): markgot(targets[s].pop(), i) elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)): markgot(len(hashes) - 1, i) else: self.places[i] = i if flag.isSet(): return numchecked += 1 statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length}) if self.amount_left == 0: finished() def _waspre(self, piece): return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece)) def _piecelen(self, piece): if piece < len(self.hashes) - 1: return self.piece_size else: return self.total_length - piece * self.piece_size def get_amount_left(self): return self.amount_left def do_I_have_anything(self): return self.amount_left < self.total_length def _make_inactive(self, index): length = min(self.piece_size, self.total_length - self.piece_size * index) l = [] x = 0 while x + self.request_size < length: l.append((x, self.request_size)) x += self.request_size l.append((x, length - x)) self.inactive_requests[index] = l def is_endgame(self): return self.endgame def get_have_list(self): return self.have.tostring() def do_I_have(self, index): return self.have[index] def do_I_have_requests(self, index): return not not self.inactive_requests[index] def new_request(self, index): # returns (begin, length) if self.inactive_requests[index] == 1: self._make_inactive(index) self.numactive[index] += 1 rs = self.inactive_requests[index] r = min(rs) rs.remove(r) self.amount_inactive -= r[1] if self.amount_inactive == 0: self.endgame = True return r def piece_came_in(self, index, begin, piece): try: return self._piece_came_in(index, begin, piece) except IOError, e: self.failed('IO Error ' + str(e)) return True