Ejemplo n.º 1
0
class StorageWrapper:
    def __init__(self,
                 storage,
                 request_size,
                 hashes,
                 piece_size,
                 finished,
                 failed,
                 statusfunc=dummy_status,
                 flag=Event(),
                 check_hashes=True,
                 data_flunked=dummy_data_flunked):
        self.storage = storage
        self.request_size = request_size
        self.hashes = hashes
        self.piece_size = piece_size
        self.data_flunked = data_flunked
        self.total_length = storage.get_total_length()
        self.amount_left = self.total_length
        if self.total_length <= piece_size * (len(hashes) - 1):
            raise ValueError, 'bad data from tracker - total too small'
        if self.total_length > piece_size * len(hashes):
            raise ValueError, 'bad data from tracker - total too big'
        self.finished = finished
        self.failed = failed
        self.numactive = [0] * len(hashes)
        self.inactive_requests = [1] * len(hashes)
        self.amount_inactive = self.total_length
        self.endgame = False
        self.have = Bitfield(len(hashes))
        self.waschecked = [check_hashes] * len(hashes)
        self.places = {}
        self.holes = []
        if len(hashes) == 0:
            finished()
            return
        targets = {}
        total = len(hashes)
        for i in xrange(len(hashes)):
            if not self._waspre(i):
                targets.setdefault(hashes[i], []).append(i)
                total -= 1
        numchecked = 0.0
        if total and check_hashes:
            statusfunc({
                "activity": 'checking existing file',
                "fractionDone": 0
            })

        def markgot(piece, pos, self=self, check_hashes=check_hashes):
            self.places[piece] = pos
            self.have[piece] = True
            self.amount_left -= self._piecelen(piece)
            self.amount_inactive -= self._piecelen(piece)
            self.inactive_requests[piece] = None
            self.waschecked[piece] = check_hashes

        lastlen = self._piecelen(len(hashes) - 1)
        for i in xrange(len(hashes)):
            if not self._waspre(i):
                self.holes.append(i)
            elif not check_hashes:
                markgot(i, i)
            else:
                sh = sha1(self.storage.read(piece_size * i, lastlen))
                sp = sh.digest()
                sh.update(
                    self.storage.read(piece_size * i + lastlen,
                                      self._piecelen(i) - lastlen))
                s = sh.digest()
                if s == hashes[i]:
                    markgot(i, i)
                elif targets.get(s) and self._piecelen(i) == self._piecelen(
                        targets[s][-1]):
                    markgot(targets[s].pop(), i)
                elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (
                        i == len(hashes) - 1
                        or not self._waspre(len(hashes) - 1)):
                    markgot(len(hashes) - 1, i)
                else:
                    self.places[i] = i
                if flag.isSet():
                    return
                numchecked += 1
                statusfunc({
                    'fractionDone':
                    1 - float(self.amount_left) / self.total_length
                })
        if self.amount_left == 0:
            finished()

    def _waspre(self, piece):
        return self.storage.was_preallocated(piece * self.piece_size,
                                             self._piecelen(piece))

    def _piecelen(self, piece):
        if piece < len(self.hashes) - 1:
            return self.piece_size
        else:
            return self.total_length - piece * self.piece_size

    def get_amount_left(self):
        return self.amount_left

    def do_I_have_anything(self):
        return self.amount_left < self.total_length

    def _make_inactive(self, index):
        length = min(self.piece_size,
                     self.total_length - self.piece_size * index)
        l = []
        x = 0
        while x + self.request_size < length:
            l.append((x, self.request_size))
            x += self.request_size
        l.append((x, length - x))
        self.inactive_requests[index] = l

    def is_endgame(self):
        return self.endgame

    def get_have_list(self):
        return self.have.tostring()

    def do_I_have(self, index):
        return self.have[index]

    def do_I_have_requests(self, index):
        return not not self.inactive_requests[index]

    def new_request(self, index):
        # returns (begin, length)
        if self.inactive_requests[index] == 1:
            self._make_inactive(index)
        self.numactive[index] += 1
        rs = self.inactive_requests[index]
        r = min(rs)
        rs.remove(r)
        self.amount_inactive -= r[1]
        if self.amount_inactive == 0:
            self.endgame = True
        return r

    def piece_came_in(self, index, begin, piece):
        try:
            return self._piece_came_in(index, begin, piece)
        except IOError, e:
            self.failed('IO Error ' + str(e))
            return True
Ejemplo n.º 2
0
class StorageWrapper:
    def __init__(self, storage, request_size, hashes, 
            piece_size, finished, failed, 
            statusfunc = dummy_status, flag = Event(), check_hashes = True,
            data_flunked = dummy_data_flunked):
        # The Storage instance.
        self.storage = storage
        # The size of blocks to request.
        self.request_size = request_size
        # An array of SHA-1 hashes for all pieces.
        self.hashes = hashes
        # The piece size, which should be a multiple of request_size.
        self.piece_size = piece_size
        # Method to call if a piece fails SHA-1 validation.
        self.data_flunked = data_flunked
        # The total bytes to download and save.
        self.total_length = storage.get_total_length()
        # The number of bytes left to download and validate.
        self.amount_left = self.total_length
        if self.total_length <= piece_size * (len(hashes) - 1):
            raise ValueError, 'bad data from tracker - total too small'
        if self.total_length > piece_size * len(hashes):
            raise ValueError, 'bad data from tracker - total too big'
        # Callback invoked once all pieces have been downloaded and validated.
        self.finished = finished
        # Callback invoked with a string describing any error.
        self.failed = failed

        # The number of outstanding requests for blocks belonging to a piece.
        self.numactive = [0] * len(hashes)
        # If an element is 1, then the piece has not been requested from peers.
        # If an elemtent is an array, it contains blocks that have not been requested from peers.
        self.inactive_requests = [1] * len(hashes)
        # The number of bytes that have not been downloaded or requested.
        # When this reaches 0, then we enter endgame mode.
        self.amount_inactive = self.total_length
        # Whether we are in endgame mode, which
        # "sends requests for all of its missing blocks to all of its peers."
        self.endgame = False
        # The bitfield of pieces we have downloaded, but not necessarily validated.
        self.have = Bitfield(len(hashes))
        # Whether each piece has been validated by computing its SHA-1 hash.
        # If check_hashes is False, then validating preallocated segments is deferred.
        self.waschecked = [check_hashes] * len(hashes)

        # Maps each piece to what piece, or segment, it occupies on disk.
        # It may not be the right segment for the piece, and the piece may be incomplete.
        self.places = {}
        # Missing segments on disk.
        self.holes = []
        if len(hashes) == 0:
            # If no hashes, then no data to download, so trivially finished.
            finished()
            return

        # Maps each SHA-1 hash to all pieces that have that hash.
        targets = {}
        # The total number of preallocated pieces.
        total = len(hashes)
        for i in xrange(len(hashes)):
            if not self._waspre(i):
                # This piece is not preallocated.
                targets.setdefault(hashes[i], []).append(i)
                total -= 1
        numchecked = 0.0
        if total and check_hashes:
            # There is at least one preallocated piece. Must determine what pieces they are.
            statusfunc({"activity" : 'checking existing file', 
                "fractionDone" : 0})

        def markgot(piece, pos, self = self, check_hashes = check_hashes):
            # Record the position of this piece and that we have it.
            self.places[piece] = pos
            self.have[piece] = True
            # This piece has been downloaded and validated if that option is enabled.
            self.amount_left -= self._piecelen(piece)
            self.amount_inactive -= self._piecelen(piece)
            # We won't be requesting this piece from peers.
            self.inactive_requests[piece] = None
            self.waschecked[piece] = check_hashes

        # Get the length of the last piece.
        lastlen = self._piecelen(len(hashes) - 1)
        for i in xrange(len(hashes)):
            if not self._waspre(i):
                # This piece is not preallocated, i.e. it has no segment of bytes on disk.
                self.holes.append(i)
            elif not check_hashes:
                # The corresponding segment on disk is full of bytes.
                # Assume that it belongs to this piece, meaning places[i] = i.
                markgot(i, i)
            else:
                # Only get here if check_hashes = True, so we called statusfunc earlier.
                # We're trying to figure out what piece is at segment i on disk.

                # The bytes in this segment on disk could belong to any piece.
                # Compute a hash of its first lastlen bytes in case it has the last piece.
                sh = sha(self.storage.read(piece_size * i, lastlen))
                sp = sh.digest()
                # Compute a hash of all its bytes in the case that it has any other piece.
                sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen))
                s = sh.digest()

                if s == hashes[i]:
                    # This is piece i, occupying its correct segment on disk.
                    markgot(i, i)
                elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]):
                    # This is not the last piece, temporarily occupying the wrong segment.
                    markgot(targets[s].pop(), i)
                elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)):
                    # This is the last piece, temporarily occupying the wrong segment.
                    markgot(len(hashes) - 1, i)
                else:
                    # This segment has been allocated but it doesn't belong to any piece.
                    # When this piece comes in, we can write to this segment directly.
                    self.places[i] = i
                if flag.isSet():
                    return
                numchecked += 1
                statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length})

        if self.amount_left == 0:
            # All data has been downloaded and validated.
            finished()

    def _waspre(self, piece):
        # Returns whether all files containing this piece were preallocated.
        return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece))

    def _piecelen(self, piece):
        # Return the length of the given piece.
        if piece < len(self.hashes) - 1:
            return self.piece_size
        else:
            return self.total_length - piece * self.piece_size

    def get_amount_left(self):
        return self.amount_left

    def do_I_have_anything(self):
        # Returns whether we've downloaded at least one piece.
        return self.amount_left < self.total_length

    def _make_inactive(self, index):
        # This assigns to length what _piecelen would return.
        length = min(self.piece_size, self.total_length - self.piece_size * index)
        # Will contain all blocks for this piece as (start byte, end byte) pairs.
        l = []
        x = 0
        while x + self.request_size < length:
            l.append((x, self.request_size))
            x += self.request_size
        l.append((x, length - x))
        # Replace the value of 1 with the array of blocks.
        self.inactive_requests[index] = l

    def is_endgame(self):
        return self.endgame

    def get_have_list(self):
        # Used when sending our bitfield to another peer.
        return self.have.tostring()

    def do_I_have(self, index):
        return self.have[index]

    def do_I_have_requests(self, index):
        # Similar to how you would convert to boolean in JavaScript,
        # this returns True if the element is 1 or a non-empty array, and
        # this returns False if the element is None or an empty array.
        return not not self.inactive_requests[index]

    def new_request(self, index):
        # returns (begin, length)
        if self.inactive_requests[index] == 1:
            # Create the blocks to request for this piece.
            self._make_inactive(index)
        # Increment count of blocks requested for this piece.
        self.numactive[index] += 1
        # Get the block with the earliest start byte.
        rs = self.inactive_requests[index]
        r = min(rs)
        rs.remove(r)
        # Deduct block length from total bytes neither downloaded nor requested.
        self.amount_inactive -= r[1]
        if self.amount_inactive == 0:
            # All bytes either downloaded or requested, so enter endgame.
            self.endgame = True
        return r

    def piece_came_in(self, index, begin, piece):
        try:
            return self._piece_came_in(index, begin, piece)
        except IOError, e:
            self.failed('IO Error ' + str(e))
            return True
from bitfield import Bitfield

bitfield = Bitfield(8, '\xee')
print 'value={0:b} complete={1}'.format(ord(bitfield.tostring()), bitfield.complete())
bitfield[3] = True
bitfield[7] = True
print 'value={0:b} complete={1}'.format(ord(bitfield.tostring()), bitfield.complete())

Ejemplo n.º 4
0
class StorageWrapper:
    def __init__(self, storage, request_size, hashes, 
            piece_size, finished, failed, 
            statusfunc = dummy_status, flag = Event(), check_hashes = True,
            data_flunked = dummy_data_flunked):
        self.storage = storage
        self.request_size = request_size
        self.hashes = hashes
        self.piece_size = piece_size
        self.data_flunked = data_flunked
        self.total_length = storage.get_total_length()
        self.amount_left = self.total_length
        if self.total_length <= piece_size * (len(hashes) - 1):
            raise ValueError, 'bad data from tracker - total too small'
        if self.total_length > piece_size * len(hashes):
            raise ValueError, 'bad data from tracker - total too big'
        self.finished = finished
        self.failed = failed
        self.numactive = [0] * len(hashes)
        self.inactive_requests = [1] * len(hashes)
        self.amount_inactive = self.total_length
        self.endgame = False
        self.have = Bitfield(len(hashes))
        self.waschecked = [check_hashes] * len(hashes)
        self.places = {}
        self.holes = []
        if len(hashes) == 0:
            finished()
            return
        targets = {}
        total = len(hashes)
        for i in xrange(len(hashes)):
            if not self._waspre(i):
                targets.setdefault(hashes[i], []).append(i)
                total -= 1
        numchecked = 0.0
        if total and check_hashes:
            statusfunc({"activity" : 'checking existing file', 
                "fractionDone" : 0})
        def markgot(piece, pos, self = self, check_hashes = check_hashes):
            self.places[piece] = pos
            self.have[piece] = True
            self.amount_left -= self._piecelen(piece)
            self.amount_inactive -= self._piecelen(piece)
            self.inactive_requests[piece] = None
            self.waschecked[piece] = check_hashes
        lastlen = self._piecelen(len(hashes) - 1)
        for i in xrange(len(hashes)):
            if not self._waspre(i):
                self.holes.append(i)
            elif not check_hashes:
                markgot(i, i)
            else:
                sh = sha(self.storage.read(piece_size * i, lastlen))
                sp = sh.digest()
                sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen))
                s = sh.digest()
                if s == hashes[i]:
                    markgot(i, i)
                elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]):
                    markgot(targets[s].pop(), i)
                elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)):
                    markgot(len(hashes) - 1, i)
                else:
                    self.places[i] = i
                if flag.isSet():
                    return
                numchecked += 1
                statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length})
        if self.amount_left == 0:
            finished()

    def _waspre(self, piece):
        return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece))

    def _piecelen(self, piece):
        if piece < len(self.hashes) - 1:
            return self.piece_size
        else:
            return self.total_length - piece * self.piece_size

    def get_amount_left(self):
        return self.amount_left

    def do_I_have_anything(self):
        return self.amount_left < self.total_length

    def _make_inactive(self, index):
        length = min(self.piece_size, self.total_length - self.piece_size * index)
        l = []
        x = 0
        while x + self.request_size < length:
            l.append((x, self.request_size))
            x += self.request_size
        l.append((x, length - x))
        self.inactive_requests[index] = l

    def is_endgame(self):
        return self.endgame

    def get_have_list(self):
        return self.have.tostring()

    def do_I_have(self, index):
        return self.have[index]

    def do_I_have_requests(self, index):
        return not not self.inactive_requests[index]

    def new_request(self, index):
        # returns (begin, length)
        if self.inactive_requests[index] == 1:
            self._make_inactive(index)
        self.numactive[index] += 1
        rs = self.inactive_requests[index]
        r = min(rs)
        rs.remove(r)
        self.amount_inactive -= r[1]
        if self.amount_inactive == 0:
            self.endgame = True
        return r

    def piece_came_in(self, index, begin, piece):
        try:
            return self._piece_came_in(index, begin, piece)
        except IOError, e:
            self.failed('IO Error ' + str(e))
            return True