def __init__(self, torrent_file):
with open(torrent_file, 'r') as t:
parsed_torrent = t.read()
self.info_dict = bencode.bdecode(parsed_torrent)
self.tracker_url = self.info_dict['announce']
self.info_hash = hashlib.sha1(bencode.bencode(
self.info_dict['info'])).digest()
self.info_hash_readable = hashlib.sha1(
bencode.bencode(self.info_dict['info'])).hexdigest()
if self.info_dict['info'].get('name'):
self.name = self.info_dict['info']['name']
self.encoding = self.info_dict.get('encoding', None)
if self.info_dict['info'].get('files'):
self.no_of_files = len(self.info_dict['info']['files'])
self.total_length = 0
for files in self.info_dict['info']['files']:
self.total_length += files['length']
else:
self.no_of_files = 1
self.total_length = self.info_dict['info']['length']
self.subpieces = self.info_dict['info']['pieces']
self.piece_length = self.info_dict['info']['piece length']
self.list_of_subpieces_hashes = []
for i in range(0, len(self.subpieces), 20):
self.list_of_subpieces_hashes.append(self.subpieces[i:i + 20])
self.no_of_subpieces = len(self.list_of_subpieces_hashes)
self.bitfield = Bitfield(self)
self.block_size = 16384
def test_stops_at_backlog():
ds = DummyStorage([[(0, 2), (2, 2), (4, 2), (6, 2)]])
events = []
d = Downloader(ds, DummyPicker(len(ds.remaining), events), 2, 15, 1,
Measure(15), 10)
sd = d.make_download(DummyConnection(events))
assert events == []
assert ds.remaining == [[(0, 2), (2, 2), (4, 2), (6, 2)]]
assert ds.active == [[]]
sd.got_have_bitfield(Bitfield(1, chr(0x80)))
assert events == ['got have', 'interested']
del events[:]
assert ds.remaining == [[(0, 2), (2, 2), (4, 2), (6, 2)]]
assert ds.active == [[]]
sd.got_unchoke()
assert events == [
'requested', ('request', 0, 6, 2), 'requested', ('request', 0, 4, 2)
]
del events[:]
assert ds.remaining == [[(0, 2), (2, 2)]]
assert ds.active == [[(4, 2), (6, 2)]]
sd.got_piece(0, 4, 'ab')
assert events == ['requested', ('request', 0, 2, 2)]
del events[:]
assert ds.remaining == [[(0, 2)]]
assert ds.active == [[(2, 2), (6, 2)]]
def __receive_bitfield__(self, peer):
data = peer.receive()
msg = Message.deserialize(data)
if msg.id != MessageCodes.BITFIELD:
raise ConnectionError("expected BITFIELD, got {}".format(
MessageCodes(msg.id).name))
return Bitfield(msg.message)
def getBitfield(self):
bfNeed = Bitfield(self.pieceNum)
for f in self.files:
for i in range(f.idx0_piece, f.idx1_piece):
bfNeed[i] = 1
bfHave = Bitfield(self.pieceNum)
for i in range(self.pieceNum):
try:
ds = self[i]
if len(ds) == 1:
beg, dat = ds[0]
if self.doHashTest(i, dat):
bfHave[i] = 1
bfNeed[i] = 0
except BTFileError as error:
pass
return bfHave, bfNeed
def _bitfield(self, data):
pm = self.pieceManager
bf = Bitfield(pm.pieces_size, data)
self.peer_bitfield = bf
if self.pieceManager.amInterested(bf):
self.interested(True)
self.__pieceRequest()
else:
self.interested(False)
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
self.choked = True
self.interested = False
self.active_requests = []
self.measure = Measure(downloader.max_rate_period)
self.have = Bitfield(downloader.numpieces)
self.last = 0
self.example_interest = None
def chunk_file(file_obj, bitfield_size):
"""
Chunk a binary file into Bitfields of the given size.
"""
# TODO: read the file in chunks
data = file_obj.read()
data = Bitfield(int.from_bytes(data, 'little'))
while data:
chunk = data[:bitfield_size]
chunk.width = bitfield_size
yield chunk
data >>= bitfield_size
def start(self):
if not self.protocol:
return
self.status = 'running'
self.btm = self.protocol.factory.btm
self.pieceManager = self.btm.pieceManager
pm = self.pieceManager
self.peer_bitfield = Bitfield(pm.pieces_size)
self.downloadSpeedMonitor.start()
self.downloadSpeedMonitor.registerObserver(
self.protocol.factory.downloadSpeedMonitor)
def test_byte_divisible_2(self):
nibbles = []
for byte in self.bytes:
bits = Bitfield(byte, width=8)
left, right = bits[:4], bits[4:]
left.width, right.width = 4, 4
nibbles.append(left)
nibbles.append(right)
with open(self.filename, 'rb') as f:
chunker = chunk_file(f, 4)
for read, actual in zip(chunker, nibbles):
self.assertEqual(read, actual)
def __init__(self, downloader, connection):
self.downloader = downloader
self.connection = connection
# Whether the peer is choking this client.
self.choked = True
# Whether this client is interested in data the peer has.
self.interested = False
# The (index, begin, length) tuples this client has requested from the peer.
self.active_requests = []
# Measures the download rate from the peer.
self.measure = Measure(downloader.max_rate_period)
# The pieces the peer has.
self.have = Bitfield(downloader.numpieces)
# The last time this client has gotten data from the peer.
self.last = 0
self.example_interest = None
def finishHandshake(self):
self.btm = self.factory.btm
self.bitfield = Bitfield(self.btm.metainfo.pieces_size)
self.upload = BTUpload(self)
self.download = BTDownload(self)
self.upload.start()
self.__uploadMonitor = self.upload._uploadMonitor
self.download.start()
self.__downloadMonitor = self.download._downloadMonitor
self.send_bitfield(self.btm.pieceManager.bitfield)
self.send_keep_alive()
if self.btm.connectionManager.isAlreadyConnected(self.peer_id):
# Already connected, dropping the connection
reactor.callLater(0, self.transport.loseConnection)
else:
self.factory.addActiveConnection(self.peer_id, self)
self.status = 'started'
def got_message(self, connection, message):
c = self.connections[connection]
t = message[0]
if t == BITFIELD and c.got_anything:
connection.close()
return
c.got_anything = True
if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED]
and len(message) != 1):
connection.close()
return
if t == CHOKE:
c.download.got_choke()
elif t == UNCHOKE:
c.download.got_unchoke()
elif t == INTERESTED:
c.upload.got_interested()
elif t == NOT_INTERESTED:
c.upload.got_not_interested()
elif t == HAVE:
if len(message) != 5:
connection.close()
return
i = toint(message[1:])
if i >= self.numpieces:
connection.close()
return
c.download.got_have(i)
elif t == BITFIELD:
try:
b = Bitfield(self.numpieces, message[1:])
except ValueError:
connection.close()
return
c.download.got_have_bitfield(b)
elif t == REQUEST:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_request(i, toint(message[5:9]), toint(message[9:]))
elif t == CANCEL:
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_cancel(i, toint(message[5:9]), toint(message[9:]))
elif t == PIECE:
if len(message) <= 9:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
if c.download.got_piece(i, toint(message[5:9]), message[9:]):
for co in self.connections.values():
co.send_have(i)
else:
connection.close()
def __init__(self,
storage,
request_size,
hashes,
piece_size,
finished,
failed,
statusfunc=dummy_status,
flag=Event(),
check_hashes=True,
data_flunked=dummy_data_flunked):
self.storage = storage
self.request_size = request_size
self.hashes = hashes
self.piece_size = piece_size
self.data_flunked = data_flunked
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= piece_size * (len(hashes) - 1):
raise ValueError, 'bad data from tracker - total too small'
if self.total_length > piece_size * len(hashes):
raise ValueError, 'bad data from tracker - total too big'
self.finished = finished
self.failed = failed
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.endgame = False
self.have = Bitfield(len(hashes))
self.waschecked = [check_hashes] * len(hashes)
self.places = {}
self.holes = []
if len(hashes) == 0:
finished()
return
targets = {}
total = len(hashes)
for i in xrange(len(hashes)):
if not self._waspre(i):
targets.setdefault(hashes[i], []).append(i)
total -= 1
numchecked = 0.0
if total and check_hashes:
statusfunc({
"activity": 'checking existing file',
"fractionDone": 0
})
def markgot(piece, pos, self=self, check_hashes=check_hashes):
self.places[piece] = pos
self.have[piece] = True
self.amount_left -= self._piecelen(piece)
self.amount_inactive -= self._piecelen(piece)
self.inactive_requests[piece] = None
self.waschecked[piece] = check_hashes
lastlen = self._piecelen(len(hashes) - 1)
for i in xrange(len(hashes)):
if not self._waspre(i):
self.holes.append(i)
elif not check_hashes:
markgot(i, i)
else:
sh = sha1(self.storage.read(piece_size * i, lastlen))
sp = sh.digest()
sh.update(
self.storage.read(piece_size * i + lastlen,
self._piecelen(i) - lastlen))
s = sh.digest()
if s == hashes[i]:
markgot(i, i)
elif targets.get(s) and self._piecelen(i) == self._piecelen(
targets[s][-1]):
markgot(targets[s].pop(), i)
elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (
i == len(hashes) - 1
or not self._waspre(len(hashes) - 1)):
markgot(len(hashes) - 1, i)
else:
self.places[i] = i
if flag.isSet():
return
numchecked += 1
statusfunc({
'fractionDone':
1 - float(self.amount_left) / self.total_length
})
if self.amount_left == 0:
finished()
class StorageWrapper:
def __init__(self,
storage,
request_size,
hashes,
piece_size,
finished,
failed,
statusfunc=dummy_status,
flag=Event(),
check_hashes=True,
data_flunked=dummy_data_flunked):
self.storage = storage
self.request_size = request_size
self.hashes = hashes
self.piece_size = piece_size
self.data_flunked = data_flunked
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= piece_size * (len(hashes) - 1):
raise ValueError, 'bad data from tracker - total too small'
if self.total_length > piece_size * len(hashes):
raise ValueError, 'bad data from tracker - total too big'
self.finished = finished
self.failed = failed
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.endgame = False
self.have = Bitfield(len(hashes))
self.waschecked = [check_hashes] * len(hashes)
self.places = {}
self.holes = []
if len(hashes) == 0:
finished()
return
targets = {}
total = len(hashes)
for i in xrange(len(hashes)):
if not self._waspre(i):
targets.setdefault(hashes[i], []).append(i)
total -= 1
numchecked = 0.0
if total and check_hashes:
statusfunc({
"activity": 'checking existing file',
"fractionDone": 0
})
def markgot(piece, pos, self=self, check_hashes=check_hashes):
self.places[piece] = pos
self.have[piece] = True
self.amount_left -= self._piecelen(piece)
self.amount_inactive -= self._piecelen(piece)
self.inactive_requests[piece] = None
self.waschecked[piece] = check_hashes
lastlen = self._piecelen(len(hashes) - 1)
for i in xrange(len(hashes)):
if not self._waspre(i):
self.holes.append(i)
elif not check_hashes:
markgot(i, i)
else:
sh = sha1(self.storage.read(piece_size * i, lastlen))
sp = sh.digest()
sh.update(
self.storage.read(piece_size * i + lastlen,
self._piecelen(i) - lastlen))
s = sh.digest()
if s == hashes[i]:
markgot(i, i)
elif targets.get(s) and self._piecelen(i) == self._piecelen(
targets[s][-1]):
markgot(targets[s].pop(), i)
elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (
i == len(hashes) - 1
or not self._waspre(len(hashes) - 1)):
markgot(len(hashes) - 1, i)
else:
self.places[i] = i
if flag.isSet():
return
numchecked += 1
statusfunc({
'fractionDone':
1 - float(self.amount_left) / self.total_length
})
if self.amount_left == 0:
finished()
def _waspre(self, piece):
return self.storage.was_preallocated(piece * self.piece_size,
self._piecelen(piece))
def _piecelen(self, piece):
if piece < len(self.hashes) - 1:
return self.piece_size
else:
return self.total_length - piece * self.piece_size
def get_amount_left(self):
return self.amount_left
def do_I_have_anything(self):
return self.amount_left < self.total_length
def _make_inactive(self, index):
length = min(self.piece_size,
self.total_length - self.piece_size * index)
l = []
x = 0
while x + self.request_size < length:
l.append((x, self.request_size))
x += self.request_size
l.append((x, length - x))
self.inactive_requests[index] = l
def is_endgame(self):
return self.endgame
def get_have_list(self):
return self.have.tostring()
def do_I_have(self, index):
return self.have[index]
def do_I_have_requests(self, index):
return not not self.inactive_requests[index]
def new_request(self, index):
# returns (begin, length)
if self.inactive_requests[index] == 1:
self._make_inactive(index)
self.numactive[index] += 1
rs = self.inactive_requests[index]
r = min(rs)
rs.remove(r)
self.amount_inactive -= r[1]
if self.amount_inactive == 0:
self.endgame = True
return r
def piece_came_in(self, index, begin, piece):
try:
return self._piece_came_in(index, begin, piece)
except IOError, e:
self.failed('IO Error ' + str(e))
return True
def __init__(self, storage, request_size, hashes,
piece_size, finished, failed,
statusfunc = dummy_status, flag = Event(), check_hashes = True,
data_flunked = dummy_data_flunked):
self.storage = storage
self.request_size = request_size
self.hashes = hashes
self.piece_size = piece_size
self.data_flunked = data_flunked
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= piece_size * (len(hashes) - 1):
raise ValueError, 'bad data from tracker - total too small'
if self.total_length > piece_size * len(hashes):
raise ValueError, 'bad data from tracker - total too big'
self.finished = finished
self.failed = failed
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.endgame = False
self.have = Bitfield(len(hashes))
self.waschecked = [check_hashes] * len(hashes)
self.places = {}
self.holes = []
if len(hashes) == 0:
finished()
return
targets = {}
total = len(hashes)
for i in xrange(len(hashes)):
if not self._waspre(i):
targets.setdefault(hashes[i], []).append(i)
total -= 1
numchecked = 0.0
if total and check_hashes:
statusfunc({"activity" : 'checking existing file',
"fractionDone" : 0})
def markgot(piece, pos, self = self, check_hashes = check_hashes):
self.places[piece] = pos
self.have[piece] = True
self.amount_left -= self._piecelen(piece)
self.amount_inactive -= self._piecelen(piece)
self.inactive_requests[piece] = None
self.waschecked[piece] = check_hashes
lastlen = self._piecelen(len(hashes) - 1)
for i in xrange(len(hashes)):
if not self._waspre(i):
self.holes.append(i)
elif not check_hashes:
markgot(i, i)
else:
sh = sha(self.storage.read(piece_size * i, lastlen))
sp = sh.digest()
sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen))
s = sh.digest()
if s == hashes[i]:
markgot(i, i)
elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]):
markgot(targets[s].pop(), i)
elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)):
markgot(len(hashes) - 1, i)
else:
self.places[i] = i
if flag.isSet():
return
numchecked += 1
statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length})
if self.amount_left == 0:
finished()
def __init__(self, storage, request_size, hashes,
piece_size, finished, failed,
statusfunc = dummy_status, flag = Event(), check_hashes = True,
data_flunked = dummy_data_flunked):
# The Storage instance.
self.storage = storage
# The size of blocks to request.
self.request_size = request_size
# An array of SHA-1 hashes for all pieces.
self.hashes = hashes
# The piece size, which should be a multiple of request_size.
self.piece_size = piece_size
# Method to call if a piece fails SHA-1 validation.
self.data_flunked = data_flunked
# The total bytes to download and save.
self.total_length = storage.get_total_length()
# The number of bytes left to download and validate.
self.amount_left = self.total_length
if self.total_length <= piece_size * (len(hashes) - 1):
raise ValueError, 'bad data from tracker - total too small'
if self.total_length > piece_size * len(hashes):
raise ValueError, 'bad data from tracker - total too big'
# Callback invoked once all pieces have been downloaded and validated.
self.finished = finished
# Callback invoked with a string describing any error.
self.failed = failed
# The number of outstanding requests for blocks belonging to a piece.
self.numactive = [0] * len(hashes)
# If an element is 1, then the piece has not been requested from peers.
# If an elemtent is an array, it contains blocks that have not been requested from peers.
self.inactive_requests = [1] * len(hashes)
# The number of bytes that have not been downloaded or requested.
# When this reaches 0, then we enter endgame mode.
self.amount_inactive = self.total_length
# Whether we are in endgame mode, which
# "sends requests for all of its missing blocks to all of its peers."
self.endgame = False
# The bitfield of pieces we have downloaded, but not necessarily validated.
self.have = Bitfield(len(hashes))
# Whether each piece has been validated by computing its SHA-1 hash.
# If check_hashes is False, then validating preallocated segments is deferred.
self.waschecked = [check_hashes] * len(hashes)
# Maps each piece to what piece, or segment, it occupies on disk.
# It may not be the right segment for the piece, and the piece may be incomplete.
self.places = {}
# Missing segments on disk.
self.holes = []
if len(hashes) == 0:
# If no hashes, then no data to download, so trivially finished.
finished()
return
# Maps each SHA-1 hash to all pieces that have that hash.
targets = {}
# The total number of preallocated pieces.
total = len(hashes)
for i in xrange(len(hashes)):
if not self._waspre(i):
# This piece is not preallocated.
targets.setdefault(hashes[i], []).append(i)
total -= 1
numchecked = 0.0
if total and check_hashes:
# There is at least one preallocated piece. Must determine what pieces they are.
statusfunc({"activity" : 'checking existing file',
"fractionDone" : 0})
def markgot(piece, pos, self = self, check_hashes = check_hashes):
# Record the position of this piece and that we have it.
self.places[piece] = pos
self.have[piece] = True
# This piece has been downloaded and validated if that option is enabled.
self.amount_left -= self._piecelen(piece)
self.amount_inactive -= self._piecelen(piece)
# We won't be requesting this piece from peers.
self.inactive_requests[piece] = None
self.waschecked[piece] = check_hashes
# Get the length of the last piece.
lastlen = self._piecelen(len(hashes) - 1)
for i in xrange(len(hashes)):
if not self._waspre(i):
# This piece is not preallocated, i.e. it has no segment of bytes on disk.
self.holes.append(i)
elif not check_hashes:
# The corresponding segment on disk is full of bytes.
# Assume that it belongs to this piece, meaning places[i] = i.
markgot(i, i)
else:
# Only get here if check_hashes = True, so we called statusfunc earlier.
# We're trying to figure out what piece is at segment i on disk.
# The bytes in this segment on disk could belong to any piece.
# Compute a hash of its first lastlen bytes in case it has the last piece.
sh = sha(self.storage.read(piece_size * i, lastlen))
sp = sh.digest()
# Compute a hash of all its bytes in the case that it has any other piece.
sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen))
s = sh.digest()
if s == hashes[i]:
# This is piece i, occupying its correct segment on disk.
markgot(i, i)
elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]):
# This is not the last piece, temporarily occupying the wrong segment.
markgot(targets[s].pop(), i)
elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)):
# This is the last piece, temporarily occupying the wrong segment.
markgot(len(hashes) - 1, i)
else:
# This segment has been allocated but it doesn't belong to any piece.
# When this piece comes in, we can write to this segment directly.
self.places[i] = i
if flag.isSet():
return
numchecked += 1
statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length})
if self.amount_left == 0:
# All data has been downloaded and validated.
finished()
class StorageWrapper:
def __init__(self, storage, request_size, hashes,
piece_size, finished, failed,
statusfunc = dummy_status, flag = Event(), check_hashes = True,
data_flunked = dummy_data_flunked):
# The Storage instance.
self.storage = storage
# The size of blocks to request.
self.request_size = request_size
# An array of SHA-1 hashes for all pieces.
self.hashes = hashes
# The piece size, which should be a multiple of request_size.
self.piece_size = piece_size
# Method to call if a piece fails SHA-1 validation.
self.data_flunked = data_flunked
# The total bytes to download and save.
self.total_length = storage.get_total_length()
# The number of bytes left to download and validate.
self.amount_left = self.total_length
if self.total_length <= piece_size * (len(hashes) - 1):
raise ValueError, 'bad data from tracker - total too small'
if self.total_length > piece_size * len(hashes):
raise ValueError, 'bad data from tracker - total too big'
# Callback invoked once all pieces have been downloaded and validated.
self.finished = finished
# Callback invoked with a string describing any error.
self.failed = failed
# The number of outstanding requests for blocks belonging to a piece.
self.numactive = [0] * len(hashes)
# If an element is 1, then the piece has not been requested from peers.
# If an elemtent is an array, it contains blocks that have not been requested from peers.
self.inactive_requests = [1] * len(hashes)
# The number of bytes that have not been downloaded or requested.
# When this reaches 0, then we enter endgame mode.
self.amount_inactive = self.total_length
# Whether we are in endgame mode, which
# "sends requests for all of its missing blocks to all of its peers."
self.endgame = False
# The bitfield of pieces we have downloaded, but not necessarily validated.
self.have = Bitfield(len(hashes))
# Whether each piece has been validated by computing its SHA-1 hash.
# If check_hashes is False, then validating preallocated segments is deferred.
self.waschecked = [check_hashes] * len(hashes)
# Maps each piece to what piece, or segment, it occupies on disk.
# It may not be the right segment for the piece, and the piece may be incomplete.
self.places = {}
# Missing segments on disk.
self.holes = []
if len(hashes) == 0:
# If no hashes, then no data to download, so trivially finished.
finished()
return
# Maps each SHA-1 hash to all pieces that have that hash.
targets = {}
# The total number of preallocated pieces.
total = len(hashes)
for i in xrange(len(hashes)):
if not self._waspre(i):
# This piece is not preallocated.
targets.setdefault(hashes[i], []).append(i)
total -= 1
numchecked = 0.0
if total and check_hashes:
# There is at least one preallocated piece. Must determine what pieces they are.
statusfunc({"activity" : 'checking existing file',
"fractionDone" : 0})
def markgot(piece, pos, self = self, check_hashes = check_hashes):
# Record the position of this piece and that we have it.
self.places[piece] = pos
self.have[piece] = True
# This piece has been downloaded and validated if that option is enabled.
self.amount_left -= self._piecelen(piece)
self.amount_inactive -= self._piecelen(piece)
# We won't be requesting this piece from peers.
self.inactive_requests[piece] = None
self.waschecked[piece] = check_hashes
# Get the length of the last piece.
lastlen = self._piecelen(len(hashes) - 1)
for i in xrange(len(hashes)):
if not self._waspre(i):
# This piece is not preallocated, i.e. it has no segment of bytes on disk.
self.holes.append(i)
elif not check_hashes:
# The corresponding segment on disk is full of bytes.
# Assume that it belongs to this piece, meaning places[i] = i.
markgot(i, i)
else:
# Only get here if check_hashes = True, so we called statusfunc earlier.
# We're trying to figure out what piece is at segment i on disk.
# The bytes in this segment on disk could belong to any piece.
# Compute a hash of its first lastlen bytes in case it has the last piece.
sh = sha(self.storage.read(piece_size * i, lastlen))
sp = sh.digest()
# Compute a hash of all its bytes in the case that it has any other piece.
sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen))
s = sh.digest()
if s == hashes[i]:
# This is piece i, occupying its correct segment on disk.
markgot(i, i)
elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]):
# This is not the last piece, temporarily occupying the wrong segment.
markgot(targets[s].pop(), i)
elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)):
# This is the last piece, temporarily occupying the wrong segment.
markgot(len(hashes) - 1, i)
else:
# This segment has been allocated but it doesn't belong to any piece.
# When this piece comes in, we can write to this segment directly.
self.places[i] = i
if flag.isSet():
return
numchecked += 1
statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length})
if self.amount_left == 0:
# All data has been downloaded and validated.
finished()
def _waspre(self, piece):
# Returns whether all files containing this piece were preallocated.
return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece))
def _piecelen(self, piece):
# Return the length of the given piece.
if piece < len(self.hashes) - 1:
return self.piece_size
else:
return self.total_length - piece * self.piece_size
def get_amount_left(self):
return self.amount_left
def do_I_have_anything(self):
# Returns whether we've downloaded at least one piece.
return self.amount_left < self.total_length
def _make_inactive(self, index):
# This assigns to length what _piecelen would return.
length = min(self.piece_size, self.total_length - self.piece_size * index)
# Will contain all blocks for this piece as (start byte, end byte) pairs.
l = []
x = 0
while x + self.request_size < length:
l.append((x, self.request_size))
x += self.request_size
l.append((x, length - x))
# Replace the value of 1 with the array of blocks.
self.inactive_requests[index] = l
def is_endgame(self):
return self.endgame
def get_have_list(self):
# Used when sending our bitfield to another peer.
return self.have.tostring()
def do_I_have(self, index):
return self.have[index]
def do_I_have_requests(self, index):
# Similar to how you would convert to boolean in JavaScript,
# this returns True if the element is 1 or a non-empty array, and
# this returns False if the element is None or an empty array.
return not not self.inactive_requests[index]
def new_request(self, index):
# returns (begin, length)
if self.inactive_requests[index] == 1:
# Create the blocks to request for this piece.
self._make_inactive(index)
# Increment count of blocks requested for this piece.
self.numactive[index] += 1
# Get the block with the earliest start byte.
rs = self.inactive_requests[index]
r = min(rs)
rs.remove(r)
# Deduct block length from total bytes neither downloaded nor requested.
self.amount_inactive -= r[1]
if self.amount_inactive == 0:
# All bytes either downloaded or requested, so enter endgame.
self.endgame = True
return r
def piece_came_in(self, index, begin, piece):
try:
return self._piece_came_in(index, begin, piece)
except IOError, e:
self.failed('IO Error ' + str(e))
return True
from bitfield import Bitfield
bitfield = Bitfield(8, '\xee')
print 'value={0:b} complete={1}'.format(ord(bitfield.tostring()), bitfield.complete())
bitfield[3] = True
bitfield[7] = True
print 'value={0:b} complete={1}'.format(ord(bitfield.tostring()), bitfield.complete())
class StorageWrapper:
def __init__(self, storage, request_size, hashes,
piece_size, finished, failed,
statusfunc = dummy_status, flag = Event(), check_hashes = True,
data_flunked = dummy_data_flunked):
self.storage = storage
self.request_size = request_size
self.hashes = hashes
self.piece_size = piece_size
self.data_flunked = data_flunked
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
if self.total_length <= piece_size * (len(hashes) - 1):
raise ValueError, 'bad data from tracker - total too small'
if self.total_length > piece_size * len(hashes):
raise ValueError, 'bad data from tracker - total too big'
self.finished = finished
self.failed = failed
self.numactive = [0] * len(hashes)
self.inactive_requests = [1] * len(hashes)
self.amount_inactive = self.total_length
self.endgame = False
self.have = Bitfield(len(hashes))
self.waschecked = [check_hashes] * len(hashes)
self.places = {}
self.holes = []
if len(hashes) == 0:
finished()
return
targets = {}
total = len(hashes)
for i in xrange(len(hashes)):
if not self._waspre(i):
targets.setdefault(hashes[i], []).append(i)
total -= 1
numchecked = 0.0
if total and check_hashes:
statusfunc({"activity" : 'checking existing file',
"fractionDone" : 0})
def markgot(piece, pos, self = self, check_hashes = check_hashes):
self.places[piece] = pos
self.have[piece] = True
self.amount_left -= self._piecelen(piece)
self.amount_inactive -= self._piecelen(piece)
self.inactive_requests[piece] = None
self.waschecked[piece] = check_hashes
lastlen = self._piecelen(len(hashes) - 1)
for i in xrange(len(hashes)):
if not self._waspre(i):
self.holes.append(i)
elif not check_hashes:
markgot(i, i)
else:
sh = sha(self.storage.read(piece_size * i, lastlen))
sp = sh.digest()
sh.update(self.storage.read(piece_size * i + lastlen, self._piecelen(i) - lastlen))
s = sh.digest()
if s == hashes[i]:
markgot(i, i)
elif targets.get(s) and self._piecelen(i) == self._piecelen(targets[s][-1]):
markgot(targets[s].pop(), i)
elif not self.have[len(hashes) - 1] and sp == hashes[-1] and (i == len(hashes) - 1 or not self._waspre(len(hashes) - 1)):
markgot(len(hashes) - 1, i)
else:
self.places[i] = i
if flag.isSet():
return
numchecked += 1
statusfunc({'fractionDone': 1 - float(self.amount_left) / self.total_length})
if self.amount_left == 0:
finished()
def _waspre(self, piece):
return self.storage.was_preallocated(piece * self.piece_size, self._piecelen(piece))
def _piecelen(self, piece):
if piece < len(self.hashes) - 1:
return self.piece_size
else:
return self.total_length - piece * self.piece_size
def get_amount_left(self):
return self.amount_left
def do_I_have_anything(self):
return self.amount_left < self.total_length
def _make_inactive(self, index):
length = min(self.piece_size, self.total_length - self.piece_size * index)
l = []
x = 0
while x + self.request_size < length:
l.append((x, self.request_size))
x += self.request_size
l.append((x, length - x))
self.inactive_requests[index] = l
def is_endgame(self):
return self.endgame
def get_have_list(self):
return self.have.tostring()
def do_I_have(self, index):
return self.have[index]
def do_I_have_requests(self, index):
return not not self.inactive_requests[index]
def new_request(self, index):
# returns (begin, length)
if self.inactive_requests[index] == 1:
self._make_inactive(index)
self.numactive[index] += 1
rs = self.inactive_requests[index]
r = min(rs)
rs.remove(r)
self.amount_inactive -= r[1]
if self.amount_inactive == 0:
self.endgame = True
return r
def piece_came_in(self, index, begin, piece):
try:
return self._piece_came_in(index, begin, piece)
except IOError, e:
self.failed('IO Error ' + str(e))
return True
def got_message(self, connection, message):
c = self.connections[connection]
t = message[0]
if t == BITFIELD and c.got_anything:
# If we are not receiving the bitfield first, close this connection.
connection.close()
return
# Got at least one message from this peer.
c.got_anything = True
if (t in [CHOKE, UNCHOKE, INTERESTED, NOT_INTERESTED]
and len(message) != 1):
connection.close()
return
if t == CHOKE:
# Choke messages affect downloading.
c.download.got_choke()
elif t == UNCHOKE:
# Unchoke messages affect downloading.
c.download.got_unchoke()
elif t == INTERESTED:
# Interested messages affect uploading.
c.upload.got_interested()
elif t == NOT_INTERESTED:
# Uninterested messages affect uploading.
c.upload.got_not_interested()
elif t == HAVE:
# A peer having a new piece affects downloading.
if len(message) != 5:
connection.close()
return
i = toint(message[1:])
if i >= self.numpieces:
connection.close()
return
c.download.got_have(i)
elif t == BITFIELD:
# A peer sending all pieces it has affects downloading.
try:
b = Bitfield(self.numpieces, message[1:])
except ValueError:
connection.close()
return
c.download.got_have_bitfield(b)
elif t == REQUEST:
# A peer requesting a block affects uploading.
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_request(i, toint(message[5:9]), toint(message[9:]))
elif t == CANCEL:
# A peer cancelling a request for a block affects uploading.
if len(message) != 13:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
c.upload.got_cancel(i, toint(message[5:9]), toint(message[9:]))
elif t == PIECE:
# A requested block sent to this client by a peer affects downloading.
if len(message) <= 9:
connection.close()
return
i = toint(message[1:5])
if i >= self.numpieces:
connection.close()
return
if c.download.got_piece(i, toint(message[5:9]), message[9:]):
for co in self.connections.values():
co.send_have(i)
else:
# This is an unknown message type, so close this connection.
connection.close()
