def test_get_peers(self): self.metainfo.announce = self.trackeraddr+"/announce" tp = TrackerProxy(self.metainfo, 6881, self.peer_id) assert tp.get_peers(10) == self.peerlist tp = TrackerProxy(self.metainfo, 6881, self.peer_id) assert tp.get_peers(2) == self.peerlist[:2] assert tp.get_peers(2) == self.peerlist[2:4] assert tp.get_peers(2) == self.peerlist[4:]
def __init__(self, client, filename, port, peer_id): self._client = client self._filename = filename self._port = port self._peer_id = peer_id # _peers is a list of peers that the TorrentManager is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} try: self._metainfo = Metainfo(filename) except (IOError, ValueError) as err: if isinstance(err, IOError): message = err.strerror+' ('+filename+')' else: message = err.message+' ('+filename+')' logger.error(message) raise TorrentManagerError(message) # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemanager = FileManager(self._metainfo) self._have = self._filemanager.have() try: self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) except TrackerError as err: logger.critical("Could not connect to tracker at {}" .format(self._metainfo.announce)) logger.debug(" TrackerError: {}".format(err.message)) raise TorrentManagerError(err.message) self._needed = {piece: (0, []) for piece in list(self._have.findall('0b0'))} self._interested = {} self._requesting = {} self._partial = [] self._reactor = Reactor() self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick = 1 print "Starting to serve torrent {} off of Cracker website...".format(filename) self._connect_to_peers(20)
def test_get_peers(self): self.metainfo.announce = self.trackeraddr + "/announce" tp = TrackerProxy(self.metainfo, 6881, self.peer_id) assert tp.get_peers(10) == self.peerlist tp = TrackerProxy(self.metainfo, 6881, self.peer_id) assert tp.get_peers(2) == self.peerlist[:2] assert tp.get_peers(2) == self.peerlist[2:4] assert tp.get_peers(2) == self.peerlist[4:]
class TorrentManager(object): def __init__(self, client, filename, port, peer_id): self._client = client self._filename = filename self._port = port self._peer_id = peer_id # _peers is a list of peers that the TorrentManager is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} try: self._metainfo = Metainfo(filename) except (IOError, ValueError) as err: if isinstance(err, IOError): message = err.strerror+' ('+filename+')' else: message = err.message+' ('+filename+')' logger.error(message) raise TorrentManagerError(message) # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemanager = FileManager(self._metainfo) self._have = self._filemanager.have() try: self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) except TrackerError as err: logger.critical("Could not connect to tracker at {}" .format(self._metainfo.announce)) logger.debug(" TrackerError: {}".format(err.message)) raise TorrentManagerError(err.message) self._needed = {piece: (0, []) for piece in list(self._have.findall('0b0'))} self._interested = {} self._requesting = {} self._partial = [] self._reactor = Reactor() self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick = 1 print "Starting to serve torrent {} off of Cracker website...".format(filename) self._connect_to_peers(20) def _connect_to_peers(self, n): # Get addresses of n peers from the tracker and try to establish # a connection with each addrs = self._tracker_proxy.get_peers(n) for addr in addrs: peer = PeerProxy(self, self._peer_id, (addr['ip'], addr['port']), info_hash=self._metainfo.info_hash) self._peers.append(peer) self._bitfields[peer] = BitArray(self._metainfo.num_pieces) def _remove_peer(self, peer): # Clean up references to the peer in various data structures self._peers.remove(peer) pieces = list(self._bitfields[peer].findall('0b1')) for piece in pieces: if piece in self._needed: occurences, peers = self._needed[piece] if peer in peers: peers.remove(peer) self._needed[piece] = (occurences-1, peers) del self._bitfields[peer] if peer in self._interested: del self._interested[peer] elif peer in self._requesting: # If the peer is in the middle of downloading a piece, save # the state in the partial list index, offset, sha1, _, _ = self._requesting[peer] self._partial.append((index, offset, sha1)) del self._requesting[peer] def _rarest(self): # Returns a list of tuples which includes a piece index sorted by # the number of peers which have the piece in ascending order return sorted([(occurences, peers, index) for (index, (occurences, peers)) in self._needed.items() if occurences != 0]) def _show_interest(self, peer): if not peer.is_interested(): logger.debug("Expressing interest in peer {}" .format(str(peer.addr()))) peer.interested() if not peer.is_peer_choked(): self._request(peer) def _check_interest(self, peer): # If the peer is not already interested or requesting, identify a piece # for it to download and show interest to the peer. if peer not in self._interested and peer not in self._requesting: # Compute the set of needed pieces which the peer has that are not # already designated for another peer needed = self._have.copy() needed.invert() of_interest = list((needed & self._bitfields[peer]).findall('0b1')) dont_consider = [i for i, _, _, _ in self._interested.values()] dont_consider.extend([i for i, _, _, _, _ in self._requesting.values()]) if len(of_interest) > 0: for index, offset, sha1 in self._partial: if index in of_interest: self._partial.remove((index, offset, sha1)) self._interested[peer] = (index, offset, sha1, self._tick) self._show_interest(peer) return for _, _, index in self._rarest(): if index in of_interest and index not in dont_consider: self._interested[peer] = (index, 0, hashlib.sha1(), self._tick) self._show_interest(peer) return if peer not in self._interested and peer.is_interested(): logger.debug("Expressing lack of interest in peer {}" .format(str(peer.addr()))) peer.not_interested() self._connect_to_peers(1) def _request(self, peer): if peer in self._interested: index, offset, sha1, _ = self._interested[peer] del self._interested[peer] self._requesting[peer] = (index, offset, sha1, self._tick, 0) index, received_bytes, _, _, _ = self._requesting[peer] bytes_to_request = self._bytes_to_request(index, received_bytes) logger.debug("Requesting pc: {} off: {} len: {} from {}" .format(index, received_bytes, bytes_to_request, str(peer.addr()))) peer.request(index, received_bytes, bytes_to_request) def _is_last_piece(self, index): return index == self._metainfo.num_pieces-1 def _length_of_last_piece(self): return (self._metainfo.total_length - (self._metainfo.num_pieces-1)*self._metainfo.piece_length) def _length_of_piece(self, index): if self._is_last_piece(index): return self._length_of_last_piece() else: return self._metainfo.piece_length def _in_last_block(self, index, offset): if self._is_last_piece(index): piece_length = self._length_of_last_piece() else: piece_length = self._metainfo.piece_length return piece_length-offset < _BLOCK_SIZE def _bytes_to_request(self, index, offset): if not self._in_last_block(index, offset): return _BLOCK_SIZE else: return self._length_of_piece(index) - offset def info_hash(self): return self._metainfo.info_hash # PeerProxy callbacks def get_bitfield(self): return self._have def peer_unconnected(self, peer): logger.info("Peer {} is unconnected".format(str(peer.addr()))) self._remove_peer(peer) self._connect_to_peers(1) def peer_bitfield(self, peer, bitfield): # Validate the bitfield length = len(bitfield) if (length < self._metainfo.num_pieces or (length > self._metainfo.num_pieces and bitfield[self._metainfo.num_pieces:length].any(1))): logger.debug("Invalid bitfield from peer {}" .format(str(peer.addr()))) peer.drop_connection() self._remove_peer(peer) self._connect_to_peers(1) return # Set the peer's bitfield and updated needed to reflect which pieces # the peer has logger.debug("Peer at {} sent bitfield".format(str(peer.addr()))) self._bitfields[peer] = bitfield[0:self._metainfo.num_pieces] pieces = list(self._bitfields[peer].findall('0b1')) for piece in pieces: if piece in self._needed: occurences, peers = self._needed[piece] if peer not in peers: peers.append(peer) self._needed[piece] = (occurences+1, peers) # Check whether there may be interest obtaining a piece from this peer self._check_interest(peer) def peer_has(self, peer, index): # Update the peer's bitfield and needed to reflect the availability # of the piece logger.debug("Peer at {} has piece {}".format(str(peer.addr()), index)) if index < self._metainfo.num_pieces: self._bitfields[peer][index] = 1 else: raise IndexError if index in self._needed: occurences, peers = self._needed[index] if peer not in peers: peers.append(peer) self._needed[index] = (occurences+1, peers) # Check whether there may be interest obtaining a piece from this # peer self._check_interest(peer) def peer_choked(self, peer): logger.debug("Peer {} choked".format(str(peer.addr()))) if peer in self._interested: del self._interested[peer] elif peer in self._requesting: # When choked in the middle of obtaining a piece, save the # progress in the partial list index, offset, sha1, _, _ = self._requesting[peer] self._partial.append((index, offset, sha1)) del self._requesting[peer] def peer_unchoked(self, peer): logger.debug("Peer {} unchoked".format(str(peer.addr()))) if peer in self._interested: self._request(peer) def peer_sent_block(self, peer, index, begin, buf): if peer not in self._requesting: # If a peer is very slow in responding, a block could come after # it has timed out. Just ignore the data at this point and # ignore the slow peer logger.debug("Received block from peer {} which has timed out" .format(str(peer.addr()))) return piece, received_bytes, sha1, _, _ = self._requesting[peer] if piece == index and begin == received_bytes: # When the next expected block is received, update the hash value # and write the block to file sha1.update(buf) self._filemanager.write_block(index, begin, buf) self._requesting[peer] = (piece, received_bytes + len(buf), sha1, self._tick, 0) if received_bytes + len(buf) < self._length_of_piece(index): # Request the next block in the piece self._request(peer) else: # On receipt of the last block in the piece, verify the hash # and update the records to reflect receipt of the piece if sha1.digest() == self._metainfo.piece_hash(index): logger.info("Successfully got piece {} from {}" .format(index, str(peer.addr()))) del self._needed[index] percent = 100 * (1 - (len(self._needed) / float(self._metainfo.num_pieces))) print "{0}: Downloaded {1:1.4f}%".format(self._filename, percent) self._have[index] = 1 else: logger.info("Unsuccessfully got piece {} from {}" .format(index, str(peer.addr()))) del self._requesting[peer] if self._needed != {}: # Try to find another piece for this peer to get self._check_interest(peer) else: logger.info("Successfully downloaded entire torrent {}" .format(self._filename)) self._client.download_complete(self._filename) def peer_interested(self, peer): pass def peer_not_interested(self, peer): pass def peer_request(self, peer, index, begin, length): pass def peer_canceled(self, peer, index, begin, length): pass # Reactor callback def timer_event(self): self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick += 1 # For any peers that have been interested but unchoked for an # excessive period of time, stop being interested, free up assigned # piece and connect to another peer for peer, (_, _, _, tick) in self._interested.items(): if tick + 4 == self._tick: logger.debug("Timed out on interest for peer {}" .format(str(peer.addr()))) peer.not_interested() del self._interested[peer] self._connect_to_peers(1) # For any peer that has an outstanding request for an excessive period # of time, resend the request message in case it got lost or is being # ignored for peer, (index, offset, sha1, tick, retries) \ in self._requesting.items(): if tick + 5 == self._tick: logger.debug("Timed out on request for peer {}" .format(str(peer.addr()))) if retries < _MAX_RETRIES: self._requesting[peer] = (index, offset, sha1, self._tick, retries+1) self._request(peer) else: self._partial.append((index, offset, sha1)) del self._requesting[peer] peer.not_interested() self._connect_to_peers(1)
def initialize(self): """ initialize() returns a deferred which fires when initialization of the TorrentMgr is complete or an error has occurred. """ if not self._state == self._States.Uninitialized: error = "TorrentMgr can't be reinitialized" logger.debug(error) d = Deferred() d.errback(TorrentMgrError(error)) return d try: self._metainfo = Metainfo(self._filename) except ValueError as err: logger.error(err) d = Deferred() d.errback(TorrentMgrError(err.message)) return d except IOError as err: logger.error(err) d = Deferred() d.errback(TorrentMgrError(err.strerror)) return d # _peers is a list of peers that the TorrentMgr is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemgr = FileMgr(self._metainfo) self._have = self._filemgr.have() # _needed is a dictionary of pieces which are still needed. # The value for each piece is a tuple of the number of peers which # have the piece and a list of those peers. self._needed = {piece: (0, []) for piece in list(self._have.findall('0b0'))} # _interested is a dictionary of peers to whom interest has been # expressed. The value for each peer is a tuple of the piece that # has been reserved for the peer, the number of bytes of the piece that # have already been received, the sha1 hash of the bytes received so # far and the value of the tick at the time interest was expressed. self._interested = {} # _requesting is a dictionary of peers to whom a block request has been # made. The value for each peer is a tuple of the piece that is being # requested, the number of bytes that have already been received, the # shal2 hash of the bytes received so far, the value of the tick at # the time the request was made and the number of retries that have # been attempted self._requesting = {} # _partial is a list which tracks pieces that were interrupted while # being downloaded. Each entry is a tuple containing the index of the # piece, the number of bytes received so far and the sha1 hash of those # bytes. self._partial = [] self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) def success(result): self._state = self._States.Initialized def failure(failure): logger.critical("Could not connect to tracker at {}" .format(self._metainfo.announce)) message = failure.value.message logger.debug(" Tracker Error: {}".format(message)) raise TorrentMgrError(message) return self._tracker_proxy.start().addCallbacks(success, failure)
class TorrentMgr(object): class _States(object): (Uninitialized, Initialized, Started) = range(3) def __init__(self, filename, port, peer_id, reactor): self._filename = filename self._port = port self._peer_id = peer_id self._reactor = reactor self._state = self._States.Uninitialized def initialize(self): """ initialize() returns a deferred which fires when initialization of the TorrentMgr is complete or an error has occurred. """ if not self._state == self._States.Uninitialized: error = "TorrentMgr can't be reinitialized" logger.debug(error) d = Deferred() d.errback(TorrentMgrError(error)) return d try: self._metainfo = Metainfo(self._filename) except ValueError as err: logger.error(err) d = Deferred() d.errback(TorrentMgrError(err.message)) return d except IOError as err: logger.error(err) d = Deferred() d.errback(TorrentMgrError(err.strerror)) return d # _peers is a list of peers that the TorrentMgr is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemgr = FileMgr(self._metainfo) self._have = self._filemgr.have() # _needed is a dictionary of pieces which are still needed. # The value for each piece is a tuple of the number of peers which # have the piece and a list of those peers. self._needed = {piece: (0, []) for piece in list(self._have.findall('0b0'))} # _interested is a dictionary of peers to whom interest has been # expressed. The value for each peer is a tuple of the piece that # has been reserved for the peer, the number of bytes of the piece that # have already been received, the sha1 hash of the bytes received so # far and the value of the tick at the time interest was expressed. self._interested = {} # _requesting is a dictionary of peers to whom a block request has been # made. The value for each peer is a tuple of the piece that is being # requested, the number of bytes that have already been received, the # shal2 hash of the bytes received so far, the value of the tick at # the time the request was made and the number of retries that have # been attempted self._requesting = {} # _partial is a list which tracks pieces that were interrupted while # being downloaded. Each entry is a tuple containing the index of the # piece, the number of bytes received so far and the sha1 hash of those # bytes. self._partial = [] self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) def success(result): self._state = self._States.Initialized def failure(failure): logger.critical("Could not connect to tracker at {}" .format(self._metainfo.announce)) message = failure.value.message logger.debug(" Tracker Error: {}".format(message)) raise TorrentMgrError(message) return self._tracker_proxy.start().addCallbacks(success, failure) def start(self): if not self._state == self._States.Initialized: raise TorrentMgrError("TorrentMgr must be initialized to be " "started") self._reactor.callLater(_TIMER_INTERVAL, self.timer_event) self._tick = 1 logger.info("Starting to serve torrent {}".format(self._filename)) print "Starting to serve torrent {}".format(self._filename) self._state = self._States.Started self._connect_to_peers(20) def percent(self): if not self._state == self._States.Uninitialized: return 100 * (1 - (len(self._needed) / float(self._metainfo.num_pieces))) else: raise TorrentMgrError("Can't get percent on uninitialized " "TorrentMgr") def info_hash(self): if not self._state == self._States.Uninitialized: return self._metainfo.info_hash else: raise TorrentMgrError("Can't get hash on uninitialized " "TorrentMgr") def name(self): return self._metainfo.name def _connect_to_peers(self, n): # Get addresses of n peers from the tracker and try to establish # a connection with each def handle_addrs(addrs): for addr in addrs: peer = PeerProxy(self, self._peer_id, (addr['ip'], addr['port']), self._reactor, info_hash=self._metainfo.info_hash) self._peers.append(peer) self._bitfields[peer] = BitArray(self._metainfo.num_pieces) self._tracker_proxy.get_peers(n).addCallback(handle_addrs) def _remove_peer(self, peer): # Clean up references to the peer in various data structures self._peers.remove(peer) pieces = list(self._bitfields[peer].findall('0b1')) for piece in pieces: if piece in self._needed: occurences, peers = self._needed[piece] if peer in peers: peers.remove(peer) self._needed[piece] = (occurences-1, peers) del self._bitfields[peer] if peer in self._interested: del self._interested[peer] elif peer in self._requesting: # If the peer is in the middle of downloading a piece, save # the state in the partial list index, offset, sha1, _, _ = self._requesting[peer] self._partial.append((index, offset, sha1)) del self._requesting[peer] def _rarest(self): # Returns a list of tuples which includes a piece index sorted by # the number of peers which have the piece in ascending order return sorted([(occurences, peers, index) for (index, (occurences, peers)) in self._needed.items() if occurences != 0]) def _show_interest(self, peer): if not peer.is_interested(): logger.debug("Expressing interest in peer {}" .format(str(peer.addr()))) peer.interested() if not peer.is_peer_choked(): self._request(peer) def _check_interest(self, peer): # If the peer is not already interested or requesting, identify a piece # for it to download and show interest to the peer. if not peer in self._interested and not peer in self._requesting: # Compute the set of needed pieces which the peer has that are not # already designated for another peer needed = self._have.copy() needed.invert() of_interest = list((needed & self._bitfields[peer]).findall('0b1')) dont_consider = [i for i, _, _, _ in self._interested.values()] dont_consider.extend([i for i, _, _, _, _ in self._requesting.values()]) # When there are potential pieces for the peer to download, give # preference to a piece that has already been partially # downloaded followed by the rarest available piece if len(of_interest) > 0: for index, offset, sha1 in self._partial: if index in of_interest: self._partial.remove((index, offset, sha1)) self._interested[peer] = (index, offset, sha1, self._tick) self._show_interest(peer) return for _, _, index in self._rarest(): if index in of_interest and not index in dont_consider: self._interested[peer] = (index, 0, hashlib.sha1(), self._tick) self._show_interest(peer) return # If there is no further piece for a peer which was previously # interested to download, make it not interested and connect to # another peer if not peer in self._interested and peer.is_interested(): logger.debug("Expressing lack of interest in peer {}" .format(str(peer.addr()))) peer.not_interested() self._connect_to_peers(1) def _request(self, peer): if peer in self._interested: index, offset, sha1, _ = self._interested[peer] del self._interested[peer] self._requesting[peer] = (index, offset, sha1, self._tick, 0) index, received_bytes, _, _, _ = self._requesting[peer] bytes_to_request = self._bytes_to_request(index, received_bytes) logger.debug("Requesting pc: {} off: {} len: {} from {}" .format(index, received_bytes, bytes_to_request, str(peer.addr()))) peer.request(index, received_bytes, bytes_to_request) def _is_last_piece(self, index): return index == self._metainfo.num_pieces-1 def _length_of_last_piece(self): return (self._metainfo.total_length - (self._metainfo.num_pieces-1)*self._metainfo.piece_length) def _length_of_piece(self, index): if self._is_last_piece(index): return self._length_of_last_piece() else: return self._metainfo.piece_length def _in_last_block(self, index, offset): if self._is_last_piece(index): piece_length = self._length_of_last_piece() else: piece_length = self._metainfo.piece_length return piece_length-offset < _BLOCK_SIZE def _bytes_to_request(self, index, offset): if not self._in_last_block(index, offset): return _BLOCK_SIZE else: return self._length_of_piece(index) - offset # PeerProxy callbacks def get_bitfield(self): return self._have def peer_unconnected(self, peer): logger.info("Peer {} is unconnected".format(str(peer.addr()))) self._remove_peer(peer) self._connect_to_peers(1) def peer_bitfield(self, peer, bitfield): # Validate the bitfield length = len(bitfield) if (length < self._metainfo.num_pieces or (length > self._metainfo.num_pieces and bitfield[self._metainfo.num_pieces:length].any(1))): logger.debug("Invalid bitfield from peer {}" .format(str(peer.addr()))) peer.drop_connection() self._remove_peer(peer) self._connect_to_peers(1) return # Set the peer's bitfield and updated needed to reflect which pieces # the peer has logger.debug("Peer at {} sent bitfield".format(str(peer.addr()))) self._bitfields[peer] = bitfield[0:self._metainfo.num_pieces] pieces = list(self._bitfields[peer].findall('0b1')) for piece in pieces: if piece in self._needed: occurences, peers = self._needed[piece] if not peer in peers: peers.append(peer) self._needed[piece] = (occurences+1, peers) # Check whether there may be interest obtaining a piece from this peer self._check_interest(peer) def peer_has(self, peer, index): # Update the peer's bitfield and needed to reflect the availability # of the piece logger.debug("Peer at {} has piece {}".format(str(peer.addr()), index)) if index < self._metainfo.num_pieces: self._bitfields[peer][index] = 1 else: raise IndexError if index in self._needed: occurences, peers = self._needed[index] if not peer in peers: peers.append(peer) self._needed[index] = (occurences+1, peers) # Check whether there may be interest obtaining a piece from this # peer self._check_interest(peer) def peer_choked(self, peer): logger.debug("Peer {} choked".format(str(peer.addr()))) if peer in self._interested: del self._interested[peer] elif peer in self._requesting: # When choked in the middle of obtaining a piece, save the # progress in the partial list index, offset, sha1, _, _ = self._requesting[peer] self._partial.append((index, offset, sha1)) del self._requesting[peer] def peer_unchoked(self, peer): logger.debug("Peer {} unchoked".format(str(peer.addr()))) if peer in self._interested: self._request(peer) def peer_sent_block(self, peer, index, begin, buf): if not peer in self._requesting: # If a peer is very slow in responding, a block could come after # it has timed out. Just ignore the data at this point and # ignore the slow peer logger.debug("Received block from peer {} which has timed out" .format(str(peer.addr()))) return piece, received_bytes, sha1, _, _ = self._requesting[peer] if piece == index and begin == received_bytes: # When the next expected block is received, update the hash value # and write the block to file sha1.update(buf) self._filemgr.write_block(index, begin, buf) self._requesting[peer] = (piece, received_bytes + len(buf), sha1, self._tick, 0) if received_bytes + len(buf) < self._length_of_piece(index): # Request the next block in the piece self._request(peer) else: # On receipt of the last block in the piece, verify the hash # and update the records to reflect receipt of the piece if sha1.digest() == self._metainfo.piece_hash(index): logger.info("Successfully received piece {} from {}" .format(index, str(peer.addr()))) del self._needed[index] print "{0}: Downloaded {1:1.4f}%".format(self._filename, self.percent()) self._have[index] = 1 else: logger.info("Unsuccessfully received piece {} from {}" .format(index, str(peer.addr()))) del self._requesting[peer] if self._needed != {}: # Try to find another piece for this peer to get self._check_interest(peer) else: logger.info("Successfully downloaded entire torrent {}" .format(self._filename)) def peer_interested(self, peer): pass def peer_not_interested(self, peer): pass def peer_request(self, peer, index, begin, length): pass def peer_canceled(self, peer, index, begin, length): pass # Reactor callback def timer_event(self): self._reactor.callLater(_TIMER_INTERVAL, self.timer_event) self._tick += 1 # For any peers that have been interested but unchoked for an # excessive period of time, stop being interested, free up assigned # piece and connect to another peer for peer, (_, _, _, tick) in self._interested.items(): if tick + 4 == self._tick: logger.debug("Timed out on interest for peer {}" .format(str(peer.addr()))) peer.not_interested() del self._interested[peer] self._connect_to_peers(1) # For any peer that has an outstanding request for an excessive period # of time, resend the request message in case it got lost or is being # ignored for peer, (index, offset, sha1, tick, retries) \ in self._requesting.items(): if tick + 5 == self._tick: logger.debug("Timed out on request for peer {}" .format(str(peer.addr()))) if retries < _MAX_RETRIES: self._requesting[peer] = (index, offset, sha1, self._tick, retries+1) self._request(peer) else: logger.debug("Giving up on peer {}" .format(str(peer.addr()))) self._partial.append((index, offset, sha1)) del self._requesting[peer] peer.not_interested() self._connect_to_peers(1)
def __init__(self, client, filename, port, peer_id): self._client = client self._filename = filename self._port = port self._peer_id = peer_id # _peers is a list of peers that the TorrentMgr is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} try: self._metainfo = Metainfo(filename) except (IOError, ValueError) as err: if isinstance(err, IOError): message = err.strerror+' ('+filename+')' else: message = err.message+' ('+filename+')' logger.error(message) raise TorrentMgrError(message) # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemgr = FileMgr(self._metainfo) self._have = self._filemgr.have() try: self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) except TrackerError as err: logger.critical("Could not connect to tracker at {}" .format(self._metainfo.announce)) logger.debug(" TrackerError: {}".format(err.message)) raise TorrentMgrError(err.message) # _needed is a dictionary of pieces which are still needed. # The value for each piece is a tuple of the number of peers which # have the piece and a list of those peers. self._needed = {piece: (0, []) for piece in list(self._have.findall('0b0'))} # _interested is a dictionary of peers to whom interest has been # expressed. The value for each peer is a tuple of the piece that # has been reserved for the peer, the number of bytes of the piece that # have already been received, the sha1 hash of the bytes received so # far and the value of the tick at the time interest was expressed. self._interested = {} # _requesting is a dictionary of peers to whom a block request has been # made. The value for each peer is a tuple of the piece that is being # requested, the number of bytes that have already been received, the # shal2 hash of the bytes received so far, the value of the tick at # the time the request was made and the number of retries that have # been attempted self._requesting = {} # _partial is a list which tracks pieces that were interrupted while # being downloaded. Each entry is a tuple containing the index of the # piece, the number of bytes received so far and the sha1 hash of those # bytes. self._partial = [] self._reactor = Reactor() self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick = 1 print "Starting to serve torrent {}".format(filename) self._connect_to_peers(20)
def test_announce(self): self.metainfo.announce = self.trackeraddr + "/announce" tp = TrackerProxy(self.metainfo, 6881, self.peer_id) assert tp.get_peers(5) == self.peerlist
def test_warning(self, capsys): self.metainfo.announce = self.trackeraddr + "/warning" tp = TrackerProxy(self.metainfo, 6881, self.peer_id) _, err = capsys.readouterr() assert "Warning:" in err assert tp.get_peers(5) == self.peerlist
def test_failure(self): with pytest.raises(TrackerError) as e: self.metainfo.announce = self.trackeraddr + "/failure" TrackerProxy(self.metainfo, 6881, self.peer_id) assert e.value.message.startswith("Failure reason:")
def test_notracker(self): with pytest.raises(TrackerError) as e: self.metainfo.announce = "http://localhost" TrackerProxy(self.metainfo, 6881, self.peer_id) assert e.value.message.startswith("Can't connect to the tracker")
def __init__(self, client, filename, port, peer_id): self._client = client self._filename = filename self._port = port self._peer_id = peer_id # _peers is a list of peers that the TorrentMgr is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} try: self._metainfo = Metainfo(filename) except (IOError, ValueError) as err: if isinstance(err, IOError): message = err.strerror + ' (' + filename + ')' else: message = err.message + ' (' + filename + ')' logger.error(message) raise TorrentMgrError(message) # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemgr = FileMgr(self._metainfo) self._have = self._filemgr.have() try: self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) except TrackerError as err: logger.critical("Could not connect to tracker at {}".format( self._metainfo.announce)) logger.debug(" TrackerError: {}".format(err.message)) raise TorrentMgrError(err.message) # _needed is a dictionary of pieces which are still needed. # The value for each piece is a tuple of the number of peers which # have the piece and a list of those peers. self._needed = { piece: (0, []) for piece in list(self._have.findall('0b0')) } # _interested is a dictionary of peers to whom interest has been # expressed. The value for each peer is a tuple of the piece that # has been reserved for the peer, the number of bytes of the piece that # have already been received, the sha1 hash of the bytes received so # far and the value of the tick at the time interest was expressed. self._interested = {} # _requesting is a dictionary of peers to whom a block request has been # made. The value for each peer is a tuple of the piece that is being # requested, the number of bytes that have already been received, the # shal2 hash of the bytes received so far, the value of the tick at # the time the request was made and the number of retries that have # been attempted self._requesting = {} # _partial is a list which tracks pieces that were interrupted while # being downloaded. Each entry is a tuple containing the index of the # piece, the number of bytes received so far and the sha1 hash of those # bytes. self._partial = [] self._reactor = Reactor() self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick = 1 print "Starting to serve torrent {}".format(filename) self._connect_to_peers(20)
class TorrentMgr(object): def __init__(self, client, filename, port, peer_id): self._client = client self._filename = filename self._port = port self._peer_id = peer_id # _peers is a list of peers that the TorrentMgr is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} try: self._metainfo = Metainfo(filename) except (IOError, ValueError) as err: if isinstance(err, IOError): message = err.strerror + ' (' + filename + ')' else: message = err.message + ' (' + filename + ')' logger.error(message) raise TorrentMgrError(message) # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemgr = FileMgr(self._metainfo) self._have = self._filemgr.have() try: self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) except TrackerError as err: logger.critical("Could not connect to tracker at {}".format( self._metainfo.announce)) logger.debug(" TrackerError: {}".format(err.message)) raise TorrentMgrError(err.message) # _needed is a dictionary of pieces which are still needed. # The value for each piece is a tuple of the number of peers which # have the piece and a list of those peers. self._needed = { piece: (0, []) for piece in list(self._have.findall('0b0')) } # _interested is a dictionary of peers to whom interest has been # expressed. The value for each peer is a tuple of the piece that # has been reserved for the peer, the number of bytes of the piece that # have already been received, the sha1 hash of the bytes received so # far and the value of the tick at the time interest was expressed. self._interested = {} # _requesting is a dictionary of peers to whom a block request has been # made. The value for each peer is a tuple of the piece that is being # requested, the number of bytes that have already been received, the # shal2 hash of the bytes received so far, the value of the tick at # the time the request was made and the number of retries that have # been attempted self._requesting = {} # _partial is a list which tracks pieces that were interrupted while # being downloaded. Each entry is a tuple containing the index of the # piece, the number of bytes received so far and the sha1 hash of those # bytes. self._partial = [] self._reactor = Reactor() self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick = 1 print "Starting to serve torrent {}".format(filename) self._connect_to_peers(20) def _connect_to_peers(self, n): # Get addresses of n peers from the tracker and try to establish # a connection with each addrs = self._tracker_proxy.get_peers(n) for addr in addrs: peer = PeerProxy(self, self._peer_id, (addr['ip'], addr['port']), info_hash=self._metainfo.info_hash) self._peers.append(peer) self._bitfields[peer] = BitArray(self._metainfo.num_pieces) def _remove_peer(self, peer): # Clean up references to the peer in various data structures self._peers.remove(peer) pieces = list(self._bitfields[peer].findall('0b1')) for piece in pieces: if piece in self._needed: occurences, peers = self._needed[piece] if peer in peers: peers.remove(peer) self._needed[piece] = (occurences - 1, peers) del self._bitfields[peer] if peer in self._interested: del self._interested[peer] elif peer in self._requesting: # If the peer is in the middle of downloading a piece, save # the state in the partial list index, offset, sha1, _, _ = self._requesting[peer] self._partial.append((index, offset, sha1)) del self._requesting[peer] def _rarest(self): # Returns a list of tuples which includes a piece index sorted by # the number of peers which have the piece in ascending order return sorted([(occurences, peers, index) for (index, (occurences, peers)) in self._needed.items() if occurences != 0]) def _show_interest(self, peer): if not peer.is_interested(): logger.debug("Expressing interest in peer {}".format( str(peer.addr()))) peer.interested() if not peer.is_peer_choked(): self._request(peer) def _check_interest(self, peer): # If the peer is not already interested or requesting, identify a piece # for it to download and show interest to the peer. if peer not in self._interested and peer not in self._requesting: # Compute the set of needed pieces which the peer has that are not # already designated for another peer needed = self._have.copy() needed.invert() of_interest = list((needed & self._bitfields[peer]).findall('0b1')) dont_consider = [i for i, _, _, _ in self._interested.values()] dont_consider.extend( [i for i, _, _, _, _ in self._requesting.values()]) # When there are potential pieces for the peer to download, give # preference to a piece that has already been partially # downloaded followed by the rarest available piece if len(of_interest) > 0: for index, offset, sha1 in self._partial: if index in of_interest: self._partial.remove((index, offset, sha1)) self._interested[peer] = (index, offset, sha1, self._tick) self._show_interest(peer) return for _, _, index in self._rarest(): if index in of_interest and index not in dont_consider: self._interested[peer] = (index, 0, hashlib.sha1(), self._tick) self._show_interest(peer) return # If there is no further piece for a peer which was previously # interested to download, make it not interested and connect to # another peer if peer not in self._interested and peer.is_interested(): logger.debug("Expressing lack of interest in peer {}".format( str(peer.addr()))) peer.not_interested() self._connect_to_peers(1) def _request(self, peer): if peer in self._interested: index, offset, sha1, _ = self._interested[peer] del self._interested[peer] self._requesting[peer] = (index, offset, sha1, self._tick, 0) index, received_bytes, _, _, _ = self._requesting[peer] bytes_to_request = self._bytes_to_request(index, received_bytes) logger.debug("Requesting pc: {} off: {} len: {} from {}".format( index, received_bytes, bytes_to_request, str(peer.addr()))) peer.request(index, received_bytes, bytes_to_request) def _is_last_piece(self, index): return index == self._metainfo.num_pieces - 1 def _length_of_last_piece(self): return (self._metainfo.total_length - (self._metainfo.num_pieces - 1) * self._metainfo.piece_length) def _length_of_piece(self, index): if self._is_last_piece(index): return self._length_of_last_piece() else: return self._metainfo.piece_length def _in_last_block(self, index, offset): if self._is_last_piece(index): piece_length = self._length_of_last_piece() else: piece_length = self._metainfo.piece_length return piece_length - offset < _BLOCK_SIZE def _bytes_to_request(self, index, offset): if not self._in_last_block(index, offset): return _BLOCK_SIZE else: return self._length_of_piece(index) - offset def info_hash(self): return self._metainfo.info_hash # PeerProxy callbacks def get_bitfield(self): return self._have def peer_unconnected(self, peer): logger.info("Peer {} is unconnected".format(str(peer.addr()))) self._remove_peer(peer) self._connect_to_peers(1) def peer_bitfield(self, peer, bitfield): # Validate the bitfield length = len(bitfield) if (length < self._metainfo.num_pieces or (length > self._metainfo.num_pieces and bitfield[self._metainfo.num_pieces:length].any(1))): logger.debug("Invalid bitfield from peer {}".format( str(peer.addr()))) peer.drop_connection() self._remove_peer(peer) self._connect_to_peers(1) return # Set the peer's bitfield and updated needed to reflect which pieces # the peer has logger.debug("Peer at {} sent bitfield".format(str(peer.addr()))) self._bitfields[peer] = bitfield[0:self._metainfo.num_pieces] pieces = list(self._bitfields[peer].findall('0b1')) for piece in pieces: if piece in self._needed: occurences, peers = self._needed[piece] if peer not in peers: peers.append(peer) self._needed[piece] = (occurences + 1, peers) # Check whether there may be interest obtaining a piece from this peer self._check_interest(peer) def peer_has(self, peer, index): # Update the peer's bitfield and needed to reflect the availability # of the piece logger.debug("Peer at {} has piece {}".format(str(peer.addr()), index)) if index < self._metainfo.num_pieces: self._bitfields[peer][index] = 1 else: raise IndexError if index in self._needed: occurences, peers = self._needed[index] if peer not in peers: peers.append(peer) self._needed[index] = (occurences + 1, peers) # Check whether there may be interest obtaining a piece from this # peer self._check_interest(peer) def peer_choked(self, peer): logger.debug("Peer {} choked".format(str(peer.addr()))) if peer in self._interested: del self._interested[peer] elif peer in self._requesting: # When choked in the middle of obtaining a piece, save the # progress in the partial list index, offset, sha1, _, _ = self._requesting[peer] self._partial.append((index, offset, sha1)) del self._requesting[peer] def peer_unchoked(self, peer): logger.debug("Peer {} unchoked".format(str(peer.addr()))) if peer in self._interested: self._request(peer) def peer_sent_block(self, peer, index, begin, buf): if peer not in self._requesting: # If a peer is very slow in responding, a block could come after # it has timed out. Just ignore the data at this point and # ignore the slow peer logger.debug( "Received block from peer {} which has timed out".format( str(peer.addr()))) return piece, received_bytes, sha1, _, _ = self._requesting[peer] if piece == index and begin == received_bytes: # When the next expected block is received, update the hash value # and write the block to file sha1.update(buf) self._filemgr.write_block(index, begin, buf) self._requesting[peer] = (piece, received_bytes + len(buf), sha1, self._tick, 0) if received_bytes + len(buf) < self._length_of_piece(index): # Request the next block in the piece self._request(peer) else: # On receipt of the last block in the piece, verify the hash # and update the records to reflect receipt of the piece if sha1.digest() == self._metainfo.piece_hash(index): logger.info("Successfully got piece {} from {}".format( index, str(peer.addr()))) del self._needed[index] percent = 100 * ( 1 - (len(self._needed) / float(self._metainfo.num_pieces))) print "{0}: Downloaded {1:1.4f}%".format( self._filename, percent) self._have[index] = 1 else: logger.info("Unsuccessfully got piece {} from {}".format( index, str(peer.addr()))) del self._requesting[peer] if self._needed != {}: # Try to find another piece for this peer to get self._check_interest(peer) else: logger.info( "Successfully downloaded entire torrent {}".format( self._filename)) self._client.download_complete(self._filename) def peer_interested(self, peer): pass def peer_not_interested(self, peer): pass def peer_request(self, peer, index, begin, length): pass def peer_canceled(self, peer, index, begin, length): pass # Reactor callback def timer_event(self): self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event) self._tick += 1 # For any peers that have been interested but unchoked for an # excessive period of time, stop being interested, free up assigned # piece and connect to another peer for peer, (_, _, _, tick) in self._interested.items(): if tick + 4 == self._tick: logger.debug("Timed out on interest for peer {}".format( str(peer.addr()))) peer.not_interested() del self._interested[peer] self._connect_to_peers(1) # For any peer that has an outstanding request for an excessive period # of time, resend the request message in case it got lost or is being # ignored for peer, (index, offset, sha1, tick, retries) \ in self._requesting.items(): if tick + 5 == self._tick: logger.debug("Timed out on request for peer {}".format( str(peer.addr()))) if retries < _MAX_RETRIES: self._requesting[peer] = (index, offset, sha1, self._tick, retries + 1) self._request(peer) else: self._partial.append((index, offset, sha1)) del self._requesting[peer] peer.not_interested() self._connect_to_peers(1)
def initialize(self): """ initialize() returns a deferred which fires when initialization of the TorrentMgr is complete or an error has occurred. """ if self._state != self._States.Uninitialized: error = "TorrentMgr can't be reinitialized" logger.debug(error) d = Deferred() d.errback(TorrentMgrError(error)) return d try: self._metainfo = Metainfo(self._filename) except ValueError as err: logger.error(err) d = Deferred() d.errback(TorrentMgrError(err.message)) return d except IOError as err: logger.error(err) d = Deferred() d.errback(TorrentMgrError(err.strerror)) return d # _peers is a list of peers that the TorrentMgr is trying # to communicate with self._peers = [] # _bitfields is a dictionary mapping peers to a bitfield of the pieces # each has self._bitfields = {} # _have is the bitfield for this torrent. It is initialized to reflect # which pieces are already available on disk. self._filemgr = FileMgr(self._metainfo) self._have = self._filemgr.have() # _needed is a dictionary of pieces which are still needed. # The value for each piece is a tuple of the number of peers which # have the piece and a list of those peers. self._needed = { piece: (0, []) for piece in list(self._have.findall('0b0')) } # _interested is a dictionary of peers to whom interest has been # expressed. The value for each peer is a tuple of the piece that # has been reserved for the peer, the number of bytes of the piece that # have already been received, the sha1 hash of the bytes received so # far and the value of the tick at the time interest was expressed. self._interested = {} # _requesting is a dictionary of peers to whom a block request has been # made. The value for each peer is a tuple of the piece that is being # requested, the number of bytes that have already been received, the # shal2 hash of the bytes received so far, the value of the tick at # the time the request was made and the number of retries that have # been attempted self._requesting = {} # _partial is a list which tracks pieces that were interrupted while # being downloaded. Each entry is a tuple containing the index of the # piece, the number of bytes received so far and the sha1 hash of those # bytes. self._partial = [] self._tracker_proxy = TrackerProxy(self._metainfo, self._port, self._peer_id) def success(result): self._state = self._States.Initialized def failure(failure): logger.critical("Could not connect to tracker at {}".format( self._metainfo.announce)) message = failure.value.message logger.debug(" Tracker Error: {}".format(message)) raise TorrentMgrError(message) return self._tracker_proxy.start().addCallbacks(success, failure)
def test_announce(self): self.metainfo.announce = self.trackeraddr+"/announce" tp = TrackerProxy(self.metainfo, 6881, self.peer_id) assert tp.get_peers(5) == self.peerlist
def test_warning(self, capsys): self.metainfo.announce = self.trackeraddr+"/warning" tp = TrackerProxy(self.metainfo, 6881, self.peer_id) _, err = capsys.readouterr() assert "Warning:" in err assert tp.get_peers(5) == self.peerlist