class Splitter(object):
def __init__(self, packet_size):
self.packet_size = packet_size
self.on_new_packet = Event()
self.on_stream_end = Event()
self._buffer = bytes()
self._packets = deque()
self._current_sequence = 0
def push_stream(self, data):
self._buffer = self._buffer + data
while len(self._buffer) >= self.packet_size:
packet_data = self._buffer[:self.packet_size]
self._buffer = self._buffer[self.packet_size:]
self._create_packet(packet_data)
def end_stream(self):
self._create_packet(self._buffer)
self.on_stream_end.call(self)
def pop_packet(self):
return self._packets.popleft()
def _create_packet(self, packet_data):
packet = DataPacketMessage.create(self._current_sequence, packet_data)
self._packets.append(packet)
self._current_sequence += 1
self.on_new_packet.call(self)
def __init__(self, url):
HTTPClientFactory.__init__(self, url)
self.deferred = defer.Deferred()
self.waiting = 1
self.on_page_part = Event()
self.on_page_end = Event()
def __init__(self, packet_size):
self.packet_size = packet_size
self.on_new_packet = Event()
self.on_stream_end = Event()
self._buffer = bytes()
self._packets = deque()
self._current_sequence = 0
def __init__(self):
self.on_data_joined = Event()
self.on_end_join = Event()
self._buffer = bytes()
self._current_sequence = 0
self._packets = {}
self.sequences = set()
class StreamClient(object):
def __init__(self):
self.on_stream_received = Event()
self.on_stream_end = Event()
def _data_received(self, data):
self.on_stream_received.call(data)
def _connection_lost(self):
self.on_stream_end.call()
class ConnectionList(object):
def __init__(self, protocol_class):
self.on_changed = Event()
self._connections = {}
self._protocol_class = protocol_class
def add(self, connection):
"""Adds a new item to the list of connections."""
assert isinstance(connection, self._protocol_class)
assert isinstance(connection.partner_id, str)
if connection.partner_id in self._connections:
logging.error('Adding existent connection')
connection.drop()
return
self._connections[connection.partner_id] = connection
self.on_changed.call(self)
def remove(self, connection):
"""Removes an item from the list of connections."""
assert isinstance(connection, self._protocol_class)
assert isinstance(connection.partner_id, str)
if connection.partner_id not in self._connections:
logging.error('Removing nonexistent connection')
return
del self._connections[connection.partner_id]
self.on_changed.call(self)
def __iter__(self):
return self._connections.itervalues()
def __contains__(self, partner_id):
return partner_id in self._connections
def __getitem__(self, partner_id):
return self._connections[partner_id]
@property
def ids(self):
return self._connections.keys()
class Joiner(object):
def __init__(self):
self.on_data_joined = Event()
self.on_end_join = Event()
self._buffer = bytes()
self._current_sequence = 0
self._packets = {}
self.sequences = set()
def push_packet(self, packet):
self._packets[packet.sequence] = packet.data
self._join_buffer()
self._update_sequences()
def end_join(self):
self.on_end_join.call(self)
def pop_stream(self):
buffer = self._buffer
self._buffer = bytes()
return buffer
def _update_sequences(self):
self.sequences = set(self._packets.keys())
def _join_buffer(self):
sequences = takewhile(lambda seq: seq in self._packets,
count(self._current_sequence))
sequences = list(sequences)
if len(sequences) > 0:
self._buffer += ''.join(self._packets[seq] for seq in sequences)
for sequence in sequences:
del self._packets[sequence]
self._current_sequence = sequences[-1] + 1
self.on_data_joined.call(self)
def __init__(self, tracker_url, peer):
"""
Initializes the manager.
"""
self._announce_repeater = TwistedRepeater(self.announce,
self.DEFAULT_ANNOUNCE)
self._utility_repeater = TwistedRepeater(self.report_utility,
self.DEFAULT_ANNOUNCE)
self.tracker_url = tracker_url
self.peer = peer
self.utility_by_peer = {}
self.peer_list = []
self.status = TrackerStatus(TrackerStatus.STANDBY, 'Ready to connect')
self.on_updated = Event()
self.on_status_changed = Event()
class _HTTPStreamDownloader(HTTPClientFactory):
protocol = HTTPPageDownloader
def __init__(self, url):
HTTPClientFactory.__init__(self, url)
self.deferred = defer.Deferred()
self.waiting = 1
self.on_page_part = Event()
self.on_page_end = Event()
def pageStart(self, partialContent):
self.waiting = 0
def pagePart(self, data):
self.on_page_part.call(data)
def pageEnd(self):
self.on_page_end.call()
self.deferred.callback(None)
def __init__(self, peer_service):
"""
Creates a new ConnectionManager object.
"""
self.on_update = Event()
self._peer_service = peer_service
self._connection_attempts = set()
self.incoming_connections = ConnectionList(IncomingProtocol)
self.incoming_connections.on_changed.add_handler(self._updated)
self.outgoing_connections = ConnectionList(OutgoingProtocol)
self.outgoing_connections.on_changed.add_handler(self._updated)
class ConnectionManager(object):
"""
Maintains a collection of incoming and outgoing connections to a peer.
"""
def __init__(self, peer_service):
"""
Creates a new ConnectionManager object.
"""
self.on_update = Event()
self._peer_service = peer_service
self._connection_attempts = set()
self.incoming_connections = ConnectionList(IncomingProtocol)
self.incoming_connections.on_changed.add_handler(self._updated)
self.outgoing_connections = ConnectionList(OutgoingProtocol)
self.outgoing_connections.on_changed.add_handler(self._updated)
@property
def all_connections(self):
"""Returns an iterator of all current connections"""
return itertools.chain(self.incoming_connections,
self.outgoing_connections)
def connection_allowed(self, peer_id):
"""
Returns true if a connection with a peer is allowed.
:param peer_id: The ID of the peer to check.
"""
if peer_id in self._connection_attempts:
logging.info('Connection attempt not allowed with ' + peer_id)
logging.info(str(self._connection_attempts))
return (peer_id not in self._connections_ids and
peer_id not in self._connection_attempts)
def listen(self, port):
"""Starts listening on the given port."""
factory = self._create_server_factory()
reactor.listenTCP(port, factory)
def get_connection(self, partner_id):
if partner_id in self.incoming_connections:
return self.incoming_connections[partner_id]
if partner_id in self.outgoing_connections:
return self.outgoing_connections[partner_id]
return None
def connect_to_peer(self, peer):
"""Establish a new connection with a given peer."""
logging.debug('Connecting to peer:' + peer.id)
self._connection_attempts.add(peer.id)
factory = self._create_client_factory(peer.id)
reactor.connectTCP(peer.ip, peer.port, factory)
def connect_to_peers(self, peers):
"""
Connect to a list of peers
"""
for peer in peers:
if peer.id in self._connections_ids:
continue
self.connect_to_peer(peer)
def heartbeat(self):
"""
Sends heartbeats
"""
for connection in self.all_connections:
connection.send_heartbeat()
def check_heartbeats(self):
"""
Check connections for last heartbeat
"""
#FIXME: Put this in a configuration file
PEER_TIMEOUT = 30
for connection in self.all_connections:
timeout = time.time() - connection.last_heartbeat
cid = connection.partner_id
logging.info('Checking peer timeout {0}'.format(cid))
if timeout > PEER_TIMEOUT:
logging.info('Peer Connection Timeout {0}'.format(cid))
connection.drop()
@property
def _connections_ids(self):
return set(self.incoming_connections.ids +
self.outgoing_connections.ids)
def _create_server_factory(self):
factory = ServerFactory()
factory.protocol = IncomingProtocol
factory.add_connection = self.incoming_connections.add
factory.remove_connection = self.incoming_connections.remove
self._init_factory(factory)
return factory
def _create_client_factory(self, target_id):
factory = ClientFactory()
factory.protocol = OutgoingProtocol
factory.target_id = target_id
factory.add_connection = self.outgoing_connections.add
factory.remove_connection = self.outgoing_connections.remove
self._init_factory(factory)
return factory
def _init_factory(self, factory):
factory.peer_service = self._peer_service
factory.connection_allowed = self.connection_allowed
factory.end_connection_attempt = self._end_connection_attempt
def _updated(self, connection):
self.on_update.call(self)
def _end_connection_attempt(self, peer_id):
if peer_id in self._connection_attempts:
self._connection_attempts.remove(peer_id)
def __init__(self, protocol_class):
self.on_changed = Event()
self._connections = {}
self._protocol_class = protocol_class
class TrackerManager(object):
"""
Manages the connection with the tracker
"""
DEFAULT_ANNOUNCE = 30
#TODO: Eliminate tracker_url parameter
def __init__(self, tracker_url, peer):
"""
Initializes the manager.
"""
self._announce_repeater = TwistedRepeater(self.announce,
self.DEFAULT_ANNOUNCE)
self._utility_repeater = TwistedRepeater(self.report_utility,
self.DEFAULT_ANNOUNCE)
self.tracker_url = tracker_url
self.peer = peer
self.utility_by_peer = {}
self.peer_list = []
self.status = TrackerStatus(TrackerStatus.STANDBY, 'Ready to connect')
self.on_updated = Event()
self.on_status_changed = Event()
def connect_to_tracker(self):
self._announce_repeater.start_now()
self._utility_repeater.start_now()
def announce(self):
"""Starts connection with the tracker."""
logging.debug('Announcing to tracker')
deferred = client.getPage(self._create_announce_url())
deferred.addCallback(self._on_announce_response)
deferred.addErrback(self._on_tracker_error)
def report_utility(self):
if len(self.utility_by_peer) == 0:
return
deferred = client.getPage(self._create_utility_url())
deferred.addCallback(self._on_utility_response)
deferred.addErrback(self._on_tracker_error)
def _create_announce_url(self):
"""Creates a request URL for the tracker with the GET query."""
query = dict(peer_id = self.peer.id,
port = self.peer.port)
if self.peer.ip is not None:
query.update(ip = self.peer.ip)
parts = list(urlparse.urlsplit(self.tracker_url))
parts[2] += '/announce' # path
parts[3] = urllib.urlencode(query) # query
return urlparse.urlunsplit(parts)
def _create_utility_url(self):
parts = list(urlparse.urlsplit(self.tracker_url))
parts[2] += '/utility'
parts[3] = urllib.urlencode(self.utility_by_peer) # query.
return urlparse.urlunsplit(parts)
def _on_tracker_error(self, error):
logging.error('Unable to contact the tracker: ' + str(error.value))
self._change_status(TrackerStatus.ERROR,
'Unable to contact the tracker')
def _on_announce_response(self, content):
try:
logging.debug('Received tracker announce response')
response = json.loads(content)
if 'failure_reason' in response:
self._on_announce_failure(response['failure_reason'])
return
if 'request_interval' in response:
self._announce_repeater.seconds = response['request_interval']
if 'peers' in response:
self._update_peers(response['peers'])
self._change_status(TrackerStatus.OK, 'OK')
except Exception as e:
self._on_announce_failure(str(e))
def _on_announce_failure(self, error):
logging.error('Tracker announce failure: ' + error)
self._change_status(TrackerStatus.ERROR, error)
def _on_utility_response(self, content):
logging.debug('Received tracker utility response')
response = json.loads(content)
if 'failure_reason' in response:
self._on_utility_failure(response['failure_reason'])
return
def _on_utility_failure(self, error):
logging.error('Tracker utility failure: ' + error)
self._change_status(TrackerStatus.ERROR, error)
def _update_peers(self, peer_list):
try:
self.peer_list = []
for peer_dict in peer_list:
id = peer_dict['id']
ip = peer_dict['ip']
port = peer_dict['port']
uf = peer_dict['utility_factor']
self.peer_list.append(Peer(id, ip, port, uf))
self.on_updated.call(self, self.peer_list)
except Exception as e:
raise TrackerManagerError('Unable to convert peer list: ' + str(e))
def _change_status(self, status, message):
if self.status.status == status and self.status.message == message:
return
self.status = TrackerStatus(status, message)
self.on_status_changed.call(self, self.status)
def __init__(self):
self.on_stream_received = Event()
self.on_stream_end = Event()