def test_strategy_multi_peer(self): """ If we have multiple peers, we should select one and send it our channel views. Also, we should still inspect our download queue. """ self.community.get_peers_return = [ Peer(default_eccrypto.generate_key(u"very-low")), Peer(default_eccrypto.generate_key(u"very-low")), Peer(default_eccrypto.generate_key(u"very-low")) ] self.strategy.take_step() self.assertEqual(1, len(self.community.send_random_to_called)) self.assertIn(self.community.send_random_to_called[0], self.community.get_peers_return)
def should_join_circuit(self, create_payload, previous_node_address): """ Check whether we should join a circuit. Returns a deferred that fires with a boolean. """ if self.settings.max_joined_circuits <= len(self.relay_from_to) + len(self.exit_sockets): self.logger.warning("too many relays (%d)", (len(self.relay_from_to) + len(self.exit_sockets))) return succeed(False) # Check whether we have a random open slot, if so, allocate this to this request. circuit_id = create_payload.circuit_id for index, slot in enumerate(self.random_slots): if not slot: self.random_slots[index] = circuit_id return succeed(True) # No random slots but this user might be allocated a competing slot. # Next, we request the token balance of the circuit initiator. balance_deferred = Deferred() self.request_cache.add(BalanceRequestCache(self, circuit_id, balance_deferred)) # Temporarily add these values, otherwise we are unable to communicate with the previous hop. self.directions[circuit_id] = EXIT_NODE shared_secret, _, _ = self.crypto.generate_diffie_shared_secret(create_payload.key) self.relay_session_keys[circuit_id] = self.crypto.generate_session_keys(shared_secret) self.send_cell([Peer(create_payload.node_public_key, previous_node_address)], u"balance-request", BalanceRequestPayload(circuit_id)) self.directions.pop(circuit_id, None) self.relay_session_keys.pop(circuit_id, None) return balance_deferred
def on_payout_block(self, source_address, data): if not self.triblerchain_community: self.logger.warning( "Got payout while not having a TriblerChain community running!" ) return _, payload = self._ez_unpack_noauth(PayoutPayload, data) peer = Peer(payload.public_key, source_address) block = self.triblerchain_community.BLOCK_CLASS.from_payload( payload, self.serializer) self.triblerchain_community.process_half_block(block, peer) # Send the next payout if payload.circuit_id in self.relay_from_to and block.transaction[ 'down'] > payload.base_amount: relay = self.relay_from_to[payload.circuit_id] circuit_peer = self.get_peer_from_mid(relay.mid) if not circuit_peer: self.logger.warning( "%s Unable to find next peer %s for payout!", self.my_peer, relay.mid.encode('hex')) return self.do_payout(circuit_peer, relay.circuit_id, block.transaction['down'] - payload.base_amount * 2, payload.base_amount)
def on_payout_block(self, source_address, data): if not self.bandwidth_wallet: self.logger.warning( "Got payout while not having a TrustChain community running!") return payload = self._ez_unpack_noauth(PayoutPayload, data, global_time=False) peer = Peer(payload.public_key, source_address) block = TriblerBandwidthBlock.from_payload(payload, self.serializer) self.bandwidth_wallet.trustchain.process_half_block(block, peer) # Send the next payout if payload.circuit_id in self.relay_from_to and block.transaction[ 'down'] > payload.base_amount: relay = self.relay_from_to[payload.circuit_id] circuit_peer = self.get_peer_from_address(relay.peer.address) if not circuit_peer: self.logger.warning( "%s Unable to find next peer %s for payout!", self.my_peer, relay.peer) return self.do_payout(circuit_peer, relay.circuit_id, block.transaction['down'] - payload.base_amount * 2, payload.base_amount)
def load_tunnel_community_in_session(self, session, exitnode=False): """ Load the tunnel community in a given session. We are using our own tunnel community here instead of the one used in Tribler. """ self.sanitize_network(session) keypair = ECCrypto().generate_key(u"curve25519") tunnel_peer = Peer(keypair) session.config.set_tunnel_community_exitnode_enabled(exitnode) overlay = self.test_class(tunnel_peer, session.lm.ipv8.endpoint, session.lm.ipv8.network, tribler_session=session, dht_provider=MockDHTProvider( session.lm.ipv8.endpoint.get_address()), settings={ "become_exitnode": exitnode, "max_circuits": 1 }) overlay._use_main_thread = False overlay.settings.remove_tunnel_delay = 0 session.lm.ipv8.overlays.append(overlay) return overlay
def load_tunnel_community_in_session(self, session, exitnode=False): """ Load the tunnel community in a given session. We are using our own tunnel community here instead of the one used in Tribler. """ keypair = ECCrypto().generate_key(u"curve25519") tunnel_peer = Peer(keypair) session.config.set_tunnel_community_exitnode_enabled(exitnode) overlay = self.test_class(tunnel_peer, session.lm.ipv8.endpoint, session.lm.ipv8.network, tribler_session=session, dht_provider=MockDHTProvider( session.lm.ipv8.endpoint.get_address())) overlay._use_main_thread = False session.lm.ipv8.overlays.append(overlay) session.lm.ipv8.strategies.append((RandomWalk(overlay), 20)) # We disable the discovery communities in this session since we don't want to walk to the live network for overlay in session.lm.ipv8.overlays: if isinstance(overlay, DiscoveryCommunity): overlay.unload() # Also reset the IPv8 network session.lm.ipv8.network = Network() return overlay
def on_payout_block(self, source_address, data): if not self.bandwidth_wallet: self.logger.warning( "Got payout while not having a TrustChain community running!") return payload = self._ez_unpack_noauth(PayoutPayload, data, global_time=False) peer = Peer(payload.public_key, source_address) def on_transaction_completed(blocks): # Send the next payout if blocks and payload.circuit_id in self.relay_from_to and block.transaction[ 'down'] > payload.base_amount: relay = self.relay_from_to[payload.circuit_id] self._logger.info("Sending next payout to peer %s", relay.peer) self.do_payout( relay.peer, relay.circuit_id, block.transaction['down'] - payload.base_amount * 2, payload.base_amount) block = TriblerBandwidthBlock.from_payload(payload, self.serializer) self.bandwidth_wallet.trustchain.process_half_block(block, peer)\ .addCallbacks(on_transaction_completed, lambda _: None) # Check whether the block has been added to the database and has been verified if not self.bandwidth_wallet.trustchain.persistence.contains(block): self.logger.warning( "Not proceeding with payout - received payout block is not valid" ) return
def test_send_random_multiple_torrents(self): """ Test whether sending a single channel with a multiple torrents to another peer works correctly """ with db_session: channel = self.nodes[ 0].overlay.metadata_store.ChannelMetadata.create_channel( "test", "bla") for _ in xrange(20): self.add_random_torrent( self.nodes[0].overlay.metadata_store.TorrentMetadata) channel.commit_channel_torrent() self.nodes[0].overlay.send_random_to( Peer(self.nodes[1].my_peer.public_key, self.nodes[1].endpoint.wan_address)) yield self.deliver_messages(timeout=0.5) with db_session: self.assertEqual( len(self.nodes[1].overlay.metadata_store.ChannelMetadata. select()), 1) channel = self.nodes[ 1].overlay.metadata_store.ChannelMetadata.select()[:][0] self.assertLess(channel.contents_len, 20)
class TriblerTunnelTestnetCommunity(TriblerTunnelCommunity): """ This community defines a testnet for the anonymous tunnels. """ master_peer = Peer("3081a7301006072a8648ce3d020106052b810400270381920004002831990fc973aaf5a8f5bd401f8771fd411d763" "0ccc4a61c4147a1200135023a2397006e0acb215783cc0245bbc69ebe66abdd13f1fa3434c630604ef2c0d99e5f98" "727e75ae7901529ba5a2dd875bab582f3f508aa7b675c9d9bd7fd6c2e2684c7fc71b72f007e080634cecf007b718f" "5cf24d24821cd08feb30d3f3059c7702615ea6f8b23823415bd1673c406e1".decode('hex'))
class TriblerTunnelTestnetCommunity(TriblerTunnelCommunity): """ This community defines a testnet for the anonymous tunnels. """ master_peer = Peer("3081a7301006072a8648ce3d020106052b81040027038192000401325e6f0a67a92a890344013914fcf9da9b0326" "bf6c845815ec8b537e356a56b2a8c27ac4918078202f60eb29ebf081436136c280316ac461daee2d04cf073d1200" "80d15628af1002b0c0273d9d94fdab08e718f568d83b9c4298117261b5647ca8295f1ba4b880a50fd2ef7041fef7" "edfbef5f02fc03827a2c09c5f9c701f87abacd022c780d76733c133363a5c46c".decode('hex'))
def on_introduction_response(self, source_address, data): super(TriblerChainCrawlerCommunity, self).on_introduction_response(source_address, data) auth, _, _ = self._ez_unpack_auth(IntroductionResponsePayload, data) peer = Peer(auth.public_key_bin, source_address) self.send_crawl_request(peer, peer.public_key.key_to_bin())
class GigaChannelTestnetCommunity(GigaChannelCommunity): """ This community defines a testnet for the giga channels, used for testing purposes. """ master_peer = Peer( unhexlify( "4c69624e61434c504b3afbd79020aa61795d1186ea505cf80fe2ac7f42dfc32b830ebade5d78479cbb4" "35bdbfda7b04e156f5515d1b0bbdafa5c67279e25937201ef6b31f7eeded20423" ))
def add_node_to_experiment(self, node): """ Add a new node to this experiment (use `create_node()`). """ for other in self.nodes: private_peer = other.my_peer public_peer = Peer(private_peer.public_key, private_peer.address) node.network.add_verified_peer(public_peer) node.network.discover_services(public_peer, node.overlay.master_peer.mid) self.nodes.append(node)
def on_tribler_started(_): # We load the TriblerChain community. triblerchain_peer = Peer(self.session.trustchain_keypair) self.triblerchain_community = TriblerChainCrawlerCommunity( triblerchain_peer, self.session.lm.ipv8.endpoint, self.session.lm.ipv8.network, tribler_session=self.session, working_directory=self.session.config.get_state_dir()) self.session.lm.ipv8.overlays.append(self.triblerchain_community) self.session.lm.ipv8.strategies.append( (RandomWalk(self.triblerchain_community), -1))
def __init__(self, crypto_curve, overlay_class, *args, **kwargs): self.endpoint = AutoMockEndpoint() self.endpoint.open() self.network = Network() self.my_peer = Peer(ECCrypto().generate_key(crypto_curve), self.endpoint.wan_address) self.overlay = overlay_class(self.my_peer, self.endpoint, self.network, *args, **kwargs) self.discovery = MockWalk(self.overlay) self.overlay.my_estimated_wan = self.endpoint.wan_address self.overlay.my_estimated_lan = self.endpoint.lan_address
def test_strategy_one_peer(self): """ If we have one peer, we should send it our channel views and inspect our download queue. """ self.community.get_peers_return = [ Peer(default_eccrypto.generate_key(u"very-low")) ] self.strategy.take_step() self.assertEqual(1, len(self.community.send_random_to_called)) self.assertEqual(self.community.get_peers_return[0], self.community.send_random_to_called[0])
def bootstrap_new_identity(self, amount): """ One-way payment channel. Create a new temporary identity, and transfer funds to the new identity. A different party can then take the result and do a transfer from the temporary identity to itself """ # Create new identity for the temporary identity crypto = ECCrypto() tmp_peer = Peer(crypto.generate_key(u"curve25519")) # Create the transaction specification transaction = {'up': 0, 'down': amount, 'type': 'tribler_bandwidth'} # Create the two half blocks that form the transaction local_half_block = TriblerBandwidthBlock.create( 'tribler_bandwidth', transaction, self.trustchain.persistence, self.trustchain.my_peer.public_key.key_to_bin(), link_pk=tmp_peer.public_key.key_to_bin()) local_half_block.sign(self.trustchain.my_peer.key) tmp_half_block = TriblerBandwidthBlock.create( 'tribler_bandwidth', transaction, self.trustchain.persistence, tmp_peer.public_key.key_to_bin(), link=local_half_block, link_pk=self.trustchain.my_peer.public_key.key_to_bin()) tmp_half_block.sign(tmp_peer.key) self.trustchain.persistence.add_block(local_half_block) self.trustchain.persistence.add_block(tmp_half_block) # Create the bootstrapped identity format block = { 'block_hash': tmp_half_block.hash.encode('base64'), 'sequence_number': tmp_half_block.sequence_number } result = { 'private_key': tmp_peer.key.key_to_bin().encode('base64'), 'transaction': { 'up': amount, 'down': 0 }, 'block': block } return result
def assign_exit_node(self, node_nr): """ Give a node a dedicated exit node to play with. """ exit_node = self.create_node() self.nodes.append(exit_node) # So it could be properly removed on exit exit_node.overlay.settings.become_exitnode = True public_peer = Peer(exit_node.my_peer.public_key, exit_node.my_peer.address) self.nodes[node_nr].network.add_verified_peer(public_peer) self.nodes[node_nr].network.discover_services(public_peer, exit_node.overlay.master_peer.mid) self.nodes[node_nr].overlay.update_exit_candidates(public_peer, True) self.nodes[node_nr].overlay.build_tunnels(1) yield self.deliver_messages() exit_sockets = exit_node.overlay.exit_sockets for exit_socket in exit_sockets: exit_sockets[exit_socket] = MockTunnelExitSocket(exit_sockets[exit_socket])
def initialize(self, overlay_class, node_count, *args, **kwargs): self.overlay_class = overlay_class self.nodes = [ self.create_node(*args, **kwargs) for _ in range(node_count) ] # Add nodes to each other for node in self.nodes: for other in self.nodes: if other == node: continue private_peer = other.my_peer public_peer = Peer(private_peer.public_key, private_peer.address) node.network.add_verified_peer(public_peer) node.network.discover_services(public_peer, overlay_class.master_peer.mid)
def setUp(self): """ Setup various variables and load the tunnel community in the main downloader session. """ yield TestAsServer.setUp(self) self.seed_tdef = None self.sessions = [] self.session2 = None self.bypass_dht = False self.seed_config = None self.tunnel_community_seeder = None self.eccrypto = ECCrypto() ec = self.eccrypto.generate_key(u"curve25519") self.test_class = TriblerTunnelCommunity self.test_class.master_peer = Peer(ec) self.tunnel_community = self.load_tunnel_community_in_session( self.session, exitnode=True) self.tunnel_communities = []
def test_send_and_get_channel_update_back(self): """ Test if sending back information on updated version of a channel works """ with db_session: # Add channel to node 0 channel = self.nodes[ 0].overlay.metadata_store.ChannelMetadata.create_channel( "test", "bla") for _ in xrange(20): self.add_random_torrent( self.nodes[0].overlay.metadata_store.TorrentMetadata) channel.commit_channel_torrent() channel_v1_dict = channel.to_dict() channel_v1_dict.pop("health") self.add_random_torrent( self.nodes[0].overlay.metadata_store.TorrentMetadata) channel.commit_channel_torrent() # Add the outdated version of the channel to node 1 self.nodes[1].overlay.metadata_store.ChannelMetadata.from_dict( channel_v1_dict) # node1 --outdated_channel--> node0 self.nodes[1].overlay.send_random_to( Peer(self.nodes[0].my_peer.public_key, self.nodes[0].endpoint.wan_address)) yield self.deliver_messages(timeout=0.5) with db_session: self.assertEqual( self.nodes[1].overlay.metadata_store.ChannelMetadata.select() [:][0].timestamp, self.nodes[0].overlay.metadata_store.ChannelMetadata.select() [:][0].timestamp)
class TriblerChainCommunity(TrustChainCommunity): """ Community for reputation based on TrustChain tamper proof interaction history. """ BLOCK_CLASS = TriblerChainBlock DB_CLASS = TriblerChainDB master_peer = Peer( "3081a7301006072a8648ce3d020106052b81040027038192000405c66d3deddb1721787a247b2285118c06ce9fb" "20ebd3546969fa2f4811fa92426637423d3bac1510f92b33e2ff5a785bf54eb3b28d29a77d557011d7d5241243c" "9c89c987cd049404c4024999e1505fa96e1d6668234bde28a666d458d67251d17ff45185515a28967ddcf50503c" "304750ae114f9bc857a79c03da1a9c9215ea07c91f166f24b6cfd1cf72309044fbd". decode('hex')) def __init__(self, *args, **kwargs): self.tribler_session = kwargs.pop('tribler_session', None) super(TriblerChainCommunity, self).__init__(*args, **kwargs) def should_sign(self, block): """ Return whether we should sign a given block. For the TriblerChain, we only sign a block when we receive bytes. In our current design, only the person that should pay bytes to others initiates a signing request. This is true when considering payouts in the tunnels and when buying bytes on the market. """ return block.transaction["down"] >= MIN_TRANSACTION_SIZE @blocking_call_on_reactor_thread def get_statistics(self, public_key=None): """ Returns a dictionary with some statistics regarding the local trustchain database :returns a dictionary with statistics """ if public_key is None: public_key = self.my_peer.public_key.key_to_bin() latest_block = self.persistence.get_latest(public_key) statistics = dict() statistics["id"] = public_key.encode("hex") interacts = self.persistence.get_num_unique_interactors(public_key) statistics["peers_that_pk_helped"] = interacts[0] if interacts[ 0] is not None else 0 statistics["peers_that_helped_pk"] = interacts[1] if interacts[ 1] is not None else 0 if latest_block: statistics["total_blocks"] = latest_block.sequence_number statistics["total_up"] = latest_block.transaction["total_up"] statistics["total_down"] = latest_block.transaction["total_down"] statistics["latest_block"] = dict(latest_block) # Set up/down statistics["latest_block"]["up"] = latest_block.transaction["up"] statistics["latest_block"]["down"] = latest_block.transaction[ "down"] else: statistics["total_blocks"] = 0 statistics["total_up"] = 0 statistics["total_down"] = 0 return statistics def get_bandwidth_tokens(self, peer=None): """ Get the bandwidth tokens for another peer. Currently this is just the difference in the amount of MBs exchanged with them. :param member: the peer we interacted with :type member: Peer :return: the amount of bandwidth tokens for this peer :rtype: int """ if peer is None: peer = self.my_peer block = self.persistence.get_latest(peer.public_key.key_to_bin()) if block: return block.transaction['total_up'] - block.transaction[ 'total_down'] return 0 def bootstrap_new_identity(self, amount): """ One-way payment channel. Create a new temporary identity, and transfer funds to the new identity. A different party can then take the result and do a transfer from the temporary identity to itself """ # Create new identity for the temporary identity crypto = ECCrypto() tmp_peer = Peer(crypto.generate_key(u"curve25519")) # Create the transaction specification transaction = {'up': 0, 'down': amount} # Create the two half blocks that form the transaction local_half_block = TriblerChainBlock.create( transaction, self.persistence, self.my_peer.public_key.key_to_bin(), link_pk=tmp_peer.public_key.key_to_bin()) local_half_block.sign(self.my_peer.key) tmp_half_block = TriblerChainBlock.create( transaction, self.persistence, tmp_peer.public_key.key_to_bin(), link=local_half_block, link_pk=self.my_peer.public_key.key_to_bin()) tmp_half_block.sign(tmp_peer.key) self.persistence.add_block(local_half_block) self.persistence.add_block(tmp_half_block) # Create the bootstrapped identity format block = { 'block_hash': tmp_half_block.hash.encode('base64'), 'sequence_number': tmp_half_block.sequence_number } result = { 'private_key': tmp_peer.key.key_to_bin().encode('base64'), 'transaction': { 'up': amount, 'down': 0 }, 'block': block } return result
def load_ipv8_overlays(self): # Discovery Community with open(self.session.config.get_permid_keypair_filename(), 'r') as key_file: content = key_file.read() content = content[31:-30].replace('\n', '').decode("BASE64") peer = Peer(M2CryptoSK(keystring=content)) discovery_community = DiscoveryCommunity(peer, self.ipv8.endpoint, self.ipv8.network) discovery_community.resolve_dns_bootstrap_addresses() self.ipv8.overlays.append(discovery_community) self.ipv8.strategies.append((RandomChurn(discovery_community), -1)) if not self.session.config.get_dispersy_enabled(): self.ipv8.strategies.append((RandomWalk(discovery_community), 20)) # TriblerChain Community if self.session.config.get_trustchain_enabled(): triblerchain_peer = Peer(self.session.trustchain_keypair) from Tribler.community.triblerchain.community import TriblerChainCommunity self.triblerchain_community = TriblerChainCommunity( triblerchain_peer, self.ipv8.endpoint, self.ipv8.network, tribler_session=self.session, working_directory=self.session.config.get_state_dir()) self.ipv8.overlays.append(self.triblerchain_community) self.ipv8.strategies.append( (EdgeWalk(self.triblerchain_community), 20)) # Tunnel Community if self.session.config.get_tunnel_community_enabled(): tunnel_peer = Peer(self.session.trustchain_keypair) from Tribler.community.triblertunnel.community import TriblerTunnelCommunity self.tunnel_community = TriblerTunnelCommunity( tunnel_peer, self.ipv8.endpoint, self.ipv8.network, tribler_session=self.session, dht_provider=MainlineDHTProvider( self.mainline_dht, self.session.config.get_dispersy_port()), triblerchain_community=self.triblerchain_community) self.ipv8.overlays.append(self.tunnel_community) self.ipv8.strategies.append( (RandomWalk(self.tunnel_community), 20)) # Market Community if self.session.config.get_market_community_enabled(): wallets = {} try: from Tribler.community.market.wallet.btc_wallet import BitcoinWallet, BitcoinTestnetWallet wallet_type = BitcoinTestnetWallet if self.session.config.get_btc_testnet( ) else BitcoinWallet btc_wallet = wallet_type( os.path.join(self.session.config.get_state_dir(), 'wallet')) wallets[btc_wallet.get_identifier()] = btc_wallet except ImportError: self._logger.error( "Electrum wallet cannot be found, Bitcoin trading not available!" ) mc_wallet = TrustchainWallet(self.triblerchain_community) wallets[mc_wallet.get_identifier()] = mc_wallet if self.session.config.get_dummy_wallets_enabled(): # For debugging purposes, we create dummy wallets dummy_wallet1 = DummyWallet1() wallets[dummy_wallet1.get_identifier()] = dummy_wallet1 dummy_wallet2 = DummyWallet2() wallets[dummy_wallet2.get_identifier()] = dummy_wallet2 from Tribler.community.market.community import MarketCommunity market_peer = Peer(self.session.tradechain_keypair) self.market_community = MarketCommunity( market_peer, self.ipv8.endpoint, self.ipv8.network, tribler_session=self.session, wallets=wallets, working_directory=self.session.config.get_state_dir()) self.ipv8.overlays.append(self.market_community) self.ipv8.strategies.append( (RandomWalk(self.market_community), 20))
def load_ipv8_overlays(self): # Discovery Community with open(self.session.config.get_permid_keypair_filename(), 'r') as key_file: content = key_file.read() content = content[31:-30].replace('\n', '').decode("BASE64") peer = Peer(M2CryptoSK(keystring=content)) discovery_community = DiscoveryCommunity(peer, self.ipv8.endpoint, self.ipv8.network) discovery_community.resolve_dns_bootstrap_addresses() self.ipv8.overlays.append(discovery_community) self.ipv8.strategies.append((RandomChurn(discovery_community), -1)) if not self.session.config.get_dispersy_enabled(): self.ipv8.strategies.append((RandomWalk(discovery_community), 20)) if self.session.config.get_testnet(): peer = Peer(self.session.trustchain_keypair) else: peer = Peer(self.session.trustchain_testnet_keypair) # TrustChain Community if self.session.config.get_trustchain_enabled(): from Tribler.pyipv8.ipv8.attestation.trustchain.community import TrustChainCommunity, \ TrustChainTestnetCommunity community_cls = TrustChainTestnetCommunity if self.session.config.get_testnet( ) else TrustChainCommunity self.trustchain_community = community_cls( peer, self.ipv8.endpoint, self.ipv8.network, working_directory=self.session.config.get_state_dir()) self.ipv8.overlays.append(self.trustchain_community) self.ipv8.strategies.append( (EdgeWalk(self.trustchain_community), 20)) tc_wallet = TrustchainWallet(self.trustchain_community) self.wallets[tc_wallet.get_identifier()] = tc_wallet # Tunnel Community if self.session.config.get_tunnel_community_enabled(): from Tribler.community.triblertunnel.community import TriblerTunnelCommunity, TriblerTunnelTestnetCommunity community_cls = TriblerTunnelTestnetCommunity if self.session.config.get_testnet() else \ TriblerTunnelCommunity self.tunnel_community = community_cls( peer, self.ipv8.endpoint, self.ipv8.network, tribler_session=self.session, dht_provider=MainlineDHTProvider( self.mainline_dht, self.session.config.get_dispersy_port()), bandwidth_wallet=self.wallets["MB"]) self.ipv8.overlays.append(self.tunnel_community) self.ipv8.strategies.append( (RandomWalk(self.tunnel_community), 20)) # Market Community if self.session.config.get_market_community_enabled(): from Tribler.community.market.community import MarketCommunity, MarketTestnetCommunity community_cls = MarketTestnetCommunity if self.session.config.get_testnet( ) else MarketCommunity self.market_community = community_cls( peer, self.ipv8.endpoint, self.ipv8.network, tribler_session=self.session, trustchain=self.trustchain_community, wallets=self.wallets, working_directory=self.session.config.get_state_dir()) self.ipv8.overlays.append(self.market_community) self.ipv8.strategies.append( (RandomWalk(self.market_community), 20)) # Popular Community if self.session.config.get_popularity_community_enabled(): from Tribler.community.popularity.community import PopularityCommunity self.popularity_community = PopularityCommunity( peer, self.ipv8.endpoint, self.ipv8.network, torrent_db=self.session.lm.torrent_db, session=self.session) self.ipv8.overlays.append(self.popularity_community) self.ipv8.strategies.append( (RandomWalk(self.popularity_community), 20)) self.popularity_community.start()
class PubSubCommunity(Community): """ This community is designed as a base community for all othe future communities that desires publish subscribe model for content dissemination. It provides a few basic primitives like subscribe/unsubscribe to publisher peers and publish/broadcast content to subscriber peers. All the derived community should implement publish_next_content() method which is responsible for publishing the next available content to all the subscribers. """ MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b8104002703819200040504278d20d6776ce7081ad57d99fe066bb2a93" \ "ce7cc92405a534ef7175bab702be557d8c7d3b725ea0eb09c686e798f6c7ad85e8781a4c3b20e54c15ede38077c" \ "8f5c801b71d13105f261da7ddcaa94ae14bd177bf1a05a66f595b9bb99117d11f73b4c8d3dcdcdc2b3f838b8ba3" \ "5a9f600d2c543e8b3ba646083307b917bbbccfc53fc5ab6ded90b711d7eeda46f5f" master_peer = Peer(unhexlify(MASTER_PUBLIC_KEY)) def __init__(self, *args, **kwargs): super(PubSubCommunity, self).__init__(*args, **kwargs) self.trustchain = kwargs.pop('trustchain_community', None) self.logger = logging.getLogger(self.__class__.__name__) self.request_cache = RequestCache() # Register messages self.decode_map.update({ chr(MSG_SUBSCRIBE): self.on_subscribe, chr(MSG_SUBSCRIPTION): self.on_subscription_status }) # A set of publisher and subscriber. # Sends data updates to subscribers, and receives updates from subscribers. self.subscribers = set() self.publishers = set() def start(self): """ Starts the community by subscribing to peers, and periodically publishing the content updates to the subscribers. """ # Subscribe peers self.subscribe_peers() def start_publishing(): # Update the publisher and subscriber list self.refresh_peer_list() # publish the new cotent from the content repository self.publish_next_content() self.register_task("start_publishing", LoopingCall(start_publishing)).start( PUBLISH_INTERVAL, False) @inlineCallbacks def unload(self): self.request_cache.clear() self.cancel_pending_task("start_publishing") yield super(PubSubCommunity, self).unload() def subscribe_peers(self): """ Subscribes to the connected peers. First, the peers are sorted based on the trust score on descending order and content subscribe request is sent to the top peers. This method is called periodically through refresh_peer_list() in start_publishing() loop so it can fill up for the disconnected peers by connecting to new peers. Note that, existing publisher peers are not disconnected even if we find new peers with higher trust score but only fill up the remaining publisher slots with new top peers. """ num_publishers = len(self.publishers) num_peers = len(self.get_peers()) # If we have some free publisher slots and there are peers available if num_publishers < MAX_PUBLISHERS and num_publishers < num_peers: available_publishers = [ peer for peer in self.get_peers() if peer not in self.publishers ] sorted_peers = sorted( available_publishers, key=lambda _peer: self.trustchain.get_trust(_peer) if self.trustchain else 1, reverse=True) for peer in sorted_peers[:MAX_PUBLISHERS - num_publishers]: self.subscribe(peer, subscribe=True) def refresh_peer_list(self): """ Updates the publishers and subscribers list by filtering out the disconnected peers. It also calls subscribe peers to replenish the available publisher slots if necessary. """ peers = self.get_peers() self.publishers = set( [peer for peer in self.publishers if peer in peers]) self.subscribers = set( [peer for peer in self.subscribers if peer in peers]) # subscribe peers if necessary self.subscribe_peers() def unsubscribe_peers(self): """ Unsubscribes from the existing publishers by sending content subscribe request with subscribe=False. It then clears up its publishers list. - Called at community unload. """ for peer in copy(self.publishers): self.subscribe(peer, subscribe=False) self.publishers.clear() def subscribe(self, peer, subscribe=True): """ Method to send content subscribe/unsubscribe message. This message is sent to each individual publisher peer we want to subscribe/unsubscribe. """ cache = self.request_cache.add( ContentRequest(self.request_cache, MSG_SUBSCRIBE, None)) # Remove the publisher peer already if user is trying to unsubscribe if not subscribe: self.publishers.remove(peer) # Create subscription packet and send it subscription = ContentSubscription(cache.number, subscribe) packet = self.create_message_packet(MSG_SUBSCRIBE, subscription) self.broadcast_message(packet, peer=peer) def on_subscribe(self, source_address, data): """ Message handler for content subscribe message. It handles both subscribe and unsubscribe requests. Upon successful subscription or unsubscription, it send the confirmation subscription message with status. In case of subscription, it also publishes a list of recently checked torrents to the subscriber. """ auth, _, payload = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) # Subscribe or unsubscribe peer subscribed = peer in self.subscribers if payload.subscribe and not subscribed: if len(self.subscribers) < MAX_SUBSCRIBERS: self.subscribers.add(peer) subscribed = True elif not payload.subscribe and subscribed: self.subscribers.remove(peer) subscribed = False # Send subscription response self.send_subscription_status(peer, payload.identifier, subscribed=subscribed) return subscribed def send_subscription_status(self, peer, identifier, subscribed=True): """ Method to send content subscription message. Content subscription message is send in response to content subscribe or unsubscribe message. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return subscription = ContentSubscription(identifier, subscribed) packet = self.create_message_packet(MSG_SUBSCRIPTION, subscription) self.broadcast_message(packet, peer=peer) def on_subscription_status(self, source_address, data): """ Message handler for content subscription message. Content subscription message is sent by the publisher stating the status of the subscription in response to subscribe or unsubscribe request. If the subscription message has subscribe=True, it means the subscription was successful, so the peer is added to the subscriber. In other case, publisher is removed if it is still present in the publishers list. """ auth, _, payload = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) if not self.request_cache.has(u'request', payload.identifier): return self.request_cache.pop(u'request', payload.identifier) if payload.subscribe: self.publishers.add(peer) elif peer in self.publishers: self.publishers.remove(peer) def create_message_packet(self, message_type, payload): """ Helper method to creates a message packet of given type with provided payload. """ auth = BinMemberAuthenticationPayload( self.my_peer.public_key.key_to_bin()).to_pack_list() dist = GlobalTimeDistributionPayload( self.claim_global_time()).to_pack_list() payload = payload if isinstance(payload, list) else payload.to_pack_list() return self._ez_pack(self._prefix, message_type, [auth, dist, payload]) def broadcast_message(self, packet, peer=None): """ Helper method to broadcast the message packet to a single peer or all the subscribers. """ if peer is not None: self.endpoint.send(peer.address, packet) return for _peer in self.subscribers: self.endpoint.send(_peer.address, packet) def get_peer_from_auth(self, auth, source_address): """ Get Peer object from the message and auth and source_address. It is used for mocking the peer in test. """ return Peer(auth.public_key_bin, source_address) def pack_sized(self, payload_list, fit_size, start_index=0): """ Packs a list of Payload objects to fit into given size limit. :param payload_list: List<Payload> list of payload objects :param fit_size: The maximum allowed size for payload field to fit into UDP packet. :param start_index: Index of list to start packing :return: packed string """ assert isinstance(payload_list, list) serialized_results = '' size = 0 current_index = start_index num_payloads = len(payload_list) while current_index < num_payloads: item = payload_list[current_index] packed_item = self.serializer.pack_multiple(item.to_pack_list())[0] packed_item_length = len(packed_item) if size + packed_item_length > fit_size: break else: size += packed_item_length serialized_results += packed_item current_index += 1 return serialized_results, current_index, current_index - start_index def publish_next_content(self): """ Method responsible for publishing content during periodic push """ pass
class TriblerTunnelCommunity(HiddenTunnelCommunity): """ This community is built upon the anonymous messaging layer in IPv8. It adds support for libtorrent anonymous downloads and bandwidth token payout when closing circuits. """ master_peer = Peer( "3081a7301006072a8648ce3d020106052b81040027038192000400965194026c987edad7c5c0364a95dfa4e961e" "ec6d1711678be182a31824363db3a54caa11e9b6f1d6051c2f33578b51c12eff3833b7c74c5a243b8705e4e032b" "14904f4d8855e2044c0408a5729cd4e5b286fec66866811d5439049448e5b8b818d8c29a84a074a13f5e7e414d4" "e5b1b115a221fe697b89bd3e63c7aecd7617f789a203a4eacf680279224b390e836". decode('hex')) def __init__(self, *args, **kwargs): self.tribler_session = kwargs.pop('tribler_session', None) num_competing_slots = kwargs.pop('competing_slots', 15) num_random_slots = kwargs.pop('random_slots', 5) self.bandwidth_wallet = kwargs.pop('bandwidth_wallet', None) socks_listen_ports = kwargs.pop('socks_listen_ports', None) self.exitnode_cache = kwargs.pop( 'exitnode_cache', (self.tribler_session.config.get_state_dir() if self.tribler_session else '') + 'exitnode_cache.dat') super(TriblerTunnelCommunity, self).__init__(*args, **kwargs) self._use_main_thread = True if self.tribler_session: self.settings.become_exitnode = self.tribler_session.config.get_tunnel_community_exitnode_enabled( ) self.tribler_session.lm.tunnel_community = self if not socks_listen_ports: socks_listen_ports = self.tribler_session.config.get_tunnel_community_socks5_listen_ports( ) elif socks_listen_ports is None: socks_listen_ports = range(1080, 1085) self.bittorrent_peers = {} self.dispatcher = TunnelDispatcher(self) self.download_states = {} self.competing_slots = [ (0, None) ] * num_competing_slots # 1st tuple item = token balance, 2nd = circuit id self.random_slots = [None] * num_random_slots # Start the SOCKS5 servers self.socks_servers = [] for port in socks_listen_ports: socks_server = Socks5Server(port, self.dispatcher) socks_server.start() self.socks_servers.append(socks_server) self.dispatcher.set_socks_servers(self.socks_servers) self.decode_map.update({ chr(23): self.on_payout_block, }) self.decode_map_private.update({ chr(24): self.on_balance_request_cell, chr(25): self.on_relay_balance_request_cell, chr(26): self.on_balance_response_cell, chr(27): self.on_relay_balance_response_cell, }) message_to_payload[u"balance-request"] = (24, BalanceRequestPayload) message_to_payload[u"relay-balance-request"] = (25, BalanceRequestPayload) message_to_payload[u"balance-response"] = (26, BalanceResponsePayload) message_to_payload[u"relay-balance-response"] = ( 27, BalanceResponsePayload) SINGLE_HOP_ENC_PACKETS.append(u"balance-request") SINGLE_HOP_ENC_PACKETS.append(u"balance-response") if self.exitnode_cache: self.restore_exitnodes_from_disk() def cache_exitnodes_to_disk(self): """ Wite a copy of self.exit_candidates to the file self.exitnode_cache. :returns: None """ exit_nodes = Network() for peer in self.exit_candidates.values(): exit_nodes.add_verified_peer(peer) self.logger.debug('Writing exit nodes to cache: %s', self.exitnode_cache) with open(self.exitnode_cache, 'w') as cache: cache.write(exit_nodes.snapshot()) def restore_exitnodes_from_disk(self): """ Send introduction requests to peers stored in the file self.exitnode_cache. :returns: None """ if os.path.isfile(self.exitnode_cache): self.logger.debug('Loading exit nodes from cache: %s', self.exitnode_cache) exit_nodes = Network() with open(self.exitnode_cache, 'r') as cache: exit_nodes.load_snapshot(cache.read()) for exit_node in exit_nodes.get_walkable_addresses(): self.endpoint.send(exit_node, self.create_introduction_request(exit_node)) else: self.logger.error( 'Could not retrieve backup exitnode cache, file does not exist!' ) def on_token_balance(self, circuit_id, balance): """ We received the token balance of a circuit initiator. Check whether we can allocate a slot to this user. """ if not self.request_cache.has(u"balance-request", circuit_id): self.logger.warning( "Received token balance without associated request cache!") return cache = self.request_cache.pop(u"balance-request", circuit_id) lowest_balance = sys.maxint lowest_index = -1 for ind, tup in enumerate(self.competing_slots): if not tup[1]: # The slot is empty, take it self.competing_slots[ind] = (balance, circuit_id) cache.balance_deferred.callback(True) return if tup[0] < lowest_balance: lowest_balance = tup[0] lowest_index = ind if balance > lowest_balance: # We kick this user out old_circuit_id = self.competing_slots[lowest_index][1] self.logger.info( "Kicked out circuit %s (balance: %s) in favor of %s (balance: %s)", old_circuit_id, lowest_balance, circuit_id, balance) self.competing_slots[lowest_index] = (balance, circuit_id) self.remove_relay(old_circuit_id, destroy=True) self.remove_exit_socket(old_circuit_id, destroy=True) cache.balance_deferred.callback(True) else: # We can't compete with the balances in the existing slots cache.balance_deferred.callback(False) def should_join_circuit(self, create_payload, previous_node_address): """ Check whether we should join a circuit. Returns a deferred that fires with a boolean. """ if self.settings.max_joined_circuits <= len(self.relay_from_to) + len( self.exit_sockets): self.logger.warning( "too many relays (%d)", (len(self.relay_from_to) + len(self.exit_sockets))) return succeed(False) # Check whether we have a random open slot, if so, allocate this to this request. circuit_id = create_payload.circuit_id for index, slot in enumerate(self.random_slots): if not slot: self.random_slots[index] = circuit_id return succeed(True) # No random slots but this user might be allocated a competing slot. # Next, we request the token balance of the circuit initiator. balance_deferred = Deferred() self.request_cache.add( BalanceRequestCache(self, circuit_id, balance_deferred)) # Temporarily add these values, otherwise we are unable to communicate with the previous hop. self.directions[circuit_id] = EXIT_NODE shared_secret, _, _ = self.crypto.generate_diffie_shared_secret( create_payload.key) self.relay_session_keys[ circuit_id] = self.crypto.generate_session_keys(shared_secret) self.send_cell( [Peer(create_payload.node_public_key, previous_node_address)], u"balance-request", BalanceRequestPayload(circuit_id)) self.directions.pop(circuit_id, None) self.relay_session_keys.pop(circuit_id, None) return balance_deferred def on_payout_block(self, source_address, data): if not self.bandwidth_wallet: self.logger.warning( "Got payout while not having a TrustChain community running!") return payload = self._ez_unpack_noauth(PayoutPayload, data, global_time=False) peer = Peer(payload.public_key, source_address) block = TriblerBandwidthBlock.from_payload(payload, self.serializer) self.bandwidth_wallet.trustchain.process_half_block(block, peer) # Send the next payout if payload.circuit_id in self.relay_from_to and block.transaction[ 'down'] > payload.base_amount: relay = self.relay_from_to[payload.circuit_id] circuit_peer = self.get_peer_from_address(relay.peer.address) if not circuit_peer: self.logger.warning( "%s Unable to find next peer %s for payout!", self.my_peer, relay.peer) return self.do_payout(circuit_peer, relay.circuit_id, block.transaction['down'] - payload.base_amount * 2, payload.base_amount) def on_balance_request_cell(self, source_address, data, _): payload = self._ez_unpack_noauth(BalanceRequestPayload, data, global_time=False) circuit_id = payload.circuit_id request = self.request_cache.get(u"anon-circuit", circuit_id) if not request: self.logger.warning("Circuit creation cache for id %s not found!", circuit_id) return if request.should_forward: forwarding_relay = RelayRoute(request.from_circuit_id, request.peer) self.send_cell([forwarding_relay.peer.address], u"relay-balance-request", BalanceRequestPayload(forwarding_relay.circuit_id)) else: self.on_balance_request(payload) def on_relay_balance_request_cell(self, source_address, data, _): payload = self._ez_unpack_noauth(BalanceRequestPayload, data, global_time=False) self.on_balance_request(payload) def on_balance_request(self, payload): """ We received a balance request from a relay or exit node. Respond with the latest block in our chain. """ if not self.bandwidth_wallet: self.logger.warn( "Bandwidth wallet is not available, not sending balance response!" ) return # Get the latest block latest_block = self.bandwidth_wallet.trustchain.persistence.get_latest( self.my_peer.public_key.key_to_bin(), block_type='tribler_bandwidth') if not latest_block: latest_block = TriblerBandwidthBlock() latest_block.public_key = EMPTY_PK # We hide the public key # We either send the response directly or relay the response to the last verified hop circuit = self.circuits[payload.circuit_id] if not circuit.hops: self.increase_bytes_sent( circuit, self.send_cell([circuit.peer.address], u"balance-response", BalanceResponsePayload.from_half_block( latest_block, circuit.circuit_id))) else: self.increase_bytes_sent( circuit, self.send_cell([circuit.peer.address], u"relay-balance-response", BalanceResponsePayload.from_half_block( latest_block, circuit.circuit_id))) def on_balance_response_cell(self, source_address, data, _): payload = self._ez_unpack_noauth(BalanceResponsePayload, data, global_time=False) block = TriblerBandwidthBlock.from_payload(payload, self.serializer) if not block.transaction: self.on_token_balance(payload.circuit_id, 0) else: self.on_token_balance( payload.circuit_id, block.transaction["total_up"] - block.transaction["total_down"]) def on_relay_balance_response_cell(self, source_address, data, _): payload = self._ez_unpack_noauth(BalanceResponsePayload, data, global_time=False) block = TriblerBandwidthBlock.from_payload(payload, self.serializer) # At this point, we don't have the circuit ID of the follow-up hop. We have to iterate over the items in the # request cache and find the link to the next hop. for cache in self.request_cache._identifiers.values(): if isinstance(cache, ExtendRequestCache ) and cache.from_circuit_id == payload.circuit_id: self.send_cell([cache.to_peer.address], u"balance-response", BalanceResponsePayload.from_half_block( block, cache.to_circuit_id)) def on_download_removed(self, download): """ This method is called when a download is removed. We check here whether we can stop building circuits for a specific number of hops in case it hasn't been finished yet. """ if download.get_hops() > 0: self.num_hops_by_downloads[download.get_hops()] -= 1 if self.num_hops_by_downloads[download.get_hops()] == 0: self.circuits_needed[download.get_hops()] = 0 def readd_bittorrent_peers(self): for torrent, peers in self.bittorrent_peers.items(): infohash = torrent.tdef.get_infohash().encode("hex") for peer in peers: self.logger.info("Re-adding peer %s to torrent %s", peer, infohash) torrent.add_peer(peer) del self.bittorrent_peers[torrent] def update_torrent(self, peers, handle, download): peers = peers.intersection(handle.get_peer_info()) if peers: if download not in self.bittorrent_peers: self.bittorrent_peers[download] = peers else: self.bittorrent_peers[ download] = peers | self.bittorrent_peers[download] # If there are active circuits, add peers immediately. Otherwise postpone. if self.active_data_circuits(): self.readd_bittorrent_peers() def get_peer_from_address(self, address): circuit_peer = None for peer in self.get_peers(): if peer.address == address: circuit_peer = peer break return circuit_peer def do_payout(self, peer, circuit_id, amount, base_amount): """ Perform a payout to a specific peer. :param peer: The peer to perform the payout to, usually the next node in the circuit. :param circuit_id: The circuit id of the payout, used by the subsequent node. :param amount: The amount to put in the transaction, multiplier of base_amount. :param base_amount: The base amount for the payouts. """ self.logger.info("Sending payout of %d (base: %d) to %s (cid: %s)", amount, base_amount, peer, circuit_id) block = TriblerBandwidthBlock.create( 'tribler_bandwidth', { 'up': 0, 'down': amount }, self.bandwidth_wallet.trustchain.persistence, self.my_peer.public_key.key_to_bin(), link_pk=peer.public_key.key_to_bin()) block.sign(self.my_peer.key) self.bandwidth_wallet.trustchain.persistence.add_block(block) payload = PayoutPayload.from_half_block(block, circuit_id, base_amount).to_pack_list() packet = self._ez_pack(self._prefix, 23, [payload], False) self.send_packet([peer], u"payout", packet) def clean_from_slots(self, circuit_id): """ Clean a specific circuit from the allocated slots. """ for ind, slot in enumerate(self.random_slots): if slot == circuit_id: self.random_slots[ind] = None for ind, tup in enumerate(self.competing_slots): if tup[1] == circuit_id: self.competing_slots[ind] = (0, None) def remove_circuit(self, circuit_id, additional_info='', remove_now=False, destroy=False): if circuit_id not in self.circuits: self.logger.warning( "Circuit %d not found when trying to remove it", circuit_id) return circuit = self.circuits[circuit_id] # Send the notification if self.tribler_session: self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE, circuit, circuit.peer.address) circuit_peer = self.get_peer_from_address(circuit.peer.address) if circuit.bytes_down >= 1024 * 1024 and self.bandwidth_wallet and circuit_peer: # We should perform a payout of the removed circuit. if circuit.ctype == CIRCUIT_TYPE_RENDEZVOUS: # We remove an e2e circuit as downloader. We pay the subsequent nodes in the downloader part of the e2e # circuit. In addition, we pay for one hop seeder anonymity since we don't know the circuit length at # the seeder side. self.do_payout( circuit_peer, circuit_id, circuit.bytes_down * ((circuit.goal_hops * 2) + 1), circuit.bytes_down) if circuit.ctype == CIRCUIT_TYPE_DATA: # We remove a regular data circuit as downloader. Pay the relay nodes and the exit nodes. self.do_payout( circuit_peer, circuit_id, circuit.bytes_down * (circuit.goal_hops * 2 - 1), circuit.bytes_down) # Reset the circuit byte counters so we do not payout again if we receive a destroy message. circuit.bytes_up = circuit.bytes_down = 0 def update_torrents(_): affected_peers = self.dispatcher.circuit_dead(circuit) ltmgr = self.tribler_session.lm.ltmgr \ if self.tribler_session and self.tribler_session.config.get_libtorrent_enabled() else None if ltmgr: for d, s in ltmgr.torrents.values(): if s == ltmgr.get_session(d.get_hops()): d.get_handle().addCallback( lambda handle, download=d: self.update_torrent( affected_peers, handle, download)) # Now we actually remove the circuit remove_deferred = super(TriblerTunnelCommunity, self)\ .remove_circuit(circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy) remove_deferred.addCallback(update_torrents) return remove_deferred def remove_relay(self, circuit_id, additional_info='', remove_now=False, destroy=False, got_destroy_from=None, both_sides=True): removed_relays = super(TriblerTunnelCommunity, self).remove_relay( circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy, got_destroy_from=got_destroy_from, both_sides=both_sides) self.clean_from_slots(circuit_id) if self.tribler_session: for removed_relay in removed_relays: self.tribler_session.notifier.notify( NTFY_TUNNEL, NTFY_REMOVE, removed_relay, removed_relay.peer.address) def remove_exit_socket(self, circuit_id, additional_info='', remove_now=False, destroy=False): if circuit_id in self.exit_sockets and self.tribler_session: exit_socket = self.exit_sockets[circuit_id] self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE, exit_socket, exit_socket.peer.address) self.clean_from_slots(circuit_id) super(TriblerTunnelCommunity, self).remove_exit_socket(circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy) def _ours_on_created_extended(self, circuit, payload): super(TriblerTunnelCommunity, self)._ours_on_created_extended(circuit, payload) if circuit.state == CIRCUIT_STATE_READY: # Re-add BitTorrent peers, if needed. self.readd_bittorrent_peers() if self.tribler_session: self.tribler_session.notifier.notify( NTFY_TUNNEL, NTFY_CREATED if len(circuit.hops) == 1 else NTFY_EXTENDED, circuit) def join_circuit(self, create_payload, previous_node_address): super(TriblerTunnelCommunity, self).join_circuit(create_payload, previous_node_address) if self.tribler_session: circuit_id = create_payload.circuit_id self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_JOINED, previous_node_address, circuit_id) def on_raw_data(self, circuit, origin, data): anon_seed = circuit.ctype == CIRCUIT_TYPE_RP self.dispatcher.on_incoming_from_tunnel(self, circuit, origin, data, anon_seed) def monitor_downloads(self, dslist): # Monitor downloads with anonymous flag set, and build rendezvous/introduction points when needed. new_states = {} hops = {} for ds in dslist: download = ds.get_download() if download.get_hops() > 0: # Convert the real infohash to the infohash used for looking up introduction points real_info_hash = download.get_def().get_infohash() info_hash = self.get_lookup_info_hash(real_info_hash) hops[info_hash] = download.get_hops() self.service_callbacks[info_hash] = download.add_peer new_states[info_hash] = ds.get_status() self.hops = hops for info_hash in set(new_states.keys() + self.download_states.keys()): new_state = new_states.get(info_hash, None) old_state = self.download_states.get(info_hash, None) state_changed = new_state != old_state # Stop creating introduction points if the download doesn't exist anymore if info_hash in self.infohash_ip_circuits and new_state is None: del self.infohash_ip_circuits[info_hash] # If the introducing circuit does not exist anymore or timed out: Build a new circuit if info_hash in self.infohash_ip_circuits: for (circuit_id, time_created) in self.infohash_ip_circuits[info_hash]: if circuit_id not in self.my_intro_points and time_created < time.time( ) - 30: self.infohash_ip_circuits[info_hash].remove( (circuit_id, time_created)) if self.tribler_session.notifier: self.tribler_session.notifier.notify( NTFY_TUNNEL, NTFY_IP_RECREATE, circuit_id, info_hash.encode('hex')[:6]) self.logger.info( 'Recreate the introducing circuit for %s', info_hash.encode('hex')) self.create_introduction_point(info_hash) time_elapsed = (time.time() - self.last_dht_lookup.get(info_hash, 0)) force_dht_lookup = time_elapsed >= self.settings.dht_lookup_interval if (state_changed or force_dht_lookup) and (new_state == DLSTATUS_DOWNLOADING): self.logger.info( 'Do dht lookup to find hidden services peers for %s', info_hash.encode('hex')) self.do_raw_dht_lookup(info_hash) if state_changed and new_state == DLSTATUS_SEEDING: self.create_introduction_point(info_hash) elif state_changed and new_state in [DLSTATUS_STOPPED, None]: if info_hash in self.infohash_pex: self.infohash_pex.pop(info_hash) for cid, info_hash_hops in self.my_download_points.items(): if info_hash_hops[0] == info_hash: self.remove_circuit(cid, 'download stopped', destroy=True) for cid, info_hash_list in self.my_intro_points.items(): for i in xrange(len(info_hash_list) - 1, -1, -1): if info_hash_list[i] == info_hash: info_hash_list.pop(i) if len(info_hash_list) == 0: self.remove_circuit(cid, 'all downloads stopped', destroy=True) self.download_states = new_states def get_download(self, lookup_info_hash): if not self.tribler_session: return None for download in self.tribler_session.get_downloads(): if lookup_info_hash == self.get_lookup_info_hash( download.get_def().get_infohash()): return download def create_introduction_point(self, info_hash, amount=1): download = self.get_download(info_hash) if download: download.add_peer(('1.1.1.1', 1024)) super(TriblerTunnelCommunity, self).create_introduction_point(info_hash, amount) def on_linked_e2e(self, source_address, data, circuit_id): payload = self._ez_unpack_noauth(LinkedE2EPayload, data, global_time=False) cache = self.request_cache.get(u"link-request", payload.identifier) if cache: download = self.get_download(cache.info_hash) if download: download.add_peer( (self.circuit_id_to_ip(cache.circuit.circuit_id), 1024)) else: self.logger.error('On linked e2e: could not find download!') super(TriblerTunnelCommunity, self).on_linked_e2e(source_address, data, circuit_id) @inlineCallbacks def unload(self): if self.bandwidth_wallet: self.bandwidth_wallet.shutdown_task_manager() for socks_server in self.socks_servers: yield socks_server.stop() if self.exitnode_cache: self.cache_exitnodes_to_disk() super(TriblerTunnelCommunity, self).unload()
def _generate_peer(): return Peer(default_eccrypto.generate_key(u"very-low"))
class TriblerTunnelCommunity(HiddenTunnelCommunity): master_peer = Peer( "3081a7301006072a8648ce3d020106052b81040027038192000402e1cd2a8158c078f5a048dd2caa4a868852e1758" "71c819947b2aabe414a6b1b6c35e89f554dd94b475c612a692a3132bbe4a30813702acd7647eb8023700dcda5b47d" "fe15f94a88049c2bb05f83f37d2cd85cce5efb8a9da6ac97dcdf97f83ae8696ffd1fab783ed28d004a99942fba756" "8a3edc2052ce379db4b3f40411d55c28e16466e9750038c677bb561eab325".decode( 'hex')) def __init__(self, *args, **kwargs): self.tribler_session = kwargs.pop('tribler_session', None) self.triblerchain_community = kwargs.pop('triblerchain_community', None) num_competing_slots = kwargs.pop('competing_slots', 15) num_random_slots = kwargs.pop('random_slots', 5) socks_listen_ports = kwargs.pop('socks_listen_ports', None) super(TriblerTunnelCommunity, self).__init__(*args, **kwargs) self._use_main_thread = True if self.tribler_session: self.settings.become_exitnode = self.tribler_session.config.get_tunnel_community_exitnode_enabled( ) self.tribler_session.lm.tunnel_community = self if not socks_listen_ports: socks_listen_ports = self.tribler_session.config.get_tunnel_community_socks5_listen_ports( ) elif socks_listen_ports is None: socks_listen_ports = range(1080, 1085) self.bittorrent_peers = {} self.dispatcher = TunnelDispatcher(self) self.download_states = {} self.competing_slots = [ (0, None) ] * num_competing_slots # 1st tuple item = token balance, 2nd = circuit id self.random_slots = [None] * num_random_slots # Start the SOCKS5 servers self.socks_servers = [] for port in socks_listen_ports: socks_server = Socks5Server(port, self.dispatcher) socks_server.start() self.socks_servers.append(socks_server) self.dispatcher.set_socks_servers(self.socks_servers) self.decode_map.update({ chr(23): self.on_payout_block, }) self.decode_map_private.update({ chr(24): self.on_balance_request_cell, chr(25): self.on_relay_balance_request_cell, chr(26): self.on_balance_response_cell, chr(27): self.on_relay_balance_response_cell, }) message_to_payload[u"balance-request"] = (24, BalanceRequestPayload) message_to_payload[u"relay-balance-request"] = (25, BalanceRequestPayload) message_to_payload[u"balance-response"] = (26, BalanceResponsePayload) message_to_payload[u"relay-balance-response"] = ( 27, BalanceResponsePayload) SINGLE_HOP_ENC_PACKETS.append(u"balance-request") SINGLE_HOP_ENC_PACKETS.append(u"balance-response") def on_token_balance(self, circuit_id, balance): """ We received the token balance of a circuit initiator. Check whether we can allocate a slot to this user. """ if not self.request_cache.has(u"balance-request", circuit_id): self.logger.warning( "Received token balance without associated request cache!") return cache = self.request_cache.pop(u"balance-request", circuit_id) lowest_balance = sys.maxint lowest_index = -1 for ind, tup in enumerate(self.competing_slots): if not tup[1]: # The slot is empty, take it self.competing_slots[ind] = (balance, circuit_id) cache.balance_deferred.callback(True) return if tup[0] < lowest_balance: lowest_balance = tup[0] lowest_index = ind if balance > lowest_balance: # We kick this user out old_circuit_id = self.competing_slots[lowest_index][1] self.logger.info( "Kicked out circuit %s (balance: %s) in favor of %s (balance: %s)", old_circuit_id, lowest_balance, circuit_id, balance) self.competing_slots[lowest_index] = (balance, circuit_id) self.remove_relay(old_circuit_id, destroy=True) self.remove_exit_socket(old_circuit_id, destroy=True) cache.balance_deferred.callback(True) else: # We can't compete with the balances in the existing slots cache.balance_deferred.callback(False) def should_join_circuit(self, create_payload, previous_node_address): """ Check whether we should join a circuit. Returns a deferred that fires with a boolean. """ if self.settings.max_joined_circuits <= len(self.relay_from_to) + len( self.exit_sockets): self.logger.warning( "too many relays (%d)", (len(self.relay_from_to) + len(self.exit_sockets))) return succeed(False) # Check whether we have a random open slot, if so, allocate this to this request. circuit_id = create_payload.circuit_id for index, slot in enumerate(self.random_slots): if not slot: self.random_slots[index] = circuit_id return succeed(True) # No random slots but this user might be allocated a competing slot. # Next, we request the token balance of the circuit initiator. balance_deferred = Deferred() self.request_cache.add( BalanceRequestCache(self, circuit_id, balance_deferred)) # Temporarily add these values, otherwise we are unable to communicate with the previous hop. self.directions[circuit_id] = EXIT_NODE shared_secret, _, _ = self.crypto.generate_diffie_shared_secret( create_payload.key) self.relay_session_keys[ circuit_id] = self.crypto.generate_session_keys(shared_secret) self.send_cell( [Peer(create_payload.node_public_key, previous_node_address)], u"balance-request", BalanceRequestPayload(circuit_id)) self.directions.pop(circuit_id, None) self.relay_session_keys.pop(circuit_id, None) return balance_deferred def on_payout_block(self, source_address, data): if not self.triblerchain_community: self.logger.warning( "Got payout while not having a TriblerChain community running!" ) return _, payload = self._ez_unpack_noauth(PayoutPayload, data) peer = Peer(payload.public_key, source_address) block = self.triblerchain_community.BLOCK_CLASS.from_payload( payload, self.serializer) self.triblerchain_community.process_half_block(block, peer) # Send the next payout if payload.circuit_id in self.relay_from_to and block.transaction[ 'down'] > payload.base_amount: relay = self.relay_from_to[payload.circuit_id] circuit_peer = self.get_peer_from_mid(relay.mid) if not circuit_peer: self.logger.warning( "%s Unable to find next peer %s for payout!", self.my_peer, relay.mid.encode('hex')) return self.do_payout(circuit_peer, relay.circuit_id, block.transaction['down'] - payload.base_amount * 2, payload.base_amount) def on_balance_request_cell(self, source_address, data, _): _, payload = self._ez_unpack_noauth(BalanceRequestPayload, data) circuit_id = payload.circuit_id request = self.request_cache.get(u"anon-circuit", circuit_id) if request.should_forward: forwarding_relay = RelayRoute(request.from_circuit_id, request.candidate_sock_addr, mid=request.candidate_mid) self.send_cell([forwarding_relay.sock_addr], u"relay-balance-request", BalanceRequestPayload(forwarding_relay.circuit_id)) else: self.on_balance_request(payload) def on_relay_balance_request_cell(self, source_address, data, _): _, payload = self._ez_unpack_noauth(BalanceRequestPayload, data) self.on_balance_request(payload) def on_balance_request(self, payload): """ We received a balance request from a relay or exit node. Respond with the latest block in our chain. """ if not self.triblerchain_community: return # Get the latest block latest_block = self.triblerchain_community.persistence.get_latest( self.my_peer.public_key.key_to_bin()) if not latest_block: latest_block = TriblerChainBlock() latest_block.public_key = EMPTY_PK # We hide the public key # We either send the response directly or relay the response to the last verified hop circuit = self.circuits[payload.circuit_id] if not circuit.hops: self.increase_bytes_sent( circuit, self.send_cell([circuit.sock_addr], u"balance-response", BalanceResponsePayload.from_half_block( latest_block, circuit.circuit_id))) else: self.increase_bytes_sent( circuit, self.send_cell([circuit.sock_addr], u"relay-balance-response", BalanceResponsePayload.from_half_block( latest_block, circuit.circuit_id))) def on_balance_response_cell(self, source_address, data, _): _, payload = self._ez_unpack_noauth(BalanceResponsePayload, data) block = TriblerChainBlock.from_payload(payload, self.serializer) if not block.transaction: self.on_token_balance(payload.circuit_id, 0) else: self.on_token_balance( payload.circuit_id, block.transaction["total_up"] - block.transaction["total_down"]) def on_relay_balance_response_cell(self, source_address, data, _): _, payload = self._ez_unpack_noauth(BalanceResponsePayload, data) block = TriblerChainBlock.from_payload(payload, self.serializer) # At this point, we don't have the circuit ID of the follow-up hop. We have to iterate over the items in the # request cache and find the link to the next hop. for cache in self.request_cache._identifiers.values(): if isinstance(cache, ExtendRequestCache ) and cache.from_circuit_id == payload.circuit_id: self.send_cell([cache.to_candidate_sock_addr], u"balance-response", BalanceResponsePayload.from_half_block( block, cache.to_circuit_id)) def on_download_removed(self, download): """ This method is called when a download is removed. We check here whether we can stop building circuits for a specific number of hops in case it hasn't been finished yet. """ if download.get_hops() > 0: self.num_hops_by_downloads[download.get_hops()] -= 1 if self.num_hops_by_downloads[download.get_hops()] == 0: self.circuits_needed[download.get_hops()] = 0 def readd_bittorrent_peers(self): for torrent, peers in self.bittorrent_peers.items(): infohash = torrent.tdef.get_infohash().encode("hex") for peer in peers: self.logger.info("Re-adding peer %s to torrent %s", peer, infohash) torrent.add_peer(peer) del self.bittorrent_peers[torrent] def update_torrent(self, peers, handle, download): peers = peers.intersection(handle.get_peer_info()) if peers: if download not in self.bittorrent_peers: self.bittorrent_peers[download] = peers else: self.bittorrent_peers[ download] = peers | self.bittorrent_peers[download] # If there are active circuits, add peers immediately. Otherwise postpone. if self.active_data_circuits(): self.readd_bittorrent_peers() def get_peer_from_mid(self, mid): circuit_peer = None for peer in self.network.verified_peers: if peer.mid == mid: circuit_peer = peer break return circuit_peer def do_payout(self, peer, circuit_id, amount, base_amount): """ Perform a payout to a specific peer. :param peer: The peer to perform the payout to, usually the next node in the circuit. :param circuit_id: The circuit id of the payout, used by the subsequent node. :param amount: The amount to put in the transaction, multiplier of base_amount. :param base_amount: The base amount for the payouts. """ self.logger.info("Sending payout of %d (base: %d) to %s (cid: %s)", amount, base_amount, peer, circuit_id) block = self.triblerchain_community.BLOCK_CLASS.create( { 'up': 0, 'down': amount }, self.triblerchain_community.persistence, self.my_peer.public_key.key_to_bin(), link_pk=peer.public_key.key_to_bin()) block.sign(self.my_peer.key) self.triblerchain_community.persistence.add_block(block) global_time = self.claim_global_time() dist = GlobalTimeDistributionPayload(global_time).to_pack_list() payload = PayoutPayload.from_half_block(block, circuit_id, base_amount).to_pack_list() packet = self._ez_pack(self._prefix, 23, [dist, payload], False) self.send_packet([peer], u"payout", packet) def clean_from_slots(self, circuit_id): """ Clean a specific circuit from the allocated slots. """ for ind, slot in enumerate(self.random_slots): if slot == circuit_id: self.random_slots[ind] = None for ind, tup in enumerate(self.competing_slots): if tup[1] == circuit_id: self.competing_slots[ind] = (0, None) def remove_circuit(self, circuit_id, additional_info='', remove_now=False, destroy=False): if circuit_id not in self.circuits: self.logger.warning( "Circuit %d not found when trying to remove it", circuit_id) return circuit = self.circuits[circuit_id] # Send the notification if self.tribler_session: self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE, circuit, circuit.sock_addr) circuit_peer = self.get_peer_from_mid(circuit.mid) if circuit.bytes_down >= 1024 * 1024 and self.triblerchain_community and circuit_peer: # We should perform a payout of the removed circuit. if circuit.ctype == CIRCUIT_TYPE_RENDEZVOUS: # We remove an e2e circuit as downloader. We pay the subsequent nodes in the downloader part of the e2e # circuit. In addition, we pay for one hop seeder anonymity since we don't know the circuit length at # the seeder side. self.do_payout( circuit_peer, circuit_id, circuit.bytes_down * ((circuit.goal_hops * 2) + 1), circuit.bytes_down) if circuit.ctype == CIRCUIT_TYPE_DATA: # We remove a regular data circuit as downloader. Pay the relay nodes and the exit nodes. self.do_payout( circuit_peer, circuit_id, circuit.bytes_down * (circuit.goal_hops * 2 - 1), circuit.bytes_down) def update_torrents(_): affected_peers = self.dispatcher.circuit_dead(circuit) ltmgr = self.tribler_session.lm.ltmgr \ if self.tribler_session and self.tribler_session.config.get_libtorrent_enabled() else None if ltmgr: for d, s in ltmgr.torrents.values(): if s == ltmgr.get_session(d.get_hops()): d.get_handle().addCallback( lambda handle, download=d: self.update_torrent( affected_peers, handle, download)) # Now we actually remove the circuit remove_deferred = super(TriblerTunnelCommunity, self)\ .remove_circuit(circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy) remove_deferred.addCallback(update_torrents) return remove_deferred def remove_relay(self, circuit_id, additional_info='', remove_now=False, destroy=False, got_destroy_from=None, both_sides=True): removed_relays = super(TriblerTunnelCommunity, self).remove_relay( circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy, got_destroy_from=got_destroy_from, both_sides=both_sides) self.clean_from_slots(circuit_id) if self.tribler_session: for removed_relay in removed_relays: self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE, removed_relay, removed_relay.sock_addr) def remove_exit_socket(self, circuit_id, additional_info='', remove_now=False, destroy=False): if circuit_id in self.exit_sockets and self.tribler_session: exit_socket = self.exit_sockets[circuit_id] self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE, exit_socket, exit_socket.sock_addr) self.clean_from_slots(circuit_id) super(TriblerTunnelCommunity, self).remove_exit_socket(circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy) def _ours_on_created_extended(self, circuit, payload): super(TriblerTunnelCommunity, self)._ours_on_created_extended(circuit, payload) if circuit.state == CIRCUIT_STATE_READY: # Re-add BitTorrent peers, if needed. self.readd_bittorrent_peers() if self.tribler_session: self.tribler_session.notifier.notify( NTFY_TUNNEL, NTFY_CREATED if len(circuit.hops) == 1 else NTFY_EXTENDED, circuit) def on_create(self, source_address, data, _): _, payload = self._ez_unpack_noauth(CreatePayload, data) if not self.check_create(payload): return circuit_id = payload.circuit_id if self.tribler_session: self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_JOINED, source_address, circuit_id) super(TriblerTunnelCommunity, self).on_create(source_address, data, _) def on_raw_data(self, circuit, origin, data): anon_seed = circuit.ctype == CIRCUIT_TYPE_RP self.dispatcher.on_incoming_from_tunnel(self, circuit, origin, data, anon_seed) @call_on_reactor_thread def monitor_downloads(self, dslist): # Monitor downloads with anonymous flag set, and build rendezvous/introduction points when needed. new_states = {} hops = {} for ds in dslist: download = ds.get_download() if download.get_hops() > 0: # Convert the real infohash to the infohash used for looking up introduction points real_info_hash = download.get_def().get_infohash() info_hash = self.get_lookup_info_hash(real_info_hash) hops[info_hash] = download.get_hops() self.service_callbacks[info_hash] = download.add_peer new_states[info_hash] = ds.get_status() self.hops = hops for info_hash in set(new_states.keys() + self.download_states.keys()): new_state = new_states.get(info_hash, None) old_state = self.download_states.get(info_hash, None) state_changed = new_state != old_state # Stop creating introduction points if the download doesn't exist anymore if info_hash in self.infohash_ip_circuits and new_state is None: del self.infohash_ip_circuits[info_hash] # If the introducing circuit does not exist anymore or timed out: Build a new circuit if info_hash in self.infohash_ip_circuits: for (circuit_id, time_created) in self.infohash_ip_circuits[info_hash]: if circuit_id not in self.my_intro_points and time_created < time.time( ) - 30: self.infohash_ip_circuits[info_hash].remove( (circuit_id, time_created)) if self.tribler_session.notifier: self.tribler_session.notifier.notify( NTFY_TUNNEL, NTFY_IP_RECREATE, circuit_id, info_hash.encode('hex')[:6]) self.logger.info( 'Recreate the introducing circuit for %s', info_hash.encode('hex')) self.create_introduction_point(info_hash) time_elapsed = (time.time() - self.last_dht_lookup.get(info_hash, 0)) force_dht_lookup = time_elapsed >= self.settings.dht_lookup_interval if (state_changed or force_dht_lookup) and \ (new_state == DLSTATUS_SEEDING or new_state == DLSTATUS_DOWNLOADING): self.logger.info( 'Do dht lookup to find hidden services peers for %s', info_hash.encode('hex')) self.do_raw_dht_lookup(info_hash) if state_changed and new_state == DLSTATUS_SEEDING: self.create_introduction_point(info_hash) elif state_changed and new_state in [DLSTATUS_STOPPED, None]: if info_hash in self.infohash_pex: self.infohash_pex.pop(info_hash) for cid, info_hash_hops in self.my_download_points.items(): if info_hash_hops[0] == info_hash: self.remove_circuit(cid, 'download stopped', destroy=True) for cid, info_hash_list in self.my_intro_points.items(): for i in xrange(len(info_hash_list) - 1, -1, -1): if info_hash_list[i] == info_hash: info_hash_list.pop(i) if len(info_hash_list) == 0: self.remove_circuit(cid, 'all downloads stopped', destroy=True) self.download_states = new_states def get_download(self, lookup_info_hash): if not self.tribler_session: return None for download in self.tribler_session.get_downloads(): if lookup_info_hash == self.get_lookup_info_hash( download.get_def().get_infohash()): return download def create_introduction_point(self, info_hash, amount=1): download = self.get_download(info_hash) if download: download.add_peer(('1.1.1.1', 1024)) super(TriblerTunnelCommunity, self).create_introduction_point(info_hash, amount) def on_linked_e2e(self, source_address, data, circuit_id): _, payload = self._ez_unpack_noauth(LinkedE2EPayload, data) cache = self.request_cache.get(u"link-request", payload.identifier) if cache: download = self.get_download(cache.info_hash) if download: download.add_peer( (self.circuit_id_to_ip(cache.circuit.circuit_id), 1024)) else: self.logger.error('On linked e2e: could not find download!') super(TriblerTunnelCommunity, self).on_linked_e2e(source_address, data, circuit_id) @inlineCallbacks def unload(self): for socks_server in self.socks_servers: yield socks_server.stop() super(TriblerTunnelCommunity, self).unload()
def get_peer_from_auth(self, auth, source_address): """ Get Peer object from the message and auth and source_address. It is used for mocking the peer in test. """ return Peer(auth.public_key_bin, source_address)
class PopularityCommunity(PubSubCommunity): """ Community for disseminating the content across the network. Follows publish-subscribe model. """ MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b810400270381920004073beff7002b6a9fc2824a3b1bbb1c4fc32546" \ "261e3ef7537874560346c5fdc0c17fe654f67d23b08cbb44141879f79a7a4c8deddf9beb4fbc7a0f02ee1586cc" \ "ebedb623eeef51710108d702f9250361c071482e83c0a4a86c8f45a0b13a19ef83eacb6267b4bfccf220ae5f6d" \ "1db7125ea1d10da3744b65679828f23376e28b76ab33132b7fa984a77f159dba7351a7" master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex')) def __init__(self, *args, **kwargs): self.torrent_db = kwargs.pop('torrent_db', None) self.channel_db = kwargs.pop('channel_db', None) self.trustchain = kwargs.pop('trustchain_community', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.torrent_db, self.channel_db) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response, chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response, chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request, chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response, chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request, chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response }) self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX')) @inlineCallbacks def unload(self): self.content_repository.cleanup() self.content_repository = None yield super(PopularityCommunity, self).unload() def on_subscribe(self, source_address, data): auth, _, _ = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data) # Publish the latest torrents to the subscriber if subscribed: self.publish_latest_torrents(peer=peer) def on_torrent_health_response(self, source_address, data): """ Message handler for torrent health response. Torrent health response is part of periodic update message from the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply dropped. In other case, a decision to accept or reject the message is made based on freshness of the message and the trustscore (check update_torrent in ContentRepository for the implementation). """ self.logger.info("Got torrent health response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return infohash = payload.infohash if not self.content_repository.has_torrent(infohash): self.send_torrent_info_request(infohash, peer=peer) peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0 self.content_repository.update_torrent_health(payload, peer_trust) def on_channel_health_response(self, source_address, data): """ Message handler for channel health response. Currently, not sure how to handle it. """ def on_torrent_info_request(self, source_address, data): """ Message handler for torrent info request. """ self.logger.info("Got torrent info request from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentInfoRequestPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.subscribers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return self.send_torrent_info_response(payload.infohash, peer=peer) def on_torrent_info_response(self, source_address, data): """ Message handler for torrent info response. """ self.logger.info("Got torrent info response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentInfoResponsePayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return self.content_repository.update_torrent_info(payload) def on_content_info_request(self, source_address, data): auth, _, payload = self._ez_unpack_auth(ContentInfoRequest, data) peer = self.get_peer_from_auth(auth, source_address) if payload.content_type == SEARCH_TORRENT_REQUEST: db_results = self.content_repository.search_torrent(payload.query_list) self.send_content_info_response(peer, payload.identifier, SEARCH_TORRENT_RESPONSE, db_results) def on_content_info_response(self, source_address, data): _, _, payload = self._ez_unpack_auth(ContentInfoResponse, data) identifier = int(payload.identifier) if not self.request_cache.has(u'request', identifier): return cache = self.request_cache.get(u'request', identifier) if payload.content_type == SEARCH_TORRENT_RESPONSE: self.process_torrent_search_response(cache.query, payload) if not payload.pagination.more: cache = self.request_cache.pop(u'request', identifier) cache.finish() def process_torrent_search_response(self, query, payload): item_format = SearchResponseItemPayload.format_list response, _ = self.serializer.unpack_multiple_as_list(item_format, payload.response) # Decode the category string to list for response_item in response: response_item[4] = decode_values(response_item[4]) self.content_repository.update_from_torrent_search_results(response) result_dict = dict() result_dict['keywords'] = query result_dict['results'] = response result_dict['candidate'] = None if self.tribler_session: self.tribler_session.notifier.notify(SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None, result_dict) # MESSAGE SENDING FUNCTIONS def send_torrent_health_response(self, payload, peer=None): """ Method to send torrent health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) def send_channel_health_response(self, payload, peer=None): """ Method to send channel health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_CHANNEL_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) def send_torrent_info_request(self, infohash, peer): """ Method to request information about a torrent with given infohash to a peer. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return info_request = TorrentInfoRequestPayload(infohash) packet = self.create_message_packet(MSG_TORRENT_INFO_REQUEST, info_request) self.broadcast_message(packet, peer=peer) def send_torrent_info_response(self, infohash, peer): """ Method to send information about a torrent with given infohash to the requesting peer. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return db_torrent = self.content_repository.get_torrent(infohash) info_response = TorrentInfoResponsePayload(infohash, db_torrent['name'], db_torrent['length'], db_torrent['creation_date'], db_torrent['num_files'], db_torrent['comment']) packet = self.create_message_packet(MSG_TORRENT_INFO_RESPONSE, info_response) self.broadcast_message(packet, peer=peer) def send_content_info_request(self, content_type, request_list, limit=25, peer=None): """ Sends the generic content request of given content_type. :param content_type: request content type :param request_list: List<string> request queries :param limit: Number of expected responses :param peer: Peer to send this request to :return a Deferred that fires when we receive the content :rtype Deferred """ cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list)) self.logger.info("Sending search request query:%s, identifier:%s", request_list, cache.number) content_request = ContentInfoRequest(cache.number, content_type, request_list, limit) packet = self.create_message_packet(MSG_CONTENT_INFO_REQUEST, content_request) if peer: self.broadcast_message(packet, peer=peer) else: for connected_peer in self.get_peers(): self.broadcast_message(packet, peer=connected_peer) return cache.deferred def send_content_info_response(self, peer, identifier, content_type, response_list): """ Sends the generic content info response with payload response list. :param peer: Receiving peer :param identifier: Request identifier :param content_type: Message content type :param response_list: Content response """ num_results = len(response_list) current_index = 0 page_num = 1 while current_index < num_results: serialized_results, current_index, page_size = self.pack_sized(response_list, MAX_PACKET_PAYLOAD_SIZE, start_index=current_index) if not serialized_results: self.logger.info("Item too big probably to fit into package. Skipping it") current_index += 1 else: pagination = Pagination(page_num, page_size, num_results, more=current_index == num_results) response_payload = ContentInfoResponse(identifier, content_type, serialized_results, pagination) packet = self.create_message_packet(MSG_CONTENT_INFO_RESPONSE, response_payload) self.broadcast_message(packet, peer=peer) def send_torrent_search_request(self, query): """ Sends torrent search query as a content info request with content_type as SEARCH_TORRENT_REQUEST. """ self.send_content_info_request(SEARCH_TORRENT_REQUEST, query) def send_channel_search_request(self, query): """ Sends channel search query to All Channel 2.0 to get a list of channels. """ # TODO: Not implemented yet. Waiting for All Channel 2.0 # CONTENT REPOSITORY STUFFS def publish_next_content(self): """ Publishes the next content from the queue to the subscribers. Does nothing if there are none subscribers. Only Torrent health response is published at the moment. """ self.logger.info("Content to publish: %d", self.content_repository.count_content()) if not self.subscribers: self.logger.info("No subscribers found. Not publishing anything") return content_type, content = self.content_repository.pop_content() if content_type is None: self.logger.error(ERROR_NO_CONTENT) return self.logger.info("Publishing content[type:%d]", content_type) if content_type == TYPE_TORRENT_HEALTH: infohash, seeders, leechers, timestamp = content payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp) self.send_torrent_health_response(payload) def publish_latest_torrents(self, peer): """ Publishes the latest torrents in local database to the given peer. """ torrents = self.content_repository.get_top_torrents() self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer) for torrent in torrents: infohash, seeders, leechers, timestamp = torrent[:4] payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp) self.send_torrent_health_response(payload, peer=peer) def queue_content(self, content_type, content): """ Basically adds a given content to the queue of content repository. """ self.content_repository.add_content(content_type, content)