def test_strategy_multi_peer(self):
"""
If we have multiple peers, we should select one and send it our channel views.
Also, we should still inspect our download queue.
"""
self.community.get_peers_return = [
Peer(default_eccrypto.generate_key(u"very-low")),
Peer(default_eccrypto.generate_key(u"very-low")),
Peer(default_eccrypto.generate_key(u"very-low"))
]
self.strategy.take_step()
self.assertEqual(1, len(self.community.send_random_to_called))
self.assertIn(self.community.send_random_to_called[0],
self.community.get_peers_return)
def should_join_circuit(self, create_payload, previous_node_address):
"""
Check whether we should join a circuit. Returns a deferred that fires with a boolean.
"""
if self.settings.max_joined_circuits <= len(self.relay_from_to) + len(self.exit_sockets):
self.logger.warning("too many relays (%d)", (len(self.relay_from_to) + len(self.exit_sockets)))
return succeed(False)
# Check whether we have a random open slot, if so, allocate this to this request.
circuit_id = create_payload.circuit_id
for index, slot in enumerate(self.random_slots):
if not slot:
self.random_slots[index] = circuit_id
return succeed(True)
# No random slots but this user might be allocated a competing slot.
# Next, we request the token balance of the circuit initiator.
balance_deferred = Deferred()
self.request_cache.add(BalanceRequestCache(self, circuit_id, balance_deferred))
# Temporarily add these values, otherwise we are unable to communicate with the previous hop.
self.directions[circuit_id] = EXIT_NODE
shared_secret, _, _ = self.crypto.generate_diffie_shared_secret(create_payload.key)
self.relay_session_keys[circuit_id] = self.crypto.generate_session_keys(shared_secret)
self.send_cell([Peer(create_payload.node_public_key, previous_node_address)], u"balance-request",
BalanceRequestPayload(circuit_id))
self.directions.pop(circuit_id, None)
self.relay_session_keys.pop(circuit_id, None)
return balance_deferred
def on_payout_block(self, source_address, data):
if not self.triblerchain_community:
self.logger.warning(
"Got payout while not having a TriblerChain community running!"
)
return
_, payload = self._ez_unpack_noauth(PayoutPayload, data)
peer = Peer(payload.public_key, source_address)
block = self.triblerchain_community.BLOCK_CLASS.from_payload(
payload, self.serializer)
self.triblerchain_community.process_half_block(block, peer)
# Send the next payout
if payload.circuit_id in self.relay_from_to and block.transaction[
'down'] > payload.base_amount:
relay = self.relay_from_to[payload.circuit_id]
circuit_peer = self.get_peer_from_mid(relay.mid)
if not circuit_peer:
self.logger.warning(
"%s Unable to find next peer %s for payout!", self.my_peer,
relay.mid.encode('hex'))
return
self.do_payout(circuit_peer, relay.circuit_id,
block.transaction['down'] - payload.base_amount * 2,
payload.base_amount)
def on_payout_block(self, source_address, data):
if not self.bandwidth_wallet:
self.logger.warning(
"Got payout while not having a TrustChain community running!")
return
payload = self._ez_unpack_noauth(PayoutPayload,
data,
global_time=False)
peer = Peer(payload.public_key, source_address)
block = TriblerBandwidthBlock.from_payload(payload, self.serializer)
self.bandwidth_wallet.trustchain.process_half_block(block, peer)
# Send the next payout
if payload.circuit_id in self.relay_from_to and block.transaction[
'down'] > payload.base_amount:
relay = self.relay_from_to[payload.circuit_id]
circuit_peer = self.get_peer_from_address(relay.peer.address)
if not circuit_peer:
self.logger.warning(
"%s Unable to find next peer %s for payout!", self.my_peer,
relay.peer)
return
self.do_payout(circuit_peer, relay.circuit_id,
block.transaction['down'] - payload.base_amount * 2,
payload.base_amount)
def load_tunnel_community_in_session(self, session, exitnode=False):
"""
Load the tunnel community in a given session. We are using our own tunnel community here instead of the one
used in Tribler.
"""
self.sanitize_network(session)
keypair = ECCrypto().generate_key(u"curve25519")
tunnel_peer = Peer(keypair)
session.config.set_tunnel_community_exitnode_enabled(exitnode)
overlay = self.test_class(tunnel_peer,
session.lm.ipv8.endpoint,
session.lm.ipv8.network,
tribler_session=session,
dht_provider=MockDHTProvider(
session.lm.ipv8.endpoint.get_address()),
settings={
"become_exitnode": exitnode,
"max_circuits": 1
})
overlay._use_main_thread = False
overlay.settings.remove_tunnel_delay = 0
session.lm.ipv8.overlays.append(overlay)
return overlay
def load_tunnel_community_in_session(self, session, exitnode=False):
"""
Load the tunnel community in a given session. We are using our own tunnel community here instead of the one
used in Tribler.
"""
keypair = ECCrypto().generate_key(u"curve25519")
tunnel_peer = Peer(keypair)
session.config.set_tunnel_community_exitnode_enabled(exitnode)
overlay = self.test_class(tunnel_peer,
session.lm.ipv8.endpoint,
session.lm.ipv8.network,
tribler_session=session,
dht_provider=MockDHTProvider(
session.lm.ipv8.endpoint.get_address()))
overlay._use_main_thread = False
session.lm.ipv8.overlays.append(overlay)
session.lm.ipv8.strategies.append((RandomWalk(overlay), 20))
# We disable the discovery communities in this session since we don't want to walk to the live network
for overlay in session.lm.ipv8.overlays:
if isinstance(overlay, DiscoveryCommunity):
overlay.unload()
# Also reset the IPv8 network
session.lm.ipv8.network = Network()
return overlay
def on_payout_block(self, source_address, data):
if not self.bandwidth_wallet:
self.logger.warning(
"Got payout while not having a TrustChain community running!")
return
payload = self._ez_unpack_noauth(PayoutPayload,
data,
global_time=False)
peer = Peer(payload.public_key, source_address)
def on_transaction_completed(blocks):
# Send the next payout
if blocks and payload.circuit_id in self.relay_from_to and block.transaction[
'down'] > payload.base_amount:
relay = self.relay_from_to[payload.circuit_id]
self._logger.info("Sending next payout to peer %s", relay.peer)
self.do_payout(
relay.peer, relay.circuit_id,
block.transaction['down'] - payload.base_amount * 2,
payload.base_amount)
block = TriblerBandwidthBlock.from_payload(payload, self.serializer)
self.bandwidth_wallet.trustchain.process_half_block(block, peer)\
.addCallbacks(on_transaction_completed, lambda _: None)
# Check whether the block has been added to the database and has been verified
if not self.bandwidth_wallet.trustchain.persistence.contains(block):
self.logger.warning(
"Not proceeding with payout - received payout block is not valid"
)
return
def test_send_random_multiple_torrents(self):
"""
Test whether sending a single channel with a multiple torrents to another peer works correctly
"""
with db_session:
channel = self.nodes[
0].overlay.metadata_store.ChannelMetadata.create_channel(
"test", "bla")
for _ in xrange(20):
self.add_random_torrent(
self.nodes[0].overlay.metadata_store.TorrentMetadata)
channel.commit_channel_torrent()
self.nodes[0].overlay.send_random_to(
Peer(self.nodes[1].my_peer.public_key,
self.nodes[1].endpoint.wan_address))
yield self.deliver_messages(timeout=0.5)
with db_session:
self.assertEqual(
len(self.nodes[1].overlay.metadata_store.ChannelMetadata.
select()), 1)
channel = self.nodes[
1].overlay.metadata_store.ChannelMetadata.select()[:][0]
self.assertLess(channel.contents_len, 20)
class TriblerTunnelTestnetCommunity(TriblerTunnelCommunity):
"""
This community defines a testnet for the anonymous tunnels.
"""
master_peer = Peer("3081a7301006072a8648ce3d020106052b810400270381920004002831990fc973aaf5a8f5bd401f8771fd411d763"
"0ccc4a61c4147a1200135023a2397006e0acb215783cc0245bbc69ebe66abdd13f1fa3434c630604ef2c0d99e5f98"
"727e75ae7901529ba5a2dd875bab582f3f508aa7b675c9d9bd7fd6c2e2684c7fc71b72f007e080634cecf007b718f"
"5cf24d24821cd08feb30d3f3059c7702615ea6f8b23823415bd1673c406e1".decode('hex'))
class TriblerTunnelTestnetCommunity(TriblerTunnelCommunity):
"""
This community defines a testnet for the anonymous tunnels.
"""
master_peer = Peer("3081a7301006072a8648ce3d020106052b81040027038192000401325e6f0a67a92a890344013914fcf9da9b0326"
"bf6c845815ec8b537e356a56b2a8c27ac4918078202f60eb29ebf081436136c280316ac461daee2d04cf073d1200"
"80d15628af1002b0c0273d9d94fdab08e718f568d83b9c4298117261b5647ca8295f1ba4b880a50fd2ef7041fef7"
"edfbef5f02fc03827a2c09c5f9c701f87abacd022c780d76733c133363a5c46c".decode('hex'))
def on_introduction_response(self, source_address, data):
super(TriblerChainCrawlerCommunity,
self).on_introduction_response(source_address, data)
auth, _, _ = self._ez_unpack_auth(IntroductionResponsePayload, data)
peer = Peer(auth.public_key_bin, source_address)
self.send_crawl_request(peer, peer.public_key.key_to_bin())
class GigaChannelTestnetCommunity(GigaChannelCommunity):
"""
This community defines a testnet for the giga channels, used for testing purposes.
"""
master_peer = Peer(
unhexlify(
"4c69624e61434c504b3afbd79020aa61795d1186ea505cf80fe2ac7f42dfc32b830ebade5d78479cbb4"
"35bdbfda7b04e156f5515d1b0bbdafa5c67279e25937201ef6b31f7eeded20423"
))
def add_node_to_experiment(self, node):
"""
Add a new node to this experiment (use `create_node()`).
"""
for other in self.nodes:
private_peer = other.my_peer
public_peer = Peer(private_peer.public_key, private_peer.address)
node.network.add_verified_peer(public_peer)
node.network.discover_services(public_peer,
node.overlay.master_peer.mid)
self.nodes.append(node)
def on_tribler_started(_):
# We load the TriblerChain community.
triblerchain_peer = Peer(self.session.trustchain_keypair)
self.triblerchain_community = TriblerChainCrawlerCommunity(
triblerchain_peer,
self.session.lm.ipv8.endpoint,
self.session.lm.ipv8.network,
tribler_session=self.session,
working_directory=self.session.config.get_state_dir())
self.session.lm.ipv8.overlays.append(self.triblerchain_community)
self.session.lm.ipv8.strategies.append(
(RandomWalk(self.triblerchain_community), -1))
def __init__(self, crypto_curve, overlay_class, *args, **kwargs):
self.endpoint = AutoMockEndpoint()
self.endpoint.open()
self.network = Network()
self.my_peer = Peer(ECCrypto().generate_key(crypto_curve),
self.endpoint.wan_address)
self.overlay = overlay_class(self.my_peer, self.endpoint, self.network,
*args, **kwargs)
self.discovery = MockWalk(self.overlay)
self.overlay.my_estimated_wan = self.endpoint.wan_address
self.overlay.my_estimated_lan = self.endpoint.lan_address
def test_strategy_one_peer(self):
"""
If we have one peer, we should send it our channel views and inspect our download queue.
"""
self.community.get_peers_return = [
Peer(default_eccrypto.generate_key(u"very-low"))
]
self.strategy.take_step()
self.assertEqual(1, len(self.community.send_random_to_called))
self.assertEqual(self.community.get_peers_return[0],
self.community.send_random_to_called[0])
def bootstrap_new_identity(self, amount):
"""
One-way payment channel.
Create a new temporary identity, and transfer funds to the new identity.
A different party can then take the result and do a transfer from the temporary identity to itself
"""
# Create new identity for the temporary identity
crypto = ECCrypto()
tmp_peer = Peer(crypto.generate_key(u"curve25519"))
# Create the transaction specification
transaction = {'up': 0, 'down': amount, 'type': 'tribler_bandwidth'}
# Create the two half blocks that form the transaction
local_half_block = TriblerBandwidthBlock.create(
'tribler_bandwidth',
transaction,
self.trustchain.persistence,
self.trustchain.my_peer.public_key.key_to_bin(),
link_pk=tmp_peer.public_key.key_to_bin())
local_half_block.sign(self.trustchain.my_peer.key)
tmp_half_block = TriblerBandwidthBlock.create(
'tribler_bandwidth',
transaction,
self.trustchain.persistence,
tmp_peer.public_key.key_to_bin(),
link=local_half_block,
link_pk=self.trustchain.my_peer.public_key.key_to_bin())
tmp_half_block.sign(tmp_peer.key)
self.trustchain.persistence.add_block(local_half_block)
self.trustchain.persistence.add_block(tmp_half_block)
# Create the bootstrapped identity format
block = {
'block_hash': tmp_half_block.hash.encode('base64'),
'sequence_number': tmp_half_block.sequence_number
}
result = {
'private_key': tmp_peer.key.key_to_bin().encode('base64'),
'transaction': {
'up': amount,
'down': 0
},
'block': block
}
return result
def assign_exit_node(self, node_nr):
"""
Give a node a dedicated exit node to play with.
"""
exit_node = self.create_node()
self.nodes.append(exit_node) # So it could be properly removed on exit
exit_node.overlay.settings.become_exitnode = True
public_peer = Peer(exit_node.my_peer.public_key, exit_node.my_peer.address)
self.nodes[node_nr].network.add_verified_peer(public_peer)
self.nodes[node_nr].network.discover_services(public_peer, exit_node.overlay.master_peer.mid)
self.nodes[node_nr].overlay.update_exit_candidates(public_peer, True)
self.nodes[node_nr].overlay.build_tunnels(1)
yield self.deliver_messages()
exit_sockets = exit_node.overlay.exit_sockets
for exit_socket in exit_sockets:
exit_sockets[exit_socket] = MockTunnelExitSocket(exit_sockets[exit_socket])
def initialize(self, overlay_class, node_count, *args, **kwargs):
self.overlay_class = overlay_class
self.nodes = [
self.create_node(*args, **kwargs) for _ in range(node_count)
]
# Add nodes to each other
for node in self.nodes:
for other in self.nodes:
if other == node:
continue
private_peer = other.my_peer
public_peer = Peer(private_peer.public_key,
private_peer.address)
node.network.add_verified_peer(public_peer)
node.network.discover_services(public_peer,
overlay_class.master_peer.mid)
def setUp(self):
"""
Setup various variables and load the tunnel community in the main downloader session.
"""
yield TestAsServer.setUp(self)
self.seed_tdef = None
self.sessions = []
self.session2 = None
self.bypass_dht = False
self.seed_config = None
self.tunnel_community_seeder = None
self.eccrypto = ECCrypto()
ec = self.eccrypto.generate_key(u"curve25519")
self.test_class = TriblerTunnelCommunity
self.test_class.master_peer = Peer(ec)
self.tunnel_community = self.load_tunnel_community_in_session(
self.session, exitnode=True)
self.tunnel_communities = []
def test_send_and_get_channel_update_back(self):
"""
Test if sending back information on updated version of a channel works
"""
with db_session:
# Add channel to node 0
channel = self.nodes[
0].overlay.metadata_store.ChannelMetadata.create_channel(
"test", "bla")
for _ in xrange(20):
self.add_random_torrent(
self.nodes[0].overlay.metadata_store.TorrentMetadata)
channel.commit_channel_torrent()
channel_v1_dict = channel.to_dict()
channel_v1_dict.pop("health")
self.add_random_torrent(
self.nodes[0].overlay.metadata_store.TorrentMetadata)
channel.commit_channel_torrent()
# Add the outdated version of the channel to node 1
self.nodes[1].overlay.metadata_store.ChannelMetadata.from_dict(
channel_v1_dict)
# node1 --outdated_channel--> node0
self.nodes[1].overlay.send_random_to(
Peer(self.nodes[0].my_peer.public_key,
self.nodes[0].endpoint.wan_address))
yield self.deliver_messages(timeout=0.5)
with db_session:
self.assertEqual(
self.nodes[1].overlay.metadata_store.ChannelMetadata.select()
[:][0].timestamp,
self.nodes[0].overlay.metadata_store.ChannelMetadata.select()
[:][0].timestamp)
class TriblerChainCommunity(TrustChainCommunity):
"""
Community for reputation based on TrustChain tamper proof interaction history.
"""
BLOCK_CLASS = TriblerChainBlock
DB_CLASS = TriblerChainDB
master_peer = Peer(
"3081a7301006072a8648ce3d020106052b81040027038192000405c66d3deddb1721787a247b2285118c06ce9fb"
"20ebd3546969fa2f4811fa92426637423d3bac1510f92b33e2ff5a785bf54eb3b28d29a77d557011d7d5241243c"
"9c89c987cd049404c4024999e1505fa96e1d6668234bde28a666d458d67251d17ff45185515a28967ddcf50503c"
"304750ae114f9bc857a79c03da1a9c9215ea07c91f166f24b6cfd1cf72309044fbd".
decode('hex'))
def __init__(self, *args, **kwargs):
self.tribler_session = kwargs.pop('tribler_session', None)
super(TriblerChainCommunity, self).__init__(*args, **kwargs)
def should_sign(self, block):
"""
Return whether we should sign a given block. For the TriblerChain, we only sign a block when we receive bytes.
In our current design, only the person that should pay bytes to others initiates a signing request.
This is true when considering payouts in the tunnels and when buying bytes on the market.
"""
return block.transaction["down"] >= MIN_TRANSACTION_SIZE
@blocking_call_on_reactor_thread
def get_statistics(self, public_key=None):
"""
Returns a dictionary with some statistics regarding the local trustchain database
:returns a dictionary with statistics
"""
if public_key is None:
public_key = self.my_peer.public_key.key_to_bin()
latest_block = self.persistence.get_latest(public_key)
statistics = dict()
statistics["id"] = public_key.encode("hex")
interacts = self.persistence.get_num_unique_interactors(public_key)
statistics["peers_that_pk_helped"] = interacts[0] if interacts[
0] is not None else 0
statistics["peers_that_helped_pk"] = interacts[1] if interacts[
1] is not None else 0
if latest_block:
statistics["total_blocks"] = latest_block.sequence_number
statistics["total_up"] = latest_block.transaction["total_up"]
statistics["total_down"] = latest_block.transaction["total_down"]
statistics["latest_block"] = dict(latest_block)
# Set up/down
statistics["latest_block"]["up"] = latest_block.transaction["up"]
statistics["latest_block"]["down"] = latest_block.transaction[
"down"]
else:
statistics["total_blocks"] = 0
statistics["total_up"] = 0
statistics["total_down"] = 0
return statistics
def get_bandwidth_tokens(self, peer=None):
"""
Get the bandwidth tokens for another peer.
Currently this is just the difference in the amount of MBs exchanged with them.
:param member: the peer we interacted with
:type member: Peer
:return: the amount of bandwidth tokens for this peer
:rtype: int
"""
if peer is None:
peer = self.my_peer
block = self.persistence.get_latest(peer.public_key.key_to_bin())
if block:
return block.transaction['total_up'] - block.transaction[
'total_down']
return 0
def bootstrap_new_identity(self, amount):
"""
One-way payment channel.
Create a new temporary identity, and transfer funds to the new identity.
A different party can then take the result and do a transfer from the temporary identity to itself
"""
# Create new identity for the temporary identity
crypto = ECCrypto()
tmp_peer = Peer(crypto.generate_key(u"curve25519"))
# Create the transaction specification
transaction = {'up': 0, 'down': amount}
# Create the two half blocks that form the transaction
local_half_block = TriblerChainBlock.create(
transaction,
self.persistence,
self.my_peer.public_key.key_to_bin(),
link_pk=tmp_peer.public_key.key_to_bin())
local_half_block.sign(self.my_peer.key)
tmp_half_block = TriblerChainBlock.create(
transaction,
self.persistence,
tmp_peer.public_key.key_to_bin(),
link=local_half_block,
link_pk=self.my_peer.public_key.key_to_bin())
tmp_half_block.sign(tmp_peer.key)
self.persistence.add_block(local_half_block)
self.persistence.add_block(tmp_half_block)
# Create the bootstrapped identity format
block = {
'block_hash': tmp_half_block.hash.encode('base64'),
'sequence_number': tmp_half_block.sequence_number
}
result = {
'private_key': tmp_peer.key.key_to_bin().encode('base64'),
'transaction': {
'up': amount,
'down': 0
},
'block': block
}
return result
def load_ipv8_overlays(self):
# Discovery Community
with open(self.session.config.get_permid_keypair_filename(),
'r') as key_file:
content = key_file.read()
content = content[31:-30].replace('\n', '').decode("BASE64")
peer = Peer(M2CryptoSK(keystring=content))
discovery_community = DiscoveryCommunity(peer, self.ipv8.endpoint,
self.ipv8.network)
discovery_community.resolve_dns_bootstrap_addresses()
self.ipv8.overlays.append(discovery_community)
self.ipv8.strategies.append((RandomChurn(discovery_community), -1))
if not self.session.config.get_dispersy_enabled():
self.ipv8.strategies.append((RandomWalk(discovery_community), 20))
# TriblerChain Community
if self.session.config.get_trustchain_enabled():
triblerchain_peer = Peer(self.session.trustchain_keypair)
from Tribler.community.triblerchain.community import TriblerChainCommunity
self.triblerchain_community = TriblerChainCommunity(
triblerchain_peer,
self.ipv8.endpoint,
self.ipv8.network,
tribler_session=self.session,
working_directory=self.session.config.get_state_dir())
self.ipv8.overlays.append(self.triblerchain_community)
self.ipv8.strategies.append(
(EdgeWalk(self.triblerchain_community), 20))
# Tunnel Community
if self.session.config.get_tunnel_community_enabled():
tunnel_peer = Peer(self.session.trustchain_keypair)
from Tribler.community.triblertunnel.community import TriblerTunnelCommunity
self.tunnel_community = TriblerTunnelCommunity(
tunnel_peer,
self.ipv8.endpoint,
self.ipv8.network,
tribler_session=self.session,
dht_provider=MainlineDHTProvider(
self.mainline_dht,
self.session.config.get_dispersy_port()),
triblerchain_community=self.triblerchain_community)
self.ipv8.overlays.append(self.tunnel_community)
self.ipv8.strategies.append(
(RandomWalk(self.tunnel_community), 20))
# Market Community
if self.session.config.get_market_community_enabled():
wallets = {}
try:
from Tribler.community.market.wallet.btc_wallet import BitcoinWallet, BitcoinTestnetWallet
wallet_type = BitcoinTestnetWallet if self.session.config.get_btc_testnet(
) else BitcoinWallet
btc_wallet = wallet_type(
os.path.join(self.session.config.get_state_dir(),
'wallet'))
wallets[btc_wallet.get_identifier()] = btc_wallet
except ImportError:
self._logger.error(
"Electrum wallet cannot be found, Bitcoin trading not available!"
)
mc_wallet = TrustchainWallet(self.triblerchain_community)
wallets[mc_wallet.get_identifier()] = mc_wallet
if self.session.config.get_dummy_wallets_enabled():
# For debugging purposes, we create dummy wallets
dummy_wallet1 = DummyWallet1()
wallets[dummy_wallet1.get_identifier()] = dummy_wallet1
dummy_wallet2 = DummyWallet2()
wallets[dummy_wallet2.get_identifier()] = dummy_wallet2
from Tribler.community.market.community import MarketCommunity
market_peer = Peer(self.session.tradechain_keypair)
self.market_community = MarketCommunity(
market_peer,
self.ipv8.endpoint,
self.ipv8.network,
tribler_session=self.session,
wallets=wallets,
working_directory=self.session.config.get_state_dir())
self.ipv8.overlays.append(self.market_community)
self.ipv8.strategies.append(
(RandomWalk(self.market_community), 20))
def load_ipv8_overlays(self):
# Discovery Community
with open(self.session.config.get_permid_keypair_filename(),
'r') as key_file:
content = key_file.read()
content = content[31:-30].replace('\n', '').decode("BASE64")
peer = Peer(M2CryptoSK(keystring=content))
discovery_community = DiscoveryCommunity(peer, self.ipv8.endpoint,
self.ipv8.network)
discovery_community.resolve_dns_bootstrap_addresses()
self.ipv8.overlays.append(discovery_community)
self.ipv8.strategies.append((RandomChurn(discovery_community), -1))
if not self.session.config.get_dispersy_enabled():
self.ipv8.strategies.append((RandomWalk(discovery_community), 20))
if self.session.config.get_testnet():
peer = Peer(self.session.trustchain_keypair)
else:
peer = Peer(self.session.trustchain_testnet_keypair)
# TrustChain Community
if self.session.config.get_trustchain_enabled():
from Tribler.pyipv8.ipv8.attestation.trustchain.community import TrustChainCommunity, \
TrustChainTestnetCommunity
community_cls = TrustChainTestnetCommunity if self.session.config.get_testnet(
) else TrustChainCommunity
self.trustchain_community = community_cls(
peer,
self.ipv8.endpoint,
self.ipv8.network,
working_directory=self.session.config.get_state_dir())
self.ipv8.overlays.append(self.trustchain_community)
self.ipv8.strategies.append(
(EdgeWalk(self.trustchain_community), 20))
tc_wallet = TrustchainWallet(self.trustchain_community)
self.wallets[tc_wallet.get_identifier()] = tc_wallet
# Tunnel Community
if self.session.config.get_tunnel_community_enabled():
from Tribler.community.triblertunnel.community import TriblerTunnelCommunity, TriblerTunnelTestnetCommunity
community_cls = TriblerTunnelTestnetCommunity if self.session.config.get_testnet() else \
TriblerTunnelCommunity
self.tunnel_community = community_cls(
peer,
self.ipv8.endpoint,
self.ipv8.network,
tribler_session=self.session,
dht_provider=MainlineDHTProvider(
self.mainline_dht,
self.session.config.get_dispersy_port()),
bandwidth_wallet=self.wallets["MB"])
self.ipv8.overlays.append(self.tunnel_community)
self.ipv8.strategies.append(
(RandomWalk(self.tunnel_community), 20))
# Market Community
if self.session.config.get_market_community_enabled():
from Tribler.community.market.community import MarketCommunity, MarketTestnetCommunity
community_cls = MarketTestnetCommunity if self.session.config.get_testnet(
) else MarketCommunity
self.market_community = community_cls(
peer,
self.ipv8.endpoint,
self.ipv8.network,
tribler_session=self.session,
trustchain=self.trustchain_community,
wallets=self.wallets,
working_directory=self.session.config.get_state_dir())
self.ipv8.overlays.append(self.market_community)
self.ipv8.strategies.append(
(RandomWalk(self.market_community), 20))
# Popular Community
if self.session.config.get_popularity_community_enabled():
from Tribler.community.popularity.community import PopularityCommunity
self.popularity_community = PopularityCommunity(
peer,
self.ipv8.endpoint,
self.ipv8.network,
torrent_db=self.session.lm.torrent_db,
session=self.session)
self.ipv8.overlays.append(self.popularity_community)
self.ipv8.strategies.append(
(RandomWalk(self.popularity_community), 20))
self.popularity_community.start()
class PubSubCommunity(Community):
"""
This community is designed as a base community for all othe future communities that desires publish subscribe model
for content dissemination. It provides a few basic primitives like subscribe/unsubscribe to publisher peers and
publish/broadcast content to subscriber peers.
All the derived community should implement publish_next_content() method which is responsible for publishing the
next available content to all the subscribers.
"""
MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b8104002703819200040504278d20d6776ce7081ad57d99fe066bb2a93" \
"ce7cc92405a534ef7175bab702be557d8c7d3b725ea0eb09c686e798f6c7ad85e8781a4c3b20e54c15ede38077c" \
"8f5c801b71d13105f261da7ddcaa94ae14bd177bf1a05a66f595b9bb99117d11f73b4c8d3dcdcdc2b3f838b8ba3" \
"5a9f600d2c543e8b3ba646083307b917bbbccfc53fc5ab6ded90b711d7eeda46f5f"
master_peer = Peer(unhexlify(MASTER_PUBLIC_KEY))
def __init__(self, *args, **kwargs):
super(PubSubCommunity, self).__init__(*args, **kwargs)
self.trustchain = kwargs.pop('trustchain_community', None)
self.logger = logging.getLogger(self.__class__.__name__)
self.request_cache = RequestCache()
# Register messages
self.decode_map.update({
chr(MSG_SUBSCRIBE):
self.on_subscribe,
chr(MSG_SUBSCRIPTION):
self.on_subscription_status
})
# A set of publisher and subscriber.
# Sends data updates to subscribers, and receives updates from subscribers.
self.subscribers = set()
self.publishers = set()
def start(self):
"""
Starts the community by subscribing to peers, and periodically publishing the content updates to
the subscribers.
"""
# Subscribe peers
self.subscribe_peers()
def start_publishing():
# Update the publisher and subscriber list
self.refresh_peer_list()
# publish the new cotent from the content repository
self.publish_next_content()
self.register_task("start_publishing",
LoopingCall(start_publishing)).start(
PUBLISH_INTERVAL, False)
@inlineCallbacks
def unload(self):
self.request_cache.clear()
self.cancel_pending_task("start_publishing")
yield super(PubSubCommunity, self).unload()
def subscribe_peers(self):
"""
Subscribes to the connected peers. First, the peers are sorted based on the trust score on descending order and
content subscribe request is sent to the top peers.
This method is called periodically through refresh_peer_list() in start_publishing() loop so it can fill up for
the disconnected peers by connecting to new peers.
Note that, existing publisher peers are not disconnected even if we find new peers with higher trust score but
only fill up the remaining publisher slots with new top peers.
"""
num_publishers = len(self.publishers)
num_peers = len(self.get_peers())
# If we have some free publisher slots and there are peers available
if num_publishers < MAX_PUBLISHERS and num_publishers < num_peers:
available_publishers = [
peer for peer in self.get_peers()
if peer not in self.publishers
]
sorted_peers = sorted(
available_publishers,
key=lambda _peer: self.trustchain.get_trust(_peer)
if self.trustchain else 1,
reverse=True)
for peer in sorted_peers[:MAX_PUBLISHERS - num_publishers]:
self.subscribe(peer, subscribe=True)
def refresh_peer_list(self):
"""
Updates the publishers and subscribers list by filtering out the disconnected peers. It also calls subscribe
peers to replenish the available publisher slots if necessary.
"""
peers = self.get_peers()
self.publishers = set(
[peer for peer in self.publishers if peer in peers])
self.subscribers = set(
[peer for peer in self.subscribers if peer in peers])
# subscribe peers if necessary
self.subscribe_peers()
def unsubscribe_peers(self):
"""
Unsubscribes from the existing publishers by sending content subscribe request with subscribe=False. It then
clears up its publishers list.
- Called at community unload.
"""
for peer in copy(self.publishers):
self.subscribe(peer, subscribe=False)
self.publishers.clear()
def subscribe(self, peer, subscribe=True):
"""
Method to send content subscribe/unsubscribe message. This message is sent to each individual publisher peer we
want to subscribe/unsubscribe.
"""
cache = self.request_cache.add(
ContentRequest(self.request_cache, MSG_SUBSCRIBE, None))
# Remove the publisher peer already if user is trying to unsubscribe
if not subscribe:
self.publishers.remove(peer)
# Create subscription packet and send it
subscription = ContentSubscription(cache.number, subscribe)
packet = self.create_message_packet(MSG_SUBSCRIBE, subscription)
self.broadcast_message(packet, peer=peer)
def on_subscribe(self, source_address, data):
"""
Message handler for content subscribe message. It handles both subscribe and unsubscribe requests.
Upon successful subscription or unsubscription, it send the confirmation subscription message with status.
In case of subscription, it also publishes a list of recently checked torrents to the subscriber.
"""
auth, _, payload = self._ez_unpack_auth(ContentSubscription, data)
peer = self.get_peer_from_auth(auth, source_address)
# Subscribe or unsubscribe peer
subscribed = peer in self.subscribers
if payload.subscribe and not subscribed:
if len(self.subscribers) < MAX_SUBSCRIBERS:
self.subscribers.add(peer)
subscribed = True
elif not payload.subscribe and subscribed:
self.subscribers.remove(peer)
subscribed = False
# Send subscription response
self.send_subscription_status(peer,
payload.identifier,
subscribed=subscribed)
return subscribed
def send_subscription_status(self, peer, identifier, subscribed=True):
"""
Method to send content subscription message. Content subscription message is send in response to content
subscribe or unsubscribe message.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
subscription = ContentSubscription(identifier, subscribed)
packet = self.create_message_packet(MSG_SUBSCRIPTION, subscription)
self.broadcast_message(packet, peer=peer)
def on_subscription_status(self, source_address, data):
"""
Message handler for content subscription message. Content subscription message is sent by the publisher stating
the status of the subscription in response to subscribe or unsubscribe request.
If the subscription message has subscribe=True, it means the subscription was successful, so the peer is added
to the subscriber. In other case, publisher is removed if it is still present in the publishers list.
"""
auth, _, payload = self._ez_unpack_auth(ContentSubscription, data)
peer = self.get_peer_from_auth(auth, source_address)
if not self.request_cache.has(u'request', payload.identifier):
return
self.request_cache.pop(u'request', payload.identifier)
if payload.subscribe:
self.publishers.add(peer)
elif peer in self.publishers:
self.publishers.remove(peer)
def create_message_packet(self, message_type, payload):
"""
Helper method to creates a message packet of given type with provided payload.
"""
auth = BinMemberAuthenticationPayload(
self.my_peer.public_key.key_to_bin()).to_pack_list()
dist = GlobalTimeDistributionPayload(
self.claim_global_time()).to_pack_list()
payload = payload if isinstance(payload,
list) else payload.to_pack_list()
return self._ez_pack(self._prefix, message_type, [auth, dist, payload])
def broadcast_message(self, packet, peer=None):
"""
Helper method to broadcast the message packet to a single peer or all the subscribers.
"""
if peer is not None:
self.endpoint.send(peer.address, packet)
return
for _peer in self.subscribers:
self.endpoint.send(_peer.address, packet)
def get_peer_from_auth(self, auth, source_address):
"""
Get Peer object from the message and auth and source_address.
It is used for mocking the peer in test.
"""
return Peer(auth.public_key_bin, source_address)
def pack_sized(self, payload_list, fit_size, start_index=0):
"""
Packs a list of Payload objects to fit into given size limit.
:param payload_list: List<Payload> list of payload objects
:param fit_size: The maximum allowed size for payload field to fit into UDP packet.
:param start_index: Index of list to start packing
:return: packed string
"""
assert isinstance(payload_list, list)
serialized_results = ''
size = 0
current_index = start_index
num_payloads = len(payload_list)
while current_index < num_payloads:
item = payload_list[current_index]
packed_item = self.serializer.pack_multiple(item.to_pack_list())[0]
packed_item_length = len(packed_item)
if size + packed_item_length > fit_size:
break
else:
size += packed_item_length
serialized_results += packed_item
current_index += 1
return serialized_results, current_index, current_index - start_index
def publish_next_content(self):
""" Method responsible for publishing content during periodic push """
pass
class TriblerTunnelCommunity(HiddenTunnelCommunity):
"""
This community is built upon the anonymous messaging layer in IPv8.
It adds support for libtorrent anonymous downloads and bandwidth token payout when closing circuits.
"""
master_peer = Peer(
"3081a7301006072a8648ce3d020106052b81040027038192000400965194026c987edad7c5c0364a95dfa4e961e"
"ec6d1711678be182a31824363db3a54caa11e9b6f1d6051c2f33578b51c12eff3833b7c74c5a243b8705e4e032b"
"14904f4d8855e2044c0408a5729cd4e5b286fec66866811d5439049448e5b8b818d8c29a84a074a13f5e7e414d4"
"e5b1b115a221fe697b89bd3e63c7aecd7617f789a203a4eacf680279224b390e836".
decode('hex'))
def __init__(self, *args, **kwargs):
self.tribler_session = kwargs.pop('tribler_session', None)
num_competing_slots = kwargs.pop('competing_slots', 15)
num_random_slots = kwargs.pop('random_slots', 5)
self.bandwidth_wallet = kwargs.pop('bandwidth_wallet', None)
socks_listen_ports = kwargs.pop('socks_listen_ports', None)
self.exitnode_cache = kwargs.pop(
'exitnode_cache',
(self.tribler_session.config.get_state_dir()
if self.tribler_session else '') + 'exitnode_cache.dat')
super(TriblerTunnelCommunity, self).__init__(*args, **kwargs)
self._use_main_thread = True
if self.tribler_session:
self.settings.become_exitnode = self.tribler_session.config.get_tunnel_community_exitnode_enabled(
)
self.tribler_session.lm.tunnel_community = self
if not socks_listen_ports:
socks_listen_ports = self.tribler_session.config.get_tunnel_community_socks5_listen_ports(
)
elif socks_listen_ports is None:
socks_listen_ports = range(1080, 1085)
self.bittorrent_peers = {}
self.dispatcher = TunnelDispatcher(self)
self.download_states = {}
self.competing_slots = [
(0, None)
] * num_competing_slots # 1st tuple item = token balance, 2nd = circuit id
self.random_slots = [None] * num_random_slots
# Start the SOCKS5 servers
self.socks_servers = []
for port in socks_listen_ports:
socks_server = Socks5Server(port, self.dispatcher)
socks_server.start()
self.socks_servers.append(socks_server)
self.dispatcher.set_socks_servers(self.socks_servers)
self.decode_map.update({
chr(23): self.on_payout_block,
})
self.decode_map_private.update({
chr(24):
self.on_balance_request_cell,
chr(25):
self.on_relay_balance_request_cell,
chr(26):
self.on_balance_response_cell,
chr(27):
self.on_relay_balance_response_cell,
})
message_to_payload[u"balance-request"] = (24, BalanceRequestPayload)
message_to_payload[u"relay-balance-request"] = (25,
BalanceRequestPayload)
message_to_payload[u"balance-response"] = (26, BalanceResponsePayload)
message_to_payload[u"relay-balance-response"] = (
27, BalanceResponsePayload)
SINGLE_HOP_ENC_PACKETS.append(u"balance-request")
SINGLE_HOP_ENC_PACKETS.append(u"balance-response")
if self.exitnode_cache:
self.restore_exitnodes_from_disk()
def cache_exitnodes_to_disk(self):
"""
Wite a copy of self.exit_candidates to the file self.exitnode_cache.
:returns: None
"""
exit_nodes = Network()
for peer in self.exit_candidates.values():
exit_nodes.add_verified_peer(peer)
self.logger.debug('Writing exit nodes to cache: %s',
self.exitnode_cache)
with open(self.exitnode_cache, 'w') as cache:
cache.write(exit_nodes.snapshot())
def restore_exitnodes_from_disk(self):
"""
Send introduction requests to peers stored in the file self.exitnode_cache.
:returns: None
"""
if os.path.isfile(self.exitnode_cache):
self.logger.debug('Loading exit nodes from cache: %s',
self.exitnode_cache)
exit_nodes = Network()
with open(self.exitnode_cache, 'r') as cache:
exit_nodes.load_snapshot(cache.read())
for exit_node in exit_nodes.get_walkable_addresses():
self.endpoint.send(exit_node,
self.create_introduction_request(exit_node))
else:
self.logger.error(
'Could not retrieve backup exitnode cache, file does not exist!'
)
def on_token_balance(self, circuit_id, balance):
"""
We received the token balance of a circuit initiator. Check whether we can allocate a slot to this user.
"""
if not self.request_cache.has(u"balance-request", circuit_id):
self.logger.warning(
"Received token balance without associated request cache!")
return
cache = self.request_cache.pop(u"balance-request", circuit_id)
lowest_balance = sys.maxint
lowest_index = -1
for ind, tup in enumerate(self.competing_slots):
if not tup[1]:
# The slot is empty, take it
self.competing_slots[ind] = (balance, circuit_id)
cache.balance_deferred.callback(True)
return
if tup[0] < lowest_balance:
lowest_balance = tup[0]
lowest_index = ind
if balance > lowest_balance:
# We kick this user out
old_circuit_id = self.competing_slots[lowest_index][1]
self.logger.info(
"Kicked out circuit %s (balance: %s) in favor of %s (balance: %s)",
old_circuit_id, lowest_balance, circuit_id, balance)
self.competing_slots[lowest_index] = (balance, circuit_id)
self.remove_relay(old_circuit_id, destroy=True)
self.remove_exit_socket(old_circuit_id, destroy=True)
cache.balance_deferred.callback(True)
else:
# We can't compete with the balances in the existing slots
cache.balance_deferred.callback(False)
def should_join_circuit(self, create_payload, previous_node_address):
"""
Check whether we should join a circuit. Returns a deferred that fires with a boolean.
"""
if self.settings.max_joined_circuits <= len(self.relay_from_to) + len(
self.exit_sockets):
self.logger.warning(
"too many relays (%d)",
(len(self.relay_from_to) + len(self.exit_sockets)))
return succeed(False)
# Check whether we have a random open slot, if so, allocate this to this request.
circuit_id = create_payload.circuit_id
for index, slot in enumerate(self.random_slots):
if not slot:
self.random_slots[index] = circuit_id
return succeed(True)
# No random slots but this user might be allocated a competing slot.
# Next, we request the token balance of the circuit initiator.
balance_deferred = Deferred()
self.request_cache.add(
BalanceRequestCache(self, circuit_id, balance_deferred))
# Temporarily add these values, otherwise we are unable to communicate with the previous hop.
self.directions[circuit_id] = EXIT_NODE
shared_secret, _, _ = self.crypto.generate_diffie_shared_secret(
create_payload.key)
self.relay_session_keys[
circuit_id] = self.crypto.generate_session_keys(shared_secret)
self.send_cell(
[Peer(create_payload.node_public_key, previous_node_address)],
u"balance-request", BalanceRequestPayload(circuit_id))
self.directions.pop(circuit_id, None)
self.relay_session_keys.pop(circuit_id, None)
return balance_deferred
def on_payout_block(self, source_address, data):
if not self.bandwidth_wallet:
self.logger.warning(
"Got payout while not having a TrustChain community running!")
return
payload = self._ez_unpack_noauth(PayoutPayload,
data,
global_time=False)
peer = Peer(payload.public_key, source_address)
block = TriblerBandwidthBlock.from_payload(payload, self.serializer)
self.bandwidth_wallet.trustchain.process_half_block(block, peer)
# Send the next payout
if payload.circuit_id in self.relay_from_to and block.transaction[
'down'] > payload.base_amount:
relay = self.relay_from_to[payload.circuit_id]
circuit_peer = self.get_peer_from_address(relay.peer.address)
if not circuit_peer:
self.logger.warning(
"%s Unable to find next peer %s for payout!", self.my_peer,
relay.peer)
return
self.do_payout(circuit_peer, relay.circuit_id,
block.transaction['down'] - payload.base_amount * 2,
payload.base_amount)
def on_balance_request_cell(self, source_address, data, _):
payload = self._ez_unpack_noauth(BalanceRequestPayload,
data,
global_time=False)
circuit_id = payload.circuit_id
request = self.request_cache.get(u"anon-circuit", circuit_id)
if not request:
self.logger.warning("Circuit creation cache for id %s not found!",
circuit_id)
return
if request.should_forward:
forwarding_relay = RelayRoute(request.from_circuit_id,
request.peer)
self.send_cell([forwarding_relay.peer.address],
u"relay-balance-request",
BalanceRequestPayload(forwarding_relay.circuit_id))
else:
self.on_balance_request(payload)
def on_relay_balance_request_cell(self, source_address, data, _):
payload = self._ez_unpack_noauth(BalanceRequestPayload,
data,
global_time=False)
self.on_balance_request(payload)
def on_balance_request(self, payload):
"""
We received a balance request from a relay or exit node. Respond with the latest block in our chain.
"""
if not self.bandwidth_wallet:
self.logger.warn(
"Bandwidth wallet is not available, not sending balance response!"
)
return
# Get the latest block
latest_block = self.bandwidth_wallet.trustchain.persistence.get_latest(
self.my_peer.public_key.key_to_bin(),
block_type='tribler_bandwidth')
if not latest_block:
latest_block = TriblerBandwidthBlock()
latest_block.public_key = EMPTY_PK # We hide the public key
# We either send the response directly or relay the response to the last verified hop
circuit = self.circuits[payload.circuit_id]
if not circuit.hops:
self.increase_bytes_sent(
circuit,
self.send_cell([circuit.peer.address], u"balance-response",
BalanceResponsePayload.from_half_block(
latest_block, circuit.circuit_id)))
else:
self.increase_bytes_sent(
circuit,
self.send_cell([circuit.peer.address],
u"relay-balance-response",
BalanceResponsePayload.from_half_block(
latest_block, circuit.circuit_id)))
def on_balance_response_cell(self, source_address, data, _):
payload = self._ez_unpack_noauth(BalanceResponsePayload,
data,
global_time=False)
block = TriblerBandwidthBlock.from_payload(payload, self.serializer)
if not block.transaction:
self.on_token_balance(payload.circuit_id, 0)
else:
self.on_token_balance(
payload.circuit_id, block.transaction["total_up"] -
block.transaction["total_down"])
def on_relay_balance_response_cell(self, source_address, data, _):
payload = self._ez_unpack_noauth(BalanceResponsePayload,
data,
global_time=False)
block = TriblerBandwidthBlock.from_payload(payload, self.serializer)
# At this point, we don't have the circuit ID of the follow-up hop. We have to iterate over the items in the
# request cache and find the link to the next hop.
for cache in self.request_cache._identifiers.values():
if isinstance(cache, ExtendRequestCache
) and cache.from_circuit_id == payload.circuit_id:
self.send_cell([cache.to_peer.address], u"balance-response",
BalanceResponsePayload.from_half_block(
block, cache.to_circuit_id))
def on_download_removed(self, download):
"""
This method is called when a download is removed. We check here whether we can stop building circuits for a
specific number of hops in case it hasn't been finished yet.
"""
if download.get_hops() > 0:
self.num_hops_by_downloads[download.get_hops()] -= 1
if self.num_hops_by_downloads[download.get_hops()] == 0:
self.circuits_needed[download.get_hops()] = 0
def readd_bittorrent_peers(self):
for torrent, peers in self.bittorrent_peers.items():
infohash = torrent.tdef.get_infohash().encode("hex")
for peer in peers:
self.logger.info("Re-adding peer %s to torrent %s", peer,
infohash)
torrent.add_peer(peer)
del self.bittorrent_peers[torrent]
def update_torrent(self, peers, handle, download):
peers = peers.intersection(handle.get_peer_info())
if peers:
if download not in self.bittorrent_peers:
self.bittorrent_peers[download] = peers
else:
self.bittorrent_peers[
download] = peers | self.bittorrent_peers[download]
# If there are active circuits, add peers immediately. Otherwise postpone.
if self.active_data_circuits():
self.readd_bittorrent_peers()
def get_peer_from_address(self, address):
circuit_peer = None
for peer in self.get_peers():
if peer.address == address:
circuit_peer = peer
break
return circuit_peer
def do_payout(self, peer, circuit_id, amount, base_amount):
"""
Perform a payout to a specific peer.
:param peer: The peer to perform the payout to, usually the next node in the circuit.
:param circuit_id: The circuit id of the payout, used by the subsequent node.
:param amount: The amount to put in the transaction, multiplier of base_amount.
:param base_amount: The base amount for the payouts.
"""
self.logger.info("Sending payout of %d (base: %d) to %s (cid: %s)",
amount, base_amount, peer, circuit_id)
block = TriblerBandwidthBlock.create(
'tribler_bandwidth', {
'up': 0,
'down': amount
},
self.bandwidth_wallet.trustchain.persistence,
self.my_peer.public_key.key_to_bin(),
link_pk=peer.public_key.key_to_bin())
block.sign(self.my_peer.key)
self.bandwidth_wallet.trustchain.persistence.add_block(block)
payload = PayoutPayload.from_half_block(block, circuit_id,
base_amount).to_pack_list()
packet = self._ez_pack(self._prefix, 23, [payload], False)
self.send_packet([peer], u"payout", packet)
def clean_from_slots(self, circuit_id):
"""
Clean a specific circuit from the allocated slots.
"""
for ind, slot in enumerate(self.random_slots):
if slot == circuit_id:
self.random_slots[ind] = None
for ind, tup in enumerate(self.competing_slots):
if tup[1] == circuit_id:
self.competing_slots[ind] = (0, None)
def remove_circuit(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False):
if circuit_id not in self.circuits:
self.logger.warning(
"Circuit %d not found when trying to remove it", circuit_id)
return
circuit = self.circuits[circuit_id]
# Send the notification
if self.tribler_session:
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE,
circuit, circuit.peer.address)
circuit_peer = self.get_peer_from_address(circuit.peer.address)
if circuit.bytes_down >= 1024 * 1024 and self.bandwidth_wallet and circuit_peer:
# We should perform a payout of the removed circuit.
if circuit.ctype == CIRCUIT_TYPE_RENDEZVOUS:
# We remove an e2e circuit as downloader. We pay the subsequent nodes in the downloader part of the e2e
# circuit. In addition, we pay for one hop seeder anonymity since we don't know the circuit length at
# the seeder side.
self.do_payout(
circuit_peer, circuit_id,
circuit.bytes_down * ((circuit.goal_hops * 2) + 1),
circuit.bytes_down)
if circuit.ctype == CIRCUIT_TYPE_DATA:
# We remove a regular data circuit as downloader. Pay the relay nodes and the exit nodes.
self.do_payout(
circuit_peer, circuit_id,
circuit.bytes_down * (circuit.goal_hops * 2 - 1),
circuit.bytes_down)
# Reset the circuit byte counters so we do not payout again if we receive a destroy message.
circuit.bytes_up = circuit.bytes_down = 0
def update_torrents(_):
affected_peers = self.dispatcher.circuit_dead(circuit)
ltmgr = self.tribler_session.lm.ltmgr \
if self.tribler_session and self.tribler_session.config.get_libtorrent_enabled() else None
if ltmgr:
for d, s in ltmgr.torrents.values():
if s == ltmgr.get_session(d.get_hops()):
d.get_handle().addCallback(
lambda handle, download=d: self.update_torrent(
affected_peers, handle, download))
# Now we actually remove the circuit
remove_deferred = super(TriblerTunnelCommunity, self)\
.remove_circuit(circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy)
remove_deferred.addCallback(update_torrents)
return remove_deferred
def remove_relay(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False,
got_destroy_from=None,
both_sides=True):
removed_relays = super(TriblerTunnelCommunity, self).remove_relay(
circuit_id,
additional_info=additional_info,
remove_now=remove_now,
destroy=destroy,
got_destroy_from=got_destroy_from,
both_sides=both_sides)
self.clean_from_slots(circuit_id)
if self.tribler_session:
for removed_relay in removed_relays:
self.tribler_session.notifier.notify(
NTFY_TUNNEL, NTFY_REMOVE, removed_relay,
removed_relay.peer.address)
def remove_exit_socket(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False):
if circuit_id in self.exit_sockets and self.tribler_session:
exit_socket = self.exit_sockets[circuit_id]
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE,
exit_socket,
exit_socket.peer.address)
self.clean_from_slots(circuit_id)
super(TriblerTunnelCommunity,
self).remove_exit_socket(circuit_id,
additional_info=additional_info,
remove_now=remove_now,
destroy=destroy)
def _ours_on_created_extended(self, circuit, payload):
super(TriblerTunnelCommunity,
self)._ours_on_created_extended(circuit, payload)
if circuit.state == CIRCUIT_STATE_READY:
# Re-add BitTorrent peers, if needed.
self.readd_bittorrent_peers()
if self.tribler_session:
self.tribler_session.notifier.notify(
NTFY_TUNNEL,
NTFY_CREATED if len(circuit.hops) == 1 else NTFY_EXTENDED,
circuit)
def join_circuit(self, create_payload, previous_node_address):
super(TriblerTunnelCommunity,
self).join_circuit(create_payload, previous_node_address)
if self.tribler_session:
circuit_id = create_payload.circuit_id
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_JOINED,
previous_node_address,
circuit_id)
def on_raw_data(self, circuit, origin, data):
anon_seed = circuit.ctype == CIRCUIT_TYPE_RP
self.dispatcher.on_incoming_from_tunnel(self, circuit, origin, data,
anon_seed)
def monitor_downloads(self, dslist):
# Monitor downloads with anonymous flag set, and build rendezvous/introduction points when needed.
new_states = {}
hops = {}
for ds in dslist:
download = ds.get_download()
if download.get_hops() > 0:
# Convert the real infohash to the infohash used for looking up introduction points
real_info_hash = download.get_def().get_infohash()
info_hash = self.get_lookup_info_hash(real_info_hash)
hops[info_hash] = download.get_hops()
self.service_callbacks[info_hash] = download.add_peer
new_states[info_hash] = ds.get_status()
self.hops = hops
for info_hash in set(new_states.keys() + self.download_states.keys()):
new_state = new_states.get(info_hash, None)
old_state = self.download_states.get(info_hash, None)
state_changed = new_state != old_state
# Stop creating introduction points if the download doesn't exist anymore
if info_hash in self.infohash_ip_circuits and new_state is None:
del self.infohash_ip_circuits[info_hash]
# If the introducing circuit does not exist anymore or timed out: Build a new circuit
if info_hash in self.infohash_ip_circuits:
for (circuit_id,
time_created) in self.infohash_ip_circuits[info_hash]:
if circuit_id not in self.my_intro_points and time_created < time.time(
) - 30:
self.infohash_ip_circuits[info_hash].remove(
(circuit_id, time_created))
if self.tribler_session.notifier:
self.tribler_session.notifier.notify(
NTFY_TUNNEL, NTFY_IP_RECREATE, circuit_id,
info_hash.encode('hex')[:6])
self.logger.info(
'Recreate the introducing circuit for %s',
info_hash.encode('hex'))
self.create_introduction_point(info_hash)
time_elapsed = (time.time() -
self.last_dht_lookup.get(info_hash, 0))
force_dht_lookup = time_elapsed >= self.settings.dht_lookup_interval
if (state_changed or
force_dht_lookup) and (new_state == DLSTATUS_DOWNLOADING):
self.logger.info(
'Do dht lookup to find hidden services peers for %s',
info_hash.encode('hex'))
self.do_raw_dht_lookup(info_hash)
if state_changed and new_state == DLSTATUS_SEEDING:
self.create_introduction_point(info_hash)
elif state_changed and new_state in [DLSTATUS_STOPPED, None]:
if info_hash in self.infohash_pex:
self.infohash_pex.pop(info_hash)
for cid, info_hash_hops in self.my_download_points.items():
if info_hash_hops[0] == info_hash:
self.remove_circuit(cid,
'download stopped',
destroy=True)
for cid, info_hash_list in self.my_intro_points.items():
for i in xrange(len(info_hash_list) - 1, -1, -1):
if info_hash_list[i] == info_hash:
info_hash_list.pop(i)
if len(info_hash_list) == 0:
self.remove_circuit(cid,
'all downloads stopped',
destroy=True)
self.download_states = new_states
def get_download(self, lookup_info_hash):
if not self.tribler_session:
return None
for download in self.tribler_session.get_downloads():
if lookup_info_hash == self.get_lookup_info_hash(
download.get_def().get_infohash()):
return download
def create_introduction_point(self, info_hash, amount=1):
download = self.get_download(info_hash)
if download:
download.add_peer(('1.1.1.1', 1024))
super(TriblerTunnelCommunity,
self).create_introduction_point(info_hash, amount)
def on_linked_e2e(self, source_address, data, circuit_id):
payload = self._ez_unpack_noauth(LinkedE2EPayload,
data,
global_time=False)
cache = self.request_cache.get(u"link-request", payload.identifier)
if cache:
download = self.get_download(cache.info_hash)
if download:
download.add_peer(
(self.circuit_id_to_ip(cache.circuit.circuit_id), 1024))
else:
self.logger.error('On linked e2e: could not find download!')
super(TriblerTunnelCommunity,
self).on_linked_e2e(source_address, data, circuit_id)
@inlineCallbacks
def unload(self):
if self.bandwidth_wallet:
self.bandwidth_wallet.shutdown_task_manager()
for socks_server in self.socks_servers:
yield socks_server.stop()
if self.exitnode_cache:
self.cache_exitnodes_to_disk()
super(TriblerTunnelCommunity, self).unload()
def _generate_peer():
return Peer(default_eccrypto.generate_key(u"very-low"))
class TriblerTunnelCommunity(HiddenTunnelCommunity):
master_peer = Peer(
"3081a7301006072a8648ce3d020106052b81040027038192000402e1cd2a8158c078f5a048dd2caa4a868852e1758"
"71c819947b2aabe414a6b1b6c35e89f554dd94b475c612a692a3132bbe4a30813702acd7647eb8023700dcda5b47d"
"fe15f94a88049c2bb05f83f37d2cd85cce5efb8a9da6ac97dcdf97f83ae8696ffd1fab783ed28d004a99942fba756"
"8a3edc2052ce379db4b3f40411d55c28e16466e9750038c677bb561eab325".decode(
'hex'))
def __init__(self, *args, **kwargs):
self.tribler_session = kwargs.pop('tribler_session', None)
self.triblerchain_community = kwargs.pop('triblerchain_community',
None)
num_competing_slots = kwargs.pop('competing_slots', 15)
num_random_slots = kwargs.pop('random_slots', 5)
socks_listen_ports = kwargs.pop('socks_listen_ports', None)
super(TriblerTunnelCommunity, self).__init__(*args, **kwargs)
self._use_main_thread = True
if self.tribler_session:
self.settings.become_exitnode = self.tribler_session.config.get_tunnel_community_exitnode_enabled(
)
self.tribler_session.lm.tunnel_community = self
if not socks_listen_ports:
socks_listen_ports = self.tribler_session.config.get_tunnel_community_socks5_listen_ports(
)
elif socks_listen_ports is None:
socks_listen_ports = range(1080, 1085)
self.bittorrent_peers = {}
self.dispatcher = TunnelDispatcher(self)
self.download_states = {}
self.competing_slots = [
(0, None)
] * num_competing_slots # 1st tuple item = token balance, 2nd = circuit id
self.random_slots = [None] * num_random_slots
# Start the SOCKS5 servers
self.socks_servers = []
for port in socks_listen_ports:
socks_server = Socks5Server(port, self.dispatcher)
socks_server.start()
self.socks_servers.append(socks_server)
self.dispatcher.set_socks_servers(self.socks_servers)
self.decode_map.update({
chr(23): self.on_payout_block,
})
self.decode_map_private.update({
chr(24):
self.on_balance_request_cell,
chr(25):
self.on_relay_balance_request_cell,
chr(26):
self.on_balance_response_cell,
chr(27):
self.on_relay_balance_response_cell,
})
message_to_payload[u"balance-request"] = (24, BalanceRequestPayload)
message_to_payload[u"relay-balance-request"] = (25,
BalanceRequestPayload)
message_to_payload[u"balance-response"] = (26, BalanceResponsePayload)
message_to_payload[u"relay-balance-response"] = (
27, BalanceResponsePayload)
SINGLE_HOP_ENC_PACKETS.append(u"balance-request")
SINGLE_HOP_ENC_PACKETS.append(u"balance-response")
def on_token_balance(self, circuit_id, balance):
"""
We received the token balance of a circuit initiator. Check whether we can allocate a slot to this user.
"""
if not self.request_cache.has(u"balance-request", circuit_id):
self.logger.warning(
"Received token balance without associated request cache!")
return
cache = self.request_cache.pop(u"balance-request", circuit_id)
lowest_balance = sys.maxint
lowest_index = -1
for ind, tup in enumerate(self.competing_slots):
if not tup[1]:
# The slot is empty, take it
self.competing_slots[ind] = (balance, circuit_id)
cache.balance_deferred.callback(True)
return
if tup[0] < lowest_balance:
lowest_balance = tup[0]
lowest_index = ind
if balance > lowest_balance:
# We kick this user out
old_circuit_id = self.competing_slots[lowest_index][1]
self.logger.info(
"Kicked out circuit %s (balance: %s) in favor of %s (balance: %s)",
old_circuit_id, lowest_balance, circuit_id, balance)
self.competing_slots[lowest_index] = (balance, circuit_id)
self.remove_relay(old_circuit_id, destroy=True)
self.remove_exit_socket(old_circuit_id, destroy=True)
cache.balance_deferred.callback(True)
else:
# We can't compete with the balances in the existing slots
cache.balance_deferred.callback(False)
def should_join_circuit(self, create_payload, previous_node_address):
"""
Check whether we should join a circuit. Returns a deferred that fires with a boolean.
"""
if self.settings.max_joined_circuits <= len(self.relay_from_to) + len(
self.exit_sockets):
self.logger.warning(
"too many relays (%d)",
(len(self.relay_from_to) + len(self.exit_sockets)))
return succeed(False)
# Check whether we have a random open slot, if so, allocate this to this request.
circuit_id = create_payload.circuit_id
for index, slot in enumerate(self.random_slots):
if not slot:
self.random_slots[index] = circuit_id
return succeed(True)
# No random slots but this user might be allocated a competing slot.
# Next, we request the token balance of the circuit initiator.
balance_deferred = Deferred()
self.request_cache.add(
BalanceRequestCache(self, circuit_id, balance_deferred))
# Temporarily add these values, otherwise we are unable to communicate with the previous hop.
self.directions[circuit_id] = EXIT_NODE
shared_secret, _, _ = self.crypto.generate_diffie_shared_secret(
create_payload.key)
self.relay_session_keys[
circuit_id] = self.crypto.generate_session_keys(shared_secret)
self.send_cell(
[Peer(create_payload.node_public_key, previous_node_address)],
u"balance-request", BalanceRequestPayload(circuit_id))
self.directions.pop(circuit_id, None)
self.relay_session_keys.pop(circuit_id, None)
return balance_deferred
def on_payout_block(self, source_address, data):
if not self.triblerchain_community:
self.logger.warning(
"Got payout while not having a TriblerChain community running!"
)
return
_, payload = self._ez_unpack_noauth(PayoutPayload, data)
peer = Peer(payload.public_key, source_address)
block = self.triblerchain_community.BLOCK_CLASS.from_payload(
payload, self.serializer)
self.triblerchain_community.process_half_block(block, peer)
# Send the next payout
if payload.circuit_id in self.relay_from_to and block.transaction[
'down'] > payload.base_amount:
relay = self.relay_from_to[payload.circuit_id]
circuit_peer = self.get_peer_from_mid(relay.mid)
if not circuit_peer:
self.logger.warning(
"%s Unable to find next peer %s for payout!", self.my_peer,
relay.mid.encode('hex'))
return
self.do_payout(circuit_peer, relay.circuit_id,
block.transaction['down'] - payload.base_amount * 2,
payload.base_amount)
def on_balance_request_cell(self, source_address, data, _):
_, payload = self._ez_unpack_noauth(BalanceRequestPayload, data)
circuit_id = payload.circuit_id
request = self.request_cache.get(u"anon-circuit", circuit_id)
if request.should_forward:
forwarding_relay = RelayRoute(request.from_circuit_id,
request.candidate_sock_addr,
mid=request.candidate_mid)
self.send_cell([forwarding_relay.sock_addr],
u"relay-balance-request",
BalanceRequestPayload(forwarding_relay.circuit_id))
else:
self.on_balance_request(payload)
def on_relay_balance_request_cell(self, source_address, data, _):
_, payload = self._ez_unpack_noauth(BalanceRequestPayload, data)
self.on_balance_request(payload)
def on_balance_request(self, payload):
"""
We received a balance request from a relay or exit node. Respond with the latest block in our chain.
"""
if not self.triblerchain_community:
return
# Get the latest block
latest_block = self.triblerchain_community.persistence.get_latest(
self.my_peer.public_key.key_to_bin())
if not latest_block:
latest_block = TriblerChainBlock()
latest_block.public_key = EMPTY_PK # We hide the public key
# We either send the response directly or relay the response to the last verified hop
circuit = self.circuits[payload.circuit_id]
if not circuit.hops:
self.increase_bytes_sent(
circuit,
self.send_cell([circuit.sock_addr], u"balance-response",
BalanceResponsePayload.from_half_block(
latest_block, circuit.circuit_id)))
else:
self.increase_bytes_sent(
circuit,
self.send_cell([circuit.sock_addr], u"relay-balance-response",
BalanceResponsePayload.from_half_block(
latest_block, circuit.circuit_id)))
def on_balance_response_cell(self, source_address, data, _):
_, payload = self._ez_unpack_noauth(BalanceResponsePayload, data)
block = TriblerChainBlock.from_payload(payload, self.serializer)
if not block.transaction:
self.on_token_balance(payload.circuit_id, 0)
else:
self.on_token_balance(
payload.circuit_id, block.transaction["total_up"] -
block.transaction["total_down"])
def on_relay_balance_response_cell(self, source_address, data, _):
_, payload = self._ez_unpack_noauth(BalanceResponsePayload, data)
block = TriblerChainBlock.from_payload(payload, self.serializer)
# At this point, we don't have the circuit ID of the follow-up hop. We have to iterate over the items in the
# request cache and find the link to the next hop.
for cache in self.request_cache._identifiers.values():
if isinstance(cache, ExtendRequestCache
) and cache.from_circuit_id == payload.circuit_id:
self.send_cell([cache.to_candidate_sock_addr],
u"balance-response",
BalanceResponsePayload.from_half_block(
block, cache.to_circuit_id))
def on_download_removed(self, download):
"""
This method is called when a download is removed. We check here whether we can stop building circuits for a
specific number of hops in case it hasn't been finished yet.
"""
if download.get_hops() > 0:
self.num_hops_by_downloads[download.get_hops()] -= 1
if self.num_hops_by_downloads[download.get_hops()] == 0:
self.circuits_needed[download.get_hops()] = 0
def readd_bittorrent_peers(self):
for torrent, peers in self.bittorrent_peers.items():
infohash = torrent.tdef.get_infohash().encode("hex")
for peer in peers:
self.logger.info("Re-adding peer %s to torrent %s", peer,
infohash)
torrent.add_peer(peer)
del self.bittorrent_peers[torrent]
def update_torrent(self, peers, handle, download):
peers = peers.intersection(handle.get_peer_info())
if peers:
if download not in self.bittorrent_peers:
self.bittorrent_peers[download] = peers
else:
self.bittorrent_peers[
download] = peers | self.bittorrent_peers[download]
# If there are active circuits, add peers immediately. Otherwise postpone.
if self.active_data_circuits():
self.readd_bittorrent_peers()
def get_peer_from_mid(self, mid):
circuit_peer = None
for peer in self.network.verified_peers:
if peer.mid == mid:
circuit_peer = peer
break
return circuit_peer
def do_payout(self, peer, circuit_id, amount, base_amount):
"""
Perform a payout to a specific peer.
:param peer: The peer to perform the payout to, usually the next node in the circuit.
:param circuit_id: The circuit id of the payout, used by the subsequent node.
:param amount: The amount to put in the transaction, multiplier of base_amount.
:param base_amount: The base amount for the payouts.
"""
self.logger.info("Sending payout of %d (base: %d) to %s (cid: %s)",
amount, base_amount, peer, circuit_id)
block = self.triblerchain_community.BLOCK_CLASS.create(
{
'up': 0,
'down': amount
},
self.triblerchain_community.persistence,
self.my_peer.public_key.key_to_bin(),
link_pk=peer.public_key.key_to_bin())
block.sign(self.my_peer.key)
self.triblerchain_community.persistence.add_block(block)
global_time = self.claim_global_time()
dist = GlobalTimeDistributionPayload(global_time).to_pack_list()
payload = PayoutPayload.from_half_block(block, circuit_id,
base_amount).to_pack_list()
packet = self._ez_pack(self._prefix, 23, [dist, payload], False)
self.send_packet([peer], u"payout", packet)
def clean_from_slots(self, circuit_id):
"""
Clean a specific circuit from the allocated slots.
"""
for ind, slot in enumerate(self.random_slots):
if slot == circuit_id:
self.random_slots[ind] = None
for ind, tup in enumerate(self.competing_slots):
if tup[1] == circuit_id:
self.competing_slots[ind] = (0, None)
def remove_circuit(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False):
if circuit_id not in self.circuits:
self.logger.warning(
"Circuit %d not found when trying to remove it", circuit_id)
return
circuit = self.circuits[circuit_id]
# Send the notification
if self.tribler_session:
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE,
circuit, circuit.sock_addr)
circuit_peer = self.get_peer_from_mid(circuit.mid)
if circuit.bytes_down >= 1024 * 1024 and self.triblerchain_community and circuit_peer:
# We should perform a payout of the removed circuit.
if circuit.ctype == CIRCUIT_TYPE_RENDEZVOUS:
# We remove an e2e circuit as downloader. We pay the subsequent nodes in the downloader part of the e2e
# circuit. In addition, we pay for one hop seeder anonymity since we don't know the circuit length at
# the seeder side.
self.do_payout(
circuit_peer, circuit_id,
circuit.bytes_down * ((circuit.goal_hops * 2) + 1),
circuit.bytes_down)
if circuit.ctype == CIRCUIT_TYPE_DATA:
# We remove a regular data circuit as downloader. Pay the relay nodes and the exit nodes.
self.do_payout(
circuit_peer, circuit_id,
circuit.bytes_down * (circuit.goal_hops * 2 - 1),
circuit.bytes_down)
def update_torrents(_):
affected_peers = self.dispatcher.circuit_dead(circuit)
ltmgr = self.tribler_session.lm.ltmgr \
if self.tribler_session and self.tribler_session.config.get_libtorrent_enabled() else None
if ltmgr:
for d, s in ltmgr.torrents.values():
if s == ltmgr.get_session(d.get_hops()):
d.get_handle().addCallback(
lambda handle, download=d: self.update_torrent(
affected_peers, handle, download))
# Now we actually remove the circuit
remove_deferred = super(TriblerTunnelCommunity, self)\
.remove_circuit(circuit_id, additional_info=additional_info, remove_now=remove_now, destroy=destroy)
remove_deferred.addCallback(update_torrents)
return remove_deferred
def remove_relay(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False,
got_destroy_from=None,
both_sides=True):
removed_relays = super(TriblerTunnelCommunity, self).remove_relay(
circuit_id,
additional_info=additional_info,
remove_now=remove_now,
destroy=destroy,
got_destroy_from=got_destroy_from,
both_sides=both_sides)
self.clean_from_slots(circuit_id)
if self.tribler_session:
for removed_relay in removed_relays:
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE,
removed_relay,
removed_relay.sock_addr)
def remove_exit_socket(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False):
if circuit_id in self.exit_sockets and self.tribler_session:
exit_socket = self.exit_sockets[circuit_id]
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE,
exit_socket,
exit_socket.sock_addr)
self.clean_from_slots(circuit_id)
super(TriblerTunnelCommunity,
self).remove_exit_socket(circuit_id,
additional_info=additional_info,
remove_now=remove_now,
destroy=destroy)
def _ours_on_created_extended(self, circuit, payload):
super(TriblerTunnelCommunity,
self)._ours_on_created_extended(circuit, payload)
if circuit.state == CIRCUIT_STATE_READY:
# Re-add BitTorrent peers, if needed.
self.readd_bittorrent_peers()
if self.tribler_session:
self.tribler_session.notifier.notify(
NTFY_TUNNEL,
NTFY_CREATED if len(circuit.hops) == 1 else NTFY_EXTENDED,
circuit)
def on_create(self, source_address, data, _):
_, payload = self._ez_unpack_noauth(CreatePayload, data)
if not self.check_create(payload):
return
circuit_id = payload.circuit_id
if self.tribler_session:
self.tribler_session.notifier.notify(NTFY_TUNNEL, NTFY_JOINED,
source_address, circuit_id)
super(TriblerTunnelCommunity, self).on_create(source_address, data, _)
def on_raw_data(self, circuit, origin, data):
anon_seed = circuit.ctype == CIRCUIT_TYPE_RP
self.dispatcher.on_incoming_from_tunnel(self, circuit, origin, data,
anon_seed)
@call_on_reactor_thread
def monitor_downloads(self, dslist):
# Monitor downloads with anonymous flag set, and build rendezvous/introduction points when needed.
new_states = {}
hops = {}
for ds in dslist:
download = ds.get_download()
if download.get_hops() > 0:
# Convert the real infohash to the infohash used for looking up introduction points
real_info_hash = download.get_def().get_infohash()
info_hash = self.get_lookup_info_hash(real_info_hash)
hops[info_hash] = download.get_hops()
self.service_callbacks[info_hash] = download.add_peer
new_states[info_hash] = ds.get_status()
self.hops = hops
for info_hash in set(new_states.keys() + self.download_states.keys()):
new_state = new_states.get(info_hash, None)
old_state = self.download_states.get(info_hash, None)
state_changed = new_state != old_state
# Stop creating introduction points if the download doesn't exist anymore
if info_hash in self.infohash_ip_circuits and new_state is None:
del self.infohash_ip_circuits[info_hash]
# If the introducing circuit does not exist anymore or timed out: Build a new circuit
if info_hash in self.infohash_ip_circuits:
for (circuit_id,
time_created) in self.infohash_ip_circuits[info_hash]:
if circuit_id not in self.my_intro_points and time_created < time.time(
) - 30:
self.infohash_ip_circuits[info_hash].remove(
(circuit_id, time_created))
if self.tribler_session.notifier:
self.tribler_session.notifier.notify(
NTFY_TUNNEL, NTFY_IP_RECREATE, circuit_id,
info_hash.encode('hex')[:6])
self.logger.info(
'Recreate the introducing circuit for %s',
info_hash.encode('hex'))
self.create_introduction_point(info_hash)
time_elapsed = (time.time() -
self.last_dht_lookup.get(info_hash, 0))
force_dht_lookup = time_elapsed >= self.settings.dht_lookup_interval
if (state_changed or force_dht_lookup) and \
(new_state == DLSTATUS_SEEDING or new_state == DLSTATUS_DOWNLOADING):
self.logger.info(
'Do dht lookup to find hidden services peers for %s',
info_hash.encode('hex'))
self.do_raw_dht_lookup(info_hash)
if state_changed and new_state == DLSTATUS_SEEDING:
self.create_introduction_point(info_hash)
elif state_changed and new_state in [DLSTATUS_STOPPED, None]:
if info_hash in self.infohash_pex:
self.infohash_pex.pop(info_hash)
for cid, info_hash_hops in self.my_download_points.items():
if info_hash_hops[0] == info_hash:
self.remove_circuit(cid,
'download stopped',
destroy=True)
for cid, info_hash_list in self.my_intro_points.items():
for i in xrange(len(info_hash_list) - 1, -1, -1):
if info_hash_list[i] == info_hash:
info_hash_list.pop(i)
if len(info_hash_list) == 0:
self.remove_circuit(cid,
'all downloads stopped',
destroy=True)
self.download_states = new_states
def get_download(self, lookup_info_hash):
if not self.tribler_session:
return None
for download in self.tribler_session.get_downloads():
if lookup_info_hash == self.get_lookup_info_hash(
download.get_def().get_infohash()):
return download
def create_introduction_point(self, info_hash, amount=1):
download = self.get_download(info_hash)
if download:
download.add_peer(('1.1.1.1', 1024))
super(TriblerTunnelCommunity,
self).create_introduction_point(info_hash, amount)
def on_linked_e2e(self, source_address, data, circuit_id):
_, payload = self._ez_unpack_noauth(LinkedE2EPayload, data)
cache = self.request_cache.get(u"link-request", payload.identifier)
if cache:
download = self.get_download(cache.info_hash)
if download:
download.add_peer(
(self.circuit_id_to_ip(cache.circuit.circuit_id), 1024))
else:
self.logger.error('On linked e2e: could not find download!')
super(TriblerTunnelCommunity,
self).on_linked_e2e(source_address, data, circuit_id)
@inlineCallbacks
def unload(self):
for socks_server in self.socks_servers:
yield socks_server.stop()
super(TriblerTunnelCommunity, self).unload()
def get_peer_from_auth(self, auth, source_address):
"""
Get Peer object from the message and auth and source_address.
It is used for mocking the peer in test.
"""
return Peer(auth.public_key_bin, source_address)
class PopularityCommunity(PubSubCommunity):
"""
Community for disseminating the content across the network. Follows publish-subscribe model.
"""
MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b810400270381920004073beff7002b6a9fc2824a3b1bbb1c4fc32546" \
"261e3ef7537874560346c5fdc0c17fe654f67d23b08cbb44141879f79a7a4c8deddf9beb4fbc7a0f02ee1586cc" \
"ebedb623eeef51710108d702f9250361c071482e83c0a4a86c8f45a0b13a19ef83eacb6267b4bfccf220ae5f6d" \
"1db7125ea1d10da3744b65679828f23376e28b76ab33132b7fa984a77f159dba7351a7"
master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex'))
def __init__(self, *args, **kwargs):
self.torrent_db = kwargs.pop('torrent_db', None)
self.channel_db = kwargs.pop('channel_db', None)
self.trustchain = kwargs.pop('trustchain_community', None)
self.tribler_session = kwargs.pop('session', None)
super(PopularityCommunity, self).__init__(*args, **kwargs)
self.content_repository = ContentRepository(self.torrent_db, self.channel_db)
self.decode_map.update({
chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response,
chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response,
chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request,
chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response,
chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request,
chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response
})
self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX'))
@inlineCallbacks
def unload(self):
self.content_repository.cleanup()
self.content_repository = None
yield super(PopularityCommunity, self).unload()
def on_subscribe(self, source_address, data):
auth, _, _ = self._ez_unpack_auth(ContentSubscription, data)
peer = self.get_peer_from_auth(auth, source_address)
subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data)
# Publish the latest torrents to the subscriber
if subscribed:
self.publish_latest_torrents(peer=peer)
def on_torrent_health_response(self, source_address, data):
"""
Message handler for torrent health response. Torrent health response is part of periodic update message from
the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply
dropped. In other case, a decision to accept or reject the message is made based on freshness of the message
and the trustscore (check update_torrent in ContentRepository for the implementation).
"""
self.logger.info("Got torrent health response from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.publishers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
infohash = payload.infohash
if not self.content_repository.has_torrent(infohash):
self.send_torrent_info_request(infohash, peer=peer)
peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0
self.content_repository.update_torrent_health(payload, peer_trust)
def on_channel_health_response(self, source_address, data):
"""
Message handler for channel health response. Currently, not sure how to handle it.
"""
def on_torrent_info_request(self, source_address, data):
"""
Message handler for torrent info request.
"""
self.logger.info("Got torrent info request from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentInfoRequestPayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.subscribers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
self.send_torrent_info_response(payload.infohash, peer=peer)
def on_torrent_info_response(self, source_address, data):
"""
Message handler for torrent info response.
"""
self.logger.info("Got torrent info response from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentInfoResponsePayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.publishers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
self.content_repository.update_torrent_info(payload)
def on_content_info_request(self, source_address, data):
auth, _, payload = self._ez_unpack_auth(ContentInfoRequest, data)
peer = self.get_peer_from_auth(auth, source_address)
if payload.content_type == SEARCH_TORRENT_REQUEST:
db_results = self.content_repository.search_torrent(payload.query_list)
self.send_content_info_response(peer, payload.identifier, SEARCH_TORRENT_RESPONSE, db_results)
def on_content_info_response(self, source_address, data):
_, _, payload = self._ez_unpack_auth(ContentInfoResponse, data)
identifier = int(payload.identifier)
if not self.request_cache.has(u'request', identifier):
return
cache = self.request_cache.get(u'request', identifier)
if payload.content_type == SEARCH_TORRENT_RESPONSE:
self.process_torrent_search_response(cache.query, payload)
if not payload.pagination.more:
cache = self.request_cache.pop(u'request', identifier)
cache.finish()
def process_torrent_search_response(self, query, payload):
item_format = SearchResponseItemPayload.format_list
response, _ = self.serializer.unpack_multiple_as_list(item_format, payload.response)
# Decode the category string to list
for response_item in response:
response_item[4] = decode_values(response_item[4])
self.content_repository.update_from_torrent_search_results(response)
result_dict = dict()
result_dict['keywords'] = query
result_dict['results'] = response
result_dict['candidate'] = None
if self.tribler_session:
self.tribler_session.notifier.notify(SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None,
result_dict)
# MESSAGE SENDING FUNCTIONS
def send_torrent_health_response(self, payload, peer=None):
"""
Method to send torrent health response. This message is sent to all the subscribers by default but if a
peer is specified then only that peer receives this message.
"""
if peer and peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload)
self.broadcast_message(packet, peer=peer)
def send_channel_health_response(self, payload, peer=None):
"""
Method to send channel health response. This message is sent to all the subscribers by default but if a
peer is specified then only that peer receives this message.
"""
if peer and peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
packet = self.create_message_packet(MSG_CHANNEL_HEALTH_RESPONSE, payload)
self.broadcast_message(packet, peer=peer)
def send_torrent_info_request(self, infohash, peer):
"""
Method to request information about a torrent with given infohash to a peer.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
info_request = TorrentInfoRequestPayload(infohash)
packet = self.create_message_packet(MSG_TORRENT_INFO_REQUEST, info_request)
self.broadcast_message(packet, peer=peer)
def send_torrent_info_response(self, infohash, peer):
"""
Method to send information about a torrent with given infohash to the requesting peer.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
db_torrent = self.content_repository.get_torrent(infohash)
info_response = TorrentInfoResponsePayload(infohash, db_torrent['name'], db_torrent['length'],
db_torrent['creation_date'], db_torrent['num_files'],
db_torrent['comment'])
packet = self.create_message_packet(MSG_TORRENT_INFO_RESPONSE, info_response)
self.broadcast_message(packet, peer=peer)
def send_content_info_request(self, content_type, request_list, limit=25, peer=None):
"""
Sends the generic content request of given content_type.
:param content_type: request content type
:param request_list: List<string> request queries
:param limit: Number of expected responses
:param peer: Peer to send this request to
:return a Deferred that fires when we receive the content
:rtype Deferred
"""
cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list))
self.logger.info("Sending search request query:%s, identifier:%s", request_list, cache.number)
content_request = ContentInfoRequest(cache.number, content_type, request_list, limit)
packet = self.create_message_packet(MSG_CONTENT_INFO_REQUEST, content_request)
if peer:
self.broadcast_message(packet, peer=peer)
else:
for connected_peer in self.get_peers():
self.broadcast_message(packet, peer=connected_peer)
return cache.deferred
def send_content_info_response(self, peer, identifier, content_type, response_list):
"""
Sends the generic content info response with payload response list.
:param peer: Receiving peer
:param identifier: Request identifier
:param content_type: Message content type
:param response_list: Content response
"""
num_results = len(response_list)
current_index = 0
page_num = 1
while current_index < num_results:
serialized_results, current_index, page_size = self.pack_sized(response_list, MAX_PACKET_PAYLOAD_SIZE,
start_index=current_index)
if not serialized_results:
self.logger.info("Item too big probably to fit into package. Skipping it")
current_index += 1
else:
pagination = Pagination(page_num, page_size, num_results, more=current_index == num_results)
response_payload = ContentInfoResponse(identifier, content_type, serialized_results, pagination)
packet = self.create_message_packet(MSG_CONTENT_INFO_RESPONSE, response_payload)
self.broadcast_message(packet, peer=peer)
def send_torrent_search_request(self, query):
"""
Sends torrent search query as a content info request with content_type as SEARCH_TORRENT_REQUEST.
"""
self.send_content_info_request(SEARCH_TORRENT_REQUEST, query)
def send_channel_search_request(self, query):
"""
Sends channel search query to All Channel 2.0 to get a list of channels.
"""
# TODO: Not implemented yet. Waiting for All Channel 2.0
# CONTENT REPOSITORY STUFFS
def publish_next_content(self):
"""
Publishes the next content from the queue to the subscribers.
Does nothing if there are none subscribers.
Only Torrent health response is published at the moment.
"""
self.logger.info("Content to publish: %d", self.content_repository.count_content())
if not self.subscribers:
self.logger.info("No subscribers found. Not publishing anything")
return
content_type, content = self.content_repository.pop_content()
if content_type is None:
self.logger.error(ERROR_NO_CONTENT)
return
self.logger.info("Publishing content[type:%d]", content_type)
if content_type == TYPE_TORRENT_HEALTH:
infohash, seeders, leechers, timestamp = content
payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp)
self.send_torrent_health_response(payload)
def publish_latest_torrents(self, peer):
"""
Publishes the latest torrents in local database to the given peer.
"""
torrents = self.content_repository.get_top_torrents()
self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer)
for torrent in torrents:
infohash, seeders, leechers, timestamp = torrent[:4]
payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp)
self.send_torrent_health_response(payload, peer=peer)
def queue_content(self, content_type, content):
"""
Basically adds a given content to the queue of content repository.
"""
self.content_repository.add_content(content_type, content)