def __init__(self, config, core_test_mode=False):
"""
A Session object is created
Only a single session instance can exist at a time in a process.
:config TriblerConfig object parametrizing the Session
"""
super().__init__()
get_event_loop().set_exception_handler(self.unhandled_error_observer)
patch_crypto_be_discovery()
self._logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.notifier = Notifier()
self.upgrader_enabled = True
self.upgrader = None
self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler
self.ipv8 = None
self.ipv8_start_time = 0
self._logger = logging.getLogger(self.__class__.__name__)
self.shutdownstarttime = None
self.bootstrap = None
# modules
self.ipv8_community_loader = IPv8CommunityLoader()
register_default_launchers(self.ipv8_community_loader)
self.api_manager = None
self.watch_folder = None
self.version_check_manager = None
self.resource_monitor = None
self.gigachannel_manager = None
self.dlmgr = None # Libtorrent Manager
self.tracker_manager = None
self.torrent_checker = None
self.tunnel_community = None
self.bandwidth_community = None
self.wallets = {}
self.popularity_community = None
self.gigachannel_community = None
self.dht_community = None
self.payout_manager = None
self.mds = None # Metadata Store
# In test mode, the Core does not communicate with the external world and the state dir is read-only
self.core_test_mode = core_test_mode
async def test_notifier_add_observer(notifier: Notifier):
def callback():
...
# test that add observer stores topics and callbacks as a set to prevent duplicates
notifier.add_observer('topic', callback)
notifier.add_observer('topic', callback)
assert len(notifier.observers['topic']) == 1
async def test_notifier_remove_observer(notifier: Notifier):
def callback1():
...
def callback2():
...
notifier.add_observer('topic', callback1)
notifier.add_observer('topic', callback2)
notifier.remove_observer('topic', callback1)
assert notifier.observers['topic'] == {callback2: True}
def __init__(self, config):
"""
A Session object is created
Only a single session instance can exist at a time in a process.
:config TriblerConfig object parametrising the Session
"""
super(Session, self).__init__()
get_event_loop().set_exception_handler(self.unhandled_error_observer)
patch_crypto_be_discovery()
self._logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.notifier = Notifier()
self.upgrader_enabled = True
self.upgrader = None
self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler
self.ipv8 = None
self.ipv8_start_time = 0
self._logger = logging.getLogger(self.__class__.__name__)
self.shutdownstarttime = None
self.bootstrap = None
# modules
self.api_manager = None
self.watch_folder = None
self.version_check_manager = None
self.resource_monitor = None
self.gigachannel_manager = None
self.dlmgr = None # Libtorrent Manager
self.tracker_manager = None
self.torrent_checker = None
self.tunnel_community = None
self.trustchain_community = None
self.wallets = {}
self.popularity_community = None
self.gigachannel_community = None
self.remote_query_community = None
self.market_community = None
self.dht_community = None
self.payout_manager = None
self.mds = None # Metadata Store
async def test_notify(notifier: Notifier):
# test that notify works as expected
normal_callback = TestCallback()
notifier.add_observer('topic', normal_callback.callback)
notifier.notify('topic', 'arg', kwarg='value')
# wait for the callback
await normal_callback.event.wait()
assert normal_callback.callback_has_been_called
assert normal_callback.callback_has_been_called_with_args == ('arg', )
assert normal_callback.callback_has_been_called_with_kwargs == {
'kwarg': 'value'
}
async def fake_dlmgr(tmpdir):
tribler_session = Mock()
tribler_session.notifier = Notifier()
tribler_session.state_dir = tmpdir
tribler_session.notify_shutdown_state = lambda _: None
tribler_session.config = Mock()
tribler_session.config.get_libtorrent_utp = lambda: True
tribler_session.config.get_libtorrent_proxy_settings = lambda: (0, None, None)
tribler_session.config.get_anon_proxy_settings = lambda: (2, ('127.0.0.1', [1338]), None)
tribler_session.config.get_libtorrent_port = lambda: 1337
tribler_session.config.get_anon_listen_port = lambda: 1338
tribler_session.config.get_state_dir = lambda: tmpdir
tribler_session.config.set_listen_port_runtime = lambda: None
tribler_session.config.get_libtorrent_max_upload_rate = lambda: 100
tribler_session.config.get_libtorrent_max_download_rate = lambda: 120
tribler_session.config.get_libtorrent_dht_enabled = lambda: False
tribler_session.config.set_libtorrent_port_runtime = lambda _: None
tribler_session.config.get_libtorrent_max_conn_download = lambda: 0
tribler_session.config.get_default_number_hops = lambda: 1
tribler_session.config.get_libtorrent_dht_readiness_timeout = lambda: 0
dlmgr = DownloadManager(tribler_session)
dlmgr.metadata_tmpdir = tmpdir
dlmgr.get_session = lambda *_, **__: Mock()
tribler_session.dlmgr = dlmgr
tribler_session.tunnel_community = None
yield dlmgr
await dlmgr.shutdown(timeout=0)
async def setUp(self):
await super(TestDownloadManager, self).setUp()
self.tribler_session = MockObject()
self.tribler_session.notifier = Notifier()
self.tribler_session.state_dir = self.session_base_dir
self.tribler_session.trustchain_keypair = MockObject()
self.tribler_session.trustchain_keypair.key_to_hash = lambda: b'a' * 20
self.tribler_session.notify_shutdown_state = lambda _: None
self.tribler_session.config = MockObject()
self.tribler_session.config.get_libtorrent_utp = lambda: True
self.tribler_session.config.get_libtorrent_proxy_settings = lambda: (0, None, None)
self.tribler_session.config.get_anon_proxy_settings = lambda: (2, ('127.0.0.1', [1338]), None)
self.tribler_session.config.get_libtorrent_port = lambda: 1337
self.tribler_session.config.get_anon_listen_port = lambda: 1338
self.tribler_session.config.get_state_dir = lambda: self.session_base_dir
self.tribler_session.config.set_listen_port_runtime = lambda: None
self.tribler_session.config.get_libtorrent_max_upload_rate = lambda: 100
self.tribler_session.config.get_libtorrent_max_download_rate = lambda: 120
self.tribler_session.config.get_libtorrent_dht_enabled = lambda: False
self.tribler_session.config.set_libtorrent_port_runtime = lambda _: None
self.tribler_session.config.get_libtorrent_max_conn_download = lambda: 0
self.tribler_session.config.get_default_number_hops = lambda: 1
self.tribler_session.config.get_libtorrent_dht_readiness_timeout = lambda: 0
self.dlmgr = DownloadManager(self.tribler_session)
self.dlmgr.metadata_tmpdir = mkdtemp(suffix=u'tribler_metainfo_tmpdir')
self.tribler_session.dlmgr = self.dlmgr
self.tribler_session.tunnel_community = None
async def test_remote_select_subscribed_channels(self):
"""
Test querying remote peers for subscribed channels and updating local votes accordingly.
"""
# We do not want the query back mechanism to interfere with this test
self.nodes[1].overlay.rqc_settings.max_channel_query_back = 0
num_channels = 5
with db_session:
# Create one channel with zero contents, to check that only non-empty channels are served
self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel sub", "")
# Create one channel that has not yet been processed (with local_version<timestamp)
incomplete_chan = self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel sub", "")
incomplete_chan.num_entries = 10
incomplete_chan.sign()
for _ in range(0, num_channels):
chan = self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel sub", "")
chan.local_version = chan.timestamp
chan.num_entries = 10
chan.sign()
for _ in range(0, num_channels):
channel_uns = self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel unsub", "")
channel_uns.subscribed = False
def mock_notify(overlay, args):
overlay.notified_results = True
self.assertTrue("results" in args)
self.nodes[1].overlay.notifier = Notifier()
self.nodes[1].overlay.notifier.notify = lambda sub, args: mock_notify(self.nodes[1].overlay, args)
peer = self.nodes[0].my_peer
await self.introduce_nodes()
await self.deliver_messages(timeout=0.5)
with db_session:
received_channels = self.nodes[1].overlay.mds.ChannelMetadata.select(lambda g: g.title == "channel sub")
self.assertEqual(num_channels, received_channels.count())
# Only subscribed channels should have been transported
received_channels_all = self.nodes[1].overlay.mds.ChannelMetadata.select()
self.assertEqual(num_channels, received_channels_all.count())
# Make sure the subscribed channels transport counted as voting
self.assertEqual(
self.nodes[1].overlay.mds.ChannelPeer.select().first().public_key, peer.public_key.key_to_bin()[10:]
)
for chan in self.nodes[1].overlay.mds.ChannelMetadata.select():
self.assertTrue(chan.votes > 0.0)
# Check that the notifier callback is called on new channel entries
self.assertTrue(self.nodes[1].overlay.notified_results)
async def test_notify_call_soon_threadsafe_with_exception(notifier: Notifier):
notifier.logger = Mock()
notifier._loop = Mock(call_soon_threadsafe=Mock(side_effect=RuntimeError))
notifier.add_observer('topic', lambda:...)
notifier.notify('topic')
notifier.logger.warning.assert_called_once()
async def test_gigachannel_search(self):
"""
Test searching several nodes for metadata entries based on title text
"""
# We do not want the query back mechanism and introduction callback to interfere with this test
for node in self.nodes:
node.overlay.settings.max_channel_query_back = 0
await self.introduce_nodes()
U_CHANNEL = "ubuntu channel"
U_TORRENT = "ubuntu torrent"
# Add test metadata to node 0
with db_session:
self.nodes[0].overlay.mds.ChannelMetadata.create_channel(
U_CHANNEL, "")
self.nodes[0].overlay.mds.ChannelMetadata.create_channel(
"debian channel", "")
# Add test metadata to node 1
with db_session:
self.nodes[1].overlay.mds.TorrentMetadata(
title=U_TORRENT, infohash=random_infohash())
self.nodes[1].overlay.mds.TorrentMetadata(
title="debian torrent", infohash=random_infohash())
notification_calls = []
def mock_notify(_, args):
notification_calls.append(args)
self.nodes[2].overlay.notifier = Notifier()
self.nodes[2].overlay.notifier.notify = lambda sub, args: mock_notify(
self.nodes[2].overlay, args)
self.nodes[2].overlay.send_search_request(**{"txt_filter": "ubuntu*"})
await self.deliver_messages(timeout=0.5)
with db_session:
assert self.nodes[2].overlay.mds.ChannelNode.select().count() == 2
assert (self.nodes[2].overlay.mds.ChannelNode.select(
lambda g: g.title in (U_CHANNEL, U_TORRENT)).count() == 2)
# Check that the notifier callback was called on both entries
assert [U_CHANNEL, U_TORRENT] == sorted(
[c["results"][0]["name"] for c in notification_calls])
async def test_notify_with_exception(notifier: Notifier):
# test that notify works as expected even if one of callbacks will raise an exception
normal_callback = TestCallback()
side_effect_callback = TestCallback(ValueError)
notifier.add_observer('topic', side_effect_callback.callback)
notifier.add_observer('topic', normal_callback.callback)
notifier.add_observer('topic', side_effect_callback.callback)
notifier.notify('topic')
# wait
await asyncio.sleep(1)
assert normal_callback.callback_has_been_called
assert side_effect_callback.callback_has_been_called
def __init__(self,
config: TriblerConfig = None,
components: List[Component] = (),
shutdown_event: Event = None,
notifier: Notifier = None,
failfast: bool = True):
# deepcode ignore unguarded~next~call: not necessary to catch StopIteration on infinite iterator
self.id = next(Session._next_session_id)
self.failfast = failfast
self.logger = logging.getLogger(self.__class__.__name__)
self.config: TriblerConfig = config or TriblerConfig()
self.shutdown_event: Event = shutdown_event or Event()
self.notifier: Notifier = notifier or Notifier()
self.components: Dict[Type[Component], Component] = {}
for component in components:
self.register(component.__class__, component)
# Reserve various (possibly) fixed ports to prevent
# components from occupying those accidentally
reserve_ports([
config.libtorrent.port, config.api.http_port,
config.api.https_port, config.ipv8.port
])
async def test_notifier_remove_nonexistent_observer(notifier: Notifier):
# test that `remove_observer` don't crash in case of calling to remove non existed topic/callback
notifier.remove_observer('nonexistent', lambda: None)
assert not notifier.observers['nonexistent']
def fixture_notifier():
return Notifier()
async def test_notifier(self):
notifier = Notifier()
notifier.add_observer(NTFY.TORRENT_FINISHED, self.callback_func)
notifier.notify(NTFY.TORRENT_FINISHED)
await self.test_future
self.assertTrue(self.called_callback)
class Session(TaskManager):
"""
A Session is a running instance of the Tribler Core and the Core's central class.
"""
__single = None
def __init__(self, config, core_test_mode=False):
"""
A Session object is created
Only a single session instance can exist at a time in a process.
:config TriblerConfig object parametrizing the Session
"""
super().__init__()
get_event_loop().set_exception_handler(self.unhandled_error_observer)
patch_crypto_be_discovery()
self._logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.notifier = Notifier()
self.upgrader_enabled = True
self.upgrader = None
self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler
self.ipv8 = None
self.ipv8_start_time = 0
self._logger = logging.getLogger(self.__class__.__name__)
self.shutdownstarttime = None
self.bootstrap = None
# modules
self.ipv8_community_loader = IPv8CommunityLoader()
register_default_launchers(self.ipv8_community_loader)
self.api_manager = None
self.watch_folder = None
self.version_check_manager = None
self.resource_monitor = None
self.gigachannel_manager = None
self.dlmgr = None # Libtorrent Manager
self.tracker_manager = None
self.torrent_checker = None
self.tunnel_community = None
self.bandwidth_community = None
self.wallets = {}
self.popularity_community = None
self.gigachannel_community = None
self.dht_community = None
self.payout_manager = None
self.mds = None # Metadata Store
# In test mode, the Core does not communicate with the external world and the state dir is read-only
self.core_test_mode = core_test_mode
def load_ipv8_overlays(self):
self.ipv8_community_loader.load(self.ipv8, self)
def enable_ipv8_statistics(self):
if self.config.get_ipv8_statistics():
for overlay in self.ipv8.overlays:
self.ipv8.endpoint.enable_community_statistics(overlay.get_prefix(), True)
def import_bootstrap_file(self):
with open(self.bootstrap.bootstrap_file) as f:
f.read()
self._logger.info("Executing bootstrap script")
# TODO we should do something here...
async def start_bootstrap_download(self):
if not self.payout_manager:
self._logger.warning("Running bootstrap without payout enabled")
from tribler_core.modules.bootstrap import Bootstrap
self.bootstrap = Bootstrap(self.config.get_state_dir(), dht=self.dht_community)
self.bootstrap.start_by_infohash(self.dlmgr.start_download, self.config.get_bootstrap_infohash())
await self.bootstrap.download.future_finished
# Temporarily disabling SQL import for experimental release
# await get_event_loop().run_in_executor(None, self.import_bootstrap_file)
self.bootstrap.bootstrap_finished = True
def create_state_directory_structure(self):
"""Create directory structure of the state directory."""
def create_dir(path):
if not path.is_dir():
os.makedirs(path)
def create_in_state_dir(path):
create_dir(self.config.get_state_dir() / path)
create_dir(self.config.get_state_dir())
create_in_state_dir(STATEDIR_DB_DIR)
create_in_state_dir(STATEDIR_CHECKPOINT_DIR)
create_in_state_dir(STATEDIR_CHANNELS_DIR)
def get_ports_in_config(self):
"""Claim all required random ports."""
if self.core_test_mode:
self.config.selected_ports = {}
return
self.config.get_libtorrent_port()
self.config.get_anon_listen_port()
self.config.get_tunnel_community_socks5_listen_ports()
def init_keypair(self):
"""
Set parameters that depend on state_dir.
"""
trustchain_pairfilename = self.config.get_trustchain_keypair_filename()
if trustchain_pairfilename.exists():
self.trustchain_keypair = permid_module.read_keypair_trustchain(trustchain_pairfilename)
else:
self.trustchain_keypair = permid_module.generate_keypair_trustchain()
# Save keypair
trustchain_pubfilename = self.config.get_state_dir() / 'ecpub_multichain.pem'
permid_module.save_keypair_trustchain(self.trustchain_keypair, trustchain_pairfilename)
permid_module.save_pub_key_trustchain(self.trustchain_keypair, trustchain_pubfilename)
trustchain_testnet_pairfilename = self.config.get_trustchain_testnet_keypair_filename()
if trustchain_testnet_pairfilename.exists():
self.trustchain_testnet_keypair = permid_module.read_keypair_trustchain(trustchain_testnet_pairfilename)
else:
self.trustchain_testnet_keypair = permid_module.generate_keypair_trustchain()
# Save keypair
trustchain_testnet_pubfilename = self.config.get_state_dir() / 'ecpub_trustchain_testnet.pem'
permid_module.save_keypair_trustchain(self.trustchain_testnet_keypair, trustchain_testnet_pairfilename)
permid_module.save_pub_key_trustchain(self.trustchain_testnet_keypair, trustchain_testnet_pubfilename)
def unhandled_error_observer(self, loop, context):
"""
This method is called when an unhandled error in Tribler is observed.
It broadcasts the tribler_exception event.
"""
try:
SentryReporter.ignore_logger(self._logger.name)
exception = context.get('exception')
ignored_message = None
try:
ignored_message = IGNORED_ERRORS.get(
(exception.__class__, exception.errno),
IGNORED_ERRORS.get(exception.__class__))
except (ValueError, AttributeError):
pass
if ignored_message is not None:
self._logger.error(ignored_message if ignored_message != "" else context.get('message'))
return
text = str(exception or context.get('message'))
# We already have a check for invalid infohash when adding a torrent, but if somehow we get this
# error then we simply log and ignore it.
if isinstance(exception, RuntimeError) and 'invalid info-hash' in text:
self._logger.error("Invalid info-hash found")
return
text_long = text
exc = context.get('exception')
if exc:
with StringIO() as buffer:
print_exception(type(exc), exc, exc.__traceback__, file=buffer)
text_long = text_long + "\n--LONG TEXT--\n" + buffer.getvalue()
text_long = text_long + "\n--CONTEXT--\n" + str(context)
self._logger.error("Unhandled exception occurred! %s", text_long, exc_info=None)
sentry_event = SentryReporter.event_from_exception(exception)
if not self.api_manager:
return
events = self.api_manager.get_endpoint('events')
events.on_tribler_exception(text_long,
sentry_event,
self.config.get_core_error_reporting_requires_user_consent())
state = self.api_manager.get_endpoint('state')
state.on_tribler_exception(text_long, sentry_event)
except Exception as ex:
SentryReporter.capture_exception(ex)
raise ex
def get_tribler_statistics(self):
"""Return a dictionary with general Tribler statistics."""
return TriblerStatistics(self).get_tribler_statistics()
def get_ipv8_statistics(self):
"""Return a dictionary with IPv8 statistics."""
return TriblerStatistics(self).get_ipv8_statistics()
async def start(self):
"""
Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
Returns a deferred that fires when the Tribler session is ready for use.
:param config: a TriblerConfig object
"""
self._logger.info("Session is using state directory: %s", self.config.get_state_dir())
self.get_ports_in_config()
self.create_state_directory_structure()
self.init_keypair()
# we have to represent `user_id` as a string to make it equal to the
# `user_id` on the GUI side
user_id_str = hexlify(self.trustchain_keypair.key.pk).encode('utf-8')
SentryReporter.set_user(user_id_str)
# Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
if self.config.get_api_http_enabled() or self.config.get_api_https_enabled():
from tribler_core.restapi.rest_manager import RESTManager
self.api_manager = RESTManager(self)
self.readable_status = STATE_START_API
await self.api_manager.start()
if self.upgrader_enabled and not self.core_test_mode:
from tribler_core.upgrade.upgrade import TriblerUpgrader
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
await self.upgrader.run()
self.tracker_manager = TrackerManager(self)
# Start torrent checker before Popularity community is loaded
if self.config.get_torrent_checking_enabled() and not self.core_test_mode:
from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem')
if self.config.get_chant_enabled():
from tribler_core.modules.metadata_store.store import MetadataStore
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db'
database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair,
notifier=self.notifier,
disable_sync=self.core_test_mode)
if self.core_test_mode:
generate_test_channels(self.mds)
# IPv8
if self.config.get_ipv8_enabled():
from ipv8.configuration import ConfigBuilder
from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint
ipv8_config_builder = (ConfigBuilder()
.set_port(self.config.get_ipv8_port())
.set_address(self.config.get_ipv8_address())
.clear_overlays()
.clear_keys() # We load the keys ourselves
.set_working_directory(str(self.config.get_state_dir()))
.set_walker_interval(self.config.get_ipv8_walk_interval()))
if self.core_test_mode:
endpoint = DispatcherEndpoint([])
else:
# IPv8 includes IPv6 support by default.
# We only load IPv4 to not kill all Tribler overlays (currently, it would instantly crash all users).
# If you want to test IPv6 in Tribler you can set ``endpoint = None`` here.
endpoint = DispatcherEndpoint(["UDPIPv4"], UDPIPv4={'port': self.config.get_ipv8_port(),
'ip': self.config.get_ipv8_address()})
self.ipv8 = IPv8(ipv8_config_builder.finalize(),
enable_statistics=self.config.get_ipv8_statistics() and not self.core_test_mode,
endpoint_override=endpoint)
await self.ipv8.start()
self.config.set_anon_proxy_settings(2, ("127.0.0.1",
self.
config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
if not self.core_test_mode:
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
await self.tunnel_community.wait_for_socks_servers()
if self.config.get_ipv8_walk_scaling_enabled():
from tribler_core.modules.ipv8_health_monitor import IPv8Monitor
IPv8Monitor(self.ipv8,
self.config.get_ipv8_walk_interval(),
self.config.get_ipv8_walk_scaling_upper_limit()).start(self)
# Note that currently we should only start libtorrent after the SOCKS5 servers have been started
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
if self.core_test_mode:
await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000")
self.readable_status = STATE_READABLE_STARTED
if self.config.get_watch_folder_enabled():
from tribler_core.modules.watch_folder import WatchFolder
self.readable_status = STATE_START_WATCH_FOLDER
self.watch_folder = WatchFolder(self)
self.watch_folder.start()
if self.config.get_resource_monitor_enabled() and not self.core_test_mode:
from tribler_core.modules.resource_monitor.core import CoreResourceMonitor
self.resource_monitor = CoreResourceMonitor(self)
self.resource_monitor.start()
if self.config.get_version_checker_enabled() and not self.core_test_mode:
from tribler_core.modules.versioncheck_manager import VersionCheckManager
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()
if self.config.get_ipv8_enabled():
from tribler_core.modules.payout_manager import PayoutManager
self.payout_manager = PayoutManager(self.bandwidth_community, self.dht_community)
if self.core_test_mode:
from ipv8.messaging.interfaces.udp.endpoint import UDPv4Address
from ipv8.dht.routing import RoutingTable
self.dht_community.routing_tables[UDPv4Address] = RoutingTable('\x00' * 20)
# GigaChannel Manager should be started *after* resuming the downloads,
# because it depends on the states of torrent downloads
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \
and self.config.get_libtorrent_enabled:
from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager
self.gigachannel_manager = GigaChannelManager(self)
if not self.core_test_mode:
self.gigachannel_manager.start()
if self.config.get_bootstrap_enabled() and not self.core_test_mode:
self.register_task('bootstrap_download', self.start_bootstrap_download)
self.notifier.notify(NTFY.TRIBLER_STARTED, self.trustchain_keypair.key.pk)
# If there is a config error, report to the user via GUI notifier
if self.config.config_error:
self.notifier.notify(NTFY.REPORT_CONFIG_ERROR, self.config.config_error)
async def shutdown(self):
"""
Checkpoints the session and closes it, stopping the download engine.
This method has to be called from the reactor thread.
"""
self.shutdownstarttime = timemod.time()
# Indicates we are shutting down core. With this environment variable set
# to 'TRUE', RESTManager will no longer accepts any new requests.
os.environ['TRIBLER_SHUTTING_DOWN'] = "TRUE"
if self.dlmgr:
self.dlmgr.stop_download_states_callback()
await self.shutdown_task_manager()
if self.torrent_checker:
self.notify_shutdown_state("Shutting down Torrent Checker...")
await self.torrent_checker.shutdown()
self.torrent_checker = None
if self.gigachannel_manager:
self.notify_shutdown_state("Shutting down Gigachannel Manager...")
await self.gigachannel_manager.shutdown()
self.gigachannel_manager = None
if self.version_check_manager:
self.notify_shutdown_state("Shutting down Version Checker...")
await self.version_check_manager.stop()
self.version_check_manager = None
if self.resource_monitor:
self.notify_shutdown_state("Shutting down Resource Monitor...")
await self.resource_monitor.stop()
self.resource_monitor = None
if self.bootstrap:
# We shutdown the bootstrap module before IPv8 since it uses the DHTCommunity.
await self.bootstrap.shutdown()
self.bootstrap = None
self.tracker_manager = None
if self.tunnel_community and self.bandwidth_community:
# We unload these overlays manually since the TrustChain has to be unloaded after the tunnel overlay.
tunnel_community = self.tunnel_community
self.tunnel_community = None
self.notify_shutdown_state("Unloading Tunnel Community...")
await self.ipv8.unload_overlay(tunnel_community)
bandwidth_community = self.bandwidth_community
self.bandwidth_community = None
self.notify_shutdown_state("Shutting down Bandwidth Community...")
await self.ipv8.unload_overlay(bandwidth_community)
if self.ipv8:
self.notify_shutdown_state("Shutting down IPv8...")
await self.ipv8.stop(stop_loop=False)
self.ipv8 = None
if self.payout_manager:
await self.payout_manager.shutdown()
self.payout_manager = None
if self.watch_folder:
self.notify_shutdown_state("Shutting down Watch Folder...")
await self.watch_folder.stop()
self.watch_folder = None
if not self.core_test_mode:
self.notify_shutdown_state("Saving configuration...")
self.config.write()
if self.dlmgr:
await self.dlmgr.shutdown()
self.dlmgr = None
if self.mds:
self.notify_shutdown_state("Shutting down Metadata Store...")
self.mds.shutdown()
self.mds = None
# We close the API manager as late as possible during shutdown.
if self.api_manager:
self.notify_shutdown_state("Shutting down API Manager...")
await self.api_manager.stop()
self.api_manager = None
def notify_shutdown_state(self, state):
self._logger.info("Tribler shutdown state notification:%s", state)
self.notifier.notify(NTFY.TRIBLER_SHUTDOWN_STATE, state)
class Session(TaskManager):
"""
A Session is a running instance of the Tribler Core and the Core's central class.
"""
__single = None
def __init__(self, config):
"""
A Session object is created
Only a single session instance can exist at a time in a process.
:config TriblerConfig object parametrising the Session
"""
super(Session, self).__init__()
get_event_loop().set_exception_handler(self.unhandled_error_observer)
patch_crypto_be_discovery()
self._logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.notifier = Notifier()
self.upgrader_enabled = True
self.upgrader = None
self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler
self.ipv8 = None
self.ipv8_start_time = 0
self._logger = logging.getLogger(self.__class__.__name__)
self.shutdownstarttime = None
self.bootstrap = None
# modules
self.api_manager = None
self.watch_folder = None
self.version_check_manager = None
self.resource_monitor = None
self.gigachannel_manager = None
self.dlmgr = None # Libtorrent Manager
self.tracker_manager = None
self.torrent_checker = None
self.tunnel_community = None
self.trustchain_community = None
self.wallets = {}
self.popularity_community = None
self.gigachannel_community = None
self.remote_query_community = None
self.market_community = None
self.dht_community = None
self.payout_manager = None
self.mds = None # Metadata Store
def load_ipv8_overlays(self):
if self.config.get_testnet():
peer = Peer(self.trustchain_testnet_keypair)
else:
peer = Peer(self.trustchain_keypair)
discovery_community = DiscoveryCommunity(peer, self.ipv8.endpoint,
self.ipv8.network)
discovery_community.resolve_dns_bootstrap_addresses()
self.ipv8.overlays.append(discovery_community)
self.ipv8.strategies.append((RandomChurn(discovery_community), -1))
self.ipv8.strategies.append(
(PeriodicSimilarity(discovery_community), -1))
self.ipv8.strategies.append((RandomWalk(discovery_community), 20))
# TrustChain Community
if self.config.get_trustchain_enabled():
from ipv8.attestation.trustchain.community import TrustChainCommunity, \
TrustChainTestnetCommunity
community_cls = TrustChainTestnetCommunity if self.config.get_testnet(
) else TrustChainCommunity
self.trustchain_community = community_cls(
peer,
self.ipv8.endpoint,
self.ipv8.network,
working_directory=self.config.get_state_dir())
self.ipv8.overlays.append(self.trustchain_community)
self.ipv8.strategies.append(
(EdgeWalk(self.trustchain_community), 20))
tc_wallet = TrustchainWallet(self.trustchain_community)
self.wallets[tc_wallet.get_identifier()] = tc_wallet
# DHT Community
if self.config.get_dht_enabled():
from ipv8.dht.discovery import DHTDiscoveryCommunity
self.dht_community = DHTDiscoveryCommunity(peer,
self.ipv8.endpoint,
self.ipv8.network)
self.ipv8.overlays.append(self.dht_community)
self.ipv8.strategies.append((RandomWalk(self.dht_community), 20))
self.ipv8.strategies.append((PingChurn(self.dht_community), -1))
# Tunnel Community
if self.config.get_tunnel_community_enabled():
from tribler_core.modules.tunnel.community.triblertunnel_community import TriblerTunnelCommunity,\
TriblerTunnelTestnetCommunity
from tribler_core.modules.tunnel.community.discovery import GoldenRatioStrategy
community_cls = TriblerTunnelTestnetCommunity if self.config.get_testnet() else \
TriblerTunnelCommunity
random_slots = self.config.get_tunnel_community_random_slots()
competing_slots = self.config.get_tunnel_community_competing_slots(
)
dht_provider = DHTCommunityProvider(self.dht_community,
self.config.get_ipv8_port())
settings = TunnelSettings()
settings.min_circuits = 3
settings.max_circuits = 10
self.tunnel_community = community_cls(
peer,
self.ipv8.endpoint,
self.ipv8.network,
tribler_session=self,
dht_provider=dht_provider,
ipv8=self.ipv8,
bandwidth_wallet=self.wallets["MB"],
random_slots=random_slots,
competing_slots=competing_slots,
settings=settings)
self.ipv8.overlays.append(self.tunnel_community)
self.ipv8.strategies.append(
(RandomWalk(self.tunnel_community), 20))
self.ipv8.strategies.append(
(GoldenRatioStrategy(self.tunnel_community), -1))
# Market Community
if self.config.get_market_community_enabled(
) and self.config.get_dht_enabled():
from anydex.core.community import MarketCommunity, MarketTestnetCommunity
community_cls = MarketTestnetCommunity if self.config.get_testnet(
) else MarketCommunity
self.market_community = community_cls(
peer,
self.ipv8.endpoint,
self.ipv8.network,
trustchain=self.trustchain_community,
dht=self.dht_community,
wallets=self.wallets,
working_directory=self.config.get_state_dir(),
record_transactions=self.config.get_record_transactions())
self.ipv8.overlays.append(self.market_community)
self.ipv8.strategies.append(
(RandomWalk(self.market_community), 20))
# Popular Community
if self.config.get_popularity_community_enabled():
from tribler_core.modules.popularity.popularity_community import PopularityCommunity
self.popularity_community = PopularityCommunity(
peer,
self.ipv8.endpoint,
self.ipv8.network,
metadata_store=self.mds,
torrent_checker=self.torrent_checker)
self.ipv8.overlays.append(self.popularity_community)
self.ipv8.strategies.append(
(RandomWalk(self.popularity_community), 20))
# Gigachannel Community
if self.config.get_chant_enabled():
from tribler_core.modules.metadata_store.community.gigachannel_community import GigaChannelCommunity, GigaChannelTestnetCommunity
from tribler_core.modules.metadata_store.community.sync_strategy import SyncChannels
community_cls = GigaChannelTestnetCommunity if self.config.get_testnet(
) else GigaChannelCommunity
self.gigachannel_community = community_cls(peer,
self.ipv8.endpoint,
self.ipv8.network,
self.mds,
notifier=self.notifier)
self.ipv8.overlays.append(self.gigachannel_community)
self.ipv8.strategies.append(
(RandomWalk(self.gigachannel_community), 20))
self.ipv8.strategies.append(
(SyncChannels(self.gigachannel_community), 20))
# Gigachannel RemoteQuery Community
from tribler_core.modules.metadata_store.community.remote_query_community \
import RemoteQueryCommunity, RemoteQueryTestnetCommunity
community_cls = RemoteQueryTestnetCommunity if self.config.get_testnet(
) else RemoteQueryCommunity
self.remote_query_community = community_cls(peer,
self.ipv8.endpoint,
self.ipv8.network,
self.mds,
notifier=self.notifier)
self.ipv8.overlays.append(self.remote_query_community)
self.ipv8.strategies.append(
(RandomWalk(self.remote_query_community), 50))
def enable_ipv8_statistics(self):
if self.config.get_ipv8_statistics():
for overlay in self.ipv8.overlays:
self.ipv8.endpoint.enable_community_statistics(
overlay.get_prefix(), True)
def import_bootstrap_file(self):
with open(self.bootstrap.bootstrap_file, 'r') as f:
sql_dumb = f.read()
self._logger.info("Executing script for trustchain bootstrap")
self.trustchain_community.persistence.executescript(sql_dumb)
self.trustchain_community.persistence.commit()
async def start_bootstrap_download(self):
if not self.payout_manager:
self._logger.warning("Running bootstrap without payout enabled")
self.bootstrap = Bootstrap(self.config.get_state_dir(),
dht=self.dht_community)
self.bootstrap.start_by_infohash(self.dlmgr.start_download,
self.config.get_bootstrap_infohash())
if self.trustchain_community:
await self.bootstrap.download.future_finished
# Temporarily disabling SQL import for experimental release
#await get_event_loop().run_in_executor(None, self.import_bootstrap_file)
self.bootstrap.bootstrap_finished = True
def create_state_directory_structure(self):
"""Create directory structure of the state directory."""
def create_dir(path):
if not path.is_dir():
os.makedirs(path)
def create_in_state_dir(path):
create_dir(self.config.get_state_dir() / path)
create_dir(self.config.get_state_dir())
create_in_state_dir(STATEDIR_DB_DIR)
create_in_state_dir(STATEDIR_CHECKPOINT_DIR)
create_in_state_dir(STATEDIR_WALLET_DIR)
create_in_state_dir(STATEDIR_CHANNELS_DIR)
def get_ports_in_config(self):
"""Claim all required random ports."""
self.config.get_libtorrent_port()
self.config.get_anon_listen_port()
self.config.get_tunnel_community_socks5_listen_ports()
def init_keypair(self):
"""
Set parameters that depend on state_dir.
"""
trustchain_pairfilename = self.config.get_trustchain_keypair_filename()
if trustchain_pairfilename.exists():
self.trustchain_keypair = permid_module.read_keypair_trustchain(
trustchain_pairfilename)
else:
self.trustchain_keypair = permid_module.generate_keypair_trustchain(
)
# Save keypair
trustchain_pubfilename = self.config.get_state_dir(
) / 'ecpub_multichain.pem'
permid_module.save_keypair_trustchain(self.trustchain_keypair,
trustchain_pairfilename)
permid_module.save_pub_key_trustchain(self.trustchain_keypair,
trustchain_pubfilename)
trustchain_testnet_pairfilename = self.config.get_trustchain_testnet_keypair_filename(
)
if trustchain_testnet_pairfilename.exists():
self.trustchain_testnet_keypair = permid_module.read_keypair_trustchain(
trustchain_testnet_pairfilename)
else:
self.trustchain_testnet_keypair = permid_module.generate_keypair_trustchain(
)
# Save keypair
trustchain_testnet_pubfilename = self.config.get_state_dir(
) / 'ecpub_trustchain_testnet.pem'
permid_module.save_keypair_trustchain(
self.trustchain_testnet_keypair,
trustchain_testnet_pairfilename)
permid_module.save_pub_key_trustchain(
self.trustchain_testnet_keypair,
trustchain_testnet_pubfilename)
def unhandled_error_observer(self, loop, context):
"""
This method is called when an unhandled error in Tribler is observed.
It broadcasts the tribler_exception event.
"""
exception = context.get('exception')
ignored_message = None
try:
ignored_message = IGNORED_ERRORS.get(
(exception.__class__, exception.errno))
except (ValueError, AttributeError):
pass
if ignored_message is not None:
self._logger.error(ignored_message if ignored_message != "" else
context.get('message'))
return
text = str(exception or context.get('message'))
# We already have a check for invalid infohash when adding a torrent, but if somehow we get this
# error then we simply log and ignore it.
if isinstance(exception, RuntimeError) and 'invalid info-hash' in text:
self._logger.error("Invalid info-hash found")
return
text_long = text
exc = context.get('exception')
if exc:
with StringIO() as buffer:
print_exception(type(exc), exc, exc.__traceback__, file=buffer)
text_long = text_long + "\n--LONG TEXT--\n" + buffer.getvalue()
text_long = text_long + "\n--CONTEXT--\n" + str(context)
self._logger.error("Unhandled exception occurred! %s", text_long)
if self.api_manager and len(text_long) > 0:
self.api_manager.get_endpoint('events').on_tribler_exception(
text_long)
self.api_manager.get_endpoint('state').on_tribler_exception(
text_long)
def get_tribler_statistics(self):
"""Return a dictionary with general Tribler statistics."""
return TriblerStatistics(self).get_tribler_statistics()
def get_ipv8_statistics(self):
"""Return a dictionary with IPv8 statistics."""
return TriblerStatistics(self).get_ipv8_statistics()
async def start(self):
"""
Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
Returns a deferred that fires when the Tribler session is ready for use.
:param config: a TriblerConfig object
"""
self._logger.info("Session is using state directory: %s",
self.config.get_state_dir())
self.get_ports_in_config()
self.create_state_directory_structure()
self.init_keypair()
# Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
if self.config.get_http_api_enabled():
self.api_manager = RESTManager(self)
self.readable_status = STATE_START_API
await self.api_manager.start()
if self.upgrader_enabled:
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
try:
await self.upgrader.run()
except Exception as e:
self._logger.error("Error in Upgrader callback chain: %s", e)
self.tracker_manager = TrackerManager(self)
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(
(get_lib_path() / 'root_certs_mac.pem'))
if self.config.get_chant_enabled():
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_testnet(
) else 'metadata_testnet.db'
database_path = self.config.get_state_dir(
) / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir,
self.trustchain_keypair)
# IPv8
if self.config.get_ipv8_enabled():
from ipv8.configuration import get_default_configuration
ipv8_config = get_default_configuration()
ipv8_config['port'] = self.config.get_ipv8_port()
ipv8_config['address'] = self.config.get_ipv8_address()
ipv8_config['overlays'] = []
ipv8_config['keys'] = [] # We load the keys ourselves
ipv8_config['working_directory'] = str(self.config.get_state_dir())
if self.config.get_ipv8_bootstrap_override():
import ipv8.community as community_file
community_file._DEFAULT_ADDRESSES = [
self.config.get_ipv8_bootstrap_override()
]
community_file._DNS_ADDRESSES = []
self.ipv8 = IPv8(
ipv8_config,
enable_statistics=self.config.get_ipv8_statistics())
await self.ipv8.start()
self.config.set_anon_proxy_settings(
2, ("127.0.0.1",
self.config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
await self.tunnel_community.wait_for_socks_servers()
# Note that currently we should only start libtorrent after the SOCKS5 servers have been started
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
self.readable_status = STATE_READABLE_STARTED
if self.config.get_torrent_checking_enabled():
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
if self.config.get_dummy_wallets_enabled():
# For debugging purposes, we create dummy wallets
dummy_wallet1 = DummyWallet1()
self.wallets[dummy_wallet1.get_identifier()] = dummy_wallet1
dummy_wallet2 = DummyWallet2()
self.wallets[dummy_wallet2.get_identifier()] = dummy_wallet2
if self.config.get_watch_folder_enabled():
self.readable_status = STATE_START_WATCH_FOLDER
self.watch_folder = WatchFolder(self)
self.watch_folder.start()
if self.config.get_resource_monitor_enabled():
self.resource_monitor = ResourceMonitor(self)
self.resource_monitor.start()
if self.config.get_version_checker_enabled():
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()
if self.config.get_ipv8_enabled(
) and self.config.get_trustchain_enabled():
self.payout_manager = PayoutManager(self.trustchain_community,
self.dht_community)
# GigaChannel Manager should be started *after* resuming the downloads,
# because it depends on the states of torrent downloads
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\
and self.config.get_libtorrent_enabled:
self.gigachannel_manager = GigaChannelManager(self)
self.gigachannel_manager.start()
if self.config.get_bootstrap_enabled():
self.register_task('bootstrap_download',
self.start_bootstrap_download)
self.notifier.notify(NTFY.TRIBLER_STARTED)
async def shutdown(self):
"""
Checkpoints the session and closes it, stopping the download engine.
This method has to be called from the reactor thread.
"""
# Indicates we are shutting down core. With this environment variable set
# to 'TRUE', RESTManager will no longer accepts any new requests.
os.environ['TRIBLER_SHUTTING_DOWN'] = "TRUE"
if self.dlmgr:
self.dlmgr.stop_download_states_callback()
await self.shutdown_task_manager()
self.shutdownstarttime = timemod.time()
if self.torrent_checker:
self.notify_shutdown_state("Shutting down Torrent Checker...")
await self.torrent_checker.shutdown()
self.torrent_checker = None
if self.gigachannel_manager:
self.notify_shutdown_state("Shutting down Gigachannel Manager...")
await self.gigachannel_manager.shutdown()
self.gigachannel_manager = None
if self.version_check_manager:
self.notify_shutdown_state("Shutting down Version Checker...")
await self.version_check_manager.stop()
self.version_check_manager = None
if self.resource_monitor:
self.notify_shutdown_state("Shutting down Resource Monitor...")
await self.resource_monitor.stop()
self.resource_monitor = None
if self.bootstrap:
# We shutdown the bootstrap module before IPv8 since it uses the DHTCommunity.
await self.bootstrap.shutdown()
self.bootstrap = None
self.tracker_manager = None
if self.tunnel_community and self.trustchain_community:
# We unload these overlays manually since the TrustChain has to be unloaded after the tunnel overlay.
tunnel_community = self.tunnel_community
self.tunnel_community = None
self.notify_shutdown_state("Unloading Tunnel Community...")
await self.ipv8.unload_overlay(tunnel_community)
trustchain_community = self.trustchain_community
self.trustchain_community = None
self.notify_shutdown_state("Shutting down TrustChain Community...")
await self.ipv8.unload_overlay(trustchain_community)
if self.ipv8:
self.notify_shutdown_state("Shutting down IPv8...")
await self.ipv8.stop(stop_loop=False)
self.ipv8 = None
if self.payout_manager:
await self.payout_manager.shutdown()
self.payout_manager = None
if self.watch_folder:
self.notify_shutdown_state("Shutting down Watch Folder...")
await self.watch_folder.stop()
self.watch_folder = None
self.notify_shutdown_state("Saving configuration...")
self.config.write()
if self.dlmgr:
await self.dlmgr.shutdown()
self.dlmgr = None
if self.mds:
self.notify_shutdown_state("Shutting down Metadata Store...")
self.mds.shutdown()
self.mds = None
# We close the API manager as late as possible during shutdown.
if self.api_manager:
self.notify_shutdown_state("Shutting down API Manager...")
await self.api_manager.stop()
self.api_manager = None
def notify_shutdown_state(self, state):
self._logger.info("Tribler shutdown state notification:%s", state)
self.notifier.notify(NTFY.TRIBLER_SHUTDOWN_STATE, state)
def notifier():
return Notifier()
