def __init__(self, config, core_test_mode=False): """ A Session object is created Only a single session instance can exist at a time in a process. :config TriblerConfig object parametrizing the Session """ super().__init__() get_event_loop().set_exception_handler(self.unhandled_error_observer) patch_crypto_be_discovery() self._logger = logging.getLogger(self.__class__.__name__) self.config = config self.notifier = Notifier() self.upgrader_enabled = True self.upgrader = None self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler self.ipv8 = None self.ipv8_start_time = 0 self._logger = logging.getLogger(self.__class__.__name__) self.shutdownstarttime = None self.bootstrap = None # modules self.ipv8_community_loader = IPv8CommunityLoader() register_default_launchers(self.ipv8_community_loader) self.api_manager = None self.watch_folder = None self.version_check_manager = None self.resource_monitor = None self.gigachannel_manager = None self.dlmgr = None # Libtorrent Manager self.tracker_manager = None self.torrent_checker = None self.tunnel_community = None self.bandwidth_community = None self.wallets = {} self.popularity_community = None self.gigachannel_community = None self.dht_community = None self.payout_manager = None self.mds = None # Metadata Store # In test mode, the Core does not communicate with the external world and the state dir is read-only self.core_test_mode = core_test_mode
async def test_notifier_add_observer(notifier: Notifier): def callback(): ... # test that add observer stores topics and callbacks as a set to prevent duplicates notifier.add_observer('topic', callback) notifier.add_observer('topic', callback) assert len(notifier.observers['topic']) == 1
async def test_notifier_remove_observer(notifier: Notifier): def callback1(): ... def callback2(): ... notifier.add_observer('topic', callback1) notifier.add_observer('topic', callback2) notifier.remove_observer('topic', callback1) assert notifier.observers['topic'] == {callback2: True}
def __init__(self, config): """ A Session object is created Only a single session instance can exist at a time in a process. :config TriblerConfig object parametrising the Session """ super(Session, self).__init__() get_event_loop().set_exception_handler(self.unhandled_error_observer) patch_crypto_be_discovery() self._logger = logging.getLogger(self.__class__.__name__) self.config = config self.notifier = Notifier() self.upgrader_enabled = True self.upgrader = None self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler self.ipv8 = None self.ipv8_start_time = 0 self._logger = logging.getLogger(self.__class__.__name__) self.shutdownstarttime = None self.bootstrap = None # modules self.api_manager = None self.watch_folder = None self.version_check_manager = None self.resource_monitor = None self.gigachannel_manager = None self.dlmgr = None # Libtorrent Manager self.tracker_manager = None self.torrent_checker = None self.tunnel_community = None self.trustchain_community = None self.wallets = {} self.popularity_community = None self.gigachannel_community = None self.remote_query_community = None self.market_community = None self.dht_community = None self.payout_manager = None self.mds = None # Metadata Store
async def test_notify(notifier: Notifier): # test that notify works as expected normal_callback = TestCallback() notifier.add_observer('topic', normal_callback.callback) notifier.notify('topic', 'arg', kwarg='value') # wait for the callback await normal_callback.event.wait() assert normal_callback.callback_has_been_called assert normal_callback.callback_has_been_called_with_args == ('arg', ) assert normal_callback.callback_has_been_called_with_kwargs == { 'kwarg': 'value' }
async def fake_dlmgr(tmpdir): tribler_session = Mock() tribler_session.notifier = Notifier() tribler_session.state_dir = tmpdir tribler_session.notify_shutdown_state = lambda _: None tribler_session.config = Mock() tribler_session.config.get_libtorrent_utp = lambda: True tribler_session.config.get_libtorrent_proxy_settings = lambda: (0, None, None) tribler_session.config.get_anon_proxy_settings = lambda: (2, ('127.0.0.1', [1338]), None) tribler_session.config.get_libtorrent_port = lambda: 1337 tribler_session.config.get_anon_listen_port = lambda: 1338 tribler_session.config.get_state_dir = lambda: tmpdir tribler_session.config.set_listen_port_runtime = lambda: None tribler_session.config.get_libtorrent_max_upload_rate = lambda: 100 tribler_session.config.get_libtorrent_max_download_rate = lambda: 120 tribler_session.config.get_libtorrent_dht_enabled = lambda: False tribler_session.config.set_libtorrent_port_runtime = lambda _: None tribler_session.config.get_libtorrent_max_conn_download = lambda: 0 tribler_session.config.get_default_number_hops = lambda: 1 tribler_session.config.get_libtorrent_dht_readiness_timeout = lambda: 0 dlmgr = DownloadManager(tribler_session) dlmgr.metadata_tmpdir = tmpdir dlmgr.get_session = lambda *_, **__: Mock() tribler_session.dlmgr = dlmgr tribler_session.tunnel_community = None yield dlmgr await dlmgr.shutdown(timeout=0)
async def setUp(self): await super(TestDownloadManager, self).setUp() self.tribler_session = MockObject() self.tribler_session.notifier = Notifier() self.tribler_session.state_dir = self.session_base_dir self.tribler_session.trustchain_keypair = MockObject() self.tribler_session.trustchain_keypair.key_to_hash = lambda: b'a' * 20 self.tribler_session.notify_shutdown_state = lambda _: None self.tribler_session.config = MockObject() self.tribler_session.config.get_libtorrent_utp = lambda: True self.tribler_session.config.get_libtorrent_proxy_settings = lambda: (0, None, None) self.tribler_session.config.get_anon_proxy_settings = lambda: (2, ('127.0.0.1', [1338]), None) self.tribler_session.config.get_libtorrent_port = lambda: 1337 self.tribler_session.config.get_anon_listen_port = lambda: 1338 self.tribler_session.config.get_state_dir = lambda: self.session_base_dir self.tribler_session.config.set_listen_port_runtime = lambda: None self.tribler_session.config.get_libtorrent_max_upload_rate = lambda: 100 self.tribler_session.config.get_libtorrent_max_download_rate = lambda: 120 self.tribler_session.config.get_libtorrent_dht_enabled = lambda: False self.tribler_session.config.set_libtorrent_port_runtime = lambda _: None self.tribler_session.config.get_libtorrent_max_conn_download = lambda: 0 self.tribler_session.config.get_default_number_hops = lambda: 1 self.tribler_session.config.get_libtorrent_dht_readiness_timeout = lambda: 0 self.dlmgr = DownloadManager(self.tribler_session) self.dlmgr.metadata_tmpdir = mkdtemp(suffix=u'tribler_metainfo_tmpdir') self.tribler_session.dlmgr = self.dlmgr self.tribler_session.tunnel_community = None
async def test_remote_select_subscribed_channels(self): """ Test querying remote peers for subscribed channels and updating local votes accordingly. """ # We do not want the query back mechanism to interfere with this test self.nodes[1].overlay.rqc_settings.max_channel_query_back = 0 num_channels = 5 with db_session: # Create one channel with zero contents, to check that only non-empty channels are served self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel sub", "") # Create one channel that has not yet been processed (with local_version<timestamp) incomplete_chan = self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel sub", "") incomplete_chan.num_entries = 10 incomplete_chan.sign() for _ in range(0, num_channels): chan = self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel sub", "") chan.local_version = chan.timestamp chan.num_entries = 10 chan.sign() for _ in range(0, num_channels): channel_uns = self.nodes[0].overlay.mds.ChannelMetadata.create_channel("channel unsub", "") channel_uns.subscribed = False def mock_notify(overlay, args): overlay.notified_results = True self.assertTrue("results" in args) self.nodes[1].overlay.notifier = Notifier() self.nodes[1].overlay.notifier.notify = lambda sub, args: mock_notify(self.nodes[1].overlay, args) peer = self.nodes[0].my_peer await self.introduce_nodes() await self.deliver_messages(timeout=0.5) with db_session: received_channels = self.nodes[1].overlay.mds.ChannelMetadata.select(lambda g: g.title == "channel sub") self.assertEqual(num_channels, received_channels.count()) # Only subscribed channels should have been transported received_channels_all = self.nodes[1].overlay.mds.ChannelMetadata.select() self.assertEqual(num_channels, received_channels_all.count()) # Make sure the subscribed channels transport counted as voting self.assertEqual( self.nodes[1].overlay.mds.ChannelPeer.select().first().public_key, peer.public_key.key_to_bin()[10:] ) for chan in self.nodes[1].overlay.mds.ChannelMetadata.select(): self.assertTrue(chan.votes > 0.0) # Check that the notifier callback is called on new channel entries self.assertTrue(self.nodes[1].overlay.notified_results)
async def test_notify_call_soon_threadsafe_with_exception(notifier: Notifier): notifier.logger = Mock() notifier._loop = Mock(call_soon_threadsafe=Mock(side_effect=RuntimeError)) notifier.add_observer('topic', lambda:...) notifier.notify('topic') notifier.logger.warning.assert_called_once()
async def test_gigachannel_search(self): """ Test searching several nodes for metadata entries based on title text """ # We do not want the query back mechanism and introduction callback to interfere with this test for node in self.nodes: node.overlay.settings.max_channel_query_back = 0 await self.introduce_nodes() U_CHANNEL = "ubuntu channel" U_TORRENT = "ubuntu torrent" # Add test metadata to node 0 with db_session: self.nodes[0].overlay.mds.ChannelMetadata.create_channel( U_CHANNEL, "") self.nodes[0].overlay.mds.ChannelMetadata.create_channel( "debian channel", "") # Add test metadata to node 1 with db_session: self.nodes[1].overlay.mds.TorrentMetadata( title=U_TORRENT, infohash=random_infohash()) self.nodes[1].overlay.mds.TorrentMetadata( title="debian torrent", infohash=random_infohash()) notification_calls = [] def mock_notify(_, args): notification_calls.append(args) self.nodes[2].overlay.notifier = Notifier() self.nodes[2].overlay.notifier.notify = lambda sub, args: mock_notify( self.nodes[2].overlay, args) self.nodes[2].overlay.send_search_request(**{"txt_filter": "ubuntu*"}) await self.deliver_messages(timeout=0.5) with db_session: assert self.nodes[2].overlay.mds.ChannelNode.select().count() == 2 assert (self.nodes[2].overlay.mds.ChannelNode.select( lambda g: g.title in (U_CHANNEL, U_TORRENT)).count() == 2) # Check that the notifier callback was called on both entries assert [U_CHANNEL, U_TORRENT] == sorted( [c["results"][0]["name"] for c in notification_calls])
async def test_notify_with_exception(notifier: Notifier): # test that notify works as expected even if one of callbacks will raise an exception normal_callback = TestCallback() side_effect_callback = TestCallback(ValueError) notifier.add_observer('topic', side_effect_callback.callback) notifier.add_observer('topic', normal_callback.callback) notifier.add_observer('topic', side_effect_callback.callback) notifier.notify('topic') # wait await asyncio.sleep(1) assert normal_callback.callback_has_been_called assert side_effect_callback.callback_has_been_called
def __init__(self, config: TriblerConfig = None, components: List[Component] = (), shutdown_event: Event = None, notifier: Notifier = None, failfast: bool = True): # deepcode ignore unguarded~next~call: not necessary to catch StopIteration on infinite iterator self.id = next(Session._next_session_id) self.failfast = failfast self.logger = logging.getLogger(self.__class__.__name__) self.config: TriblerConfig = config or TriblerConfig() self.shutdown_event: Event = shutdown_event or Event() self.notifier: Notifier = notifier or Notifier() self.components: Dict[Type[Component], Component] = {} for component in components: self.register(component.__class__, component) # Reserve various (possibly) fixed ports to prevent # components from occupying those accidentally reserve_ports([ config.libtorrent.port, config.api.http_port, config.api.https_port, config.ipv8.port ])
async def test_notifier_remove_nonexistent_observer(notifier: Notifier): # test that `remove_observer` don't crash in case of calling to remove non existed topic/callback notifier.remove_observer('nonexistent', lambda: None) assert not notifier.observers['nonexistent']
def fixture_notifier(): return Notifier()
async def test_notifier(self): notifier = Notifier() notifier.add_observer(NTFY.TORRENT_FINISHED, self.callback_func) notifier.notify(NTFY.TORRENT_FINISHED) await self.test_future self.assertTrue(self.called_callback)
class Session(TaskManager): """ A Session is a running instance of the Tribler Core and the Core's central class. """ __single = None def __init__(self, config, core_test_mode=False): """ A Session object is created Only a single session instance can exist at a time in a process. :config TriblerConfig object parametrizing the Session """ super().__init__() get_event_loop().set_exception_handler(self.unhandled_error_observer) patch_crypto_be_discovery() self._logger = logging.getLogger(self.__class__.__name__) self.config = config self.notifier = Notifier() self.upgrader_enabled = True self.upgrader = None self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler self.ipv8 = None self.ipv8_start_time = 0 self._logger = logging.getLogger(self.__class__.__name__) self.shutdownstarttime = None self.bootstrap = None # modules self.ipv8_community_loader = IPv8CommunityLoader() register_default_launchers(self.ipv8_community_loader) self.api_manager = None self.watch_folder = None self.version_check_manager = None self.resource_monitor = None self.gigachannel_manager = None self.dlmgr = None # Libtorrent Manager self.tracker_manager = None self.torrent_checker = None self.tunnel_community = None self.bandwidth_community = None self.wallets = {} self.popularity_community = None self.gigachannel_community = None self.dht_community = None self.payout_manager = None self.mds = None # Metadata Store # In test mode, the Core does not communicate with the external world and the state dir is read-only self.core_test_mode = core_test_mode def load_ipv8_overlays(self): self.ipv8_community_loader.load(self.ipv8, self) def enable_ipv8_statistics(self): if self.config.get_ipv8_statistics(): for overlay in self.ipv8.overlays: self.ipv8.endpoint.enable_community_statistics(overlay.get_prefix(), True) def import_bootstrap_file(self): with open(self.bootstrap.bootstrap_file) as f: f.read() self._logger.info("Executing bootstrap script") # TODO we should do something here... async def start_bootstrap_download(self): if not self.payout_manager: self._logger.warning("Running bootstrap without payout enabled") from tribler_core.modules.bootstrap import Bootstrap self.bootstrap = Bootstrap(self.config.get_state_dir(), dht=self.dht_community) self.bootstrap.start_by_infohash(self.dlmgr.start_download, self.config.get_bootstrap_infohash()) await self.bootstrap.download.future_finished # Temporarily disabling SQL import for experimental release # await get_event_loop().run_in_executor(None, self.import_bootstrap_file) self.bootstrap.bootstrap_finished = True def create_state_directory_structure(self): """Create directory structure of the state directory.""" def create_dir(path): if not path.is_dir(): os.makedirs(path) def create_in_state_dir(path): create_dir(self.config.get_state_dir() / path) create_dir(self.config.get_state_dir()) create_in_state_dir(STATEDIR_DB_DIR) create_in_state_dir(STATEDIR_CHECKPOINT_DIR) create_in_state_dir(STATEDIR_CHANNELS_DIR) def get_ports_in_config(self): """Claim all required random ports.""" if self.core_test_mode: self.config.selected_ports = {} return self.config.get_libtorrent_port() self.config.get_anon_listen_port() self.config.get_tunnel_community_socks5_listen_ports() def init_keypair(self): """ Set parameters that depend on state_dir. """ trustchain_pairfilename = self.config.get_trustchain_keypair_filename() if trustchain_pairfilename.exists(): self.trustchain_keypair = permid_module.read_keypair_trustchain(trustchain_pairfilename) else: self.trustchain_keypair = permid_module.generate_keypair_trustchain() # Save keypair trustchain_pubfilename = self.config.get_state_dir() / 'ecpub_multichain.pem' permid_module.save_keypair_trustchain(self.trustchain_keypair, trustchain_pairfilename) permid_module.save_pub_key_trustchain(self.trustchain_keypair, trustchain_pubfilename) trustchain_testnet_pairfilename = self.config.get_trustchain_testnet_keypair_filename() if trustchain_testnet_pairfilename.exists(): self.trustchain_testnet_keypair = permid_module.read_keypair_trustchain(trustchain_testnet_pairfilename) else: self.trustchain_testnet_keypair = permid_module.generate_keypair_trustchain() # Save keypair trustchain_testnet_pubfilename = self.config.get_state_dir() / 'ecpub_trustchain_testnet.pem' permid_module.save_keypair_trustchain(self.trustchain_testnet_keypair, trustchain_testnet_pairfilename) permid_module.save_pub_key_trustchain(self.trustchain_testnet_keypair, trustchain_testnet_pubfilename) def unhandled_error_observer(self, loop, context): """ This method is called when an unhandled error in Tribler is observed. It broadcasts the tribler_exception event. """ try: SentryReporter.ignore_logger(self._logger.name) exception = context.get('exception') ignored_message = None try: ignored_message = IGNORED_ERRORS.get( (exception.__class__, exception.errno), IGNORED_ERRORS.get(exception.__class__)) except (ValueError, AttributeError): pass if ignored_message is not None: self._logger.error(ignored_message if ignored_message != "" else context.get('message')) return text = str(exception or context.get('message')) # We already have a check for invalid infohash when adding a torrent, but if somehow we get this # error then we simply log and ignore it. if isinstance(exception, RuntimeError) and 'invalid info-hash' in text: self._logger.error("Invalid info-hash found") return text_long = text exc = context.get('exception') if exc: with StringIO() as buffer: print_exception(type(exc), exc, exc.__traceback__, file=buffer) text_long = text_long + "\n--LONG TEXT--\n" + buffer.getvalue() text_long = text_long + "\n--CONTEXT--\n" + str(context) self._logger.error("Unhandled exception occurred! %s", text_long, exc_info=None) sentry_event = SentryReporter.event_from_exception(exception) if not self.api_manager: return events = self.api_manager.get_endpoint('events') events.on_tribler_exception(text_long, sentry_event, self.config.get_core_error_reporting_requires_user_consent()) state = self.api_manager.get_endpoint('state') state.on_tribler_exception(text_long, sentry_event) except Exception as ex: SentryReporter.capture_exception(ex) raise ex def get_tribler_statistics(self): """Return a dictionary with general Tribler statistics.""" return TriblerStatistics(self).get_tribler_statistics() def get_ipv8_statistics(self): """Return a dictionary with IPv8 statistics.""" return TriblerStatistics(self).get_ipv8_statistics() async def start(self): """ Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader. Returns a deferred that fires when the Tribler session is ready for use. :param config: a TriblerConfig object """ self._logger.info("Session is using state directory: %s", self.config.get_state_dir()) self.get_ports_in_config() self.create_state_directory_structure() self.init_keypair() # we have to represent `user_id` as a string to make it equal to the # `user_id` on the GUI side user_id_str = hexlify(self.trustchain_keypair.key.pk).encode('utf-8') SentryReporter.set_user(user_id_str) # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.config.get_api_http_enabled() or self.config.get_api_https_enabled(): from tribler_core.restapi.rest_manager import RESTManager self.api_manager = RESTManager(self) self.readable_status = STATE_START_API await self.api_manager.start() if self.upgrader_enabled and not self.core_test_mode: from tribler_core.upgrade.upgrade import TriblerUpgrader self.upgrader = TriblerUpgrader(self) self.readable_status = STATE_UPGRADING_READABLE await self.upgrader.run() self.tracker_manager = TrackerManager(self) # Start torrent checker before Popularity community is loaded if self.config.get_torrent_checking_enabled() and not self.core_test_mode: from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker self.readable_status = STATE_START_TORRENT_CHECKER self.torrent_checker = TorrentChecker(self) await self.torrent_checker.initialize() # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root # certificates provided by the system trust store. if sys.platform == 'darwin': os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem') if self.config.get_chant_enabled(): from tribler_core.modules.metadata_store.store import MetadataStore channels_dir = self.config.get_chant_channels_dir() metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db' database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair, notifier=self.notifier, disable_sync=self.core_test_mode) if self.core_test_mode: generate_test_channels(self.mds) # IPv8 if self.config.get_ipv8_enabled(): from ipv8.configuration import ConfigBuilder from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint ipv8_config_builder = (ConfigBuilder() .set_port(self.config.get_ipv8_port()) .set_address(self.config.get_ipv8_address()) .clear_overlays() .clear_keys() # We load the keys ourselves .set_working_directory(str(self.config.get_state_dir())) .set_walker_interval(self.config.get_ipv8_walk_interval())) if self.core_test_mode: endpoint = DispatcherEndpoint([]) else: # IPv8 includes IPv6 support by default. # We only load IPv4 to not kill all Tribler overlays (currently, it would instantly crash all users). # If you want to test IPv6 in Tribler you can set ``endpoint = None`` here. endpoint = DispatcherEndpoint(["UDPIPv4"], UDPIPv4={'port': self.config.get_ipv8_port(), 'ip': self.config.get_ipv8_address()}) self.ipv8 = IPv8(ipv8_config_builder.finalize(), enable_statistics=self.config.get_ipv8_statistics() and not self.core_test_mode, endpoint_override=endpoint) await self.ipv8.start() self.config.set_anon_proxy_settings(2, ("127.0.0.1", self. config.get_tunnel_community_socks5_listen_ports())) self.ipv8_start_time = timemod.time() self.load_ipv8_overlays() if not self.core_test_mode: self.enable_ipv8_statistics() if self.api_manager: self.api_manager.set_ipv8_session(self.ipv8) if self.config.get_tunnel_community_enabled(): await self.tunnel_community.wait_for_socks_servers() if self.config.get_ipv8_walk_scaling_enabled(): from tribler_core.modules.ipv8_health_monitor import IPv8Monitor IPv8Monitor(self.ipv8, self.config.get_ipv8_walk_interval(), self.config.get_ipv8_walk_scaling_upper_limit()).start(self) # Note that currently we should only start libtorrent after the SOCKS5 servers have been started if self.config.get_libtorrent_enabled(): self.readable_status = STATE_START_LIBTORRENT from tribler_core.modules.libtorrent.download_manager import DownloadManager self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode) self.dlmgr.initialize() self.readable_status = STATE_LOAD_CHECKPOINTS await self.dlmgr.load_checkpoints() if self.core_test_mode: await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000") self.readable_status = STATE_READABLE_STARTED if self.config.get_watch_folder_enabled(): from tribler_core.modules.watch_folder import WatchFolder self.readable_status = STATE_START_WATCH_FOLDER self.watch_folder = WatchFolder(self) self.watch_folder.start() if self.config.get_resource_monitor_enabled() and not self.core_test_mode: from tribler_core.modules.resource_monitor.core import CoreResourceMonitor self.resource_monitor = CoreResourceMonitor(self) self.resource_monitor.start() if self.config.get_version_checker_enabled() and not self.core_test_mode: from tribler_core.modules.versioncheck_manager import VersionCheckManager self.version_check_manager = VersionCheckManager(self) self.version_check_manager.start() if self.config.get_ipv8_enabled(): from tribler_core.modules.payout_manager import PayoutManager self.payout_manager = PayoutManager(self.bandwidth_community, self.dht_community) if self.core_test_mode: from ipv8.messaging.interfaces.udp.endpoint import UDPv4Address from ipv8.dht.routing import RoutingTable self.dht_community.routing_tables[UDPv4Address] = RoutingTable('\x00' * 20) # GigaChannel Manager should be started *after* resuming the downloads, # because it depends on the states of torrent downloads if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \ and self.config.get_libtorrent_enabled: from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager self.gigachannel_manager = GigaChannelManager(self) if not self.core_test_mode: self.gigachannel_manager.start() if self.config.get_bootstrap_enabled() and not self.core_test_mode: self.register_task('bootstrap_download', self.start_bootstrap_download) self.notifier.notify(NTFY.TRIBLER_STARTED, self.trustchain_keypair.key.pk) # If there is a config error, report to the user via GUI notifier if self.config.config_error: self.notifier.notify(NTFY.REPORT_CONFIG_ERROR, self.config.config_error) async def shutdown(self): """ Checkpoints the session and closes it, stopping the download engine. This method has to be called from the reactor thread. """ self.shutdownstarttime = timemod.time() # Indicates we are shutting down core. With this environment variable set # to 'TRUE', RESTManager will no longer accepts any new requests. os.environ['TRIBLER_SHUTTING_DOWN'] = "TRUE" if self.dlmgr: self.dlmgr.stop_download_states_callback() await self.shutdown_task_manager() if self.torrent_checker: self.notify_shutdown_state("Shutting down Torrent Checker...") await self.torrent_checker.shutdown() self.torrent_checker = None if self.gigachannel_manager: self.notify_shutdown_state("Shutting down Gigachannel Manager...") await self.gigachannel_manager.shutdown() self.gigachannel_manager = None if self.version_check_manager: self.notify_shutdown_state("Shutting down Version Checker...") await self.version_check_manager.stop() self.version_check_manager = None if self.resource_monitor: self.notify_shutdown_state("Shutting down Resource Monitor...") await self.resource_monitor.stop() self.resource_monitor = None if self.bootstrap: # We shutdown the bootstrap module before IPv8 since it uses the DHTCommunity. await self.bootstrap.shutdown() self.bootstrap = None self.tracker_manager = None if self.tunnel_community and self.bandwidth_community: # We unload these overlays manually since the TrustChain has to be unloaded after the tunnel overlay. tunnel_community = self.tunnel_community self.tunnel_community = None self.notify_shutdown_state("Unloading Tunnel Community...") await self.ipv8.unload_overlay(tunnel_community) bandwidth_community = self.bandwidth_community self.bandwidth_community = None self.notify_shutdown_state("Shutting down Bandwidth Community...") await self.ipv8.unload_overlay(bandwidth_community) if self.ipv8: self.notify_shutdown_state("Shutting down IPv8...") await self.ipv8.stop(stop_loop=False) self.ipv8 = None if self.payout_manager: await self.payout_manager.shutdown() self.payout_manager = None if self.watch_folder: self.notify_shutdown_state("Shutting down Watch Folder...") await self.watch_folder.stop() self.watch_folder = None if not self.core_test_mode: self.notify_shutdown_state("Saving configuration...") self.config.write() if self.dlmgr: await self.dlmgr.shutdown() self.dlmgr = None if self.mds: self.notify_shutdown_state("Shutting down Metadata Store...") self.mds.shutdown() self.mds = None # We close the API manager as late as possible during shutdown. if self.api_manager: self.notify_shutdown_state("Shutting down API Manager...") await self.api_manager.stop() self.api_manager = None def notify_shutdown_state(self, state): self._logger.info("Tribler shutdown state notification:%s", state) self.notifier.notify(NTFY.TRIBLER_SHUTDOWN_STATE, state)
class Session(TaskManager): """ A Session is a running instance of the Tribler Core and the Core's central class. """ __single = None def __init__(self, config): """ A Session object is created Only a single session instance can exist at a time in a process. :config TriblerConfig object parametrising the Session """ super(Session, self).__init__() get_event_loop().set_exception_handler(self.unhandled_error_observer) patch_crypto_be_discovery() self._logger = logging.getLogger(self.__class__.__name__) self.config = config self.notifier = Notifier() self.upgrader_enabled = True self.upgrader = None self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler self.ipv8 = None self.ipv8_start_time = 0 self._logger = logging.getLogger(self.__class__.__name__) self.shutdownstarttime = None self.bootstrap = None # modules self.api_manager = None self.watch_folder = None self.version_check_manager = None self.resource_monitor = None self.gigachannel_manager = None self.dlmgr = None # Libtorrent Manager self.tracker_manager = None self.torrent_checker = None self.tunnel_community = None self.trustchain_community = None self.wallets = {} self.popularity_community = None self.gigachannel_community = None self.remote_query_community = None self.market_community = None self.dht_community = None self.payout_manager = None self.mds = None # Metadata Store def load_ipv8_overlays(self): if self.config.get_testnet(): peer = Peer(self.trustchain_testnet_keypair) else: peer = Peer(self.trustchain_keypair) discovery_community = DiscoveryCommunity(peer, self.ipv8.endpoint, self.ipv8.network) discovery_community.resolve_dns_bootstrap_addresses() self.ipv8.overlays.append(discovery_community) self.ipv8.strategies.append((RandomChurn(discovery_community), -1)) self.ipv8.strategies.append( (PeriodicSimilarity(discovery_community), -1)) self.ipv8.strategies.append((RandomWalk(discovery_community), 20)) # TrustChain Community if self.config.get_trustchain_enabled(): from ipv8.attestation.trustchain.community import TrustChainCommunity, \ TrustChainTestnetCommunity community_cls = TrustChainTestnetCommunity if self.config.get_testnet( ) else TrustChainCommunity self.trustchain_community = community_cls( peer, self.ipv8.endpoint, self.ipv8.network, working_directory=self.config.get_state_dir()) self.ipv8.overlays.append(self.trustchain_community) self.ipv8.strategies.append( (EdgeWalk(self.trustchain_community), 20)) tc_wallet = TrustchainWallet(self.trustchain_community) self.wallets[tc_wallet.get_identifier()] = tc_wallet # DHT Community if self.config.get_dht_enabled(): from ipv8.dht.discovery import DHTDiscoveryCommunity self.dht_community = DHTDiscoveryCommunity(peer, self.ipv8.endpoint, self.ipv8.network) self.ipv8.overlays.append(self.dht_community) self.ipv8.strategies.append((RandomWalk(self.dht_community), 20)) self.ipv8.strategies.append((PingChurn(self.dht_community), -1)) # Tunnel Community if self.config.get_tunnel_community_enabled(): from tribler_core.modules.tunnel.community.triblertunnel_community import TriblerTunnelCommunity,\ TriblerTunnelTestnetCommunity from tribler_core.modules.tunnel.community.discovery import GoldenRatioStrategy community_cls = TriblerTunnelTestnetCommunity if self.config.get_testnet() else \ TriblerTunnelCommunity random_slots = self.config.get_tunnel_community_random_slots() competing_slots = self.config.get_tunnel_community_competing_slots( ) dht_provider = DHTCommunityProvider(self.dht_community, self.config.get_ipv8_port()) settings = TunnelSettings() settings.min_circuits = 3 settings.max_circuits = 10 self.tunnel_community = community_cls( peer, self.ipv8.endpoint, self.ipv8.network, tribler_session=self, dht_provider=dht_provider, ipv8=self.ipv8, bandwidth_wallet=self.wallets["MB"], random_slots=random_slots, competing_slots=competing_slots, settings=settings) self.ipv8.overlays.append(self.tunnel_community) self.ipv8.strategies.append( (RandomWalk(self.tunnel_community), 20)) self.ipv8.strategies.append( (GoldenRatioStrategy(self.tunnel_community), -1)) # Market Community if self.config.get_market_community_enabled( ) and self.config.get_dht_enabled(): from anydex.core.community import MarketCommunity, MarketTestnetCommunity community_cls = MarketTestnetCommunity if self.config.get_testnet( ) else MarketCommunity self.market_community = community_cls( peer, self.ipv8.endpoint, self.ipv8.network, trustchain=self.trustchain_community, dht=self.dht_community, wallets=self.wallets, working_directory=self.config.get_state_dir(), record_transactions=self.config.get_record_transactions()) self.ipv8.overlays.append(self.market_community) self.ipv8.strategies.append( (RandomWalk(self.market_community), 20)) # Popular Community if self.config.get_popularity_community_enabled(): from tribler_core.modules.popularity.popularity_community import PopularityCommunity self.popularity_community = PopularityCommunity( peer, self.ipv8.endpoint, self.ipv8.network, metadata_store=self.mds, torrent_checker=self.torrent_checker) self.ipv8.overlays.append(self.popularity_community) self.ipv8.strategies.append( (RandomWalk(self.popularity_community), 20)) # Gigachannel Community if self.config.get_chant_enabled(): from tribler_core.modules.metadata_store.community.gigachannel_community import GigaChannelCommunity, GigaChannelTestnetCommunity from tribler_core.modules.metadata_store.community.sync_strategy import SyncChannels community_cls = GigaChannelTestnetCommunity if self.config.get_testnet( ) else GigaChannelCommunity self.gigachannel_community = community_cls(peer, self.ipv8.endpoint, self.ipv8.network, self.mds, notifier=self.notifier) self.ipv8.overlays.append(self.gigachannel_community) self.ipv8.strategies.append( (RandomWalk(self.gigachannel_community), 20)) self.ipv8.strategies.append( (SyncChannels(self.gigachannel_community), 20)) # Gigachannel RemoteQuery Community from tribler_core.modules.metadata_store.community.remote_query_community \ import RemoteQueryCommunity, RemoteQueryTestnetCommunity community_cls = RemoteQueryTestnetCommunity if self.config.get_testnet( ) else RemoteQueryCommunity self.remote_query_community = community_cls(peer, self.ipv8.endpoint, self.ipv8.network, self.mds, notifier=self.notifier) self.ipv8.overlays.append(self.remote_query_community) self.ipv8.strategies.append( (RandomWalk(self.remote_query_community), 50)) def enable_ipv8_statistics(self): if self.config.get_ipv8_statistics(): for overlay in self.ipv8.overlays: self.ipv8.endpoint.enable_community_statistics( overlay.get_prefix(), True) def import_bootstrap_file(self): with open(self.bootstrap.bootstrap_file, 'r') as f: sql_dumb = f.read() self._logger.info("Executing script for trustchain bootstrap") self.trustchain_community.persistence.executescript(sql_dumb) self.trustchain_community.persistence.commit() async def start_bootstrap_download(self): if not self.payout_manager: self._logger.warning("Running bootstrap without payout enabled") self.bootstrap = Bootstrap(self.config.get_state_dir(), dht=self.dht_community) self.bootstrap.start_by_infohash(self.dlmgr.start_download, self.config.get_bootstrap_infohash()) if self.trustchain_community: await self.bootstrap.download.future_finished # Temporarily disabling SQL import for experimental release #await get_event_loop().run_in_executor(None, self.import_bootstrap_file) self.bootstrap.bootstrap_finished = True def create_state_directory_structure(self): """Create directory structure of the state directory.""" def create_dir(path): if not path.is_dir(): os.makedirs(path) def create_in_state_dir(path): create_dir(self.config.get_state_dir() / path) create_dir(self.config.get_state_dir()) create_in_state_dir(STATEDIR_DB_DIR) create_in_state_dir(STATEDIR_CHECKPOINT_DIR) create_in_state_dir(STATEDIR_WALLET_DIR) create_in_state_dir(STATEDIR_CHANNELS_DIR) def get_ports_in_config(self): """Claim all required random ports.""" self.config.get_libtorrent_port() self.config.get_anon_listen_port() self.config.get_tunnel_community_socks5_listen_ports() def init_keypair(self): """ Set parameters that depend on state_dir. """ trustchain_pairfilename = self.config.get_trustchain_keypair_filename() if trustchain_pairfilename.exists(): self.trustchain_keypair = permid_module.read_keypair_trustchain( trustchain_pairfilename) else: self.trustchain_keypair = permid_module.generate_keypair_trustchain( ) # Save keypair trustchain_pubfilename = self.config.get_state_dir( ) / 'ecpub_multichain.pem' permid_module.save_keypair_trustchain(self.trustchain_keypair, trustchain_pairfilename) permid_module.save_pub_key_trustchain(self.trustchain_keypair, trustchain_pubfilename) trustchain_testnet_pairfilename = self.config.get_trustchain_testnet_keypair_filename( ) if trustchain_testnet_pairfilename.exists(): self.trustchain_testnet_keypair = permid_module.read_keypair_trustchain( trustchain_testnet_pairfilename) else: self.trustchain_testnet_keypair = permid_module.generate_keypair_trustchain( ) # Save keypair trustchain_testnet_pubfilename = self.config.get_state_dir( ) / 'ecpub_trustchain_testnet.pem' permid_module.save_keypair_trustchain( self.trustchain_testnet_keypair, trustchain_testnet_pairfilename) permid_module.save_pub_key_trustchain( self.trustchain_testnet_keypair, trustchain_testnet_pubfilename) def unhandled_error_observer(self, loop, context): """ This method is called when an unhandled error in Tribler is observed. It broadcasts the tribler_exception event. """ exception = context.get('exception') ignored_message = None try: ignored_message = IGNORED_ERRORS.get( (exception.__class__, exception.errno)) except (ValueError, AttributeError): pass if ignored_message is not None: self._logger.error(ignored_message if ignored_message != "" else context.get('message')) return text = str(exception or context.get('message')) # We already have a check for invalid infohash when adding a torrent, but if somehow we get this # error then we simply log and ignore it. if isinstance(exception, RuntimeError) and 'invalid info-hash' in text: self._logger.error("Invalid info-hash found") return text_long = text exc = context.get('exception') if exc: with StringIO() as buffer: print_exception(type(exc), exc, exc.__traceback__, file=buffer) text_long = text_long + "\n--LONG TEXT--\n" + buffer.getvalue() text_long = text_long + "\n--CONTEXT--\n" + str(context) self._logger.error("Unhandled exception occurred! %s", text_long) if self.api_manager and len(text_long) > 0: self.api_manager.get_endpoint('events').on_tribler_exception( text_long) self.api_manager.get_endpoint('state').on_tribler_exception( text_long) def get_tribler_statistics(self): """Return a dictionary with general Tribler statistics.""" return TriblerStatistics(self).get_tribler_statistics() def get_ipv8_statistics(self): """Return a dictionary with IPv8 statistics.""" return TriblerStatistics(self).get_ipv8_statistics() async def start(self): """ Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader. Returns a deferred that fires when the Tribler session is ready for use. :param config: a TriblerConfig object """ self._logger.info("Session is using state directory: %s", self.config.get_state_dir()) self.get_ports_in_config() self.create_state_directory_structure() self.init_keypair() # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.config.get_http_api_enabled(): self.api_manager = RESTManager(self) self.readable_status = STATE_START_API await self.api_manager.start() if self.upgrader_enabled: self.upgrader = TriblerUpgrader(self) self.readable_status = STATE_UPGRADING_READABLE try: await self.upgrader.run() except Exception as e: self._logger.error("Error in Upgrader callback chain: %s", e) self.tracker_manager = TrackerManager(self) # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root # certificates provided by the system trust store. if sys.platform == 'darwin': os.environ['SSL_CERT_FILE'] = str( (get_lib_path() / 'root_certs_mac.pem')) if self.config.get_chant_enabled(): channels_dir = self.config.get_chant_channels_dir() metadata_db_name = 'metadata.db' if not self.config.get_testnet( ) else 'metadata_testnet.db' database_path = self.config.get_state_dir( ) / 'sqlite' / metadata_db_name self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair) # IPv8 if self.config.get_ipv8_enabled(): from ipv8.configuration import get_default_configuration ipv8_config = get_default_configuration() ipv8_config['port'] = self.config.get_ipv8_port() ipv8_config['address'] = self.config.get_ipv8_address() ipv8_config['overlays'] = [] ipv8_config['keys'] = [] # We load the keys ourselves ipv8_config['working_directory'] = str(self.config.get_state_dir()) if self.config.get_ipv8_bootstrap_override(): import ipv8.community as community_file community_file._DEFAULT_ADDRESSES = [ self.config.get_ipv8_bootstrap_override() ] community_file._DNS_ADDRESSES = [] self.ipv8 = IPv8( ipv8_config, enable_statistics=self.config.get_ipv8_statistics()) await self.ipv8.start() self.config.set_anon_proxy_settings( 2, ("127.0.0.1", self.config.get_tunnel_community_socks5_listen_ports())) self.ipv8_start_time = timemod.time() self.load_ipv8_overlays() self.enable_ipv8_statistics() if self.api_manager: self.api_manager.set_ipv8_session(self.ipv8) if self.config.get_tunnel_community_enabled(): await self.tunnel_community.wait_for_socks_servers() # Note that currently we should only start libtorrent after the SOCKS5 servers have been started if self.config.get_libtorrent_enabled(): self.readable_status = STATE_START_LIBTORRENT from tribler_core.modules.libtorrent.download_manager import DownloadManager self.dlmgr = DownloadManager(self) self.dlmgr.initialize() self.readable_status = STATE_LOAD_CHECKPOINTS await self.dlmgr.load_checkpoints() self.readable_status = STATE_READABLE_STARTED if self.config.get_torrent_checking_enabled(): self.readable_status = STATE_START_TORRENT_CHECKER self.torrent_checker = TorrentChecker(self) await self.torrent_checker.initialize() if self.config.get_dummy_wallets_enabled(): # For debugging purposes, we create dummy wallets dummy_wallet1 = DummyWallet1() self.wallets[dummy_wallet1.get_identifier()] = dummy_wallet1 dummy_wallet2 = DummyWallet2() self.wallets[dummy_wallet2.get_identifier()] = dummy_wallet2 if self.config.get_watch_folder_enabled(): self.readable_status = STATE_START_WATCH_FOLDER self.watch_folder = WatchFolder(self) self.watch_folder.start() if self.config.get_resource_monitor_enabled(): self.resource_monitor = ResourceMonitor(self) self.resource_monitor.start() if self.config.get_version_checker_enabled(): self.version_check_manager = VersionCheckManager(self) self.version_check_manager.start() if self.config.get_ipv8_enabled( ) and self.config.get_trustchain_enabled(): self.payout_manager = PayoutManager(self.trustchain_community, self.dht_community) # GigaChannel Manager should be started *after* resuming the downloads, # because it depends on the states of torrent downloads if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\ and self.config.get_libtorrent_enabled: self.gigachannel_manager = GigaChannelManager(self) self.gigachannel_manager.start() if self.config.get_bootstrap_enabled(): self.register_task('bootstrap_download', self.start_bootstrap_download) self.notifier.notify(NTFY.TRIBLER_STARTED) async def shutdown(self): """ Checkpoints the session and closes it, stopping the download engine. This method has to be called from the reactor thread. """ # Indicates we are shutting down core. With this environment variable set # to 'TRUE', RESTManager will no longer accepts any new requests. os.environ['TRIBLER_SHUTTING_DOWN'] = "TRUE" if self.dlmgr: self.dlmgr.stop_download_states_callback() await self.shutdown_task_manager() self.shutdownstarttime = timemod.time() if self.torrent_checker: self.notify_shutdown_state("Shutting down Torrent Checker...") await self.torrent_checker.shutdown() self.torrent_checker = None if self.gigachannel_manager: self.notify_shutdown_state("Shutting down Gigachannel Manager...") await self.gigachannel_manager.shutdown() self.gigachannel_manager = None if self.version_check_manager: self.notify_shutdown_state("Shutting down Version Checker...") await self.version_check_manager.stop() self.version_check_manager = None if self.resource_monitor: self.notify_shutdown_state("Shutting down Resource Monitor...") await self.resource_monitor.stop() self.resource_monitor = None if self.bootstrap: # We shutdown the bootstrap module before IPv8 since it uses the DHTCommunity. await self.bootstrap.shutdown() self.bootstrap = None self.tracker_manager = None if self.tunnel_community and self.trustchain_community: # We unload these overlays manually since the TrustChain has to be unloaded after the tunnel overlay. tunnel_community = self.tunnel_community self.tunnel_community = None self.notify_shutdown_state("Unloading Tunnel Community...") await self.ipv8.unload_overlay(tunnel_community) trustchain_community = self.trustchain_community self.trustchain_community = None self.notify_shutdown_state("Shutting down TrustChain Community...") await self.ipv8.unload_overlay(trustchain_community) if self.ipv8: self.notify_shutdown_state("Shutting down IPv8...") await self.ipv8.stop(stop_loop=False) self.ipv8 = None if self.payout_manager: await self.payout_manager.shutdown() self.payout_manager = None if self.watch_folder: self.notify_shutdown_state("Shutting down Watch Folder...") await self.watch_folder.stop() self.watch_folder = None self.notify_shutdown_state("Saving configuration...") self.config.write() if self.dlmgr: await self.dlmgr.shutdown() self.dlmgr = None if self.mds: self.notify_shutdown_state("Shutting down Metadata Store...") self.mds.shutdown() self.mds = None # We close the API manager as late as possible during shutdown. if self.api_manager: self.notify_shutdown_state("Shutting down API Manager...") await self.api_manager.stop() self.api_manager = None def notify_shutdown_state(self, state): self._logger.info("Tribler shutdown state notification:%s", state) self.notifier.notify(NTFY.TRIBLER_SHUTDOWN_STATE, state)
def notifier(): return Notifier()