async def start_components(self): self.logger.info("Session is using state directory: %s", self.config.state_dir) create_state_directory_structure(self.config.state_dir) patch_crypto_be_discovery() # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root # certificates provided by the system trust store. if sys.platform == 'darwin': os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem') coros = [comp.start() for comp in self.components.values()] await gather(*coros, return_exceptions=not self.failfast) if self._startup_exception: self._reraise_startup_exception_in_separate_task()
from pathlib import Path from configobj import ConfigObj from validate import Validator from tribler_core.exceptions import InvalidConfigException from tribler_core.utilities import path_util from tribler_core.utilities.install_dir import get_lib_path from tribler_core.utilities.libtorrent_helper import libtorrent as lt from tribler_core.utilities.osutils import get_home_dir from tribler_core.utilities.path_util import str_path from tribler_core.utilities.utilities import bdecode_compat SPEC_FILENAME = 'download_config.spec' CONFIG_SPEC_PATH = get_lib_path() / 'modules' / 'libtorrent' / SPEC_FILENAME NONPERSISTENT_DEFAULTS = {} class DownloadConfig: def __init__(self, config=None, state_dir=None): self.config = config or ConfigObj(configspec=str(CONFIG_SPEC_PATH), default_encoding='utf8') # Values that should not be stored and should be initialized to some default value self.nonpersistent = NONPERSISTENT_DEFAULTS.copy() self.state_dir = state_dir self.validate() def validate(self): """ Validate the ConfigObj using Validator.
""" Category. Author(s): Yuan Yuan, Jelle Roozenburg """ import logging import re from functools import cmp_to_key from tribler_core.modules.category_filter.family_filter import default_xxx_filter from tribler_core.modules.category_filter.init_category import getCategoryInfo from tribler_core.utilities.install_dir import get_lib_path from tribler_core.utilities.unicode import recursive_unicode CATEGORY_CONFIG_FILE = get_lib_path() / 'modules/category_filter/category.conf' def cmp_rank(a, b): if 'rank' not in a: return 1 if 'rank' not in b: return -1 if a['rank'] == b['rank']: return 0 if a['rank'] == -1: return 1 if b['rank'] == -1: return -1 if a['rank'] < b['rank']: return -1 return 1
async def start(self): """ Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader. Returns a deferred that fires when the Tribler session is ready for use. :param config: a TriblerConfig object """ self._logger.info("Session is using state directory: %s", self.config.get_state_dir()) self.get_ports_in_config() self.create_state_directory_structure() self.init_keypair() # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.config.get_http_api_enabled(): self.api_manager = RESTManager(self) self.readable_status = STATE_START_API await self.api_manager.start() if self.upgrader_enabled: self.upgrader = TriblerUpgrader(self) self.readable_status = STATE_UPGRADING_READABLE try: await self.upgrader.run() except Exception as e: self._logger.error("Error in Upgrader callback chain: %s", e) self.tracker_manager = TrackerManager(self) # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root # certificates provided by the system trust store. if sys.platform == 'darwin': os.environ['SSL_CERT_FILE'] = str( (get_lib_path() / 'root_certs_mac.pem')) if self.config.get_chant_enabled(): channels_dir = self.config.get_chant_channels_dir() metadata_db_name = 'metadata.db' if not self.config.get_testnet( ) else 'metadata_testnet.db' database_path = self.config.get_state_dir( ) / 'sqlite' / metadata_db_name self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair) # IPv8 if self.config.get_ipv8_enabled(): from ipv8.configuration import get_default_configuration ipv8_config = get_default_configuration() ipv8_config['port'] = self.config.get_ipv8_port() ipv8_config['address'] = self.config.get_ipv8_address() ipv8_config['overlays'] = [] ipv8_config['keys'] = [] # We load the keys ourselves ipv8_config['working_directory'] = str(self.config.get_state_dir()) if self.config.get_ipv8_bootstrap_override(): import ipv8.community as community_file community_file._DEFAULT_ADDRESSES = [ self.config.get_ipv8_bootstrap_override() ] community_file._DNS_ADDRESSES = [] self.ipv8 = IPv8( ipv8_config, enable_statistics=self.config.get_ipv8_statistics()) await self.ipv8.start() self.config.set_anon_proxy_settings( 2, ("127.0.0.1", self.config.get_tunnel_community_socks5_listen_ports())) self.ipv8_start_time = timemod.time() self.load_ipv8_overlays() self.enable_ipv8_statistics() if self.api_manager: self.api_manager.set_ipv8_session(self.ipv8) if self.config.get_tunnel_community_enabled(): await self.tunnel_community.wait_for_socks_servers() # Note that currently we should only start libtorrent after the SOCKS5 servers have been started if self.config.get_libtorrent_enabled(): self.readable_status = STATE_START_LIBTORRENT from tribler_core.modules.libtorrent.download_manager import DownloadManager self.dlmgr = DownloadManager(self) self.dlmgr.initialize() self.readable_status = STATE_LOAD_CHECKPOINTS await self.dlmgr.load_checkpoints() self.readable_status = STATE_READABLE_STARTED if self.config.get_torrent_checking_enabled(): self.readable_status = STATE_START_TORRENT_CHECKER self.torrent_checker = TorrentChecker(self) await self.torrent_checker.initialize() if self.config.get_dummy_wallets_enabled(): # For debugging purposes, we create dummy wallets dummy_wallet1 = DummyWallet1() self.wallets[dummy_wallet1.get_identifier()] = dummy_wallet1 dummy_wallet2 = DummyWallet2() self.wallets[dummy_wallet2.get_identifier()] = dummy_wallet2 if self.config.get_watch_folder_enabled(): self.readable_status = STATE_START_WATCH_FOLDER self.watch_folder = WatchFolder(self) self.watch_folder.start() if self.config.get_resource_monitor_enabled(): self.resource_monitor = ResourceMonitor(self) self.resource_monitor.start() if self.config.get_version_checker_enabled(): self.version_check_manager = VersionCheckManager(self) self.version_check_manager.start() if self.config.get_ipv8_enabled( ) and self.config.get_trustchain_enabled(): self.payout_manager = PayoutManager(self.trustchain_community, self.dht_community) # GigaChannel Manager should be started *after* resuming the downloads, # because it depends on the states of torrent downloads if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\ and self.config.get_libtorrent_enabled: self.gigachannel_manager = GigaChannelManager(self) self.gigachannel_manager.start() if self.config.get_bootstrap_enabled(): self.register_task('bootstrap_download', self.start_bootstrap_download) self.notifier.notify(NTFY.TRIBLER_STARTED)
from pathlib import Path import pytest from tribler_core.exceptions import OperationNotPossibleAtRuntimeException from tribler_core.utilities.configparser import CallbackConfigParser from tribler_core.utilities.install_dir import get_lib_path CONFIG_FILES_DIR = get_lib_path() / "tests/tools/data/config_files/" def test_configparser_config1(): ccp = CallbackConfigParser() ccp.read_file(CONFIG_FILES_DIR / 'config1.conf') assert ccp.get('general', 'version') == 11 assert ccp.get('search_community', 'enabled') assert isinstance(ccp.get('tunnel_community', 'socks5_listen_ports'), list) assert not ccp.get('foo', 'bar') def test_configparser_copy(): ccp = CallbackConfigParser() ccp.read_file(CONFIG_FILES_DIR / 'config1.conf') copy_ccp = ccp.copy() assert copy_ccp.get('general', 'version') == 11 assert copy_ccp.get('search_community', 'enabled') def test_configparser_set_callback():
class TestConfigParser(TriblerCoreTest): CONFIG_FILES_DIR = get_lib_path() / "tests/tools/data/config_files/" def test_configparser_config1(self): ccp = CallbackConfigParser() ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf') self.assertEqual(ccp.get('general', 'version'), 11) self.assertTrue(ccp.get('search_community', 'enabled')) self.assertIsInstance(ccp.get('tunnel_community', 'socks5_listen_ports'), list) self.assertFalse(ccp.get('foo', 'bar')) def test_configparser_copy(self): ccp = CallbackConfigParser() ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf') copy_ccp = ccp.copy() self.assertEqual(copy_ccp.get('general', 'version'), 11) self.assertTrue(copy_ccp.get('search_community', 'enabled')) def test_configparser_set_callback(self): def parser_callback(section, option, old_value, new_value): return True ccp = CallbackConfigParser() ccp.set_callback(parser_callback) ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf') ccp.set('search_community', 'enabled', False) ccp.set('search_community', 'bar', 42) self.assertFalse(ccp.get('search_community', 'enabled')) self.assertEqual(ccp.get('search_community', 'bar'), 42) @raises(OperationNotPossibleAtRuntimeException) def test_configparser_false_callback(self): def parser_callback(section, option, old_value, new_value): return False ccp = CallbackConfigParser() ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf') ccp.set_callback(parser_callback) ccp.set('search_community', 'enabled', False) def test_configparser_write_file(self): ccp = CallbackConfigParser() ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf') new_path = self.session_base_dir / 'config_new.conf' ccp.write_file(new_path) self.assertTrue(new_path.is_file()) ccp.read_file(new_path) self.assertEqual(ccp.get('general', 'version'), 11) self.assertTrue(ccp.get('search_community', 'enabled')) self.assertIsInstance(ccp.get('tunnel_community', 'socks5_listen_ports'), list) self.assertFalse(ccp.get('foo', 'bar')) def test_configparser_write_file_defaults(self): ccp = CallbackConfigParser(defaults={'foo': 'bar'}) new_path = self.session_base_dir / 'config_new.conf' ccp.write_file(new_path) self.assertTrue(new_path.is_file()) ccp.read_file(new_path) self.assertEqual(ccp.get('DEFAULT', 'foo'), 'bar')
import base64 from configobj import ConfigObj from validate import Validator from tribler_common.osutils import get_home_dir from tribler_core.components.libtorrent.utils.libtorrent_helper import libtorrent as lt from tribler_core.exceptions import InvalidConfigException from tribler_core.utilities.install_dir import get_lib_path from tribler_core.utilities.path_util import Path from tribler_core.utilities.utilities import bdecode_compat SPEC_FILENAME = 'download_config.spec' CONFIG_SPEC_PATH = get_lib_path( ) / 'components/libtorrent/download_manager' / SPEC_FILENAME NONPERSISTENT_DEFAULTS = {} class DownloadConfig: def __init__(self, config=None, state_dir=None): self.config = config or ConfigObj(configspec=str(CONFIG_SPEC_PATH), default_encoding='utf8') # Values that should not be stored and should be initialized to some default value self.nonpersistent = NONPERSISTENT_DEFAULTS.copy() self.state_dir = state_dir self.validate() def validate(self): """ Validate the ConfigObj using Validator.
from configobj import ConfigObj, ParseError from validate import Validator from tribler_common.network_utils import get_random_port from tribler_common.simpledefs import MAX_LIBTORRENT_RATE_LIMIT from tribler_core.exceptions import InvalidConfigException from tribler_core.modules.libtorrent.download_config import get_default_dest_dir from tribler_core.utilities import path_util from tribler_core.utilities.install_dir import get_lib_path from tribler_core.utilities.path_util import Path CONFIG_FILENAME = 'triblerd.conf' SPEC_FILENAME = 'tribler_config.spec' CONFIG_SPEC_PATH = get_lib_path() / 'config' / SPEC_FILENAME class TriblerConfig: """ Holds all Tribler Core configurable variables. This class is a wrapper around a ConfigObj. It has a specification of it's configuration sections and fields, their allowed values and default value in `config.spec`. """ def __init__(self, state_dir, config_file=None, reset_config_on_error=False): """ Create a new TriblerConfig instance. :param config_file: path to existing config file
import re from tribler_core.utilities.install_dir import get_lib_path # !ACHTUNG! We must first read the line into a file, then release the lock, and only then pass it to regex compiler. # Otherwise, there is an annoying race condition that reads in an empty file! with open(get_lib_path() / 'components' / 'metadata_store' / 'category_filter' / 'level2.regex', encoding="utf-8") as f: regex = f.read().strip() stoplist_expression = re.compile(regex, re.IGNORECASE) def is_forbidden(txt): return bool(stoplist_expression.search(txt))
async def start(self): """ Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader. Returns a deferred that fires when the Tribler session is ready for use. :param config: a TriblerConfig object """ self._logger.info("Session is using state directory: %s", self.config.get_state_dir()) self.get_ports_in_config() self.create_state_directory_structure() self.init_keypair() # we have to represent `user_id` as a string to make it equal to the # `user_id` on the GUI side user_id_str = hexlify(self.trustchain_keypair.key.pk).encode('utf-8') SentryReporter.set_user(user_id_str) # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.config.get_api_http_enabled() or self.config.get_api_https_enabled(): from tribler_core.restapi.rest_manager import RESTManager self.api_manager = RESTManager(self) self.readable_status = STATE_START_API await self.api_manager.start() if self.upgrader_enabled and not self.core_test_mode: from tribler_core.upgrade.upgrade import TriblerUpgrader self.upgrader = TriblerUpgrader(self) self.readable_status = STATE_UPGRADING_READABLE await self.upgrader.run() self.tracker_manager = TrackerManager(self) # Start torrent checker before Popularity community is loaded if self.config.get_torrent_checking_enabled() and not self.core_test_mode: from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker self.readable_status = STATE_START_TORRENT_CHECKER self.torrent_checker = TorrentChecker(self) await self.torrent_checker.initialize() # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root # certificates provided by the system trust store. if sys.platform == 'darwin': os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem') if self.config.get_chant_enabled(): from tribler_core.modules.metadata_store.store import MetadataStore channels_dir = self.config.get_chant_channels_dir() metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db' database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair, notifier=self.notifier, disable_sync=self.core_test_mode) if self.core_test_mode: generate_test_channels(self.mds) # IPv8 if self.config.get_ipv8_enabled(): from ipv8.configuration import ConfigBuilder from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint ipv8_config_builder = (ConfigBuilder() .set_port(self.config.get_ipv8_port()) .set_address(self.config.get_ipv8_address()) .clear_overlays() .clear_keys() # We load the keys ourselves .set_working_directory(str(self.config.get_state_dir())) .set_walker_interval(self.config.get_ipv8_walk_interval())) if self.core_test_mode: endpoint = DispatcherEndpoint([]) else: # IPv8 includes IPv6 support by default. # We only load IPv4 to not kill all Tribler overlays (currently, it would instantly crash all users). # If you want to test IPv6 in Tribler you can set ``endpoint = None`` here. endpoint = DispatcherEndpoint(["UDPIPv4"], UDPIPv4={'port': self.config.get_ipv8_port(), 'ip': self.config.get_ipv8_address()}) self.ipv8 = IPv8(ipv8_config_builder.finalize(), enable_statistics=self.config.get_ipv8_statistics() and not self.core_test_mode, endpoint_override=endpoint) await self.ipv8.start() self.config.set_anon_proxy_settings(2, ("127.0.0.1", self. config.get_tunnel_community_socks5_listen_ports())) self.ipv8_start_time = timemod.time() self.load_ipv8_overlays() if not self.core_test_mode: self.enable_ipv8_statistics() if self.api_manager: self.api_manager.set_ipv8_session(self.ipv8) if self.config.get_tunnel_community_enabled(): await self.tunnel_community.wait_for_socks_servers() if self.config.get_ipv8_walk_scaling_enabled(): from tribler_core.modules.ipv8_health_monitor import IPv8Monitor IPv8Monitor(self.ipv8, self.config.get_ipv8_walk_interval(), self.config.get_ipv8_walk_scaling_upper_limit()).start(self) # Note that currently we should only start libtorrent after the SOCKS5 servers have been started if self.config.get_libtorrent_enabled(): self.readable_status = STATE_START_LIBTORRENT from tribler_core.modules.libtorrent.download_manager import DownloadManager self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode) self.dlmgr.initialize() self.readable_status = STATE_LOAD_CHECKPOINTS await self.dlmgr.load_checkpoints() if self.core_test_mode: await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000") self.readable_status = STATE_READABLE_STARTED if self.config.get_watch_folder_enabled(): from tribler_core.modules.watch_folder import WatchFolder self.readable_status = STATE_START_WATCH_FOLDER self.watch_folder = WatchFolder(self) self.watch_folder.start() if self.config.get_resource_monitor_enabled() and not self.core_test_mode: from tribler_core.modules.resource_monitor.core import CoreResourceMonitor self.resource_monitor = CoreResourceMonitor(self) self.resource_monitor.start() if self.config.get_version_checker_enabled() and not self.core_test_mode: from tribler_core.modules.versioncheck_manager import VersionCheckManager self.version_check_manager = VersionCheckManager(self) self.version_check_manager.start() if self.config.get_ipv8_enabled(): from tribler_core.modules.payout_manager import PayoutManager self.payout_manager = PayoutManager(self.bandwidth_community, self.dht_community) if self.core_test_mode: from ipv8.messaging.interfaces.udp.endpoint import UDPv4Address from ipv8.dht.routing import RoutingTable self.dht_community.routing_tables[UDPv4Address] = RoutingTable('\x00' * 20) # GigaChannel Manager should be started *after* resuming the downloads, # because it depends on the states of torrent downloads if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \ and self.config.get_libtorrent_enabled: from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager self.gigachannel_manager = GigaChannelManager(self) if not self.core_test_mode: self.gigachannel_manager.start() if self.config.get_bootstrap_enabled() and not self.core_test_mode: self.register_task('bootstrap_download', self.start_bootstrap_download) self.notifier.notify(NTFY.TRIBLER_STARTED, self.trustchain_keypair.key.pk) # If there is a config error, report to the user via GUI notifier if self.config.config_error: self.notifier.notify(NTFY.REPORT_CONFIG_ERROR, self.config.config_error)
""" The FamilyFilter filters out nsfw content if enabled. Author(s): Jelle Roozenburg """ import logging import re from tribler_core.utilities.install_dir import get_lib_path WORDS_REGEXP = re.compile('[a-zA-Z0-9]+') termfilename = get_lib_path( ) / 'modules' / 'category_filter' / 'filter_terms.filter' def initTerms(filename): terms = set() searchterms = set() try: with open(filename) as f: lines = f.read().lower().splitlines() for line in lines: if line.startswith('*'): searchterms.add(line[1:]) else: terms.add(line) except OSError: raise OSError("Could not open %s, initTerms failed.", filename)
""" The FamilyFilter filters out nsfw content if enabled. Author(s): Jelle Roozenburg """ import logging import re from tribler_core.utilities.install_dir import get_lib_path WORDS_REGEXP = re.compile('[a-zA-Z0-9]+') termfilename = get_lib_path() / 'components' / 'metadata_store' /'category_filter' / 'filter_terms.filter' def initTerms(filename): terms = set() searchterms = set() try: with open(filename) as f: lines = f.read().lower().splitlines() for line in lines: if line.startswith('*'): searchterms.add(line[1:]) else: terms.add(line) except OSError: raise OSError("Could not open %s, initTerms failed.", filename)
""" Category. Author(s): Yuan Yuan, Jelle Roozenburg """ import logging import re from functools import cmp_to_key from tribler_core.components.metadata_store.category_filter.family_filter import default_xxx_filter from tribler_core.components.metadata_store.category_filter.init_category import getCategoryInfo from tribler_core.utilities.install_dir import get_lib_path from tribler_core.utilities.unicode import recursive_unicode CATEGORY_CONFIG_FILE = get_lib_path( ) / 'components/metadata_store/category_filter/category.conf' def cmp_rank(a, b): if 'rank' not in a: return 1 if 'rank' not in b: return -1 if a['rank'] == b['rank']: return 0 if a['rank'] == -1: return 1 if b['rank'] == -1: return -1 if a['rank'] < b['rank']: return -1
import re from tribler_core.utilities.install_dir import get_lib_path # !ACHTUNG! We must first read the line into a file, then release the lock, and only then pass it to regex compiler. # Otherwise, there is an annoying race condition that reads in an empty file! with open(get_lib_path() / 'modules' / 'category_filter' / 'level2.regex', encoding="utf-8") as f: regex = f.read().strip() stoplist_expression = re.compile(regex, re.IGNORECASE) def is_forbidden(txt): return bool(stoplist_expression.search(txt))