async def start_components(self):
self.logger.info("Session is using state directory: %s",
self.config.state_dir)
create_state_directory_structure(self.config.state_dir)
patch_crypto_be_discovery()
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(get_lib_path() /
'root_certs_mac.pem')
coros = [comp.start() for comp in self.components.values()]
await gather(*coros, return_exceptions=not self.failfast)
if self._startup_exception:
self._reraise_startup_exception_in_separate_task()
from pathlib import Path
from configobj import ConfigObj
from validate import Validator
from tribler_core.exceptions import InvalidConfigException
from tribler_core.utilities import path_util
from tribler_core.utilities.install_dir import get_lib_path
from tribler_core.utilities.libtorrent_helper import libtorrent as lt
from tribler_core.utilities.osutils import get_home_dir
from tribler_core.utilities.path_util import str_path
from tribler_core.utilities.utilities import bdecode_compat
SPEC_FILENAME = 'download_config.spec'
CONFIG_SPEC_PATH = get_lib_path() / 'modules' / 'libtorrent' / SPEC_FILENAME
NONPERSISTENT_DEFAULTS = {}
class DownloadConfig:
def __init__(self, config=None, state_dir=None):
self.config = config or ConfigObj(configspec=str(CONFIG_SPEC_PATH),
default_encoding='utf8')
# Values that should not be stored and should be initialized to some default value
self.nonpersistent = NONPERSISTENT_DEFAULTS.copy()
self.state_dir = state_dir
self.validate()
def validate(self):
"""
Validate the ConfigObj using Validator.
"""
Category.
Author(s): Yuan Yuan, Jelle Roozenburg
"""
import logging
import re
from functools import cmp_to_key
from tribler_core.modules.category_filter.family_filter import default_xxx_filter
from tribler_core.modules.category_filter.init_category import getCategoryInfo
from tribler_core.utilities.install_dir import get_lib_path
from tribler_core.utilities.unicode import recursive_unicode
CATEGORY_CONFIG_FILE = get_lib_path() / 'modules/category_filter/category.conf'
def cmp_rank(a, b):
if 'rank' not in a:
return 1
if 'rank' not in b:
return -1
if a['rank'] == b['rank']:
return 0
if a['rank'] == -1:
return 1
if b['rank'] == -1:
return -1
if a['rank'] < b['rank']:
return -1
return 1
async def start(self):
"""
Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
Returns a deferred that fires when the Tribler session is ready for use.
:param config: a TriblerConfig object
"""
self._logger.info("Session is using state directory: %s",
self.config.get_state_dir())
self.get_ports_in_config()
self.create_state_directory_structure()
self.init_keypair()
# Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
if self.config.get_http_api_enabled():
self.api_manager = RESTManager(self)
self.readable_status = STATE_START_API
await self.api_manager.start()
if self.upgrader_enabled:
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
try:
await self.upgrader.run()
except Exception as e:
self._logger.error("Error in Upgrader callback chain: %s", e)
self.tracker_manager = TrackerManager(self)
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(
(get_lib_path() / 'root_certs_mac.pem'))
if self.config.get_chant_enabled():
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_testnet(
) else 'metadata_testnet.db'
database_path = self.config.get_state_dir(
) / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir,
self.trustchain_keypair)
# IPv8
if self.config.get_ipv8_enabled():
from ipv8.configuration import get_default_configuration
ipv8_config = get_default_configuration()
ipv8_config['port'] = self.config.get_ipv8_port()
ipv8_config['address'] = self.config.get_ipv8_address()
ipv8_config['overlays'] = []
ipv8_config['keys'] = [] # We load the keys ourselves
ipv8_config['working_directory'] = str(self.config.get_state_dir())
if self.config.get_ipv8_bootstrap_override():
import ipv8.community as community_file
community_file._DEFAULT_ADDRESSES = [
self.config.get_ipv8_bootstrap_override()
]
community_file._DNS_ADDRESSES = []
self.ipv8 = IPv8(
ipv8_config,
enable_statistics=self.config.get_ipv8_statistics())
await self.ipv8.start()
self.config.set_anon_proxy_settings(
2, ("127.0.0.1",
self.config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
await self.tunnel_community.wait_for_socks_servers()
# Note that currently we should only start libtorrent after the SOCKS5 servers have been started
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
self.readable_status = STATE_READABLE_STARTED
if self.config.get_torrent_checking_enabled():
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
if self.config.get_dummy_wallets_enabled():
# For debugging purposes, we create dummy wallets
dummy_wallet1 = DummyWallet1()
self.wallets[dummy_wallet1.get_identifier()] = dummy_wallet1
dummy_wallet2 = DummyWallet2()
self.wallets[dummy_wallet2.get_identifier()] = dummy_wallet2
if self.config.get_watch_folder_enabled():
self.readable_status = STATE_START_WATCH_FOLDER
self.watch_folder = WatchFolder(self)
self.watch_folder.start()
if self.config.get_resource_monitor_enabled():
self.resource_monitor = ResourceMonitor(self)
self.resource_monitor.start()
if self.config.get_version_checker_enabled():
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()
if self.config.get_ipv8_enabled(
) and self.config.get_trustchain_enabled():
self.payout_manager = PayoutManager(self.trustchain_community,
self.dht_community)
# GigaChannel Manager should be started *after* resuming the downloads,
# because it depends on the states of torrent downloads
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\
and self.config.get_libtorrent_enabled:
self.gigachannel_manager = GigaChannelManager(self)
self.gigachannel_manager.start()
if self.config.get_bootstrap_enabled():
self.register_task('bootstrap_download',
self.start_bootstrap_download)
self.notifier.notify(NTFY.TRIBLER_STARTED)
from pathlib import Path
import pytest
from tribler_core.exceptions import OperationNotPossibleAtRuntimeException
from tribler_core.utilities.configparser import CallbackConfigParser
from tribler_core.utilities.install_dir import get_lib_path
CONFIG_FILES_DIR = get_lib_path() / "tests/tools/data/config_files/"
def test_configparser_config1():
ccp = CallbackConfigParser()
ccp.read_file(CONFIG_FILES_DIR / 'config1.conf')
assert ccp.get('general', 'version') == 11
assert ccp.get('search_community', 'enabled')
assert isinstance(ccp.get('tunnel_community', 'socks5_listen_ports'), list)
assert not ccp.get('foo', 'bar')
def test_configparser_copy():
ccp = CallbackConfigParser()
ccp.read_file(CONFIG_FILES_DIR / 'config1.conf')
copy_ccp = ccp.copy()
assert copy_ccp.get('general', 'version') == 11
assert copy_ccp.get('search_community', 'enabled')
def test_configparser_set_callback():
class TestConfigParser(TriblerCoreTest):
CONFIG_FILES_DIR = get_lib_path() / "tests/tools/data/config_files/"
def test_configparser_config1(self):
ccp = CallbackConfigParser()
ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf')
self.assertEqual(ccp.get('general', 'version'), 11)
self.assertTrue(ccp.get('search_community', 'enabled'))
self.assertIsInstance(ccp.get('tunnel_community', 'socks5_listen_ports'), list)
self.assertFalse(ccp.get('foo', 'bar'))
def test_configparser_copy(self):
ccp = CallbackConfigParser()
ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf')
copy_ccp = ccp.copy()
self.assertEqual(copy_ccp.get('general', 'version'), 11)
self.assertTrue(copy_ccp.get('search_community', 'enabled'))
def test_configparser_set_callback(self):
def parser_callback(section, option, old_value, new_value):
return True
ccp = CallbackConfigParser()
ccp.set_callback(parser_callback)
ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf')
ccp.set('search_community', 'enabled', False)
ccp.set('search_community', 'bar', 42)
self.assertFalse(ccp.get('search_community', 'enabled'))
self.assertEqual(ccp.get('search_community', 'bar'), 42)
@raises(OperationNotPossibleAtRuntimeException)
def test_configparser_false_callback(self):
def parser_callback(section, option, old_value, new_value):
return False
ccp = CallbackConfigParser()
ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf')
ccp.set_callback(parser_callback)
ccp.set('search_community', 'enabled', False)
def test_configparser_write_file(self):
ccp = CallbackConfigParser()
ccp.read_file(self.CONFIG_FILES_DIR / 'config1.conf')
new_path = self.session_base_dir / 'config_new.conf'
ccp.write_file(new_path)
self.assertTrue(new_path.is_file())
ccp.read_file(new_path)
self.assertEqual(ccp.get('general', 'version'), 11)
self.assertTrue(ccp.get('search_community', 'enabled'))
self.assertIsInstance(ccp.get('tunnel_community', 'socks5_listen_ports'), list)
self.assertFalse(ccp.get('foo', 'bar'))
def test_configparser_write_file_defaults(self):
ccp = CallbackConfigParser(defaults={'foo': 'bar'})
new_path = self.session_base_dir / 'config_new.conf'
ccp.write_file(new_path)
self.assertTrue(new_path.is_file())
ccp.read_file(new_path)
self.assertEqual(ccp.get('DEFAULT', 'foo'), 'bar')
import base64
from configobj import ConfigObj
from validate import Validator
from tribler_common.osutils import get_home_dir
from tribler_core.components.libtorrent.utils.libtorrent_helper import libtorrent as lt
from tribler_core.exceptions import InvalidConfigException
from tribler_core.utilities.install_dir import get_lib_path
from tribler_core.utilities.path_util import Path
from tribler_core.utilities.utilities import bdecode_compat
SPEC_FILENAME = 'download_config.spec'
CONFIG_SPEC_PATH = get_lib_path(
) / 'components/libtorrent/download_manager' / SPEC_FILENAME
NONPERSISTENT_DEFAULTS = {}
class DownloadConfig:
def __init__(self, config=None, state_dir=None):
self.config = config or ConfigObj(configspec=str(CONFIG_SPEC_PATH),
default_encoding='utf8')
# Values that should not be stored and should be initialized to some default value
self.nonpersistent = NONPERSISTENT_DEFAULTS.copy()
self.state_dir = state_dir
self.validate()
def validate(self):
"""
Validate the ConfigObj using Validator.
from configobj import ConfigObj, ParseError
from validate import Validator
from tribler_common.network_utils import get_random_port
from tribler_common.simpledefs import MAX_LIBTORRENT_RATE_LIMIT
from tribler_core.exceptions import InvalidConfigException
from tribler_core.modules.libtorrent.download_config import get_default_dest_dir
from tribler_core.utilities import path_util
from tribler_core.utilities.install_dir import get_lib_path
from tribler_core.utilities.path_util import Path
CONFIG_FILENAME = 'triblerd.conf'
SPEC_FILENAME = 'tribler_config.spec'
CONFIG_SPEC_PATH = get_lib_path() / 'config' / SPEC_FILENAME
class TriblerConfig:
"""
Holds all Tribler Core configurable variables.
This class is a wrapper around a ConfigObj. It has a specification of it's configuration sections and fields,
their allowed values and default value in `config.spec`.
"""
def __init__(self, state_dir, config_file=None, reset_config_on_error=False):
"""
Create a new TriblerConfig instance.
:param config_file: path to existing config file
import re
from tribler_core.utilities.install_dir import get_lib_path
# !ACHTUNG! We must first read the line into a file, then release the lock, and only then pass it to regex compiler.
# Otherwise, there is an annoying race condition that reads in an empty file!
with open(get_lib_path() / 'components' / 'metadata_store' / 'category_filter' / 'level2.regex', encoding="utf-8") as f:
regex = f.read().strip()
stoplist_expression = re.compile(regex, re.IGNORECASE)
def is_forbidden(txt):
return bool(stoplist_expression.search(txt))
async def start(self):
"""
Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
Returns a deferred that fires when the Tribler session is ready for use.
:param config: a TriblerConfig object
"""
self._logger.info("Session is using state directory: %s", self.config.get_state_dir())
self.get_ports_in_config()
self.create_state_directory_structure()
self.init_keypair()
# we have to represent `user_id` as a string to make it equal to the
# `user_id` on the GUI side
user_id_str = hexlify(self.trustchain_keypair.key.pk).encode('utf-8')
SentryReporter.set_user(user_id_str)
# Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
if self.config.get_api_http_enabled() or self.config.get_api_https_enabled():
from tribler_core.restapi.rest_manager import RESTManager
self.api_manager = RESTManager(self)
self.readable_status = STATE_START_API
await self.api_manager.start()
if self.upgrader_enabled and not self.core_test_mode:
from tribler_core.upgrade.upgrade import TriblerUpgrader
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
await self.upgrader.run()
self.tracker_manager = TrackerManager(self)
# Start torrent checker before Popularity community is loaded
if self.config.get_torrent_checking_enabled() and not self.core_test_mode:
from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem')
if self.config.get_chant_enabled():
from tribler_core.modules.metadata_store.store import MetadataStore
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db'
database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair,
notifier=self.notifier,
disable_sync=self.core_test_mode)
if self.core_test_mode:
generate_test_channels(self.mds)
# IPv8
if self.config.get_ipv8_enabled():
from ipv8.configuration import ConfigBuilder
from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint
ipv8_config_builder = (ConfigBuilder()
.set_port(self.config.get_ipv8_port())
.set_address(self.config.get_ipv8_address())
.clear_overlays()
.clear_keys() # We load the keys ourselves
.set_working_directory(str(self.config.get_state_dir()))
.set_walker_interval(self.config.get_ipv8_walk_interval()))
if self.core_test_mode:
endpoint = DispatcherEndpoint([])
else:
# IPv8 includes IPv6 support by default.
# We only load IPv4 to not kill all Tribler overlays (currently, it would instantly crash all users).
# If you want to test IPv6 in Tribler you can set ``endpoint = None`` here.
endpoint = DispatcherEndpoint(["UDPIPv4"], UDPIPv4={'port': self.config.get_ipv8_port(),
'ip': self.config.get_ipv8_address()})
self.ipv8 = IPv8(ipv8_config_builder.finalize(),
enable_statistics=self.config.get_ipv8_statistics() and not self.core_test_mode,
endpoint_override=endpoint)
await self.ipv8.start()
self.config.set_anon_proxy_settings(2, ("127.0.0.1",
self.
config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
if not self.core_test_mode:
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
await self.tunnel_community.wait_for_socks_servers()
if self.config.get_ipv8_walk_scaling_enabled():
from tribler_core.modules.ipv8_health_monitor import IPv8Monitor
IPv8Monitor(self.ipv8,
self.config.get_ipv8_walk_interval(),
self.config.get_ipv8_walk_scaling_upper_limit()).start(self)
# Note that currently we should only start libtorrent after the SOCKS5 servers have been started
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
if self.core_test_mode:
await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000")
self.readable_status = STATE_READABLE_STARTED
if self.config.get_watch_folder_enabled():
from tribler_core.modules.watch_folder import WatchFolder
self.readable_status = STATE_START_WATCH_FOLDER
self.watch_folder = WatchFolder(self)
self.watch_folder.start()
if self.config.get_resource_monitor_enabled() and not self.core_test_mode:
from tribler_core.modules.resource_monitor.core import CoreResourceMonitor
self.resource_monitor = CoreResourceMonitor(self)
self.resource_monitor.start()
if self.config.get_version_checker_enabled() and not self.core_test_mode:
from tribler_core.modules.versioncheck_manager import VersionCheckManager
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()
if self.config.get_ipv8_enabled():
from tribler_core.modules.payout_manager import PayoutManager
self.payout_manager = PayoutManager(self.bandwidth_community, self.dht_community)
if self.core_test_mode:
from ipv8.messaging.interfaces.udp.endpoint import UDPv4Address
from ipv8.dht.routing import RoutingTable
self.dht_community.routing_tables[UDPv4Address] = RoutingTable('\x00' * 20)
# GigaChannel Manager should be started *after* resuming the downloads,
# because it depends on the states of torrent downloads
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \
and self.config.get_libtorrent_enabled:
from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager
self.gigachannel_manager = GigaChannelManager(self)
if not self.core_test_mode:
self.gigachannel_manager.start()
if self.config.get_bootstrap_enabled() and not self.core_test_mode:
self.register_task('bootstrap_download', self.start_bootstrap_download)
self.notifier.notify(NTFY.TRIBLER_STARTED, self.trustchain_keypair.key.pk)
# If there is a config error, report to the user via GUI notifier
if self.config.config_error:
self.notifier.notify(NTFY.REPORT_CONFIG_ERROR, self.config.config_error)
"""
The FamilyFilter filters out nsfw content if enabled.
Author(s): Jelle Roozenburg
"""
import logging
import re
from tribler_core.utilities.install_dir import get_lib_path
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
termfilename = get_lib_path(
) / 'modules' / 'category_filter' / 'filter_terms.filter'
def initTerms(filename):
terms = set()
searchterms = set()
try:
with open(filename) as f:
lines = f.read().lower().splitlines()
for line in lines:
if line.startswith('*'):
searchterms.add(line[1:])
else:
terms.add(line)
except OSError:
raise OSError("Could not open %s, initTerms failed.", filename)
"""
The FamilyFilter filters out nsfw content if enabled.
Author(s): Jelle Roozenburg
"""
import logging
import re
from tribler_core.utilities.install_dir import get_lib_path
WORDS_REGEXP = re.compile('[a-zA-Z0-9]+')
termfilename = get_lib_path() / 'components' / 'metadata_store' /'category_filter' / 'filter_terms.filter'
def initTerms(filename):
terms = set()
searchterms = set()
try:
with open(filename) as f:
lines = f.read().lower().splitlines()
for line in lines:
if line.startswith('*'):
searchterms.add(line[1:])
else:
terms.add(line)
except OSError:
raise OSError("Could not open %s, initTerms failed.", filename)
"""
Category.
Author(s): Yuan Yuan, Jelle Roozenburg
"""
import logging
import re
from functools import cmp_to_key
from tribler_core.components.metadata_store.category_filter.family_filter import default_xxx_filter
from tribler_core.components.metadata_store.category_filter.init_category import getCategoryInfo
from tribler_core.utilities.install_dir import get_lib_path
from tribler_core.utilities.unicode import recursive_unicode
CATEGORY_CONFIG_FILE = get_lib_path(
) / 'components/metadata_store/category_filter/category.conf'
def cmp_rank(a, b):
if 'rank' not in a:
return 1
if 'rank' not in b:
return -1
if a['rank'] == b['rank']:
return 0
if a['rank'] == -1:
return 1
if b['rank'] == -1:
return -1
if a['rank'] < b['rank']:
return -1
import re
from tribler_core.utilities.install_dir import get_lib_path
# !ACHTUNG! We must first read the line into a file, then release the lock, and only then pass it to regex compiler.
# Otherwise, there is an annoying race condition that reads in an empty file!
with open(get_lib_path() / 'modules' / 'category_filter' / 'level2.regex', encoding="utf-8") as f:
regex = f.read().strip()
stoplist_expression = re.compile(regex, re.IGNORECASE)
def is_forbidden(txt):
return bool(stoplist_expression.search(txt))
