def upgrade_state_dir(root_state_dir: Path,
update_status_callback=None,
interrupt_upgrade_event=None):
# Before any upgrade, prepare a separate state directory for the update version so it does not
# affect the older version state directory. This allows for safe rollback.
version_history = VersionHistory(root_state_dir)
version_history.fork_state_directory_if_necessary()
version_history.save_if_necessary()
state_dir = version_history.code_version.directory
if not state_dir.exists():
return
config = TriblerConfig.load(file=state_dir / CONFIG_FILE_NAME,
state_dir=state_dir,
reset_config_on_error=True)
channels_dir = config.chant.get_path_as_absolute('channels_dir',
config.state_dir)
primary_private_key_path = config.state_dir / KeyComponent.get_private_key_filename(
config)
primary_public_key_path = config.state_dir / config.trustchain.ec_keypair_pubfilename
primary_key = KeyComponent.load_or_create(primary_private_key_path,
primary_public_key_path)
upgrader = TriblerUpgrader(state_dir,
channels_dir,
primary_key,
update_status_callback=update_status_callback,
interrupt_upgrade_event=interrupt_upgrade_event)
upgrader.run()
def test_upgrade_pony13to14(upgrader: TriblerUpgrader, state_dir, channels_dir,
trustchain_keypair, mds_path):
tags_path = state_dir / 'sqlite/tags.db'
_copy(source_name='pony_v13.db', target=mds_path)
_copy(source_name='tags_v13.db', target=tags_path)
upgrader.upgrade_pony_db_13to14()
mds = MetadataStore(mds_path,
channels_dir,
trustchain_keypair,
check_tables=False)
tags = TagDatabase(str(tags_path), create_tables=False, check_tables=False)
with db_session:
assert upgrader.column_exists_in_table(mds._db, 'ChannelNode',
'tag_processor_version')
assert upgrader.column_exists_in_table(tags.instance, 'TorrentTagOp',
'auto_generated')
assert mds.get_value('db_version') == '14'
async def start(self):
"""
Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
Returns a deferred that fires when the Tribler session is ready for use.
:param config: a TriblerConfig object
"""
self._logger.info("Session is using state directory: %s",
self.config.get_state_dir())
self.get_ports_in_config()
self.create_state_directory_structure()
self.init_keypair()
# Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
if self.config.get_http_api_enabled():
self.api_manager = RESTManager(self)
self.readable_status = STATE_START_API
await self.api_manager.start()
if self.upgrader_enabled:
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
try:
await self.upgrader.run()
except Exception as e:
self._logger.error("Error in Upgrader callback chain: %s", e)
self.tracker_manager = TrackerManager(self)
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(
(get_lib_path() / 'root_certs_mac.pem'))
if self.config.get_chant_enabled():
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_testnet(
) else 'metadata_testnet.db'
database_path = self.config.get_state_dir(
) / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir,
self.trustchain_keypair)
# IPv8
if self.config.get_ipv8_enabled():
from ipv8.configuration import get_default_configuration
ipv8_config = get_default_configuration()
ipv8_config['port'] = self.config.get_ipv8_port()
ipv8_config['address'] = self.config.get_ipv8_address()
ipv8_config['overlays'] = []
ipv8_config['keys'] = [] # We load the keys ourselves
ipv8_config['working_directory'] = str(self.config.get_state_dir())
if self.config.get_ipv8_bootstrap_override():
import ipv8.community as community_file
community_file._DEFAULT_ADDRESSES = [
self.config.get_ipv8_bootstrap_override()
]
community_file._DNS_ADDRESSES = []
self.ipv8 = IPv8(
ipv8_config,
enable_statistics=self.config.get_ipv8_statistics())
await self.ipv8.start()
self.config.set_anon_proxy_settings(
2, ("127.0.0.1",
self.config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
await self.tunnel_community.wait_for_socks_servers()
# Note that currently we should only start libtorrent after the SOCKS5 servers have been started
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
self.readable_status = STATE_READABLE_STARTED
if self.config.get_torrent_checking_enabled():
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
if self.config.get_dummy_wallets_enabled():
# For debugging purposes, we create dummy wallets
dummy_wallet1 = DummyWallet1()
self.wallets[dummy_wallet1.get_identifier()] = dummy_wallet1
dummy_wallet2 = DummyWallet2()
self.wallets[dummy_wallet2.get_identifier()] = dummy_wallet2
if self.config.get_watch_folder_enabled():
self.readable_status = STATE_START_WATCH_FOLDER
self.watch_folder = WatchFolder(self)
self.watch_folder.start()
if self.config.get_resource_monitor_enabled():
self.resource_monitor = ResourceMonitor(self)
self.resource_monitor.start()
if self.config.get_version_checker_enabled():
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()
if self.config.get_ipv8_enabled(
) and self.config.get_trustchain_enabled():
self.payout_manager = PayoutManager(self.trustchain_community,
self.dht_community)
# GigaChannel Manager should be started *after* resuming the downloads,
# because it depends on the states of torrent downloads
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\
and self.config.get_libtorrent_enabled:
self.gigachannel_manager = GigaChannelManager(self)
self.gigachannel_manager.start()
if self.config.get_bootstrap_enabled():
self.register_task('bootstrap_download',
self.start_bootstrap_download)
self.notifier.notify(NTFY.TRIBLER_STARTED)
def upgrader(session):
return TriblerUpgrader(session)
def upgrader(state_dir, channels_dir, trustchain_keypair):
return TriblerUpgrader(state_dir, channels_dir, trustchain_keypair)
async def setUp(self):
await super(TestUpgrader, self).setUp()
self.upgrader = TriblerUpgrader(self.session)
class TestUpgrader(TestAsServer):
async def setUp(self):
await super(TestUpgrader, self).setUp()
self.upgrader = TriblerUpgrader(self.session)
@timeout(10)
async def test_update_status_text(self):
test_future = Future()
def on_upgrade_tick(status_text):
self.assertEqual(status_text, "12345")
test_future.set_result(None)
self.session.notifier.add_observer(NTFY.UPGRADER_TICK, on_upgrade_tick)
self.upgrader.update_status("12345")
await test_future
@timeout(10)
async def test_upgrade_72_to_pony(self):
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'tribler_v29.sdb'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'tribler.sdb'
new_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
await self.upgrader.run()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(new_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 24)
mds.shutdown()
def test_upgrade_pony_db_6to7(self):
"""
Test that channels and torrents with forbidden words are cleaned up during upgrade from Pony db ver 6 to 7.
Also, check that the DB version is upgraded.
:return:
"""
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
self.upgrader.upgrade_pony_db_6to7()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 23)
self.assertEqual(mds.ChannelMetadata.select().count(), 2)
self.assertEqual(int(mds.MiscData.get(name="db_version").value), 7)
mds.shutdown()
def test_upgrade_pony_db_7to8(self):
"""
Test that proper additionald index is created.
Also, check that the DB version is upgraded.
"""
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v7.db'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
self.upgrader.upgrade_pony_db_7to8()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(int(mds.MiscData.get(name="db_version").value), 8)
self.assertEqual(mds.Vsids[0].exp_period, 24.0 * 60 * 60 * 3)
self.assertTrue(
list(
mds._db.execute(
'PRAGMA index_info("idx_channelnode__metadata_type")'))
)
mds.shutdown()
@timeout(10)
async def test_upgrade_pony_db_complete(self):
"""
Test complete update sequence for Pony DB (e.g. 6->7->8)
"""
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
await self.upgrader.run()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 23)
self.assertEqual(mds.ChannelMetadata.select().count(), 2)
self.assertEqual(int(mds.MiscData.get(name="db_version").value), 8)
self.assertTrue(
list(
mds._db.execute(
'PRAGMA index_info("idx_channelnode__metadata_type")'))
)
mds.shutdown()
@timeout(10)
async def test_skip_upgrade_72_to_pony(self):
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'tribler_v29.sdb'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'tribler.sdb'
new_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
channels_dir = self.session.config.get_chant_channels_dir()
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
self.upgrader.skip()
await self.upgrader.run()
mds = MetadataStore(new_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 0)
self.assertEqual(mds.ChannelMetadata.select().count(), 0)
mds.shutdown()
def test_delete_noncompliant_state(self):
STATE_DIR = TESTS_DATA_DIR / 'noncompliant_state_dir'
tmpdir = self.temporary_directory() / STATE_DIR.name
shutil.copytree(str_path(STATE_DIR), str_path(tmpdir))
cleanup_noncompliant_channel_torrents(tmpdir)
# Check cleanup of the channels dir
dir_listing = list((tmpdir / "channels").iterdir())
self.assertEqual(3, len(dir_listing))
for f in (tmpdir / "channels").iterdir():
self.assertEqual(CHANNEL_DIR_NAME_LENGTH, len(f.stem))
# Check cleanup of torrent state dir
checkpoints_dir = tmpdir / "dlcheckpoints"
dir_listing = os.listdir(checkpoints_dir)
self.assertEqual(1, len(dir_listing))
file_path = checkpoints_dir / dir_listing[0]
pstate = CallbackConfigParser()
pstate.read_file(file_path)
self.assertEqual(CHANNEL_DIR_NAME_LENGTH,
len(pstate.get('state', 'metainfo')['info']['name']))
async def setUp(self):
await super(TestUpgraderStateDirectory, self).setUp()
self.upgrader = TriblerUpgrader(self.session)
self.original_version = version.version_id
async def start(self):
"""
Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
Returns a deferred that fires when the Tribler session is ready for use.
:param config: a TriblerConfig object
"""
self._logger.info("Session is using state directory: %s", self.config.get_state_dir())
self.get_ports_in_config()
self.create_state_directory_structure()
self.init_keypair()
# we have to represent `user_id` as a string to make it equal to the
# `user_id` on the GUI side
user_id_str = hexlify(self.trustchain_keypair.key.pk).encode('utf-8')
SentryReporter.set_user(user_id_str)
# Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
if self.config.get_api_http_enabled() or self.config.get_api_https_enabled():
from tribler_core.restapi.rest_manager import RESTManager
self.api_manager = RESTManager(self)
self.readable_status = STATE_START_API
await self.api_manager.start()
if self.upgrader_enabled and not self.core_test_mode:
from tribler_core.upgrade.upgrade import TriblerUpgrader
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
await self.upgrader.run()
self.tracker_manager = TrackerManager(self)
# Start torrent checker before Popularity community is loaded
if self.config.get_torrent_checking_enabled() and not self.core_test_mode:
from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem')
if self.config.get_chant_enabled():
from tribler_core.modules.metadata_store.store import MetadataStore
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db'
database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair,
notifier=self.notifier,
disable_sync=self.core_test_mode)
if self.core_test_mode:
generate_test_channels(self.mds)
# IPv8
if self.config.get_ipv8_enabled():
from ipv8.configuration import ConfigBuilder
from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint
ipv8_config_builder = (ConfigBuilder()
.set_port(self.config.get_ipv8_port())
.set_address(self.config.get_ipv8_address())
.clear_overlays()
.clear_keys() # We load the keys ourselves
.set_working_directory(str(self.config.get_state_dir()))
.set_walker_interval(self.config.get_ipv8_walk_interval()))
if self.core_test_mode:
endpoint = DispatcherEndpoint([])
else:
# IPv8 includes IPv6 support by default.
# We only load IPv4 to not kill all Tribler overlays (currently, it would instantly crash all users).
# If you want to test IPv6 in Tribler you can set ``endpoint = None`` here.
endpoint = DispatcherEndpoint(["UDPIPv4"], UDPIPv4={'port': self.config.get_ipv8_port(),
'ip': self.config.get_ipv8_address()})
self.ipv8 = IPv8(ipv8_config_builder.finalize(),
enable_statistics=self.config.get_ipv8_statistics() and not self.core_test_mode,
endpoint_override=endpoint)
await self.ipv8.start()
self.config.set_anon_proxy_settings(2, ("127.0.0.1",
self.
config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
if not self.core_test_mode:
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
await self.tunnel_community.wait_for_socks_servers()
if self.config.get_ipv8_walk_scaling_enabled():
from tribler_core.modules.ipv8_health_monitor import IPv8Monitor
IPv8Monitor(self.ipv8,
self.config.get_ipv8_walk_interval(),
self.config.get_ipv8_walk_scaling_upper_limit()).start(self)
# Note that currently we should only start libtorrent after the SOCKS5 servers have been started
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
if self.core_test_mode:
await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000")
self.readable_status = STATE_READABLE_STARTED
if self.config.get_watch_folder_enabled():
from tribler_core.modules.watch_folder import WatchFolder
self.readable_status = STATE_START_WATCH_FOLDER
self.watch_folder = WatchFolder(self)
self.watch_folder.start()
if self.config.get_resource_monitor_enabled() and not self.core_test_mode:
from tribler_core.modules.resource_monitor.core import CoreResourceMonitor
self.resource_monitor = CoreResourceMonitor(self)
self.resource_monitor.start()
if self.config.get_version_checker_enabled() and not self.core_test_mode:
from tribler_core.modules.versioncheck_manager import VersionCheckManager
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()
if self.config.get_ipv8_enabled():
from tribler_core.modules.payout_manager import PayoutManager
self.payout_manager = PayoutManager(self.bandwidth_community, self.dht_community)
if self.core_test_mode:
from ipv8.messaging.interfaces.udp.endpoint import UDPv4Address
from ipv8.dht.routing import RoutingTable
self.dht_community.routing_tables[UDPv4Address] = RoutingTable('\x00' * 20)
# GigaChannel Manager should be started *after* resuming the downloads,
# because it depends on the states of torrent downloads
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \
and self.config.get_libtorrent_enabled:
from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager
self.gigachannel_manager = GigaChannelManager(self)
if not self.core_test_mode:
self.gigachannel_manager.start()
if self.config.get_bootstrap_enabled() and not self.core_test_mode:
self.register_task('bootstrap_download', self.start_bootstrap_download)
self.notifier.notify(NTFY.TRIBLER_STARTED, self.trustchain_keypair.key.pk)
# If there is a config error, report to the user via GUI notifier
if self.config.config_error:
self.notifier.notify(NTFY.REPORT_CONFIG_ERROR, self.config.config_error)