コード例 #1
0
def upgrade_state_dir(root_state_dir: Path,
                      update_status_callback=None,
                      interrupt_upgrade_event=None):
    # Before any upgrade, prepare a separate state directory for the update version so it does not
    # affect the older version state directory. This allows for safe rollback.
    version_history = VersionHistory(root_state_dir)
    version_history.fork_state_directory_if_necessary()
    version_history.save_if_necessary()
    state_dir = version_history.code_version.directory
    if not state_dir.exists():
        return

    config = TriblerConfig.load(file=state_dir / CONFIG_FILE_NAME,
                                state_dir=state_dir,
                                reset_config_on_error=True)
    channels_dir = config.chant.get_path_as_absolute('channels_dir',
                                                     config.state_dir)

    primary_private_key_path = config.state_dir / KeyComponent.get_private_key_filename(
        config)
    primary_public_key_path = config.state_dir / config.trustchain.ec_keypair_pubfilename
    primary_key = KeyComponent.load_or_create(primary_private_key_path,
                                              primary_public_key_path)

    upgrader = TriblerUpgrader(state_dir,
                               channels_dir,
                               primary_key,
                               update_status_callback=update_status_callback,
                               interrupt_upgrade_event=interrupt_upgrade_event)
    upgrader.run()
コード例 #2
0
def test_upgrade_pony13to14(upgrader: TriblerUpgrader, state_dir, channels_dir,
                            trustchain_keypair, mds_path):
    tags_path = state_dir / 'sqlite/tags.db'

    _copy(source_name='pony_v13.db', target=mds_path)
    _copy(source_name='tags_v13.db', target=tags_path)

    upgrader.upgrade_pony_db_13to14()
    mds = MetadataStore(mds_path,
                        channels_dir,
                        trustchain_keypair,
                        check_tables=False)
    tags = TagDatabase(str(tags_path), create_tables=False, check_tables=False)

    with db_session:
        assert upgrader.column_exists_in_table(mds._db, 'ChannelNode',
                                               'tag_processor_version')
        assert upgrader.column_exists_in_table(tags.instance, 'TorrentTagOp',
                                               'auto_generated')
        assert mds.get_value('db_version') == '14'
コード例 #3
0
    async def start(self):
        """
        Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
        Returns a deferred that fires when the Tribler session is ready for use.

        :param config: a TriblerConfig object
        """

        self._logger.info("Session is using state directory: %s",
                          self.config.get_state_dir())
        self.get_ports_in_config()
        self.create_state_directory_structure()
        self.init_keypair()

        # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
        if self.config.get_http_api_enabled():
            self.api_manager = RESTManager(self)
            self.readable_status = STATE_START_API
            await self.api_manager.start()

        if self.upgrader_enabled:
            self.upgrader = TriblerUpgrader(self)
            self.readable_status = STATE_UPGRADING_READABLE
            try:
                await self.upgrader.run()
            except Exception as e:
                self._logger.error("Error in Upgrader callback chain: %s", e)

        self.tracker_manager = TrackerManager(self)

        # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
        # certificates provided by the system trust store.
        if sys.platform == 'darwin':
            os.environ['SSL_CERT_FILE'] = str(
                (get_lib_path() / 'root_certs_mac.pem'))

        if self.config.get_chant_enabled():
            channels_dir = self.config.get_chant_channels_dir()
            metadata_db_name = 'metadata.db' if not self.config.get_testnet(
            ) else 'metadata_testnet.db'
            database_path = self.config.get_state_dir(
            ) / 'sqlite' / metadata_db_name
            self.mds = MetadataStore(database_path, channels_dir,
                                     self.trustchain_keypair)

        # IPv8
        if self.config.get_ipv8_enabled():
            from ipv8.configuration import get_default_configuration
            ipv8_config = get_default_configuration()
            ipv8_config['port'] = self.config.get_ipv8_port()
            ipv8_config['address'] = self.config.get_ipv8_address()
            ipv8_config['overlays'] = []
            ipv8_config['keys'] = []  # We load the keys ourselves
            ipv8_config['working_directory'] = str(self.config.get_state_dir())

            if self.config.get_ipv8_bootstrap_override():
                import ipv8.community as community_file
                community_file._DEFAULT_ADDRESSES = [
                    self.config.get_ipv8_bootstrap_override()
                ]
                community_file._DNS_ADDRESSES = []

            self.ipv8 = IPv8(
                ipv8_config,
                enable_statistics=self.config.get_ipv8_statistics())
            await self.ipv8.start()

            self.config.set_anon_proxy_settings(
                2, ("127.0.0.1",
                    self.config.get_tunnel_community_socks5_listen_ports()))
            self.ipv8_start_time = timemod.time()
            self.load_ipv8_overlays()
            self.enable_ipv8_statistics()
            if self.api_manager:
                self.api_manager.set_ipv8_session(self.ipv8)
            if self.config.get_tunnel_community_enabled():
                await self.tunnel_community.wait_for_socks_servers()

        # Note that currently we should only start libtorrent after the SOCKS5 servers have been started
        if self.config.get_libtorrent_enabled():
            self.readable_status = STATE_START_LIBTORRENT
            from tribler_core.modules.libtorrent.download_manager import DownloadManager
            self.dlmgr = DownloadManager(self)
            self.dlmgr.initialize()
            self.readable_status = STATE_LOAD_CHECKPOINTS
            await self.dlmgr.load_checkpoints()
        self.readable_status = STATE_READABLE_STARTED

        if self.config.get_torrent_checking_enabled():
            self.readable_status = STATE_START_TORRENT_CHECKER
            self.torrent_checker = TorrentChecker(self)
            await self.torrent_checker.initialize()

        if self.config.get_dummy_wallets_enabled():
            # For debugging purposes, we create dummy wallets
            dummy_wallet1 = DummyWallet1()
            self.wallets[dummy_wallet1.get_identifier()] = dummy_wallet1

            dummy_wallet2 = DummyWallet2()
            self.wallets[dummy_wallet2.get_identifier()] = dummy_wallet2

        if self.config.get_watch_folder_enabled():
            self.readable_status = STATE_START_WATCH_FOLDER
            self.watch_folder = WatchFolder(self)
            self.watch_folder.start()

        if self.config.get_resource_monitor_enabled():
            self.resource_monitor = ResourceMonitor(self)
            self.resource_monitor.start()

        if self.config.get_version_checker_enabled():
            self.version_check_manager = VersionCheckManager(self)
            self.version_check_manager.start()

        if self.config.get_ipv8_enabled(
        ) and self.config.get_trustchain_enabled():
            self.payout_manager = PayoutManager(self.trustchain_community,
                                                self.dht_community)

        # GigaChannel Manager should be started *after* resuming the downloads,
        # because it depends on the states of torrent downloads
        if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\
                and self.config.get_libtorrent_enabled:
            self.gigachannel_manager = GigaChannelManager(self)
            self.gigachannel_manager.start()

        if self.config.get_bootstrap_enabled():
            self.register_task('bootstrap_download',
                               self.start_bootstrap_download)

        self.notifier.notify(NTFY.TRIBLER_STARTED)
コード例 #4
0
def upgrader(session):
    return TriblerUpgrader(session)
コード例 #5
0
ファイル: test_upgrader.py プロジェクト: overflw/tribler
def upgrader(state_dir, channels_dir, trustchain_keypair):
    return TriblerUpgrader(state_dir, channels_dir, trustchain_keypair)
コード例 #6
0
ファイル: test_upgrader.py プロジェクト: hbiyik/tribler
 async def setUp(self):
     await super(TestUpgrader, self).setUp()
     self.upgrader = TriblerUpgrader(self.session)
コード例 #7
0
ファイル: test_upgrader.py プロジェクト: hbiyik/tribler
class TestUpgrader(TestAsServer):
    async def setUp(self):
        await super(TestUpgrader, self).setUp()
        self.upgrader = TriblerUpgrader(self.session)

    @timeout(10)
    async def test_update_status_text(self):
        test_future = Future()

        def on_upgrade_tick(status_text):
            self.assertEqual(status_text, "12345")
            test_future.set_result(None)

        self.session.notifier.add_observer(NTFY.UPGRADER_TICK, on_upgrade_tick)
        self.upgrader.update_status("12345")
        await test_future

    @timeout(10)
    async def test_upgrade_72_to_pony(self):
        OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'tribler_v29.sdb'
        old_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'tribler.sdb'
        new_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'metadata.db'
        shutil.copyfile(OLD_DB_SAMPLE, old_database_path)

        await self.upgrader.run()
        channels_dir = self.session.config.get_chant_channels_dir()
        mds = MetadataStore(new_database_path, channels_dir,
                            self.session.trustchain_keypair)
        with db_session:
            self.assertEqual(mds.TorrentMetadata.select().count(), 24)
        mds.shutdown()

    def test_upgrade_pony_db_6to7(self):
        """
        Test that channels and torrents with forbidden words are cleaned up during upgrade from Pony db ver 6 to 7.
        Also, check that the DB version is upgraded.
        :return:
        """
        OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
        old_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'metadata.db'
        shutil.copyfile(OLD_DB_SAMPLE, old_database_path)

        self.upgrader.upgrade_pony_db_6to7()
        channels_dir = self.session.config.get_chant_channels_dir()
        mds = MetadataStore(old_database_path, channels_dir,
                            self.session.trustchain_keypair)
        with db_session:
            self.assertEqual(mds.TorrentMetadata.select().count(), 23)
            self.assertEqual(mds.ChannelMetadata.select().count(), 2)
            self.assertEqual(int(mds.MiscData.get(name="db_version").value), 7)
        mds.shutdown()

    def test_upgrade_pony_db_7to8(self):
        """
        Test that proper additionald index is created.
        Also, check that the DB version is upgraded.
        """
        OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v7.db'
        old_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'metadata.db'
        shutil.copyfile(OLD_DB_SAMPLE, old_database_path)

        self.upgrader.upgrade_pony_db_7to8()
        channels_dir = self.session.config.get_chant_channels_dir()
        mds = MetadataStore(old_database_path, channels_dir,
                            self.session.trustchain_keypair)
        with db_session:
            self.assertEqual(int(mds.MiscData.get(name="db_version").value), 8)
            self.assertEqual(mds.Vsids[0].exp_period, 24.0 * 60 * 60 * 3)
            self.assertTrue(
                list(
                    mds._db.execute(
                        'PRAGMA index_info("idx_channelnode__metadata_type")'))
            )
        mds.shutdown()

    @timeout(10)
    async def test_upgrade_pony_db_complete(self):
        """
        Test complete update sequence for Pony DB (e.g. 6->7->8)
        """
        OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
        old_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'metadata.db'
        shutil.copyfile(OLD_DB_SAMPLE, old_database_path)

        await self.upgrader.run()
        channels_dir = self.session.config.get_chant_channels_dir()
        mds = MetadataStore(old_database_path, channels_dir,
                            self.session.trustchain_keypair)
        with db_session:
            self.assertEqual(mds.TorrentMetadata.select().count(), 23)
            self.assertEqual(mds.ChannelMetadata.select().count(), 2)
            self.assertEqual(int(mds.MiscData.get(name="db_version").value), 8)
            self.assertTrue(
                list(
                    mds._db.execute(
                        'PRAGMA index_info("idx_channelnode__metadata_type")'))
            )
        mds.shutdown()

    @timeout(10)
    async def test_skip_upgrade_72_to_pony(self):
        OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'tribler_v29.sdb'
        old_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'tribler.sdb'
        new_database_path = self.session.config.get_state_dir(
        ) / 'sqlite' / 'metadata.db'
        channels_dir = self.session.config.get_chant_channels_dir()

        shutil.copyfile(OLD_DB_SAMPLE, old_database_path)

        self.upgrader.skip()
        await self.upgrader.run()
        mds = MetadataStore(new_database_path, channels_dir,
                            self.session.trustchain_keypair)
        with db_session:
            self.assertEqual(mds.TorrentMetadata.select().count(), 0)
            self.assertEqual(mds.ChannelMetadata.select().count(), 0)
        mds.shutdown()

    def test_delete_noncompliant_state(self):
        STATE_DIR = TESTS_DATA_DIR / 'noncompliant_state_dir'
        tmpdir = self.temporary_directory() / STATE_DIR.name
        shutil.copytree(str_path(STATE_DIR), str_path(tmpdir))
        cleanup_noncompliant_channel_torrents(tmpdir)

        # Check cleanup of the channels dir
        dir_listing = list((tmpdir / "channels").iterdir())
        self.assertEqual(3, len(dir_listing))
        for f in (tmpdir / "channels").iterdir():
            self.assertEqual(CHANNEL_DIR_NAME_LENGTH, len(f.stem))

        # Check cleanup of torrent state dir
        checkpoints_dir = tmpdir / "dlcheckpoints"
        dir_listing = os.listdir(checkpoints_dir)
        self.assertEqual(1, len(dir_listing))
        file_path = checkpoints_dir / dir_listing[0]
        pstate = CallbackConfigParser()
        pstate.read_file(file_path)
        self.assertEqual(CHANNEL_DIR_NAME_LENGTH,
                         len(pstate.get('state', 'metainfo')['info']['name']))
コード例 #8
0
ファイル: test_upgrader.py プロジェクト: hbiyik/tribler
 async def setUp(self):
     await super(TestUpgraderStateDirectory, self).setUp()
     self.upgrader = TriblerUpgrader(self.session)
     self.original_version = version.version_id
コード例 #9
0
    async def start(self):
        """
        Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader.
        Returns a deferred that fires when the Tribler session is ready for use.

        :param config: a TriblerConfig object
        """
        self._logger.info("Session is using state directory: %s", self.config.get_state_dir())
        self.get_ports_in_config()
        self.create_state_directory_structure()
        self.init_keypair()

        # we have to represent `user_id` as a string to make it equal to the
        # `user_id` on the GUI side
        user_id_str = hexlify(self.trustchain_keypair.key.pk).encode('utf-8')
        SentryReporter.set_user(user_id_str)

        # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket
        if self.config.get_api_http_enabled() or self.config.get_api_https_enabled():
            from tribler_core.restapi.rest_manager import RESTManager
            self.api_manager = RESTManager(self)
            self.readable_status = STATE_START_API
            await self.api_manager.start()

        if self.upgrader_enabled and not self.core_test_mode:
            from tribler_core.upgrade.upgrade import TriblerUpgrader
            self.upgrader = TriblerUpgrader(self)
            self.readable_status = STATE_UPGRADING_READABLE
            await self.upgrader.run()

        self.tracker_manager = TrackerManager(self)

        # Start torrent checker before Popularity community is loaded
        if self.config.get_torrent_checking_enabled() and not self.core_test_mode:
            from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker
            self.readable_status = STATE_START_TORRENT_CHECKER
            self.torrent_checker = TorrentChecker(self)
            await self.torrent_checker.initialize()

        # On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
        # certificates provided by the system trust store.
        if sys.platform == 'darwin':
            os.environ['SSL_CERT_FILE'] = str(get_lib_path() / 'root_certs_mac.pem')

        if self.config.get_chant_enabled():
            from tribler_core.modules.metadata_store.store import MetadataStore
            channels_dir = self.config.get_chant_channels_dir()
            metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db'
            database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name
            self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair,
                                     notifier=self.notifier,
                                     disable_sync=self.core_test_mode)
            if self.core_test_mode:
                generate_test_channels(self.mds)

        # IPv8
        if self.config.get_ipv8_enabled():
            from ipv8.configuration import ConfigBuilder
            from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint
            ipv8_config_builder = (ConfigBuilder()
                                   .set_port(self.config.get_ipv8_port())
                                   .set_address(self.config.get_ipv8_address())
                                   .clear_overlays()
                                   .clear_keys()  # We load the keys ourselves
                                   .set_working_directory(str(self.config.get_state_dir()))
                                   .set_walker_interval(self.config.get_ipv8_walk_interval()))

            if self.core_test_mode:
                endpoint = DispatcherEndpoint([])
            else:
                # IPv8 includes IPv6 support by default.
                # We only load IPv4 to not kill all Tribler overlays (currently, it would instantly crash all users).
                # If you want to test IPv6 in Tribler you can set ``endpoint = None`` here.
                endpoint = DispatcherEndpoint(["UDPIPv4"], UDPIPv4={'port': self.config.get_ipv8_port(),
                                                                    'ip': self.config.get_ipv8_address()})
            self.ipv8 = IPv8(ipv8_config_builder.finalize(),
                             enable_statistics=self.config.get_ipv8_statistics() and not self.core_test_mode,
                             endpoint_override=endpoint)
            await self.ipv8.start()

            self.config.set_anon_proxy_settings(2, ("127.0.0.1",
                                                    self.
                                                    config.get_tunnel_community_socks5_listen_ports()))
            self.ipv8_start_time = timemod.time()
            self.load_ipv8_overlays()
            if not self.core_test_mode:
                self.enable_ipv8_statistics()
            if self.api_manager:
                self.api_manager.set_ipv8_session(self.ipv8)
            if self.config.get_tunnel_community_enabled():
                await self.tunnel_community.wait_for_socks_servers()
            if self.config.get_ipv8_walk_scaling_enabled():
                from tribler_core.modules.ipv8_health_monitor import IPv8Monitor
                IPv8Monitor(self.ipv8,
                            self.config.get_ipv8_walk_interval(),
                            self.config.get_ipv8_walk_scaling_upper_limit()).start(self)

        # Note that currently we should only start libtorrent after the SOCKS5 servers have been started
        if self.config.get_libtorrent_enabled():
            self.readable_status = STATE_START_LIBTORRENT
            from tribler_core.modules.libtorrent.download_manager import DownloadManager
            self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode)
            self.dlmgr.initialize()
            self.readable_status = STATE_LOAD_CHECKPOINTS
            await self.dlmgr.load_checkpoints()
            if self.core_test_mode:
                await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000")
        self.readable_status = STATE_READABLE_STARTED

        if self.config.get_watch_folder_enabled():
            from tribler_core.modules.watch_folder import WatchFolder
            self.readable_status = STATE_START_WATCH_FOLDER
            self.watch_folder = WatchFolder(self)
            self.watch_folder.start()

        if self.config.get_resource_monitor_enabled() and not self.core_test_mode:
            from tribler_core.modules.resource_monitor.core import CoreResourceMonitor
            self.resource_monitor = CoreResourceMonitor(self)
            self.resource_monitor.start()

        if self.config.get_version_checker_enabled() and not self.core_test_mode:
            from tribler_core.modules.versioncheck_manager import VersionCheckManager
            self.version_check_manager = VersionCheckManager(self)
            self.version_check_manager.start()

        if self.config.get_ipv8_enabled():
            from tribler_core.modules.payout_manager import PayoutManager
            self.payout_manager = PayoutManager(self.bandwidth_community, self.dht_community)

            if self.core_test_mode:
                from ipv8.messaging.interfaces.udp.endpoint import UDPv4Address
                from ipv8.dht.routing import RoutingTable
                self.dht_community.routing_tables[UDPv4Address] = RoutingTable('\x00' * 20)

        # GigaChannel Manager should be started *after* resuming the downloads,
        # because it depends on the states of torrent downloads
        if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \
                and self.config.get_libtorrent_enabled:
            from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager
            self.gigachannel_manager = GigaChannelManager(self)
            if not self.core_test_mode:
                self.gigachannel_manager.start()

        if self.config.get_bootstrap_enabled() and not self.core_test_mode:
            self.register_task('bootstrap_download', self.start_bootstrap_download)

        self.notifier.notify(NTFY.TRIBLER_STARTED, self.trustchain_keypair.key.pk)

        # If there is a config error, report to the user via GUI notifier
        if self.config.config_error:
            self.notifier.notify(NTFY.REPORT_CONFIG_ERROR, self.config.config_error)