예제 #1
0
    def initialize(self):
        # Start upnp
        self.get_session().start_upnp()

        if has_bep33_support():
            # Also listen to DHT log notifications - we need the dht_pkt_alert and extract the BEP33 bloom filters
            dht_health_session = self.get_session(
                self.tribler_session.config.get_default_number_hops())
            dht_health_session.set_alert_mask(
                self.default_alert_mask
                | lt.alert.category_t.dht_log_notification)
            self.dht_health_manager = DHTHealthManager(dht_health_session)

        # Make temporary directory for metadata collecting through DHT
        self.metadata_tmpdir = mkdtemp(suffix='tribler_metainfo_tmpdir')

        # Register tasks
        self.register_task("process_alerts",
                           self._task_process_alerts,
                           interval=1)
        if self.dht_readiness_timeout > 0:
            self._dht_ready_task = self.register_task("check_dht_ready",
                                                      self._check_dht_ready)
        self.register_task("request_torrent_updates",
                           self._request_torrent_updates,
                           interval=1)
        self.register_task('task_cleanup_metacache',
                           self._task_cleanup_metainfo_cache,
                           interval=60,
                           delay=0)

        self.set_download_states_callback(self.sesscb_states_callback)
예제 #2
0
    async def setUp(self):
        await super(TestDHTHealthManager, self).setUp()

        self.mock_lt_session = MockObject()
        self.mock_lt_session.dht_get_peers = lambda _: None

        self.dht_health_manager = DHTHealthManager(self.mock_lt_session)
예제 #3
0
class DownloadManager(TaskManager):
    def __init__(self, tribler_session, dummy_mode=False):
        super().__init__()
        self.dummy_mode = dummy_mode
        self._logger = logging.getLogger(self.__class__.__name__)

        self.tribler_session = tribler_session
        self.ltsettings = {
        }  # Stores a copy of the settings dict for each libtorrent session
        self.ltsessions = {}
        self.dht_health_manager = None
        self.listen_ports = {}

        self.notifier = tribler_session.notifier

        self.set_upload_rate_limit(0)
        self.set_download_rate_limit(0)

        self.downloads = {}

        self.metadata_tmpdir = None
        # Dictionary that maps infohashes to download instances. These include only downloads that have
        # been made specifically for fetching metainfo, and will be removed afterwards.
        self.metainfo_requests = {}
        self.metainfo_cache = {
        }  # Dictionary that maps infohashes to cached metainfo items

        self.default_alert_mask = lt.alert.category_t.error_notification | lt.alert.category_t.status_notification | \
                                  lt.alert.category_t.storage_notification | lt.alert.category_t.performance_warning | \
                                  lt.alert.category_t.tracker_notification | lt.alert.category_t.debug_notification
        self.session_stats_callback = None
        self.state_cb_count = 0

        # Status of libtorrent session to indicate if it can safely close and no pending writes to disk exists.
        self.lt_session_shutdown_ready = {}
        self._dht_ready_task = None
        self.dht_readiness_timeout = tribler_session.config.get_libtorrent_dht_readiness_timeout() \
            if not self.dummy_mode else 0

    async def _check_dht_ready(self, min_dht_peers=60):
        # Checks whether we got enough DHT peers. If the number of DHT peers is low,
        # checking for a bunch of torrents in a short period of time may result in several consecutive requests
        # sent to the same peers. This can trigger those peers' flood protection mechanism,
        # which results in DHT checks stuck for hours.
        # See https://github.com/Tribler/tribler/issues/5319
        while not (self.get_session()
                   and self.get_session().status().dht_nodes > min_dht_peers):
            await asyncio.sleep(1)

    def initialize(self):
        # Start upnp
        self.get_session().start_upnp()

        if has_bep33_support():
            # Also listen to DHT log notifications - we need the dht_pkt_alert and extract the BEP33 bloom filters
            dht_health_session = self.get_session(
                self.tribler_session.config.get_default_number_hops())
            dht_health_session.set_alert_mask(
                self.default_alert_mask
                | lt.alert.category_t.dht_log_notification)
            self.dht_health_manager = DHTHealthManager(dht_health_session)

        # Make temporary directory for metadata collecting through DHT
        self.metadata_tmpdir = mkdtemp(suffix='tribler_metainfo_tmpdir')

        # Register tasks
        self.register_task("process_alerts",
                           self._task_process_alerts,
                           interval=1)
        if self.dht_readiness_timeout > 0:
            self._dht_ready_task = self.register_task("check_dht_ready",
                                                      self._check_dht_ready)
        self.register_task("request_torrent_updates",
                           self._request_torrent_updates,
                           interval=1)
        self.register_task('task_cleanup_metacache',
                           self._task_cleanup_metainfo_cache,
                           interval=60,
                           delay=0)

        self.set_download_states_callback(self.sesscb_states_callback)

    async def shutdown(self, timeout=30):
        if self.downloads:
            self.tribler_session.notify_shutdown_state(
                "Checkpointing Downloads...")
            await gather(
                *[download.stop() for download in self.downloads.values()],
                return_exceptions=True)
            self.tribler_session.notify_shutdown_state(
                "Shutting down Downloads...")
            await gather(
                *[download.shutdown() for download in self.downloads.values()],
                return_exceptions=True)

        self.tribler_session.notify_shutdown_state(
            "Shutting down Libtorrent Manager...")
        # If libtorrent session has pending disk io, wait until timeout (default: 30 seconds) to let it finish.
        # In between ask for session stats to check if state is clean for shutdown.
        while not self.is_shutdown_ready() and timeout >= 1:
            self.tribler_session.notify_shutdown_state(
                "Waiting for Libtorrent to finish...")
            self.post_session_stats()
            timeout -= 1
            await asyncio.sleep(1)

        await self.shutdown_task_manager()

        if self.dht_health_manager:
            await self.dht_health_manager.shutdown_task_manager()

        # Save libtorrent state
        if self.has_session():
            with open(
                    self.tribler_session.config.get_state_dir() /
                    LTSTATE_FILENAME, 'wb') as ltstate_file:
                ltstate_file.write(lt.bencode(self.get_session().save_state()))

        if self.has_session():
            self.get_session().stop_upnp()

        for ltsession in self.ltsessions.values():
            del ltsession
        self.ltsessions = None

        # Remove metadata temporary directory
        if self.metadata_tmpdir:
            rmtree(self.metadata_tmpdir)
            self.metadata_tmpdir = None

        self.tribler_session = None

    def is_shutdown_ready(self):
        return all(self.lt_session_shutdown_ready.values())

    def create_session(self, hops=0, store_listen_port=True):
        # Due to a bug in Libtorrent 0.16.18, the outgoing_port and num_outgoing_ports value should be set in
        # the settings dictionary
        settings = {
            'outgoing_port': 0,
            'num_outgoing_ports': 1,
            'allow_multiple_connections_per_ip': 0
        }

        # Copy construct so we don't modify the default list
        extensions = list(DEFAULT_LT_EXTENSIONS)

        # Elric: Strip out the -rcX, -beta, -whatever tail on the version string.
        fingerprint = ['TL'] + [
            int(x) for x in version_id.split('-')[0].split('.')
        ] + [0]
        if self.dummy_mode:
            from unittest.mock import Mock
            ltsession = Mock()
            ltsession.pop_alerts = lambda: {}
            ltsession.listen_port = lambda: 123
            ltsession.get_settings = lambda: {"peer_fingerprint": "000"}
        else:
            ltsession = lt.session(lt.fingerprint(
                *fingerprint), flags=0) if hops == 0 else lt.session(flags=0)

        if hops == 0:
            settings['user_agent'] = 'Tribler/' + version_id
            enable_utp = self.tribler_session.config.get_libtorrent_utp()
            settings['enable_outgoing_utp'] = enable_utp
            settings['enable_incoming_utp'] = enable_utp

            if LooseVersion(
                    self.get_libtorrent_version()) >= LooseVersion("1.1.0"):
                settings['prefer_rc4'] = True
                settings[
                    "listen_interfaces"] = "0.0.0.0:%d" % self.tribler_session.config.get_libtorrent_port(
                    )
            else:
                pe_settings = lt.pe_settings()
                pe_settings.prefer_rc4 = True
                ltsession.set_pe_settings(pe_settings)

            mid = self.tribler_session.trustchain_keypair.key_to_hash()
            settings['peer_fingerprint'] = mid
            settings[
                'handshake_client_version'] = 'Tribler/' + version_id + '/' + hexlify(
                    mid)
        else:
            settings['enable_outgoing_utp'] = True
            settings['enable_incoming_utp'] = True
            settings['enable_outgoing_tcp'] = False
            settings['enable_incoming_tcp'] = False
            settings['anonymous_mode'] = True
            settings['force_proxy'] = True

            if LooseVersion(
                    self.get_libtorrent_version()) >= LooseVersion("1.1.0"):
                settings[
                    "listen_interfaces"] = "0.0.0.0:%d" % self.tribler_session.config.get_anon_listen_port(
                    )

            # By default block all IPs except 1.1.1.1 (which is used to ensure libtorrent makes a connection to us)
            self.update_ip_filter(ltsession, ['1.1.1.1'])

        self.set_session_settings(ltsession, settings)
        ltsession.set_alert_mask(self.default_alert_mask)

        # Load proxy settings
        if hops == 0:
            proxy_settings = self.tribler_session.config.get_libtorrent_proxy_settings(
            )
        else:
            proxy_settings = list(
                self.tribler_session.config.get_anon_proxy_settings())
            proxy_host, proxy_ports = proxy_settings[1]
            proxy_settings[1] = (proxy_host, proxy_ports[hops - 1])
        self.set_proxy_settings(ltsession, *proxy_settings)

        for extension in extensions:
            ltsession.add_extension(extension)

        # Set listen port & start the DHT
        if hops == 0:
            listen_port = self.tribler_session.config.get_libtorrent_port()
            ltsession.listen_on(listen_port, listen_port + 10)
            if listen_port != ltsession.listen_port() and store_listen_port:
                self.tribler_session.config.set_libtorrent_port_runtime(
                    ltsession.listen_port())
            try:
                with open(
                        self.tribler_session.config.get_state_dir() /
                        LTSTATE_FILENAME, 'rb') as fp:
                    lt_state = bdecode_compat(fp.read())
                if lt_state is not None:
                    ltsession.load_state(lt_state)
                else:
                    self._logger.warning(
                        "the lt.state appears to be corrupt, writing new data on shutdown"
                    )
            except Exception as exc:
                self._logger.info(
                    f"could not load libtorrent state, got exception: {exc!r}. starting from scratch"
                )
        else:
            ltsession.listen_on(
                self.tribler_session.config.get_anon_listen_port(),
                self.tribler_session.config.get_anon_listen_port() + 20)

            settings = {
                'upload_rate_limit':
                self.tribler_session.config.get_libtorrent_max_upload_rate(),
                'download_rate_limit':
                self.tribler_session.config.get_libtorrent_max_download_rate()
            }
            self.set_session_settings(ltsession, settings)

        if self.tribler_session.config.get_libtorrent_dht_enabled(
        ) and not self.dummy_mode:
            ltsession.start_dht()
            for router in DEFAULT_DHT_ROUTERS:
                ltsession.add_dht_router(*router)
            ltsession.start_lsd()

        self._logger.debug("Started libtorrent session for %d hops on port %d",
                           hops, ltsession.listen_port())
        self.lt_session_shutdown_ready[hops] = False

        return ltsession

    def has_session(self, hops=0):
        return hops in self.ltsessions

    def get_session(self, hops=0):
        if hops not in self.ltsessions:
            self.ltsessions[hops] = self.create_session(hops)

        return self.ltsessions[hops]

    def set_proxy_settings(self, ltsession, ptype, server=None, auth=None):
        """
        Apply the proxy settings to a libtorrent session. This mechanism changed significantly in libtorrent 1.1.0.
        """
        if LooseVersion(
                self.get_libtorrent_version()) >= LooseVersion("1.1.0"):
            settings = {}
            settings["proxy_type"] = ptype
            settings["proxy_hostnames"] = True
            settings["proxy_peer_connections"] = True
            if server and server[0] and server[1]:
                settings["proxy_hostname"] = server[0]
                settings["proxy_port"] = int(server[1])
            if auth:
                settings["proxy_username"] = auth[0]
                settings["proxy_password"] = auth[1]
            self.set_session_settings(ltsession, settings)
        else:
            proxy_settings = lt.proxy_settings()
            proxy_settings.type = lt.proxy_type(ptype)
            if server and server[0] and server[1]:
                proxy_settings.hostname = server[0]
                proxy_settings.port = int(server[1])
            if auth:
                proxy_settings.username = auth[0]
                proxy_settings.password = auth[1]
            proxy_settings.proxy_hostnames = True
            proxy_settings.proxy_peer_connections = True

            ltsession.set_proxy(proxy_settings)

    def set_max_connections(self, conns, hops=None):
        self._map_call_on_ltsessions(hops, 'set_max_connections', conns)

    def set_upload_rate_limit(self, rate, hops=None):
        # Rate conversion due to the fact that we had a different system with Swift
        # and the old python BitTorrent core: unlimited == 0, stop == -1, else rate in kbytes
        libtorrent_rate = int(-1 if rate == 0 else (1 if rate == -1 else rate *
                                                    1024))

        # Pass outgoing_port and num_outgoing_ports to dict due to bug in libtorrent 0.16.18
        settings_dict = {
            'upload_rate_limit': libtorrent_rate,
            'outgoing_port': 0,
            'num_outgoing_ports': 1
        }
        for session in self.ltsessions.values():
            self.set_session_settings(session, settings_dict)

    def get_upload_rate_limit(self, hops=None):
        # Rate conversion due to the fact that we had a different system with Swift
        # and the old python BitTorrent core: unlimited == 0, stop == -1, else rate in kbytes
        libtorrent_rate = self.get_session(hops).upload_rate_limit()
        return 0 if libtorrent_rate == -1 else (
            -1 if libtorrent_rate == 1 else libtorrent_rate / 1024)

    def set_download_rate_limit(self, rate, hops=None):
        libtorrent_rate = int(-1 if rate == 0 else (1 if rate == -1 else rate *
                                                    1024))

        # Pass outgoing_port and num_outgoing_ports to dict due to bug in libtorrent 0.16.18
        settings_dict = {'download_rate_limit': libtorrent_rate}
        for session in self.ltsessions.values():
            self.set_session_settings(session, settings_dict)

    def get_download_rate_limit(self, hops=0):
        libtorrent_rate = self.get_session(hops).download_rate_limit()
        return 0 if libtorrent_rate == -1 else (
            -1 if libtorrent_rate == 1 else libtorrent_rate / 1024)

    def process_alert(self, alert, hops=0):
        alert_type = alert.__class__.__name__

        # Periodically, libtorrent will send us a state_update_alert, which contains the torrent status of
        # all torrents changed since the last time we received this alert.
        if alert_type == 'state_update_alert':
            for status in alert.status:
                infohash = unhexlify(str(status.info_hash))
                if infohash not in self.downloads:
                    self._logger.debug(
                        "Got state_update for unknown torrent %s",
                        hexlify(infohash))
                    continue
                self.downloads[infohash].update_lt_status(status)

        infohash = unhexlify(
            str(alert.handle.info_hash() if hasattr(alert, 'handle') and alert.
                handle.is_valid() else getattr(alert, 'info_hash', '')))
        download = self.downloads.get(infohash)
        if download:
            if (download.handle and download.handle.is_valid())\
                    or (not download.handle and alert_type == 'add_torrent_alert') \
                    or (download.handle and alert_type == 'torrent_removed_alert'):
                download.process_alert(alert, alert_type)
            else:
                self._logger.debug(
                    "Got alert for download without handle %s: %s",
                    hexlify(infohash), alert)
        elif infohash:
            self._logger.debug("Got alert for unknown download %s: %s",
                               hexlify(infohash), alert)

        if alert_type == 'listen_succeeded_alert':
            # The ``port`` attribute was added in libtorrent 1.1.14.
            # Older versions (most notably libtorrent 1.1.13 - the default  on Ubuntu 20.04) do not have this attribute.
            # We use the now-deprecated ``endpoint`` attribute for these older versions.
            self.listen_ports[hops] = getattr(alert, "port", alert.endpoint[1])

        elif alert_type == 'peer_disconnected_alert' and \
                self.tribler_session and self.tribler_session.payout_manager:
            self.tribler_session.payout_manager.do_payout(alert.pid.to_bytes())

        elif alert_type == 'session_stats_alert':
            queued_disk_jobs = alert.values['disk.queued_disk_jobs']
            queued_write_bytes = alert.values['disk.queued_write_bytes']
            num_write_jobs = alert.values['disk.num_write_jobs']

            if queued_disk_jobs == queued_write_bytes == num_write_jobs == 0:
                self.lt_session_shutdown_ready[hops] = True

            if self.session_stats_callback:
                self.session_stats_callback(alert)

        elif alert_type == "dht_pkt_alert":
            # Unfortunately, the Python bindings don't have a direction attribute.
            # So, we'll have to resort to using the string representation of the alert instead.
            incoming = str(alert).startswith('<==')
            decoded = bdecode_compat(alert.pkt_buf)
            if not decoded:
                return

            # We are sending a raw DHT message - notify the DHTHealthManager of the outstanding request.
            if not incoming and decoded.get(b'y') == b'q' \
               and decoded.get(b'q') == b'get_peers' and decoded[b'a'].get(b'scrape') == 1:
                self.dht_health_manager.requesting_bloomfilters(
                    decoded[b't'], decoded[b'a'][b'info_hash'])

            # We received a raw DHT message - decode it and check whether it is a BEP33 message.
            if incoming and b'r' in decoded and b'BFsd' in decoded[
                    b'r'] and b'BFpe' in decoded[b'r']:
                self.dht_health_manager.received_bloomfilters(
                    decoded[b't'], bytearray(decoded[b'r'][b'BFsd']),
                    bytearray(decoded[b'r'][b'BFpe']))

    def update_ip_filter(self, lt_session, ip_addresses):
        self._logger.debug('Updating IP filter %s', ip_addresses)
        ip_filter = lt.ip_filter()
        ip_filter.add_rule('0.0.0.0', '255.255.255.255', 1)
        for ip in ip_addresses:
            ip_filter.add_rule(ip, ip, 0)
        lt_session.set_ip_filter(ip_filter)

    async def get_metainfo(self, infohash, timeout=30, hops=None, url=None):
        """
        Lookup metainfo for a given infohash. The mechanism works by joining the swarm for the infohash connecting
        to a few peers, and downloading the metadata for the torrent.
        :param infohash: The (binary) infohash to lookup metainfo for.
        :param timeout: A timeout in seconds.
        :param hops: the number of tunnel hops to use for this lookup. If None, use config default.
        :param url: Optional URL. Can contain trackers info, etc.
        :return: The metainfo
        """
        infohash_hex = hexlify(infohash)
        if infohash in self.metainfo_cache:
            self._logger.info('Returning metainfo from cache for %s',
                              infohash_hex)
            return self.metainfo_cache[infohash]['meta_info']

        self._logger.info('Trying to fetch metainfo for %s', infohash_hex)
        if infohash in self.metainfo_requests:
            download = self.metainfo_requests[infohash][0]
            self.metainfo_requests[infohash][1] += 1
        elif infohash in self.downloads:
            download = self.downloads[infohash]
        else:
            tdef = TorrentDefNoMetainfo(infohash, 'metainfo request', url=url)
            dcfg = DownloadConfig()
            dcfg.set_hops(self.tribler_session.config.get_default_number_hops(
            ) if hops is None else hops)
            dcfg.set_upload_mode(
                True
            )  # Upload mode should prevent libtorrent from creating files
            dcfg.set_dest_dir(self.metadata_tmpdir)
            try:
                download = self.start_download(tdef=tdef,
                                               config=dcfg,
                                               hidden=True,
                                               checkpoint_disabled=True)
            except TypeError:
                return
            self.metainfo_requests[infohash] = [download, 1]

        try:
            metainfo = download.tdef.get_metainfo() or await wait_for(
                shield(download.future_metainfo), timeout)
            self._logger.info('Successfully retrieved metainfo for %s',
                              infohash_hex)
            self.metainfo_cache[infohash] = {
                'time': timemod.time(),
                'meta_info': metainfo
            }
        except (CancelledError, asyncio.TimeoutError):
            metainfo = None
            self._logger.info('Failed to retrieve metainfo for %s',
                              infohash_hex)

        if infohash in self.metainfo_requests:
            self.metainfo_requests[infohash][1] -= 1
            if self.metainfo_requests[infohash][1] <= 0:
                await self.remove_download(download, remove_content=True)
                self.metainfo_requests.pop(infohash)

        return metainfo

    def _task_cleanup_metainfo_cache(self):
        oldest_time = timemod.time() - METAINFO_CACHE_PERIOD

        for info_hash, cache_entry in list(self.metainfo_cache.items()):
            last_time = cache_entry['time']
            if last_time < oldest_time:
                del self.metainfo_cache[info_hash]

    def _request_torrent_updates(self):
        for ltsession in self.ltsessions.values():
            if ltsession:
                # Newer version of libtorrent require the flags argument in the post_torrent_updates call.
                if LooseVersion(self.get_libtorrent_version()) >= LooseVersion(
                        "1.1.0"):
                    ltsession.post_torrent_updates(0xffffffff)
                else:
                    ltsession.post_torrent_updates()

    def _task_process_alerts(self):
        for hops, ltsession in list(self.ltsessions.items()):
            if ltsession:
                for alert in ltsession.pop_alerts():
                    self.process_alert(alert, hops=hops)

    def _map_call_on_ltsessions(self, hops, funcname, *args, **kwargs):
        if hops is None:
            for session in self.ltsessions.values():
                getattr(session, funcname)(*args, **kwargs)
        else:
            getattr(self.get_session(hops), funcname)(*args, **kwargs)

    async def start_download_from_uri(self, uri, config=None):
        if uri.startswith("http"):
            tdef = await TorrentDef.load_from_url(uri)
            return self.start_download(tdef=tdef, config=config)
        if uri.startswith("magnet:"):
            name, infohash, _ = parse_magnetlink(uri)
            if infohash is None:
                raise RuntimeError("Missing infohash")
            if infohash in self.metainfo_cache:
                tdef = TorrentDef.load_from_dict(
                    self.metainfo_cache[infohash]['meta_info'])
            else:
                tdef = TorrentDefNoMetainfo(
                    infohash,
                    "Unknown name" if name is None else name,
                    url=uri)
            return self.start_download(tdef=tdef, config=config)
        if uri.startswith("file:"):
            argument = url2pathname(uri[5:])
            return self.start_download(torrent_file=argument, config=config)
        raise Exception("invalid uri")

    def start_download(self,
                       torrent_file=None,
                       tdef=None,
                       config=None,
                       checkpoint_disabled=False,
                       hidden=False):
        self._logger.debug("Starting download: filename: %s, torrent def: %s",
                           torrent_file, tdef)

        # the priority of the parameters is: (1) tdef, (2) torrent_file.
        # so if we have tdef, and torrent_file will be ignored, and so on.
        if tdef is None:
            if torrent_file is None:
                raise ValueError(
                    "Torrent file must be provided if tdef is not given")
            # try to get the torrent from the given torrent file
            tdef = TorrentDef.load(torrent_file)

        assert tdef is not None, "tdef MUST not be None after loading torrent"

        config = config or DownloadConfig()
        infohash = tdef.get_infohash()
        download = self.get_download(infohash)

        if download and infohash not in self.metainfo_requests:
            new_trackers = list(
                set(tdef.get_trackers_as_single_tuple()) -
                set(download.get_def().get_trackers_as_single_tuple()))
            if new_trackers:
                self.update_trackers(tdef.get_infohash(), new_trackers)
            return download

        # Create the destination directory if it does not exist yet
        try:
            if not config.get_dest_dir().is_dir():
                os.makedirs(config.get_dest_dir())
        except OSError:
            self._logger.error(
                "Unable to create the download destination directory.")

        if config.get_time_added() == 0:
            config.set_time_added(int(timemod.time()))

        # Create the download
        download = Download(self.tribler_session, tdef, dummy=self.dummy_mode)
        atp = download.setup(config,
                             checkpoint_disabled=checkpoint_disabled,
                             hidden=hidden or config.get_bootstrap_download())
        # Keep metainfo downloads in self.downloads for now because we will need to remove it later,
        # and removing the download at this point will stop us from receiving any further alerts.
        if infohash not in self.metainfo_requests or self.metainfo_requests[
                infohash][0] == download:
            self.downloads[infohash] = download
        if not self.dummy_mode:
            self.start_handle(download, atp)
        return download

    @task
    async def start_handle(self, download, atp):
        ltsession = self.get_session(download.config.get_hops())
        infohash = download.get_def().get_infohash()

        if infohash in self.metainfo_requests and self.metainfo_requests[
                infohash][0] != download:
            self._logger.info("Cancelling metainfo request(s) for infohash:%s",
                              hexlify(infohash))
            metainfo_dl, _ = self.metainfo_requests.pop(infohash)
            # Leave the checkpoint. Any checkpoint that exists will belong to the download we are currently starting.
            await self.remove_download(metainfo_dl,
                                       remove_content=True,
                                       remove_checkpoint=False)
            self.downloads[infohash] = download

        known = {
            unhexlify(str(h.info_hash())): h
            for h in ltsession.get_torrents()
        }
        existing_handle = known.get(infohash)
        if existing_handle:
            # Reuse existing handle
            self._logger.debug("Reusing handle %s", hexlify(infohash))
            download.post_alert('add_torrent_alert',
                                dict(handle=existing_handle))
        else:
            # Otherwise, add it anew
            self._logger.debug("Adding handle %s", hexlify(infohash))
            # To prevent flooding the DHT with a short burst of queries and triggering
            # flood protection, we postpone adding torrents until we get enough DHT peers.
            # The asynchronous wait should be done as close as possible to the actual
            # Libtorrent calls, so the higher-level download-adding logic does not block.
            # Otherwise, e.g. if added to the Session init sequence, this results in startup
            # time increasing by 10-20 seconds.
            # See https://github.com/Tribler/tribler/issues/5319
            if self.dht_readiness_timeout > 0 and self._dht_ready_task is not None:
                try:
                    await wait_for(shield(self._dht_ready_task),
                                   timeout=self.dht_readiness_timeout)
                except asyncio.TimeoutError:
                    self._logger.warning(
                        "Timeout waiting for libtorrent DHT getting enough peers"
                    )
            ltsession.async_add_torrent(encode_atp(atp))
        return await download.future_added

    def get_libtorrent_version(self):
        try:
            return lt.__version__
        except AttributeError:
            return lt.version

    def set_session_settings(self, lt_session, new_settings):
        """
        Apply/set new sessions in a libtorrent session.
        :param lt_session: The libtorrent session to apply the settings to.
        :param new_settings: The new settings to apply.
        """

        # Keeping a copy of the settings because subsequent calls to get_settings are likely to fail
        # when libtorrent will try to decode peer_fingerprint to unicode.
        if lt_session not in self.ltsettings:
            self.ltsettings[lt_session] = lt_session.get_settings()
        self.ltsettings[lt_session].update(new_settings)

        try:
            if hasattr(lt_session, "apply_settings"):
                lt_session.apply_settings(new_settings)
            else:
                lt_session.set_settings(new_settings)
        except OverflowError:
            raise OverflowError(
                f"Overflow error when setting libtorrent sessions with settings: {new_settings}"
            )

    def get_session_settings(self, lt_session):
        return deepcopy(self.ltsettings.get(lt_session, {}))

    def update_max_rates_from_config(self):
        """
        Set the maximum download and maximum upload rate limits with the value in the config.

        This is the extra step necessary to apply a new maximum download/upload rate setting.
        :return:
        """
        for lt_session in self.ltsessions.values():
            settings = {
                'download_rate_limit':
                self.tribler_session.config.get_libtorrent_max_download_rate(),
                'upload_rate_limit':
                self.tribler_session.config.get_libtorrent_max_upload_rate()
            }
            self.set_session_settings(lt_session, settings)

    def post_session_stats(self, hops=None):
        for ltsession in self.ltsessions.values() if hops is None else [
                self.ltsessions[hops]
        ]:
            if hasattr(ltsession, "post_session_stats"):
                ltsession.post_session_stats()

    async def remove_download(self,
                              download,
                              remove_content=False,
                              remove_checkpoint=True):
        infohash = download.get_def().get_infohash()
        handle = download.handle

        # Note that the following block of code needs to be able to deal with multiple simultaneous
        # calls using the same download object. We need to make sure that we don't return without
        # the removal having finished.
        if handle:
            if handle.is_valid():
                download.stream.disable()
                self._logger.debug("Removing handle %s", hexlify(infohash))
                ltsession = self.get_session(download.config.get_hops())
                ltsession.remove_torrent(handle, int(remove_content))
            # We need to wait even if the handle is invalid. It's important to synchronize
            # here because the upcoming call to shutdown will also cancel future_removed.
            await download.future_removed
        else:
            self._logger.debug(
                "Cannot remove handle %s because it does not exists",
                hexlify(infohash))
        await download.shutdown()

        if infohash in self.downloads and self.downloads[infohash] == download:
            self.downloads.pop(infohash)
            if remove_checkpoint:
                self.remove_config(infohash)
        else:
            self._logger.debug("Cannot remove unknown download")

    def get_download(self, infohash):
        return self.downloads.get(infohash, None)

    def get_downloads(self):
        return list(self.downloads.values())

    def get_channel_downloads(self):
        return [
            download for download in self.downloads.values()
            if download.config.get_channel_download()
        ]

    def download_exists(self, infohash):
        return infohash in self.downloads

    async def update_hops(self, download, new_hops):
        """
        Update the amount of hops for a specified download. This can be done on runtime.
        """
        infohash = hexlify(download.tdef.get_infohash())
        self._logger.info("Updating the amount of hops of download %s",
                          infohash)
        await download.save_resume_data()
        await self.remove_download(download)

        # copy the old download_config and change the hop count
        config = download.config.copy()
        config.set_hops(new_hops)
        # If the user wants to change the hop count to 0, don't automatically bump this up to 1 anymore
        config.set_safe_seeding(False)

        self.start_download(tdef=download.tdef, config=config)

    def update_trackers(self, infohash, trackers):
        """ Update the trackers for a download.
        :param infohash: infohash of the torrent that needs to be updated
        :param trackers: A list of tracker urls.
        """
        download = self.get_download(infohash)
        if download:
            old_def = download.get_def()
            old_trackers = old_def.get_trackers_as_single_tuple()
            new_trackers = list(set(trackers) - set(old_trackers))
            all_trackers = list(old_trackers) + new_trackers

            if new_trackers:
                # Add new trackers to the download
                download.add_trackers(new_trackers)

                # Create a new TorrentDef
                if isinstance(old_def, TorrentDefNoMetainfo):
                    new_def = TorrentDefNoMetainfo(old_def.get_infohash(),
                                                   old_def.get_name(),
                                                   download.get_magnet_link())
                else:
                    metainfo = old_def.get_metainfo()
                    if len(all_trackers) > 1:
                        metainfo["announce-list"] = [all_trackers]
                    else:
                        metainfo["announce"] = all_trackers[0]
                    new_def = TorrentDef.load_from_dict(metainfo)

                # Set TorrentDef + checkpoint
                download.set_def(new_def)
                download.checkpoint()

    def set_download_states_callback(self, user_callback, interval=1.0):
        """
        Set the download state callback. Remove any old callback if it's present.
        Calls user_callback with a list of
        DownloadStates, one for each Download in the Session as first argument.
        The user_callback must return a tuple (when, getpeerlist) that indicates
        when to invoke the callback again (as a number of seconds from now,
        or < 0.0 if not at all) and whether to also include the details of
        the connected peers in the DownloadStates on that next call.

        :param user_callback: a function adhering to the above spec
        :param interval: time in between the download states callback's
        """
        self._logger.debug(
            "Starting the download state callback with interval %f", interval)
        self.replace_task("download_states_lc",
                          self._invoke_states_cb,
                          user_callback,
                          interval=interval)

    def stop_download_states_callback(self):
        return self.cancel_pending_task("download_states_lc")

    async def _invoke_states_cb(self, callback):
        """
        Invoke the download states callback with a list of the download states.
        """
        result = callback(
            [download.get_state() for download in self.downloads.values()])
        if iscoroutine(result):
            await result

    async def sesscb_states_callback(self, states_list):
        """
        This method is periodically (every second) called with a list of the download states of the active downloads.
        """
        # TODO: refactor this method. It is too long and tightly coupled with higher-level modules.
        self.state_cb_count += 1

        for ds in states_list:
            download = ds.get_download()
            infohash = download.get_def().get_infohash()

            if ds.get_status() == DLSTATUS_SEEDING:
                if download.config.get_hops(
                ) == 0 and download.config.get_safe_seeding():
                    # Re-add the download with anonymity enabled
                    hops = self.tribler_session.config.get_default_number_hops(
                    )
                    await self.update_hops(download, hops)

            # Check the peers of this download every five seconds and add them to the payout manager when
            # this peer runs a Tribler instance
            if self.state_cb_count % 5 == 0 and download.config.get_hops(
            ) == 0 and self.tribler_session.payout_manager:
                for peer in download.get_peerlist():
                    if str(peer["extended_version"]).startswith('Tribler'):
                        self.tribler_session.payout_manager.update_peer(
                            unhexlify(peer["id"]), infohash, peer["dtotal"])

        if self.state_cb_count % 4 == 0:
            if self.tribler_session.tunnel_community:
                self.tribler_session.tunnel_community.monitor_downloads(
                    states_list)

    async def load_checkpoints(self):
        for filename in self.get_checkpoint_dir().glob('*.conf'):
            self.load_checkpoint(filename)
            await sleep(.01)

    def load_checkpoint(self, filename):
        try:
            config = DownloadConfig.load(filename)
        except Exception:
            self._logger.exception("Could not open checkpoint file %s",
                                   filename)
            return

        metainfo = config.get_metainfo()
        if not metainfo:
            self._logger.error(
                "Could not resume checkpoint %s; metainfo not found", filename)
            return
        if not isinstance(metainfo, dict):
            self._logger.error(
                "Could not resume checkpoint %s; metainfo is not dict %s %s",
                filename, type(metainfo), repr(metainfo))
            return

        try:
            url = metainfo.get(b'url', None)
            url = url.decode('utf-8') if url else url
            tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'],
                                         metainfo[b'name'], url) if b'infohash'
                    in metainfo else TorrentDef.load_from_dict(metainfo))
        except (KeyError, ValueError) as e:
            self._logger.exception(
                "Could not restore tdef from metainfo dict: %s %s ", e,
                metainfo)
            return

        if config.get_bootstrap_download():
            if hexlify(tdef.get_infohash(
            )) != self.tribler_session.config.get_bootstrap_infohash():
                self.remove_config(tdef.get_infohash())
                return

        config.state_dir = self.tribler_session.config.get_state_dir()
        if config.get_dest_dir() == '':  # removed torrent ignoring
            self._logger.info("Removing checkpoint %s destdir is %s", filename,
                              config.get_dest_dir())
            os.remove(filename)
            return

        try:
            if self.download_exists(tdef.get_infohash()):
                self._logger.info(
                    "Not resuming checkpoint because download has already been added"
                )
            else:
                self.start_download(tdef=tdef, config=config)
        except Exception:
            self._logger.exception(
                "Not resume checkpoint due to exception while adding download")

    def remove_config(self, infohash):
        if infohash not in self.downloads:
            try:
                basename = hexlify(infohash) + '.conf'
                filename = self.get_checkpoint_dir() / basename
                self._logger.debug("Removing download checkpoint %s", filename)
                if os.access(filename, os.F_OK):
                    os.remove(filename)
            except:
                # Show must go on
                self._logger.exception("Could not remove state")
        else:
            self._logger.warning(
                "Download is back, restarted? Cancelling removal! %s",
                hexlify(infohash))

    def get_checkpoint_dir(self):
        """
        Returns the directory in which to checkpoint the Downloads in this Session.
        """
        return self.tribler_session.config.get_state_dir(
        ) / STATEDIR_CHECKPOINT_DIR

    @staticmethod
    async def create_torrent_file(file_path_list, params=None):
        """
        Creates a torrent file.

        :param file_path_list: files to add in torrent file
        :param params: optional parameters for torrent file
        :return: a Deferred that fires when the torrent file has been created
        """
        return await asyncio.get_event_loop().run_in_executor(
            None, torrent_utils.create_torrent_file, file_path_list, params
            or {})

    def get_downloads_by_name(self, torrent_name, channels_only=False):
        downloads = (self.get_channel_downloads()
                     if channels_only else self.get_downloads())
        return [
            d for d in downloads if d.get_def().get_name_utf8() == torrent_name
        ]
예제 #4
0
async def dht_health_manager(session):
    session.dht_get_peers = lambda _: None
    manager = DHTHealthManager(session)
    yield manager
    await manager.shutdown_task_manager()
예제 #5
0
class TestDHTHealthManager(TriblerCoreTest):
    """
    This class contains various tests for the DHT health manager.
    """
    async def setUp(self):
        await super(TestDHTHealthManager, self).setUp()

        self.mock_lt_session = MockObject()
        self.mock_lt_session.dht_get_peers = lambda _: None

        self.dht_health_manager = DHTHealthManager(self.mock_lt_session)

    async def tearDown(self):
        await self.dht_health_manager.shutdown_task_manager()
        await super(TestDHTHealthManager, self).tearDown()

    @timeout(10)
    async def test_get_health(self):
        """
        Test fetching the health of a trackerless torrent.
        """
        response = await self.dht_health_manager.get_health(b'a' * 20,
                                                            timeout=0.1)
        self.assertIsInstance(response, dict)
        self.assertIn('DHT', response)
        self.assertEqual(response['DHT'][0]['infohash'], hexlify(b'a' * 20))

    @timeout(10)
    async def test_existing_get_health(self):
        lookup_future = self.dht_health_manager.get_health(b'a' * 20,
                                                           timeout=0.1)
        self.assertEqual(
            self.dht_health_manager.get_health(b'a' * 20, timeout=0.1),
            lookup_future)
        await lookup_future

    @timeout(10)
    async def test_combine_bloom_filters(self):
        """
        Test combining two bloom filters
        """
        bf1 = bytearray(b'a' * 256)
        bf2 = bytearray(b'a' * 256)
        self.assertEqual(
            self.dht_health_manager.combine_bloomfilters(bf1, bf2), bf1)

        bf1 = bytearray(b'\0' * 256)
        bf2 = bytearray(b'b' * 256)
        self.assertEqual(
            self.dht_health_manager.combine_bloomfilters(bf1, bf2), bf2)

    @timeout(10)
    async def test_get_size_from_bloom_filter(self):
        """
        Test whether we can successfully estimate the size from a bloom filter
        """
        # See http://www.bittorrent.org/beps/bep_0033.html
        bf = bytearray(
            unhexlify(
                """F6C3F5EA A07FFD91 BDE89F77 7F26FB2B FF37BDB8 FB2BBAA2 FD3DDDE7 BACFFF75 EE7CCBAE
                                    FE5EEDB1 FBFAFF67 F6ABFF5E 43DDBCA3 FD9B9FFD F4FFD3E9 DFF12D1B DF59DB53 DBE9FA5B
                                    7FF3B8FD FCDE1AFB 8BEDD7BE 2F3EE71E BBBFE93B CDEEFE14 8246C2BC 5DBFF7E7 EFDCF24F
                                    D8DC7ADF FD8FFFDF DDFFF7A4 BBEEDF5C B95CE81F C7FCFF1F F4FFFFDF E5F7FDCB B7FD79B3
                                    FA1FC77B FE07FFF9 05B7B7FF C7FEFEFF E0B8370B B0CD3F5B 7F2BD93F EB4386CF DD6F7FD5
                                    BFAF2E9E BFFFFEEC D67ADBF7 C67F17EF D5D75EBA 6FFEBA7F FF47A91E B1BFBB53 E8ABFB57
                                    62ABE8FF 237279BF EFBFEEF5 FFC5FEBF DFE5ADFF ADFEE1FB 737FFFFB FD9F6AEF FEEE76B6
                                    FD8F72EF""".replace(' ',
                                                        '').replace('\n', '')))
        self.assertEqual(self.dht_health_manager.get_size_from_bloomfilter(bf),
                         1224)

        # Maximum capacity
        bf = bytearray(b'\xff' * 256)
        self.assertEqual(self.dht_health_manager.get_size_from_bloomfilter(bf),
                         6000)

    @timeout(10)
    async def test_receive_bloomfilters(self):
        """
        Test whether the right operations happen when receiving a bloom filter
        """
        infohash = b'a' * 20
        self.dht_health_manager.received_bloomfilters(
            infohash)  # It should not do anything
        self.assertFalse(self.dht_health_manager.bf_seeders)
        self.assertFalse(self.dht_health_manager.bf_peers)

        self.dht_health_manager.lookup_futures[infohash] = Future()
        self.dht_health_manager.bf_seeders[infohash] = bytearray(256)
        self.dht_health_manager.bf_peers[infohash] = bytearray(256)
        self.dht_health_manager.received_bloomfilters(
            b'b' * 20,
            bf_seeds=bytearray(b'\xee' * 256),
            bf_peers=bytearray(b'\xff' * 256))
        self.assertEqual(self.dht_health_manager.bf_seeders[infohash],
                         bytearray(b'\xee' * 256))
        self.assertEqual(self.dht_health_manager.bf_peers[infohash],
                         bytearray(b'\xff' * 256))