async def test_on_metadata_received_alert(mock_handle, test_download): """ Testing whether the right operations happen when we receive metadata """ test_future = Future() mocked_file = Mock() mocked_file.path = 'test' test_download.handle.trackers = lambda: [] test_download.handle.get_peer_info = lambda: [] test_download.handle.save_resume_data = lambda: test_future test_download.handle.rename_file = lambda *_: None with open(TESTS_DATA_DIR / "bak_single.torrent", mode='rb') as torrent_file: encoded_metainfo = torrent_file.read() decoded_metainfo = bdecode_compat(encoded_metainfo) get_info_from_handle(test_download.handle).metadata = lambda: bencode( decoded_metainfo[b'info']) get_info_from_handle(test_download.handle).files = lambda: [mocked_file] test_download.checkpoint = lambda: test_future.set_result(None) test_download.session = MockObject() test_download.session.torrent_db = None test_download.handle.save_path = lambda: None test_download.handle.prioritize_files = lambda _: None test_download.get_share_mode = lambda: False test_download.on_metadata_received_alert(None) await test_future
def on_metadata_received_alert(self, _): torrent_info = get_info_from_handle(self.handle) if not torrent_info: return metadata = { b'info': bdecode_compat(torrent_info.metadata()), b'leechers': 0, b'seeders': 0 } trackers = [ tracker['url'].encode('utf-8') for tracker in self.handle.trackers() ] if len(trackers) > 1: metadata[b"announce-list"] = [trackers] elif trackers: metadata[b"announce"] = trackers[0] for peer in self.handle.get_peer_info(): if peer.progress == 1: metadata[b"seeders"] += 1 else: metadata[b"leechers"] += 1 try: self.tdef = TorrentDef.load_from_dict(metadata) except ValueError as ve: self._logger.exception(ve) return self.set_selected_files() self.checkpoint()
def _process_scrape_response(self, body): """ This function handles the response body of a HTTP tracker, parsing the results. """ # parse the retrieved results if body is None: self.failed(msg="no response body") response_dict = bdecode_compat(body) if not response_dict: self.failed(msg="no valid response") response_list = [] unprocessed_infohash_list = self.infohash_list[:] if b'files' in response_dict and isinstance(response_dict[b'files'], dict): for infohash in response_dict[b'files']: complete = 0 incomplete = 0 if isinstance(response_dict[b'files'][infohash], dict): complete = response_dict[b'files'][infohash].get( b'complete', 0) incomplete = response_dict[b'files'][infohash].get( b'incomplete', 0) # Sow complete as seeders. "complete: number of peers with the entire file, i.e. seeders (integer)" # - https://wiki.theory.org/BitTorrentSpecification#Tracker_.27scrape.27_Convention seeders = complete leechers = incomplete # Store the information in the dictionary response_list.append({ 'infohash': hexlify(infohash), 'seeders': seeders, 'leechers': leechers }) # remove this infohash in the infohash list of this session if infohash in unprocessed_infohash_list: unprocessed_infohash_list.remove(infohash) elif b'failure reason' in response_dict: self._logger.info("%s Failure as reported by tracker [%s]", self, repr(response_dict[b'failure reason'])) self.failed(msg=repr(response_dict[b'failure reason'])) # handle the infohashes with no result (seeders/leechers = 0/0) for infohash in unprocessed_infohash_list: response_list.append({ 'infohash': hexlify(infohash), 'seeders': 0, 'leechers': 0 }) self.is_finished = True return {self.tracker_url: response_list}
def save(self, torrent_filepath=None): """ Generate the metainfo and save the torrent file. :param torrent_filepath: An optional absolute path to where to save the generated .torrent file. """ torrent_dict = create_torrent_file(self.files_list, self.torrent_parameters, torrent_filepath=torrent_filepath) self.metainfo = bdecode_compat(torrent_dict['metainfo']) self.copy_metainfo_to_torrent_parameters() self.infohash = torrent_dict['infohash']
def load_from_memory(bencoded_data): """ Load some bencoded data into a TorrentDef. :param bencoded_data: The bencoded data to decode and use as metainfo """ metainfo = bdecode_compat(bencoded_data) # Some versions of libtorrent will not raise an exception when providing invalid data. # This issue is present in 1.0.8 (included with Tribler 7.3.0), but has been fixed since at least 1.2.1. if metainfo is None: raise ValueError("Data is not a bencoded string") return TorrentDef.load_from_dict(metainfo)
def process_alert(self, alert, hops=0): alert_type = alert.__class__.__name__ # Periodically, libtorrent will send us a state_update_alert, which contains the torrent status of # all torrents changed since the last time we received this alert. if alert_type == 'state_update_alert': for status in alert.status: infohash = unhexlify(str(status.info_hash)) if infohash not in self.downloads: self._logger.debug("Got state_update for unknown torrent %s", hexlify(infohash)) continue self.downloads[infohash].update_lt_status(status) infohash = unhexlify(str(alert.handle.info_hash() if hasattr(alert, 'handle') and alert.handle.is_valid() else getattr(alert, 'info_hash', ''))) download = self.downloads.get(infohash) if download: if (download.handle and download.handle.is_valid())\ or (not download.handle and alert_type == 'add_torrent_alert') \ or (download.handle and alert_type == 'torrent_removed_alert'): download.process_alert(alert, alert_type) else: self._logger.debug("Got alert for download without handle %s: %s", hexlify(infohash), alert) elif infohash: self._logger.debug("Got alert for unknown download %s: %s", hexlify(infohash), alert) if alert_type == 'peer_disconnected_alert' and \ self.tribler_session and self.tribler_session.payout_manager: self.tribler_session.payout_manager.do_payout(alert.pid.to_bytes()) elif alert_type == 'session_stats_alert': queued_disk_jobs = alert.values['disk.queued_disk_jobs'] queued_write_bytes = alert.values['disk.queued_write_bytes'] num_write_jobs = alert.values['disk.num_write_jobs'] if queued_disk_jobs == queued_write_bytes == num_write_jobs == 0: self.lt_session_shutdown_ready[hops] = True if self.session_stats_callback: self.session_stats_callback(alert) elif alert_type == "dht_pkt_alert": # We received a raw DHT message - decode it and check whether it is a BEP33 message. decoded = bdecode_compat(alert.pkt_buf) if decoded and b'r' in decoded: if b'BFsd' in decoded[b'r'] and b'BFpe' in decoded[b'r']: self.dht_health_manager.received_bloomfilters(decoded[b'r'][b'id'], bytearray(decoded[b'r'][b'BFsd']), bytearray(decoded[b'r'][b'BFpe']))
def process_alert(self, alert, hops=0): alert_type = alert.__class__.__name__ # Periodically, libtorrent will send us a state_update_alert, which contains the torrent status of # all torrents changed since the last time we received this alert. if alert_type == 'state_update_alert': for status in alert.status: infohash = unhexlify(str(status.info_hash)) if infohash not in self.downloads: self._logger.debug( "Got state_update for unknown torrent %s", hexlify(infohash)) continue self.downloads[infohash].update_lt_status(status) infohash = unhexlify( str(alert.handle.info_hash() if hasattr(alert, 'handle') and alert. handle.is_valid() else getattr(alert, 'info_hash', ''))) download = self.downloads.get(infohash) if download: if (download.handle and download.handle.is_valid())\ or (not download.handle and alert_type == 'add_torrent_alert') \ or (download.handle and alert_type == 'torrent_removed_alert'): download.process_alert(alert, alert_type) else: self._logger.debug( "Got alert for download without handle %s: %s", hexlify(infohash), alert) elif infohash: self._logger.debug("Got alert for unknown download %s: %s", hexlify(infohash), alert) if alert_type == 'listen_succeeded_alert': # The ``port`` attribute was added in libtorrent 1.1.14. # Older versions (most notably libtorrent 1.1.13 - the default on Ubuntu 20.04) do not have this attribute. # We use the now-deprecated ``endpoint`` attribute for these older versions. self.listen_ports[hops] = getattr(alert, "port", alert.endpoint[1]) elif alert_type == 'peer_disconnected_alert' and \ self.tribler_session and self.tribler_session.payout_manager: self.tribler_session.payout_manager.do_payout(alert.pid.to_bytes()) elif alert_type == 'session_stats_alert': queued_disk_jobs = alert.values['disk.queued_disk_jobs'] queued_write_bytes = alert.values['disk.queued_write_bytes'] num_write_jobs = alert.values['disk.num_write_jobs'] if queued_disk_jobs == queued_write_bytes == num_write_jobs == 0: self.lt_session_shutdown_ready[hops] = True if self.session_stats_callback: self.session_stats_callback(alert) elif alert_type == "dht_pkt_alert": # Unfortunately, the Python bindings don't have a direction attribute. # So, we'll have to resort to using the string representation of the alert instead. incoming = str(alert).startswith('<==') decoded = bdecode_compat(alert.pkt_buf) if not decoded: return # We are sending a raw DHT message - notify the DHTHealthManager of the outstanding request. if not incoming and decoded.get(b'y') == b'q' \ and decoded.get(b'q') == b'get_peers' and decoded[b'a'].get(b'scrape') == 1: self.dht_health_manager.requesting_bloomfilters( decoded[b't'], decoded[b'a'][b'info_hash']) # We received a raw DHT message - decode it and check whether it is a BEP33 message. if incoming and b'r' in decoded and b'BFsd' in decoded[ b'r'] and b'BFpe' in decoded[b'r']: self.dht_health_manager.received_bloomfilters( decoded[b't'], bytearray(decoded[b'r'][b'BFsd']), bytearray(decoded[b'r'][b'BFpe']))
def create_session(self, hops=0, store_listen_port=True): # Due to a bug in Libtorrent 0.16.18, the outgoing_port and num_outgoing_ports value should be set in # the settings dictionary settings = { 'outgoing_port': 0, 'num_outgoing_ports': 1, 'allow_multiple_connections_per_ip': 0 } # Copy construct so we don't modify the default list extensions = list(DEFAULT_LT_EXTENSIONS) # Elric: Strip out the -rcX, -beta, -whatever tail on the version string. fingerprint = ['TL'] + [ int(x) for x in version_id.split('-')[0].split('.') ] + [0] if self.dummy_mode: from unittest.mock import Mock ltsession = Mock() ltsession.pop_alerts = lambda: {} ltsession.listen_port = lambda: 123 ltsession.get_settings = lambda: {"peer_fingerprint": "000"} else: ltsession = lt.session(lt.fingerprint( *fingerprint), flags=0) if hops == 0 else lt.session(flags=0) if hops == 0: settings['user_agent'] = 'Tribler/' + version_id enable_utp = self.tribler_session.config.get_libtorrent_utp() settings['enable_outgoing_utp'] = enable_utp settings['enable_incoming_utp'] = enable_utp if LooseVersion( self.get_libtorrent_version()) >= LooseVersion("1.1.0"): settings['prefer_rc4'] = True settings[ "listen_interfaces"] = "0.0.0.0:%d" % self.tribler_session.config.get_libtorrent_port( ) else: pe_settings = lt.pe_settings() pe_settings.prefer_rc4 = True ltsession.set_pe_settings(pe_settings) mid = self.tribler_session.trustchain_keypair.key_to_hash() settings['peer_fingerprint'] = mid settings[ 'handshake_client_version'] = 'Tribler/' + version_id + '/' + hexlify( mid) else: settings['enable_outgoing_utp'] = True settings['enable_incoming_utp'] = True settings['enable_outgoing_tcp'] = False settings['enable_incoming_tcp'] = False settings['anonymous_mode'] = True settings['force_proxy'] = True if LooseVersion( self.get_libtorrent_version()) >= LooseVersion("1.1.0"): settings[ "listen_interfaces"] = "0.0.0.0:%d" % self.tribler_session.config.get_anon_listen_port( ) # By default block all IPs except 1.1.1.1 (which is used to ensure libtorrent makes a connection to us) self.update_ip_filter(ltsession, ['1.1.1.1']) self.set_session_settings(ltsession, settings) ltsession.set_alert_mask(self.default_alert_mask) # Load proxy settings if hops == 0: proxy_settings = self.tribler_session.config.get_libtorrent_proxy_settings( ) else: proxy_settings = list( self.tribler_session.config.get_anon_proxy_settings()) proxy_host, proxy_ports = proxy_settings[1] proxy_settings[1] = (proxy_host, proxy_ports[hops - 1]) self.set_proxy_settings(ltsession, *proxy_settings) for extension in extensions: ltsession.add_extension(extension) # Set listen port & start the DHT if hops == 0: listen_port = self.tribler_session.config.get_libtorrent_port() ltsession.listen_on(listen_port, listen_port + 10) if listen_port != ltsession.listen_port() and store_listen_port: self.tribler_session.config.set_libtorrent_port_runtime( ltsession.listen_port()) try: with open( self.tribler_session.config.get_state_dir() / LTSTATE_FILENAME, 'rb') as fp: lt_state = bdecode_compat(fp.read()) if lt_state is not None: ltsession.load_state(lt_state) else: self._logger.warning( "the lt.state appears to be corrupt, writing new data on shutdown" ) except Exception as exc: self._logger.info( f"could not load libtorrent state, got exception: {exc!r}. starting from scratch" ) else: ltsession.listen_on( self.tribler_session.config.get_anon_listen_port(), self.tribler_session.config.get_anon_listen_port() + 20) settings = { 'upload_rate_limit': self.tribler_session.config.get_libtorrent_max_upload_rate(), 'download_rate_limit': self.tribler_session.config.get_libtorrent_max_download_rate() } self.set_session_settings(ltsession, settings) if self.tribler_session.config.get_libtorrent_dht_enabled( ) and not self.dummy_mode: ltsession.start_dht() for router in DEFAULT_DHT_ROUTERS: ltsession.add_dht_router(*router) ltsession.start_lsd() self._logger.debug("Started libtorrent session for %d hops on port %d", hops, ltsession.listen_port()) self.lt_session_shutdown_ready[hops] = False return ltsession
def get_engineresumedata(self): return bdecode_compat( base64.b64decode( self.config['state']['engineresumedata'].encode('utf-8')))
def get_metainfo(self): return bdecode_compat( base64.b64decode(self.config['state']['metainfo'].encode('utf-8')))
def create_session(self, hops=0, store_listen_port=True): # Due to a bug in Libtorrent 0.16.18, the outgoing_port and num_outgoing_ports value should be set in # the settings dictionary self._logger.info('Creating a session') settings = { 'outgoing_port': 0, 'num_outgoing_ports': 1, 'allow_multiple_connections_per_ip': 0, 'enable_upnp': int(self.config.upnp), 'enable_dht': int(self.config.dht), 'enable_lsd': int(self.config.lsd), 'enable_natpmp': int(self.config.natpmp) } # Copy construct so we don't modify the default list extensions = list(DEFAULT_LT_EXTENSIONS) self._logger.info(f'Dummy mode: {self.dummy_mode}. Hops: {hops}.') # Elric: Strip out the -rcX, -beta, -whatever tail on the version string. fingerprint = ['TL'] + [ int(x) for x in version_id.split('-')[0].split('.') ] + [0] if self.dummy_mode: from unittest.mock import Mock ltsession = Mock() ltsession.pop_alerts = lambda: {} ltsession.listen_port = lambda: 123 ltsession.get_settings = lambda: {"peer_fingerprint": "000"} else: ltsession = lt.session(lt.fingerprint( *fingerprint), flags=0) if hops == 0 else lt.session(flags=0) libtorrent_port = self.config.port or default_network_utils.get_random_free_port( ) self._libtorrent_port = libtorrent_port self._logger.info(f'Libtorrent port: {libtorrent_port}') if hops == 0: settings['user_agent'] = 'Tribler/' + version_id enable_utp = self.config.utp settings['enable_outgoing_utp'] = enable_utp settings['enable_incoming_utp'] = enable_utp settings['prefer_rc4'] = True settings["listen_interfaces"] = "0.0.0.0:%d" % libtorrent_port settings['peer_fingerprint'] = self.peer_mid settings[ 'handshake_client_version'] = 'Tribler/' + version_id + '/' + hexlify( self.peer_mid) else: settings['enable_outgoing_utp'] = True settings['enable_incoming_utp'] = True settings['enable_outgoing_tcp'] = False settings['enable_incoming_tcp'] = False settings['anonymous_mode'] = True settings['force_proxy'] = True # Anon listen port is never used anywhere, so we let Libtorrent set it # settings["listen_interfaces"] = "0.0.0.0:%d" % anon_port # By default block all IPs except 1.1.1.1 (which is used to ensure libtorrent makes a connection to us) self.update_ip_filter(ltsession, ['1.1.1.1']) self.set_session_settings(ltsession, settings) ltsession.set_alert_mask(self.default_alert_mask) if hops == 0: proxy_settings = DownloadManager.get_libtorrent_proxy_settings( self.config) else: proxy_settings = [ SOCKS5_PROXY_DEF, ("127.0.0.1", self.socks_listen_ports[hops - 1]), None ] self.set_proxy_settings(ltsession, *proxy_settings) for extension in extensions: ltsession.add_extension(extension) # Set listen port & start the DHT if hops == 0: ltsession.listen_on(libtorrent_port, libtorrent_port + 10) if libtorrent_port != ltsession.listen_port( ) and store_listen_port: self.config.port = ltsession.listen_port() try: with open(self.state_dir / LTSTATE_FILENAME, 'rb') as fp: lt_state = bdecode_compat(fp.read()) if lt_state is not None: ltsession.load_state(lt_state) else: self._logger.warning( "the lt.state appears to be corrupt, writing new data on shutdown" ) except Exception as exc: self._logger.info( f"could not load libtorrent state, got exception: {exc!r}. starting from scratch" ) else: # ltsession.listen_on(anon_port, anon_port + 20) rate = DownloadManager.get_libtorrent_max_upload_rate(self.config) download_rate = DownloadManager.get_libtorrent_max_download_rate( self.config) settings = { 'upload_rate_limit': rate, 'download_rate_limit': download_rate } self.set_session_settings(ltsession, settings) if self.config.dht and not self.dummy_mode: ltsession.start_dht() for router in DEFAULT_DHT_ROUTERS: ltsession.add_dht_router(*router) ltsession.start_lsd() self._logger.debug("Started libtorrent session for %d hops on port %d", hops, ltsession.listen_port()) self.lt_session_shutdown_ready[hops] = False return ltsession
async def get_torrent_info(self, request): args = request.query hops = None if 'hops' in args: try: hops = int(args['hops']) except ValueError: return RESTResponse( { "error": f"wrong value of 'hops' parameter: {repr(args['hops'])}" }, status=HTTP_BAD_REQUEST) if 'uri' not in args or not args['uri']: return RESTResponse({"error": "uri parameter missing"}, status=HTTP_BAD_REQUEST) uri = args['uri'] if uri.startswith('file:'): try: filename = url2pathname(uri[5:]) tdef = TorrentDef.load(filename) metainfo = tdef.get_metainfo() except (TypeError, RuntimeError): return RESTResponse( {"error": "error while decoding torrent file"}, status=HTTP_INTERNAL_SERVER_ERROR) elif uri.startswith('http'): try: async with ClientSession(raise_for_status=True) as session: response = await session.get(uri) response = await response.read() except (ServerConnectionError, ClientResponseError) as e: return RESTResponse({"error": str(e)}, status=HTTP_INTERNAL_SERVER_ERROR) if response.startswith(b'magnet'): _, infohash, _ = parse_magnetlink(response) if infohash: metainfo = await self.session.dlmgr.get_metainfo( infohash, timeout=60, hops=hops, url=response) else: metainfo = bdecode_compat(response) elif uri.startswith('magnet'): infohash = parse_magnetlink(uri)[1] if infohash is None: return RESTResponse({"error": "missing infohash"}, status=HTTP_BAD_REQUEST) metainfo = await self.session.dlmgr.get_metainfo(infohash, timeout=60, hops=hops, url=uri) else: return RESTResponse({"error": "invalid uri"}, status=HTTP_BAD_REQUEST) if not metainfo: return RESTResponse({"error": "metainfo error"}, status=HTTP_INTERNAL_SERVER_ERROR) if not isinstance(metainfo, dict) or b'info' not in metainfo: self._logger.warning("Received metainfo is not a valid dictionary") return RESTResponse({"error": "invalid response"}, status=HTTP_INTERNAL_SERVER_ERROR) # Add the torrent to GigaChannel as a free-for-all entry, so others can search it self.session.mds.TorrentMetadata.add_ffa_from_dict( tdef_to_metadata_dict(TorrentDef.load_from_dict(metainfo))) # TODO(Martijn): store the stuff in a database!!! # TODO(Vadim): this means cache the downloaded torrent in a binary storage, like LevelDB infohash = hashlib.sha1(bencode(metainfo[b'info'])).digest() download = self.session.dlmgr.downloads.get(infohash) metainfo_request = self.session.dlmgr.metainfo_requests.get( infohash, [None])[0] download_is_metainfo_request = download == metainfo_request # Check if the torrent is already in the downloads encoded_metainfo = deepcopy(metainfo) # FIXME: json.dumps garbles binary data that is used by the 'pieces' field # However, this is fine as long as the GUI does not use this field. encoded_metainfo[b'info'][b'pieces'] = hexlify( encoded_metainfo[b'info'][b'pieces']).encode('utf-8') encoded_metainfo = hexlify( json.dumps(recursive_unicode(encoded_metainfo, ignore_errors=True), ensure_ascii=False).encode('utf-8')) return RESTResponse({ "metainfo": encoded_metainfo, "download_exists": download and not download_is_metainfo_request })
def test_load_from_dict(): with open(TESTS_DATA_DIR / "bak_single.torrent", mode='rb') as torrent_file: encoded_metainfo = torrent_file.read() assert TorrentDef.load_from_dict(bdecode_compat(encoded_metainfo))
async def create_torrent(self, request): parameters = await request.json() params = {} if 'files' in parameters and parameters['files']: file_path_list = [ ensure_unicode(f, 'utf-8') for f in parameters['files'] ] else: return RESTResponse({"error": "files parameter missing"}, status=HTTP_BAD_REQUEST) if 'description' in parameters and parameters['description']: params['comment'] = parameters['description'] if 'trackers' in parameters and parameters['trackers']: tracker_url_list = parameters['trackers'] params['announce'] = tracker_url_list[0] params['announce-list'] = tracker_url_list name = 'unknown' if 'name' in parameters and parameters['name']: name = parameters['name'] params['name'] = name export_dir = None if 'export_dir' in parameters and parameters['export_dir']: export_dir = Path(parameters['export_dir']) from tribler_core.version import version_id params['created by'] = '%s version: %s' % ('Tribler', version_id) params['nodes'] = False params['httpseeds'] = False params['encoding'] = False params['piece length'] = 0 # auto try: result = await self.session.dlmgr.create_torrent_file( file_path_list, recursive_bytes(params)) except (IOError, UnicodeDecodeError, RuntimeError) as e: self._logger.exception(e) return return_handled_exception(request, e) metainfo_dict = bdecode_compat(result['metainfo']) if export_dir and export_dir.exists(): save_path = export_dir / ("%s.torrent" % name) with open(save_path, "wb") as fd: fd.write(result['metainfo']) # Download this torrent if specified if 'download' in request.query and request.query[ 'download'] and request.query['download'] == "1": download_config = DownloadConfig() download_config.set_dest_dir(result['base_path'] if len( file_path_list) == 1 else result['base_dir']) try: self.session.dlmgr.start_download( tdef=TorrentDef(metainfo_dict), config=download_config) except DuplicateDownloadException: self._logger.warning( "The created torrent is already being downloaded.") return RESTResponse( json.dumps({ "torrent": base64.b64encode(result['metainfo']).decode('utf-8') }))