def update_trackers(self, infohash, trackers): """ Update the trackers for a download. :param infohash: infohash of the torrent that needs to be updated :param trackers: A list of tracker urls. """ download = self.get_download(infohash) if download: old_def = download.get_def() old_trackers = old_def.get_trackers_as_single_tuple() new_trackers = list(set(trackers) - set(old_trackers)) all_trackers = list(old_trackers) + new_trackers if new_trackers: # Add new trackers to the download download.add_trackers(new_trackers) # Create a new TorrentDef if isinstance(old_def, TorrentDefNoMetainfo): new_def = TorrentDefNoMetainfo(old_def.get_infohash(), old_def.get_name(), download.get_magnet_link()) else: metainfo = old_def.get_metainfo() if len(all_trackers) > 1: metainfo["announce-list"] = [all_trackers] else: metainfo["announce"] = all_trackers[0] new_def = TorrentDef.load_from_dict(metainfo) # Set TorrentDef + checkpoint download.set_def(new_def) download.checkpoint()
async def test_move_storage(self): """ Test that move_storage method works as expected by Libtorrent """ result = [] def mock_move(s): result.append(s) tdef = self.create_tdef() dl = Download(self.session, tdef) dl.setup() dl.handle = Mock() dl.handle.move_storage = mock_move dl.move_storage(Path("some_path")) self.assertEqual("some_path", result[0]) self.assertTrue("some_path", dl.config.get_dest_dir().name) await dl.shutdown() # Check the same thing, this time for TorrentDefNoMetainfo dl = Download(self.session, TorrentDefNoMetainfo(random_infohash(), "some_torrent")) dl.setup() dl.move_storage(Path("some_path")) self.assertEqual("some_path", result[0]) self.assertTrue("some_path", dl.config.get_dest_dir().name) await dl.shutdown()
async def test_start_download(fake_dlmgr): """ Testing the addition of a torrent to the libtorrent manager """ infohash = b'a' * 20 mock_handle = Mock() mock_handle.info_hash = lambda: hexlify(infohash) mock_handle.is_valid = lambda: True mock_error = Mock() mock_error.value = lambda: None mock_alert = type('add_torrent_alert', (object,), dict(handle=mock_handle, error=mock_error, category=lambda _: None))() mock_ltsession = Mock() mock_ltsession.get_torrents = lambda: [] mock_ltsession.async_add_torrent = lambda _: fake_dlmgr.register_task('post_alert', fake_dlmgr.process_alert, mock_alert, delay=0.1) fake_dlmgr.get_session = lambda *_: mock_ltsession download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, ''), checkpoint_disabled=True) handle = await download.get_handle() assert handle == mock_handle fake_dlmgr.downloads.clear() await download.shutdown() # Test waiting on DHT getting enough nodes and adding the torrent after timing out fake_dlmgr.dht_readiness_timeout = 0.5 flag = [] check_was_run = Mock() async def mock_check(): while not flag: check_was_run() await sleep(0.1) fake_dlmgr._check_dht_ready = mock_check fake_dlmgr.initialize() mock_download = Mock() mock_download.get_def().get_infohash = lambda: b"1"*20 mock_download.future_added = succeed(True) mock_ltsession.async_add_torrent = Mock() await fake_dlmgr.start_handle(mock_download, {}) check_was_run.assert_called() fake_dlmgr.downloads.clear() # Test waiting on DHT getting enough nodes fake_dlmgr.dht_readiness_timeout = 100 flag.append(True) mock_download.future_added = succeed(True) await fake_dlmgr.start_handle(mock_download, {}) fake_dlmgr.downloads.clear()
def load_checkpoint(self, filename): try: config = DownloadConfig.load(filename) except Exception: self._logger.exception("Could not open checkpoint file %s", filename) return metainfo = config.get_metainfo() if not metainfo: self._logger.error( "Could not resume checkpoint %s; metainfo not found", filename) return if not isinstance(metainfo, dict): self._logger.error( "Could not resume checkpoint %s; metainfo is not dict %s %s", filename, type(metainfo), repr(metainfo)) return try: url = metainfo.get(b'url', None) url = url.decode('utf-8') if url else url tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'], metainfo[b'name'], url) if b'infohash' in metainfo else TorrentDef.load_from_dict(metainfo)) except (KeyError, ValueError) as e: self._logger.exception( "Could not restore tdef from metainfo dict: %s %s ", e, metainfo) return if config.get_bootstrap_download(): if hexlify(tdef.get_infohash( )) != self.tribler_session.config.get_bootstrap_infohash(): self.remove_config(tdef.get_infohash()) return config.state_dir = self.tribler_session.config.get_state_dir() if config.get_dest_dir() == '': # removed torrent ignoring self._logger.info("Removing checkpoint %s destdir is %s", filename, config.get_dest_dir()) os.remove(filename) return try: if self.download_exists(tdef.get_infohash()): self._logger.info( "Not resuming checkpoint because download has already been added" ) else: self.start_download(tdef=tdef, config=config) except Exception: self._logger.exception( "Not resume checkpoint due to exception while adding download")
def start_by_infohash(self, download_function, infohash): """ Download bootstrap file from current seeders :param download_function: function to download via tdef :return: download on bootstrap file """ self._logger.debug("Starting bootstrap downloading %s", infohash) tdef = TorrentDefNoMetainfo(unhexlify(infohash), name='bootstrap.blocks') self.download = download_function(tdef=tdef, config=self.dcfg, hidden=True) self.infohash = infohash
async def get_metainfo(self, infohash, timeout=30, hops=None, url=None): """ Lookup metainfo for a given infohash. The mechanism works by joining the swarm for the infohash connecting to a few peers, and downloading the metadata for the torrent. :param infohash: The (binary) infohash to lookup metainfo for. :param timeout: A timeout in seconds. :param hops: the number of tunnel hops to use for this lookup. If None, use config default. :param url: Optional URL. Can contain trackers info, etc. :return: The metainfo """ infohash_hex = hexlify(infohash) if infohash in self.metainfo_cache: self._logger.info('Returning metainfo from cache for %s', infohash_hex) return self.metainfo_cache[infohash]['meta_info'] self._logger.info('Trying to fetch metainfo for %s', infohash_hex) if infohash in self.metainfo_requests: download = self.metainfo_requests[infohash][0] self.metainfo_requests[infohash][1] += 1 elif infohash in self.downloads: download = self.downloads[infohash] else: tdef = TorrentDefNoMetainfo(infohash, 'metainfo request', url=url) dcfg = DownloadConfig() dcfg.set_hops(self.tribler_session.config.get_default_number_hops() if hops is None else hops) dcfg.set_upload_mode(True) # Upload mode should prevent libtorrent from creating files dcfg.set_dest_dir(self.metadata_tmpdir) try: download = self.start_download(tdef=tdef, config=dcfg, hidden=True, checkpoint_disabled=True) except TypeError: return self.metainfo_requests[infohash] = [download, 1] try: metainfo = download.tdef.get_metainfo() or await wait_for(shield(download.future_metainfo), timeout) self._logger.info('Successfully retrieved metainfo for %s', infohash_hex) self.metainfo_cache[infohash] = {'time': timemod.time(), 'meta_info': metainfo} except (CancelledError, asyncio.TimeoutError): metainfo = None self._logger.info('Failed to retrieve metainfo for %s', infohash_hex) if infohash in self.metainfo_requests: self.metainfo_requests[infohash][1] -= 1 if self.metainfo_requests[infohash][1] <= 0: await self.remove_download(download, remove_content=True) self.metainfo_requests.pop(infohash) return metainfo
async def start_download_from_uri(self, uri, config=None): if uri.startswith("http"): tdef = await TorrentDef.load_from_url(uri) return self.start_download(tdef=tdef, config=config) if uri.startswith("magnet:"): name, infohash, _ = parse_magnetlink(uri) if infohash is None: raise RuntimeError("Missing infohash") if infohash in self.metainfo_cache: tdef = TorrentDef.load_from_dict(self.metainfo_cache[infohash]['meta_info']) else: tdef = TorrentDefNoMetainfo(infohash, "Unknown name" if name is None else name, url=uri) return self.start_download(tdef=tdef, config=config) if uri.startswith("file:"): argument = url2pathname(uri[5:]) return self.start_download(torrent_file=argument, config=config) raise Exception("invalid uri")
async def test_start_download_existing_download(fake_dlmgr): """ Testing the addition of a torrent to the libtorrent manager, if there is a pre-existing download. """ infohash = b'a' * 20 mock_download = Mock() mock_download.get_def = lambda: Mock(get_trackers_as_single_tuple=lambda: ()) mock_ltsession = Mock() fake_dlmgr.downloads[infohash] = mock_download fake_dlmgr.get_session = lambda *_: mock_ltsession download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True) assert download == mock_download fake_dlmgr.downloads.clear()
async def test_start_download_existing_handle(self): """ Testing the addition of a torrent to the libtorrent manager, if there is a pre-existing handle. """ infohash = b'a' * 20 mock_handle = Mock() mock_handle.info_hash = lambda: hexlify(infohash) mock_handle.is_valid = lambda: True mock_ltsession = Mock() mock_ltsession.get_torrents = lambda: [mock_handle] self.dlmgr.get_session = lambda *_: mock_ltsession download = self.dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True) handle = await download.get_handle() self.assertEqual(handle, mock_handle)
async def test_start_download_while_getting_metainfo(fake_dlmgr): """ Testing adding a torrent while a metainfo request is running. """ infohash = b"a" * 20 metainfo_session = Mock() metainfo_session.get_torrents = lambda: [] metainfo_dl = Mock() metainfo_dl.get_def = lambda: Mock(get_infohash=lambda: infohash) fake_dlmgr.initialize() fake_dlmgr.get_session = lambda *_: metainfo_session fake_dlmgr.downloads[infohash] = metainfo_dl fake_dlmgr.metainfo_requests[infohash] = [metainfo_dl, 1] fake_dlmgr.remove_download = Mock(return_value=succeed(None)) tdef = TorrentDefNoMetainfo(infohash, 'name', f'magnet:?xt=urn:btih:{hexlify(infohash)}&') download = fake_dlmgr.start_download(tdef=tdef, checkpoint_disabled=True) assert metainfo_dl != download await sleep(.1) assert fake_dlmgr.downloads[infohash] == download fake_dlmgr.remove_download.assert_called_once_with(metainfo_dl, remove_content=True, remove_checkpoint=False)
async def test_start_download_while_getting_metainfo(self): """ Testing adding a torrent while a metainfo request is running. """ infohash = b"a" * 20 metainfo_session = Mock() metainfo_session.get_torrents = lambda: [] metainfo_dl = Mock() metainfo_dl.get_def = lambda: Mock(get_infohash=lambda: infohash) self.dlmgr.initialize() self.dlmgr.get_session = lambda *_: metainfo_session self.dlmgr.downloads[infohash] = metainfo_dl self.dlmgr.metainfo_requests[infohash] = [metainfo_dl, 1] self.dlmgr.remove_download = Mock(return_value=succeed(None)) tdef = TorrentDefNoMetainfo(infohash, 'name', 'magnet:?xt=urn:btih:%s&' % hexlify(infohash)) download = self.dlmgr.start_download(tdef=tdef, checkpoint_disabled=True) self.assertNotEqual(metainfo_dl, download) await sleep(.1) self.assertEqual(self.dlmgr.downloads[infohash], download) self.dlmgr.remove_download.assert_called_once_with(metainfo_dl, remove_content=True, remove_checkpoint=False)
def test_torrent_no_metainfo(): tdef = TorrentDefNoMetainfo(b"12345678901234567890", VIDEO_FILE_NAME, "http://google.com") assert tdef.get_name() == VIDEO_FILE_NAME assert tdef.get_infohash() == b"12345678901234567890" assert tdef.get_length() == 0 # there are no files assert not tdef.get_metainfo() assert tdef.get_url() == "http://google.com" assert not tdef.is_multifile_torrent() assert tdef.get_name_as_unicode() == VIDEO_FILE_NAME assert not tdef.get_files() assert tdef.get_files_with_length() == [] assert not tdef.get_trackers_as_single_tuple() assert not tdef.is_private() assert tdef.get_name_utf8() == "video.avi" assert tdef.get_nr_pieces() == 0 torrent2 = TorrentDefNoMetainfo(b"12345678901234567890", VIDEO_FILE_NAME, "magnet:") assert not torrent2.get_trackers_as_single_tuple()
async def add_torrent_to_channel(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: channel = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id) if not channel: return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND) parameters = await request.json() extra_info = {} if parameters.get('description', None): extra_info = {'description': parameters['description']} # First, check whether we did upload a magnet link or URL if parameters.get('uri', None): uri = parameters['uri'] if uri.startswith("http:") or uri.startswith("https:"): async with ClientSession() as session: response = await session.get(uri) data = await response.read() tdef = TorrentDef.load_from_memory(data) elif uri.startswith("magnet:"): _, xt, _ = parse_magnetlink(uri) if ( xt and is_infohash(codecs.encode(xt, 'hex')) and (channel.torrent_exists(xt) or channel.copy_torrent_from_infohash(xt)) ): return RESTResponse({"added": 1}) filesize = parameters.get("filesize") if filesize and not (isinstance(filesize, int) or int is None): return RESTResponse({"error": "filesize must be an integer"}, status=HTTP_BAD_REQUEST,) if filesize: dn, xt, _ = parse_magnetlink(uri) tdef = TorrentDefNoMetainfo(xt, dn, uri, filesize) else: meta_info = await self.session.dlmgr.get_metainfo(xt, timeout=30, url=uri) if not meta_info: raise RuntimeError("Metainfo timeout") tdef = TorrentDef.load_from_dict(meta_info) else: return RESTResponse({"error": "unknown uri type"}, status=HTTP_BAD_REQUEST) added = 0 if tdef: channel.add_torrent_to_channel(tdef, extra_info) added = 1 return RESTResponse({"added": added}) torrents_dir = None if parameters.get('torrents_dir', None): torrents_dir = parameters['torrents_dir'] if not path_util.isabs(torrents_dir): return RESTResponse({"error": "the torrents_dir should point to a directory"}, status=HTTP_BAD_REQUEST) recursive = False if parameters.get('recursive'): recursive = parameters['recursive'] if not torrents_dir: return RESTResponse( {"error": "the torrents_dir parameter should be provided when the recursive parameter is set"}, status=HTTP_BAD_REQUEST, ) if torrents_dir: torrents_list, errors_list = channel.add_torrents_from_dir(torrents_dir, recursive) return RESTResponse({"added": len(torrents_list), "errors": errors_list}) if not parameters.get('torrent', None): return RESTResponse({"error": "torrent parameter missing"}, status=HTTP_BAD_REQUEST) # Try to parse the torrent data # Any errors will be handled by the error_middleware torrent = base64.b64decode(parameters['torrent']) torrent_def = TorrentDef.load_from_memory(torrent) channel.add_torrent_to_channel(torrent_def, extra_info) return RESTResponse({"added": 1})
def test_tdef_no_metainfo(state_dir): tdef = TorrentDefNoMetainfo(b"1" * 20, "test") return tdef
def test_torrent_no_metainfo(self): torrent = TorrentDefNoMetainfo(b"12345678901234567890", VIDEO_FILE_NAME, "http://google.com") self.assertEqual(torrent.get_name(), VIDEO_FILE_NAME) self.assertEqual(torrent.get_infohash(), b"12345678901234567890") self.assertEqual(torrent.get_length(), 0) # there are no files self.assertFalse(torrent.get_metainfo()) self.assertEqual(torrent.get_url(), "http://google.com") self.assertFalse(torrent.is_multifile_torrent()) self.assertEqual(torrent.get_name_as_unicode(), VIDEO_FILE_NAME) self.assertFalse(torrent.get_files()) self.assertFalse(torrent.get_files_with_length()) self.assertFalse(torrent.get_trackers_as_single_tuple()) self.assertFalse(torrent.is_private()) torrent2 = TorrentDefNoMetainfo(b"12345678901234567890", VIDEO_FILE_NAME, "magnet:") self.assertFalse(torrent2.get_trackers_as_single_tuple())