def create_dconfig_from_params(parameters): """ Create a download configuration based on some given parameters. Possible parameters are: - anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download - safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on) - destination: the destination path of the torrent (where it is saved on disk) """ download_config = DownloadConfig() anon_hops = parameters.get('anon_hops', 0) safe_seeding = bool(parameters.get('safe_seeding', 0)) if anon_hops > 0 and not safe_seeding: return None, "Cannot set anonymous download without safe seeding enabled" if anon_hops > 0: download_config.set_hops(anon_hops) if safe_seeding: download_config.set_safe_seeding(True) if 'destination' in parameters: download_config.set_dest_dir(parameters['destination']) if 'selected_files' in parameters: download_config.set_selected_files(parameters['selected_files']) return download_config, None
async def get_metainfo(self, infohash, timeout=30, hops=None, url=None): """ Lookup metainfo for a given infohash. The mechanism works by joining the swarm for the infohash connecting to a few peers, and downloading the metadata for the torrent. :param infohash: The (binary) infohash to lookup metainfo for. :param timeout: A timeout in seconds. :param hops: the number of tunnel hops to use for this lookup. If None, use config default. :param url: Optional URL. Can contain trackers info, etc. :return: The metainfo """ infohash_hex = hexlify(infohash) if infohash in self.metainfo_cache: self._logger.info('Returning metainfo from cache for %s', infohash_hex) return self.metainfo_cache[infohash]['meta_info'] self._logger.info('Trying to fetch metainfo for %s', infohash_hex) if infohash in self.metainfo_requests: download = self.metainfo_requests[infohash][0] self.metainfo_requests[infohash][1] += 1 elif infohash in self.downloads: download = self.downloads[infohash] else: tdef = TorrentDefNoMetainfo(infohash, 'metainfo request', url=url) dcfg = DownloadConfig() dcfg.set_hops(hops or self.download_defaults.number_hops) dcfg.set_upload_mode( True ) # Upload mode should prevent libtorrent from creating files dcfg.set_dest_dir(self.metadata_tmpdir) try: download = self.start_download(tdef=tdef, config=dcfg, hidden=True, checkpoint_disabled=True) except TypeError: return self.metainfo_requests[infohash] = [download, 1] try: metainfo = download.tdef.get_metainfo() or await wait_for( shield(download.future_metainfo), timeout) self._logger.info('Successfully retrieved metainfo for %s', infohash_hex) self.metainfo_cache[infohash] = { 'time': timemod.time(), 'meta_info': metainfo } except (CancelledError, asyncio.TimeoutError): metainfo = None self._logger.info('Failed to retrieve metainfo for %s', infohash_hex) if infohash in self.metainfo_requests: self.metainfo_requests[infohash][1] -= 1 if self.metainfo_requests[infohash][1] <= 0: await self.remove_download(download, remove_content=True) self.metainfo_requests.pop(infohash, None) return metainfo
async def test_seeding(download_manager, video_seeder, video_tdef, tmp_path): """ Test whether a torrent is correctly seeded """ dscfg = DownloadConfig() dscfg.set_dest_dir(tmp_path) download = download_manager.start_download(tdef=video_tdef, config=dscfg) download.add_peer(("127.0.0.1", video_seeder.libtorrent_port)) await download.wait_for_status(DLSTATUS_SEEDING) with open(tmp_path / "video.avi", "rb") as f: realdata = f.read() with open(TESTS_DATA_DIR / 'video.avi', "rb") as f: expdata = f.read() assert realdata == expdata
async def video_seeder(tmp_path_factory, video_tdef): config = LibtorrentSettings() config.dht = False config.upnp = False config.natpmp = False config.lsd = False seeder_state_dir = tmp_path_factory.mktemp('video_seeder_state_dir') dlmgr = DownloadManager(config=config, state_dir=seeder_state_dir, notifier=Mock(), peer_mid=b"0000") dlmgr.metadata_tmpdir = tmp_path_factory.mktemp('metadata_tmpdir') dlmgr.initialize() dscfg_seed = DownloadConfig() dscfg_seed.set_dest_dir(TESTS_DATA_DIR) upload = dlmgr.start_download(tdef=video_tdef, config=dscfg_seed) await upload.wait_for_status(DLSTATUS_SEEDING) yield dlmgr await dlmgr.shutdown()
async def channel_seeder(channel_tdef, loop, tmp_path_factory): # pylint: disable=unused-argument, redefined-outer-name config = LibtorrentSettings() config.dht = False config.upnp = False config.natpmp = False config.lsd = False seeder_dlmgr = DownloadManager( state_dir=tmp_path_factory.mktemp('state_dir'), config=config, notifier=Mock(), peer_mid=b"0000") seeder_dlmgr.metadata_tmpdir = tmp_path_factory.mktemp('metadata_tmpdir') seeder_dlmgr.initialize() dscfg_seed = DownloadConfig() dscfg_seed.set_dest_dir(TESTS_DATA_DIR / 'sample_channel') upload = seeder_dlmgr.start_download(tdef=channel_tdef, config=dscfg_seed) await upload.wait_for_status(DLSTATUS_SEEDING) yield seeder_dlmgr await seeder_dlmgr.shutdown()
def load_checkpoint(self, filename): try: config = DownloadConfig.load(filename) except Exception: self._logger.exception("Could not open checkpoint file %s", filename) return metainfo = config.get_metainfo() if not metainfo: self._logger.error( "Could not resume checkpoint %s; metainfo not found", filename) return if not isinstance(metainfo, dict): self._logger.error( "Could not resume checkpoint %s; metainfo is not dict %s %s", filename, type(metainfo), repr(metainfo)) return try: url = metainfo.get(b'url', None) url = url.decode('utf-8') if url else url tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'], metainfo[b'name'], url) if b'infohash' in metainfo else TorrentDef.load_from_dict(metainfo)) except (KeyError, ValueError) as e: self._logger.exception( "Could not restore tdef from metainfo dict: %s %s ", e, metainfo) return if config.get_bootstrap_download(): # In case the download is marked as bootstrap, remove it if its infohash does not # match the configured bootstrap infohash if hexlify(tdef.get_infohash()) != self.bootstrap_infohash: self.remove_config(tdef.get_infohash()) return config.state_dir = self.state_dir if config.get_dest_dir() == '': # removed torrent ignoring self._logger.info("Removing checkpoint %s destdir is %s", filename, config.get_dest_dir()) os.remove(filename) return try: if self.download_exists(tdef.get_infohash()): self._logger.info( "Not resuming checkpoint because download has already been added" ) else: self.start_download(tdef=tdef, config=config) except Exception: self._logger.exception( "Not resume checkpoint due to exception while adding download")
async def hidden_seeder_comm(proxy_factory, video_tdef): # Also load the tunnel community in the seeder session comm = await proxy_factory.get(start_lt=True) comm.build_tunnels(1) dscfg_seed = DownloadConfig() dscfg_seed.set_dest_dir(TESTS_DATA_DIR) dscfg_seed.set_hops(1) upload = comm.dlmgr.start_download(tdef=video_tdef, config=dscfg_seed) def seeder_state_callback(ds): """ The callback of the seeder download. For now, this only logs the state of the download that's seeder and is useful for debugging purposes. """ comm.monitor_downloads([ds]) d = ds.get_download() print( f"seeder: {repr(d.get_def().get_name())} {dlstatus_strings[ds.get_status()]} {ds.get_progress()}" ) return 2 upload.set_state_callback(seeder_state_callback) await upload.wait_for_status(DLSTATUS_SEEDING) return comm
def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_disabled=False, hidden=False): self._logger.debug("Starting download: filename: %s, torrent def: %s", torrent_file, tdef) # the priority of the parameters is: (1) tdef, (2) torrent_file. # so if we have tdef, and torrent_file will be ignored, and so on. if tdef is None: if torrent_file is None: raise ValueError("Torrent file must be provided if tdef is not given") # try to get the torrent from the given torrent file tdef = TorrentDef.load(torrent_file) assert tdef is not None, "tdef MUST not be None after loading torrent" config = config or DownloadConfig() infohash = tdef.get_infohash() download = self.get_download(infohash) if download and infohash not in self.metainfo_requests: new_trackers = list(set(tdef.get_trackers_as_single_tuple()) - set(download.get_def().get_trackers_as_single_tuple())) if new_trackers: self.update_trackers(tdef.get_infohash(), new_trackers) return download # Create the destination directory if it does not exist yet try: if not config.get_dest_dir().is_dir(): os.makedirs(config.get_dest_dir()) except OSError: self._logger.error("Unable to create the download destination directory.") if config.get_time_added() == 0: config.set_time_added(int(timemod.time())) # Create the download download = Download(tdef=tdef, config=config, download_defaults=self.download_defaults, checkpoint_disabled=checkpoint_disabled, hidden=hidden or config.get_bootstrap_download(), notifier=self.notifier, state_dir=self.state_dir, download_manager=self, dummy=self.dummy_mode) atp = download.get_atp() # Keep metainfo downloads in self.downloads for now because we will need to remove it later, # and removing the download at this point will stop us from receiving any further alerts. if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download: self.downloads[infohash] = download if not self.dummy_mode: self.start_handle(download, atp) return download
async def test_save_resume(mock_handle, test_download, test_tdef): """ testing call resume data alert """ mock_handle.is_valid = lambda: True mock_handle.save_resume_data = lambda: test_download.register_task( 'post_alert', test_download.process_alert, alert, 'save_resume_data_alert', delay=0.1) alert = Mock(resume_data={b'info-hash': test_tdef.get_infohash()}) await test_download.save_resume_data() basename = hexlify(test_tdef.get_infohash()) + '.conf' filename = test_download.dlmgr.get_checkpoint_dir() / basename dcfg = DownloadConfig.load(str(filename)) assert test_tdef.get_infohash(), dcfg.get_engineresumedata().get( b'info-hash')
def start_anon_download(tunnel_community: TriblerTunnelCommunity, seeder_port, tdef: TorrentDef, hops=1): """ Start an anonymous download in the main Tribler session. """ download_manager = tunnel_community.dlmgr dscfg = DownloadConfig() dscfg.set_dest_dir(download_manager.state_dir) dscfg.set_hops(hops) download = download_manager.start_download(tdef=tdef, config=dscfg) tunnel_community.bittorrent_peers[download] = [("127.0.0.1", seeder_port)] return download
def updated_my_channel(self, tdef): """ Notify the core that we updated our channel. """ with db_session: my_channel = self.mds.ChannelMetadata.get( infohash=tdef.get_infohash()) if (my_channel and my_channel.status == COMMITTED and not self.download_manager.download_exists( bytes(my_channel.infohash))): dcfg = DownloadConfig(state_dir=self.state_dir) dcfg.set_dest_dir(self.mds.channels_dir) dcfg.set_channel_download(True) return self.download_manager.start_download(tdef=tdef, config=dcfg)
async def download_channel(self, channel): """ Download a channel with a given infohash and title. :param channel: The channel metadata ORM object. """ metainfo = await self.download_manager.get_metainfo(bytes( channel.infohash), timeout=60, hops=0) if metainfo is None: # Timeout looking for the channel metainfo. Probably, there are no seeds. # TODO: count the number of tries we had with the channel, so we can stop trying eventually return try: if metainfo[b'info'][b'name'].decode('utf-8') != channel.dirname: # Malformed channel # TODO: stop trying to download this channel until it is updated with a new infohash return except (KeyError, TypeError): return dcfg = DownloadConfig(state_dir=self.state_dir) dcfg.set_dest_dir(self.mds.channels_dir) dcfg.set_channel_download(True) tdef = TorrentDef(metainfo=metainfo) download = self.download_manager.start_download(tdef=tdef, config=dcfg, hidden=True) try: await download.future_finished except CancelledError: pass else: self.channels_processing_queue[channel.infohash] = ( PROCESS_CHANNEL_DIR, channel) return download
def __init__(self, tdef: TorrentDef, config: DownloadConfig = None, download_defaults: DownloadDefaultsSettings = None, notifier: Notifier = None, state_dir: Path = None, download_manager=None, checkpoint_disabled=False, hidden=False, dummy=False): super().__init__() self._logger = logging.getLogger(self.__class__.__name__) self.dummy = dummy self.tdef = tdef self.handle = None self.state_dir = state_dir self.dlmgr = download_manager self.download_defaults = download_defaults or DownloadDefaultsSettings( ) self.notifier = notifier # With hidden True download will not be in GET/downloads set, as a result will not be shown in GUI self.hidden = False # Libtorrent status self.lt_status = None self.error = None self.pause_after_next_hashcheck = False self.checkpoint_after_next_hashcheck = False self.tracker_status = {} # {url: [num_peers, status_str]} self.checkpoint_disabled = self.dummy self.futures = defaultdict(list) self.alert_handlers = defaultdict(list) self.future_added = self.wait_for_alert('add_torrent_alert', lambda a: a.handle) self.future_removed = self.wait_for_alert('torrent_removed_alert') self.future_finished = self.wait_for_alert('torrent_finished_alert') self.future_metainfo = self.wait_for_alert( 'metadata_received_alert', lambda a: self.tdef.get_metainfo()) alert_handlers = { 'tracker_reply_alert': self.on_tracker_reply_alert, 'tracker_error_alert': self.on_tracker_error_alert, 'tracker_warning_alert': self.on_tracker_warning_alert, 'metadata_received_alert': self.on_metadata_received_alert, 'performance_alert': self.on_performance_alert, 'torrent_checked_alert': self.on_torrent_checked_alert, 'torrent_finished_alert': self.on_torrent_finished_alert, 'save_resume_data_alert': self.on_save_resume_data_alert, 'state_changed_alert': self.on_state_changed_alert, 'torrent_error_alert': self.on_torrent_error_alert, 'add_torrent_alert': self.on_add_torrent_alert, 'torrent_removed_alert': self.on_torrent_removed_alert } for alert_type, alert_handler in alert_handlers.items(): self.register_alert_handler(alert_type, alert_handler) self.stream: Optional[Stream] = None self.hidden = hidden self.checkpoint_disabled = checkpoint_disabled or self.dummy self.config = config or DownloadConfig(state_dir=self.state_dir) self._logger.debug("Setup: %s", hexlify(self.tdef.get_infohash())) self.checkpoint()
def test_default_download_config_load(tmpdir): with open(tmpdir / "dlconfig.conf", 'wb') as conf_file: conf_file.write(b"[Tribler]\nabc=def") dcfg = DownloadConfig.load(tmpdir / "dlconfig.conf") assert dcfg.config['Tribler']['abc'] == 'def'
async def test_download(mock_dlmgr, test_tdef): config = DownloadConfig(state_dir=mock_dlmgr.state_dir) download = Download(test_tdef, download_manager=mock_dlmgr, config=config) download.infohash = hexlify(test_tdef.get_infohash()) yield download await download.shutdown()
async def create_torrent(self, request): parameters = await request.json() params = {} if 'files' in parameters and parameters['files']: file_path_list = [ ensure_unicode(f, 'utf-8') for f in parameters['files'] ] else: return RESTResponse({"error": "files parameter missing"}, status=HTTP_BAD_REQUEST) if 'description' in parameters and parameters['description']: params['comment'] = parameters['description'] if 'trackers' in parameters and parameters['trackers']: tracker_url_list = parameters['trackers'] params['announce'] = tracker_url_list[0] params['announce-list'] = tracker_url_list name = 'unknown' if 'name' in parameters and parameters['name']: name = parameters['name'] params['name'] = name export_dir = None if 'export_dir' in parameters and parameters['export_dir']: export_dir = Path(parameters['export_dir']) from tribler_core.version import version_id params['created by'] = f"Tribler version: {version_id}" params['nodes'] = False params['httpseeds'] = False params['encoding'] = False params['piece length'] = 0 # auto try: result = await self.download_manager.create_torrent_file( file_path_list, recursive_bytes(params)) except (OSError, UnicodeDecodeError, RuntimeError) as e: self._logger.exception(e) return return_handled_exception(request, e) metainfo_dict = bdecode_compat(result['metainfo']) if export_dir and export_dir.exists(): save_path = export_dir / (f"{name}.torrent") with open(save_path, "wb") as fd: fd.write(result['metainfo']) # Download this torrent if specified if 'download' in request.query and request.query[ 'download'] and request.query['download'] == "1": download_config = DownloadConfig() download_config.set_dest_dir(result['base_dir']) download_config.set_hops( self.download_manager.download_defaults.number_hops) self.download_manager.start_download( tdef=TorrentDef(metainfo_dict), config=download_config) return RESTResponse( json.dumps({ "torrent": base64.b64encode(result['metainfo']).decode('utf-8') }))
def download_config(): return DownloadConfig()