def create_dconfig_from_params(parameters):
"""
Create a download configuration based on some given parameters. Possible parameters are:
- anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download
- safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on)
- destination: the destination path of the torrent (where it is saved on disk)
"""
download_config = DownloadConfig()
anon_hops = parameters.get('anon_hops', 0)
safe_seeding = bool(parameters.get('safe_seeding', 0))
if anon_hops > 0 and not safe_seeding:
return None, "Cannot set anonymous download without safe seeding enabled"
if anon_hops > 0:
download_config.set_hops(anon_hops)
if safe_seeding:
download_config.set_safe_seeding(True)
if 'destination' in parameters:
download_config.set_dest_dir(parameters['destination'])
if 'selected_files' in parameters:
download_config.set_selected_files(parameters['selected_files'])
return download_config, None
async def get_metainfo(self, infohash, timeout=30, hops=None, url=None):
"""
Lookup metainfo for a given infohash. The mechanism works by joining the swarm for the infohash connecting
to a few peers, and downloading the metadata for the torrent.
:param infohash: The (binary) infohash to lookup metainfo for.
:param timeout: A timeout in seconds.
:param hops: the number of tunnel hops to use for this lookup. If None, use config default.
:param url: Optional URL. Can contain trackers info, etc.
:return: The metainfo
"""
infohash_hex = hexlify(infohash)
if infohash in self.metainfo_cache:
self._logger.info('Returning metainfo from cache for %s',
infohash_hex)
return self.metainfo_cache[infohash]['meta_info']
self._logger.info('Trying to fetch metainfo for %s', infohash_hex)
if infohash in self.metainfo_requests:
download = self.metainfo_requests[infohash][0]
self.metainfo_requests[infohash][1] += 1
elif infohash in self.downloads:
download = self.downloads[infohash]
else:
tdef = TorrentDefNoMetainfo(infohash, 'metainfo request', url=url)
dcfg = DownloadConfig()
dcfg.set_hops(hops or self.download_defaults.number_hops)
dcfg.set_upload_mode(
True
) # Upload mode should prevent libtorrent from creating files
dcfg.set_dest_dir(self.metadata_tmpdir)
try:
download = self.start_download(tdef=tdef,
config=dcfg,
hidden=True,
checkpoint_disabled=True)
except TypeError:
return
self.metainfo_requests[infohash] = [download, 1]
try:
metainfo = download.tdef.get_metainfo() or await wait_for(
shield(download.future_metainfo), timeout)
self._logger.info('Successfully retrieved metainfo for %s',
infohash_hex)
self.metainfo_cache[infohash] = {
'time': timemod.time(),
'meta_info': metainfo
}
except (CancelledError, asyncio.TimeoutError):
metainfo = None
self._logger.info('Failed to retrieve metainfo for %s',
infohash_hex)
if infohash in self.metainfo_requests:
self.metainfo_requests[infohash][1] -= 1
if self.metainfo_requests[infohash][1] <= 0:
await self.remove_download(download, remove_content=True)
self.metainfo_requests.pop(infohash, None)
return metainfo
async def test_seeding(download_manager, video_seeder, video_tdef, tmp_path):
"""
Test whether a torrent is correctly seeded
"""
dscfg = DownloadConfig()
dscfg.set_dest_dir(tmp_path)
download = download_manager.start_download(tdef=video_tdef, config=dscfg)
download.add_peer(("127.0.0.1", video_seeder.libtorrent_port))
await download.wait_for_status(DLSTATUS_SEEDING)
with open(tmp_path / "video.avi", "rb") as f:
realdata = f.read()
with open(TESTS_DATA_DIR / 'video.avi', "rb") as f:
expdata = f.read()
assert realdata == expdata
async def video_seeder(tmp_path_factory, video_tdef):
config = LibtorrentSettings()
config.dht = False
config.upnp = False
config.natpmp = False
config.lsd = False
seeder_state_dir = tmp_path_factory.mktemp('video_seeder_state_dir')
dlmgr = DownloadManager(config=config,
state_dir=seeder_state_dir,
notifier=Mock(),
peer_mid=b"0000")
dlmgr.metadata_tmpdir = tmp_path_factory.mktemp('metadata_tmpdir')
dlmgr.initialize()
dscfg_seed = DownloadConfig()
dscfg_seed.set_dest_dir(TESTS_DATA_DIR)
upload = dlmgr.start_download(tdef=video_tdef, config=dscfg_seed)
await upload.wait_for_status(DLSTATUS_SEEDING)
yield dlmgr
await dlmgr.shutdown()
async def channel_seeder(channel_tdef, loop, tmp_path_factory): # pylint: disable=unused-argument, redefined-outer-name
config = LibtorrentSettings()
config.dht = False
config.upnp = False
config.natpmp = False
config.lsd = False
seeder_dlmgr = DownloadManager(
state_dir=tmp_path_factory.mktemp('state_dir'),
config=config,
notifier=Mock(),
peer_mid=b"0000")
seeder_dlmgr.metadata_tmpdir = tmp_path_factory.mktemp('metadata_tmpdir')
seeder_dlmgr.initialize()
dscfg_seed = DownloadConfig()
dscfg_seed.set_dest_dir(TESTS_DATA_DIR / 'sample_channel')
upload = seeder_dlmgr.start_download(tdef=channel_tdef, config=dscfg_seed)
await upload.wait_for_status(DLSTATUS_SEEDING)
yield seeder_dlmgr
await seeder_dlmgr.shutdown()
def load_checkpoint(self, filename):
try:
config = DownloadConfig.load(filename)
except Exception:
self._logger.exception("Could not open checkpoint file %s",
filename)
return
metainfo = config.get_metainfo()
if not metainfo:
self._logger.error(
"Could not resume checkpoint %s; metainfo not found", filename)
return
if not isinstance(metainfo, dict):
self._logger.error(
"Could not resume checkpoint %s; metainfo is not dict %s %s",
filename, type(metainfo), repr(metainfo))
return
try:
url = metainfo.get(b'url', None)
url = url.decode('utf-8') if url else url
tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'],
metainfo[b'name'], url) if b'infohash'
in metainfo else TorrentDef.load_from_dict(metainfo))
except (KeyError, ValueError) as e:
self._logger.exception(
"Could not restore tdef from metainfo dict: %s %s ", e,
metainfo)
return
if config.get_bootstrap_download():
# In case the download is marked as bootstrap, remove it if its infohash does not
# match the configured bootstrap infohash
if hexlify(tdef.get_infohash()) != self.bootstrap_infohash:
self.remove_config(tdef.get_infohash())
return
config.state_dir = self.state_dir
if config.get_dest_dir() == '': # removed torrent ignoring
self._logger.info("Removing checkpoint %s destdir is %s", filename,
config.get_dest_dir())
os.remove(filename)
return
try:
if self.download_exists(tdef.get_infohash()):
self._logger.info(
"Not resuming checkpoint because download has already been added"
)
else:
self.start_download(tdef=tdef, config=config)
except Exception:
self._logger.exception(
"Not resume checkpoint due to exception while adding download")
async def hidden_seeder_comm(proxy_factory, video_tdef):
# Also load the tunnel community in the seeder session
comm = await proxy_factory.get(start_lt=True)
comm.build_tunnels(1)
dscfg_seed = DownloadConfig()
dscfg_seed.set_dest_dir(TESTS_DATA_DIR)
dscfg_seed.set_hops(1)
upload = comm.dlmgr.start_download(tdef=video_tdef, config=dscfg_seed)
def seeder_state_callback(ds):
"""
The callback of the seeder download. For now, this only logs the state of the download that's seeder and is
useful for debugging purposes.
"""
comm.monitor_downloads([ds])
d = ds.get_download()
print(
f"seeder: {repr(d.get_def().get_name())} {dlstatus_strings[ds.get_status()]} {ds.get_progress()}"
)
return 2
upload.set_state_callback(seeder_state_callback)
await upload.wait_for_status(DLSTATUS_SEEDING)
return comm
def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_disabled=False, hidden=False):
self._logger.debug("Starting download: filename: %s, torrent def: %s", torrent_file, tdef)
# the priority of the parameters is: (1) tdef, (2) torrent_file.
# so if we have tdef, and torrent_file will be ignored, and so on.
if tdef is None:
if torrent_file is None:
raise ValueError("Torrent file must be provided if tdef is not given")
# try to get the torrent from the given torrent file
tdef = TorrentDef.load(torrent_file)
assert tdef is not None, "tdef MUST not be None after loading torrent"
config = config or DownloadConfig()
infohash = tdef.get_infohash()
download = self.get_download(infohash)
if download and infohash not in self.metainfo_requests:
new_trackers = list(set(tdef.get_trackers_as_single_tuple()) -
set(download.get_def().get_trackers_as_single_tuple()))
if new_trackers:
self.update_trackers(tdef.get_infohash(), new_trackers)
return download
# Create the destination directory if it does not exist yet
try:
if not config.get_dest_dir().is_dir():
os.makedirs(config.get_dest_dir())
except OSError:
self._logger.error("Unable to create the download destination directory.")
if config.get_time_added() == 0:
config.set_time_added(int(timemod.time()))
# Create the download
download = Download(tdef=tdef,
config=config,
download_defaults=self.download_defaults,
checkpoint_disabled=checkpoint_disabled,
hidden=hidden or config.get_bootstrap_download(),
notifier=self.notifier,
state_dir=self.state_dir,
download_manager=self,
dummy=self.dummy_mode)
atp = download.get_atp()
# Keep metainfo downloads in self.downloads for now because we will need to remove it later,
# and removing the download at this point will stop us from receiving any further alerts.
if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download:
self.downloads[infohash] = download
if not self.dummy_mode:
self.start_handle(download, atp)
return download
async def test_save_resume(mock_handle, test_download, test_tdef):
"""
testing call resume data alert
"""
mock_handle.is_valid = lambda: True
mock_handle.save_resume_data = lambda: test_download.register_task(
'post_alert',
test_download.process_alert,
alert,
'save_resume_data_alert',
delay=0.1)
alert = Mock(resume_data={b'info-hash': test_tdef.get_infohash()})
await test_download.save_resume_data()
basename = hexlify(test_tdef.get_infohash()) + '.conf'
filename = test_download.dlmgr.get_checkpoint_dir() / basename
dcfg = DownloadConfig.load(str(filename))
assert test_tdef.get_infohash(), dcfg.get_engineresumedata().get(
b'info-hash')
def start_anon_download(tunnel_community: TriblerTunnelCommunity,
seeder_port,
tdef: TorrentDef,
hops=1):
"""
Start an anonymous download in the main Tribler session.
"""
download_manager = tunnel_community.dlmgr
dscfg = DownloadConfig()
dscfg.set_dest_dir(download_manager.state_dir)
dscfg.set_hops(hops)
download = download_manager.start_download(tdef=tdef, config=dscfg)
tunnel_community.bittorrent_peers[download] = [("127.0.0.1", seeder_port)]
return download
def updated_my_channel(self, tdef):
"""
Notify the core that we updated our channel.
"""
with db_session:
my_channel = self.mds.ChannelMetadata.get(
infohash=tdef.get_infohash())
if (my_channel and my_channel.status == COMMITTED
and not self.download_manager.download_exists(
bytes(my_channel.infohash))):
dcfg = DownloadConfig(state_dir=self.state_dir)
dcfg.set_dest_dir(self.mds.channels_dir)
dcfg.set_channel_download(True)
return self.download_manager.start_download(tdef=tdef, config=dcfg)
async def download_channel(self, channel):
"""
Download a channel with a given infohash and title.
:param channel: The channel metadata ORM object.
"""
metainfo = await self.download_manager.get_metainfo(bytes(
channel.infohash),
timeout=60,
hops=0)
if metainfo is None:
# Timeout looking for the channel metainfo. Probably, there are no seeds.
# TODO: count the number of tries we had with the channel, so we can stop trying eventually
return
try:
if metainfo[b'info'][b'name'].decode('utf-8') != channel.dirname:
# Malformed channel
# TODO: stop trying to download this channel until it is updated with a new infohash
return
except (KeyError, TypeError):
return
dcfg = DownloadConfig(state_dir=self.state_dir)
dcfg.set_dest_dir(self.mds.channels_dir)
dcfg.set_channel_download(True)
tdef = TorrentDef(metainfo=metainfo)
download = self.download_manager.start_download(tdef=tdef,
config=dcfg,
hidden=True)
try:
await download.future_finished
except CancelledError:
pass
else:
self.channels_processing_queue[channel.infohash] = (
PROCESS_CHANNEL_DIR, channel)
return download
def __init__(self,
tdef: TorrentDef,
config: DownloadConfig = None,
download_defaults: DownloadDefaultsSettings = None,
notifier: Notifier = None,
state_dir: Path = None,
download_manager=None,
checkpoint_disabled=False,
hidden=False,
dummy=False):
super().__init__()
self._logger = logging.getLogger(self.__class__.__name__)
self.dummy = dummy
self.tdef = tdef
self.handle = None
self.state_dir = state_dir
self.dlmgr = download_manager
self.download_defaults = download_defaults or DownloadDefaultsSettings(
)
self.notifier = notifier
# With hidden True download will not be in GET/downloads set, as a result will not be shown in GUI
self.hidden = False
# Libtorrent status
self.lt_status = None
self.error = None
self.pause_after_next_hashcheck = False
self.checkpoint_after_next_hashcheck = False
self.tracker_status = {} # {url: [num_peers, status_str]}
self.checkpoint_disabled = self.dummy
self.futures = defaultdict(list)
self.alert_handlers = defaultdict(list)
self.future_added = self.wait_for_alert('add_torrent_alert',
lambda a: a.handle)
self.future_removed = self.wait_for_alert('torrent_removed_alert')
self.future_finished = self.wait_for_alert('torrent_finished_alert')
self.future_metainfo = self.wait_for_alert(
'metadata_received_alert', lambda a: self.tdef.get_metainfo())
alert_handlers = {
'tracker_reply_alert': self.on_tracker_reply_alert,
'tracker_error_alert': self.on_tracker_error_alert,
'tracker_warning_alert': self.on_tracker_warning_alert,
'metadata_received_alert': self.on_metadata_received_alert,
'performance_alert': self.on_performance_alert,
'torrent_checked_alert': self.on_torrent_checked_alert,
'torrent_finished_alert': self.on_torrent_finished_alert,
'save_resume_data_alert': self.on_save_resume_data_alert,
'state_changed_alert': self.on_state_changed_alert,
'torrent_error_alert': self.on_torrent_error_alert,
'add_torrent_alert': self.on_add_torrent_alert,
'torrent_removed_alert': self.on_torrent_removed_alert
}
for alert_type, alert_handler in alert_handlers.items():
self.register_alert_handler(alert_type, alert_handler)
self.stream: Optional[Stream] = None
self.hidden = hidden
self.checkpoint_disabled = checkpoint_disabled or self.dummy
self.config = config or DownloadConfig(state_dir=self.state_dir)
self._logger.debug("Setup: %s", hexlify(self.tdef.get_infohash()))
self.checkpoint()
def test_default_download_config_load(tmpdir):
with open(tmpdir / "dlconfig.conf", 'wb') as conf_file:
conf_file.write(b"[Tribler]\nabc=def")
dcfg = DownloadConfig.load(tmpdir / "dlconfig.conf")
assert dcfg.config['Tribler']['abc'] == 'def'
async def test_download(mock_dlmgr, test_tdef):
config = DownloadConfig(state_dir=mock_dlmgr.state_dir)
download = Download(test_tdef, download_manager=mock_dlmgr, config=config)
download.infohash = hexlify(test_tdef.get_infohash())
yield download
await download.shutdown()
async def create_torrent(self, request):
parameters = await request.json()
params = {}
if 'files' in parameters and parameters['files']:
file_path_list = [
ensure_unicode(f, 'utf-8') for f in parameters['files']
]
else:
return RESTResponse({"error": "files parameter missing"},
status=HTTP_BAD_REQUEST)
if 'description' in parameters and parameters['description']:
params['comment'] = parameters['description']
if 'trackers' in parameters and parameters['trackers']:
tracker_url_list = parameters['trackers']
params['announce'] = tracker_url_list[0]
params['announce-list'] = tracker_url_list
name = 'unknown'
if 'name' in parameters and parameters['name']:
name = parameters['name']
params['name'] = name
export_dir = None
if 'export_dir' in parameters and parameters['export_dir']:
export_dir = Path(parameters['export_dir'])
from tribler_core.version import version_id
params['created by'] = f"Tribler version: {version_id}"
params['nodes'] = False
params['httpseeds'] = False
params['encoding'] = False
params['piece length'] = 0 # auto
try:
result = await self.download_manager.create_torrent_file(
file_path_list, recursive_bytes(params))
except (OSError, UnicodeDecodeError, RuntimeError) as e:
self._logger.exception(e)
return return_handled_exception(request, e)
metainfo_dict = bdecode_compat(result['metainfo'])
if export_dir and export_dir.exists():
save_path = export_dir / (f"{name}.torrent")
with open(save_path, "wb") as fd:
fd.write(result['metainfo'])
# Download this torrent if specified
if 'download' in request.query and request.query[
'download'] and request.query['download'] == "1":
download_config = DownloadConfig()
download_config.set_dest_dir(result['base_dir'])
download_config.set_hops(
self.download_manager.download_defaults.number_hops)
self.download_manager.start_download(
tdef=TorrentDef(metainfo_dict), config=download_config)
return RESTResponse(
json.dumps({
"torrent":
base64.b64encode(result['metainfo']).decode('utf-8')
}))
def download_config():
return DownloadConfig()
