def update_trackers(self, infohash, trackers):
""" Update the trackers for a download.
:param infohash: infohash of the torrent that needs to be updated
:param trackers: A list of tracker urls.
"""
download = self.get_download(infohash)
if download:
old_def = download.get_def()
old_trackers = old_def.get_trackers_as_single_tuple()
new_trackers = list(set(trackers) - set(old_trackers))
all_trackers = list(old_trackers) + new_trackers
if new_trackers:
# Add new trackers to the download
download.add_trackers(new_trackers)
# Create a new TorrentDef
if isinstance(old_def, TorrentDefNoMetainfo):
new_def = TorrentDefNoMetainfo(old_def.get_infohash(),
old_def.get_name(),
download.get_magnet_link())
else:
metainfo = old_def.get_metainfo()
if len(all_trackers) > 1:
metainfo["announce-list"] = [all_trackers]
else:
metainfo["announce"] = all_trackers[0]
new_def = TorrentDef.load_from_dict(metainfo)
# Set TorrentDef + checkpoint
download.set_def(new_def)
download.checkpoint()
async def test_move_storage(self):
"""
Test that move_storage method works as expected by Libtorrent
"""
result = []
def mock_move(s):
result.append(s)
tdef = self.create_tdef()
dl = Download(self.session, tdef)
dl.setup()
dl.handle = Mock()
dl.handle.move_storage = mock_move
dl.move_storage(Path("some_path"))
self.assertEqual("some_path", result[0])
self.assertTrue("some_path", dl.config.get_dest_dir().name)
await dl.shutdown()
# Check the same thing, this time for TorrentDefNoMetainfo
dl = Download(self.session,
TorrentDefNoMetainfo(random_infohash(), "some_torrent"))
dl.setup()
dl.move_storage(Path("some_path"))
self.assertEqual("some_path", result[0])
self.assertTrue("some_path", dl.config.get_dest_dir().name)
await dl.shutdown()
async def test_start_download(fake_dlmgr):
"""
Testing the addition of a torrent to the libtorrent manager
"""
infohash = b'a' * 20
mock_handle = Mock()
mock_handle.info_hash = lambda: hexlify(infohash)
mock_handle.is_valid = lambda: True
mock_error = Mock()
mock_error.value = lambda: None
mock_alert = type('add_torrent_alert', (object,), dict(handle=mock_handle,
error=mock_error,
category=lambda _: None))()
mock_ltsession = Mock()
mock_ltsession.get_torrents = lambda: []
mock_ltsession.async_add_torrent = lambda _: fake_dlmgr.register_task('post_alert',
fake_dlmgr.process_alert,
mock_alert, delay=0.1)
fake_dlmgr.get_session = lambda *_: mock_ltsession
download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, ''), checkpoint_disabled=True)
handle = await download.get_handle()
assert handle == mock_handle
fake_dlmgr.downloads.clear()
await download.shutdown()
# Test waiting on DHT getting enough nodes and adding the torrent after timing out
fake_dlmgr.dht_readiness_timeout = 0.5
flag = []
check_was_run = Mock()
async def mock_check():
while not flag:
check_was_run()
await sleep(0.1)
fake_dlmgr._check_dht_ready = mock_check
fake_dlmgr.initialize()
mock_download = Mock()
mock_download.get_def().get_infohash = lambda: b"1"*20
mock_download.future_added = succeed(True)
mock_ltsession.async_add_torrent = Mock()
await fake_dlmgr.start_handle(mock_download, {})
check_was_run.assert_called()
fake_dlmgr.downloads.clear()
# Test waiting on DHT getting enough nodes
fake_dlmgr.dht_readiness_timeout = 100
flag.append(True)
mock_download.future_added = succeed(True)
await fake_dlmgr.start_handle(mock_download, {})
fake_dlmgr.downloads.clear()
def load_checkpoint(self, filename):
try:
config = DownloadConfig.load(filename)
except Exception:
self._logger.exception("Could not open checkpoint file %s",
filename)
return
metainfo = config.get_metainfo()
if not metainfo:
self._logger.error(
"Could not resume checkpoint %s; metainfo not found", filename)
return
if not isinstance(metainfo, dict):
self._logger.error(
"Could not resume checkpoint %s; metainfo is not dict %s %s",
filename, type(metainfo), repr(metainfo))
return
try:
url = metainfo.get(b'url', None)
url = url.decode('utf-8') if url else url
tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'],
metainfo[b'name'], url) if b'infohash'
in metainfo else TorrentDef.load_from_dict(metainfo))
except (KeyError, ValueError) as e:
self._logger.exception(
"Could not restore tdef from metainfo dict: %s %s ", e,
metainfo)
return
if config.get_bootstrap_download():
if hexlify(tdef.get_infohash(
)) != self.tribler_session.config.get_bootstrap_infohash():
self.remove_config(tdef.get_infohash())
return
config.state_dir = self.tribler_session.config.get_state_dir()
if config.get_dest_dir() == '': # removed torrent ignoring
self._logger.info("Removing checkpoint %s destdir is %s", filename,
config.get_dest_dir())
os.remove(filename)
return
try:
if self.download_exists(tdef.get_infohash()):
self._logger.info(
"Not resuming checkpoint because download has already been added"
)
else:
self.start_download(tdef=tdef, config=config)
except Exception:
self._logger.exception(
"Not resume checkpoint due to exception while adding download")
def start_by_infohash(self, download_function, infohash):
"""
Download bootstrap file from current seeders
:param download_function: function to download via tdef
:return: download on bootstrap file
"""
self._logger.debug("Starting bootstrap downloading %s", infohash)
tdef = TorrentDefNoMetainfo(unhexlify(infohash),
name='bootstrap.blocks')
self.download = download_function(tdef=tdef,
config=self.dcfg,
hidden=True)
self.infohash = infohash
async def get_metainfo(self, infohash, timeout=30, hops=None, url=None):
"""
Lookup metainfo for a given infohash. The mechanism works by joining the swarm for the infohash connecting
to a few peers, and downloading the metadata for the torrent.
:param infohash: The (binary) infohash to lookup metainfo for.
:param timeout: A timeout in seconds.
:param hops: the number of tunnel hops to use for this lookup. If None, use config default.
:param url: Optional URL. Can contain trackers info, etc.
:return: The metainfo
"""
infohash_hex = hexlify(infohash)
if infohash in self.metainfo_cache:
self._logger.info('Returning metainfo from cache for %s', infohash_hex)
return self.metainfo_cache[infohash]['meta_info']
self._logger.info('Trying to fetch metainfo for %s', infohash_hex)
if infohash in self.metainfo_requests:
download = self.metainfo_requests[infohash][0]
self.metainfo_requests[infohash][1] += 1
elif infohash in self.downloads:
download = self.downloads[infohash]
else:
tdef = TorrentDefNoMetainfo(infohash, 'metainfo request', url=url)
dcfg = DownloadConfig()
dcfg.set_hops(self.tribler_session.config.get_default_number_hops() if hops is None else hops)
dcfg.set_upload_mode(True) # Upload mode should prevent libtorrent from creating files
dcfg.set_dest_dir(self.metadata_tmpdir)
try:
download = self.start_download(tdef=tdef, config=dcfg, hidden=True, checkpoint_disabled=True)
except TypeError:
return
self.metainfo_requests[infohash] = [download, 1]
try:
metainfo = download.tdef.get_metainfo() or await wait_for(shield(download.future_metainfo), timeout)
self._logger.info('Successfully retrieved metainfo for %s', infohash_hex)
self.metainfo_cache[infohash] = {'time': timemod.time(), 'meta_info': metainfo}
except (CancelledError, asyncio.TimeoutError):
metainfo = None
self._logger.info('Failed to retrieve metainfo for %s', infohash_hex)
if infohash in self.metainfo_requests:
self.metainfo_requests[infohash][1] -= 1
if self.metainfo_requests[infohash][1] <= 0:
await self.remove_download(download, remove_content=True)
self.metainfo_requests.pop(infohash)
return metainfo
async def start_download_from_uri(self, uri, config=None):
if uri.startswith("http"):
tdef = await TorrentDef.load_from_url(uri)
return self.start_download(tdef=tdef, config=config)
if uri.startswith("magnet:"):
name, infohash, _ = parse_magnetlink(uri)
if infohash is None:
raise RuntimeError("Missing infohash")
if infohash in self.metainfo_cache:
tdef = TorrentDef.load_from_dict(self.metainfo_cache[infohash]['meta_info'])
else:
tdef = TorrentDefNoMetainfo(infohash, "Unknown name" if name is None else name, url=uri)
return self.start_download(tdef=tdef, config=config)
if uri.startswith("file:"):
argument = url2pathname(uri[5:])
return self.start_download(torrent_file=argument, config=config)
raise Exception("invalid uri")
async def test_start_download_existing_download(fake_dlmgr):
"""
Testing the addition of a torrent to the libtorrent manager, if there is a pre-existing download.
"""
infohash = b'a' * 20
mock_download = Mock()
mock_download.get_def = lambda: Mock(get_trackers_as_single_tuple=lambda: ())
mock_ltsession = Mock()
fake_dlmgr.downloads[infohash] = mock_download
fake_dlmgr.get_session = lambda *_: mock_ltsession
download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True)
assert download == mock_download
fake_dlmgr.downloads.clear()
async def test_start_download_existing_handle(self):
"""
Testing the addition of a torrent to the libtorrent manager, if there is a pre-existing handle.
"""
infohash = b'a' * 20
mock_handle = Mock()
mock_handle.info_hash = lambda: hexlify(infohash)
mock_handle.is_valid = lambda: True
mock_ltsession = Mock()
mock_ltsession.get_torrents = lambda: [mock_handle]
self.dlmgr.get_session = lambda *_: mock_ltsession
download = self.dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True)
handle = await download.get_handle()
self.assertEqual(handle, mock_handle)
async def test_start_download_while_getting_metainfo(fake_dlmgr):
"""
Testing adding a torrent while a metainfo request is running.
"""
infohash = b"a" * 20
metainfo_session = Mock()
metainfo_session.get_torrents = lambda: []
metainfo_dl = Mock()
metainfo_dl.get_def = lambda: Mock(get_infohash=lambda: infohash)
fake_dlmgr.initialize()
fake_dlmgr.get_session = lambda *_: metainfo_session
fake_dlmgr.downloads[infohash] = metainfo_dl
fake_dlmgr.metainfo_requests[infohash] = [metainfo_dl, 1]
fake_dlmgr.remove_download = Mock(return_value=succeed(None))
tdef = TorrentDefNoMetainfo(infohash, 'name', f'magnet:?xt=urn:btih:{hexlify(infohash)}&')
download = fake_dlmgr.start_download(tdef=tdef, checkpoint_disabled=True)
assert metainfo_dl != download
await sleep(.1)
assert fake_dlmgr.downloads[infohash] == download
fake_dlmgr.remove_download.assert_called_once_with(metainfo_dl, remove_content=True, remove_checkpoint=False)
async def test_start_download_while_getting_metainfo(self):
"""
Testing adding a torrent while a metainfo request is running.
"""
infohash = b"a" * 20
metainfo_session = Mock()
metainfo_session.get_torrents = lambda: []
metainfo_dl = Mock()
metainfo_dl.get_def = lambda: Mock(get_infohash=lambda: infohash)
self.dlmgr.initialize()
self.dlmgr.get_session = lambda *_: metainfo_session
self.dlmgr.downloads[infohash] = metainfo_dl
self.dlmgr.metainfo_requests[infohash] = [metainfo_dl, 1]
self.dlmgr.remove_download = Mock(return_value=succeed(None))
tdef = TorrentDefNoMetainfo(infohash, 'name', 'magnet:?xt=urn:btih:%s&' % hexlify(infohash))
download = self.dlmgr.start_download(tdef=tdef, checkpoint_disabled=True)
self.assertNotEqual(metainfo_dl, download)
await sleep(.1)
self.assertEqual(self.dlmgr.downloads[infohash], download)
self.dlmgr.remove_download.assert_called_once_with(metainfo_dl, remove_content=True, remove_checkpoint=False)
def test_torrent_no_metainfo():
tdef = TorrentDefNoMetainfo(b"12345678901234567890", VIDEO_FILE_NAME,
"http://google.com")
assert tdef.get_name() == VIDEO_FILE_NAME
assert tdef.get_infohash() == b"12345678901234567890"
assert tdef.get_length() == 0 # there are no files
assert not tdef.get_metainfo()
assert tdef.get_url() == "http://google.com"
assert not tdef.is_multifile_torrent()
assert tdef.get_name_as_unicode() == VIDEO_FILE_NAME
assert not tdef.get_files()
assert tdef.get_files_with_length() == []
assert not tdef.get_trackers_as_single_tuple()
assert not tdef.is_private()
assert tdef.get_name_utf8() == "video.avi"
assert tdef.get_nr_pieces() == 0
torrent2 = TorrentDefNoMetainfo(b"12345678901234567890", VIDEO_FILE_NAME,
"magnet:")
assert not torrent2.get_trackers_as_single_tuple()
async def add_torrent_to_channel(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id)
if not channel:
return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND)
parameters = await request.json()
extra_info = {}
if parameters.get('description', None):
extra_info = {'description': parameters['description']}
# First, check whether we did upload a magnet link or URL
if parameters.get('uri', None):
uri = parameters['uri']
if uri.startswith("http:") or uri.startswith("https:"):
async with ClientSession() as session:
response = await session.get(uri)
data = await response.read()
tdef = TorrentDef.load_from_memory(data)
elif uri.startswith("magnet:"):
_, xt, _ = parse_magnetlink(uri)
if (
xt
and is_infohash(codecs.encode(xt, 'hex'))
and (channel.torrent_exists(xt) or channel.copy_torrent_from_infohash(xt))
):
return RESTResponse({"added": 1})
filesize = parameters.get("filesize")
if filesize and not (isinstance(filesize, int) or int is None):
return RESTResponse({"error": "filesize must be an integer"},
status=HTTP_BAD_REQUEST,)
if filesize:
dn, xt, _ = parse_magnetlink(uri)
tdef = TorrentDefNoMetainfo(xt, dn, uri, filesize)
else:
meta_info = await self.session.dlmgr.get_metainfo(xt, timeout=30, url=uri)
if not meta_info:
raise RuntimeError("Metainfo timeout")
tdef = TorrentDef.load_from_dict(meta_info)
else:
return RESTResponse({"error": "unknown uri type"}, status=HTTP_BAD_REQUEST)
added = 0
if tdef:
channel.add_torrent_to_channel(tdef, extra_info)
added = 1
return RESTResponse({"added": added})
torrents_dir = None
if parameters.get('torrents_dir', None):
torrents_dir = parameters['torrents_dir']
if not path_util.isabs(torrents_dir):
return RESTResponse({"error": "the torrents_dir should point to a directory"}, status=HTTP_BAD_REQUEST)
recursive = False
if parameters.get('recursive'):
recursive = parameters['recursive']
if not torrents_dir:
return RESTResponse(
{"error": "the torrents_dir parameter should be provided when the recursive parameter is set"},
status=HTTP_BAD_REQUEST,
)
if torrents_dir:
torrents_list, errors_list = channel.add_torrents_from_dir(torrents_dir, recursive)
return RESTResponse({"added": len(torrents_list), "errors": errors_list})
if not parameters.get('torrent', None):
return RESTResponse({"error": "torrent parameter missing"}, status=HTTP_BAD_REQUEST)
# Try to parse the torrent data
# Any errors will be handled by the error_middleware
torrent = base64.b64decode(parameters['torrent'])
torrent_def = TorrentDef.load_from_memory(torrent)
channel.add_torrent_to_channel(torrent_def, extra_info)
return RESTResponse({"added": 1})
def test_tdef_no_metainfo(state_dir):
tdef = TorrentDefNoMetainfo(b"1" * 20, "test")
return tdef
def test_torrent_no_metainfo(self):
torrent = TorrentDefNoMetainfo(b"12345678901234567890",
VIDEO_FILE_NAME, "http://google.com")
self.assertEqual(torrent.get_name(), VIDEO_FILE_NAME)
self.assertEqual(torrent.get_infohash(), b"12345678901234567890")
self.assertEqual(torrent.get_length(), 0) # there are no files
self.assertFalse(torrent.get_metainfo())
self.assertEqual(torrent.get_url(), "http://google.com")
self.assertFalse(torrent.is_multifile_torrent())
self.assertEqual(torrent.get_name_as_unicode(), VIDEO_FILE_NAME)
self.assertFalse(torrent.get_files())
self.assertFalse(torrent.get_files_with_length())
self.assertFalse(torrent.get_trackers_as_single_tuple())
self.assertFalse(torrent.is_private())
torrent2 = TorrentDefNoMetainfo(b"12345678901234567890",
VIDEO_FILE_NAME, "magnet:")
self.assertFalse(torrent2.get_trackers_as_single_tuple())