def create_fake_download_and_state():
"""
Create a fake download and state which can be passed to the global download callback.
"""
tdef = TorrentDef()
tdef.get_infohash = lambda: b'aaaa'
fake_peer = {
'extended_version': 'Tribler',
'id': 'a' * 20,
'dtotal': 10 * 1024 * 1024
}
fake_download = MagicMock()
fake_download.get_def = lambda: tdef
fake_download.get_def().get_name_as_unicode = lambda: "test.iso"
fake_download.get_peerlist = lambda: [fake_peer]
fake_download.hidden = False
fake_download.checkpoint = lambda: succeed(None)
fake_download.stop = lambda: succeed(None)
fake_download.shutdown = lambda: succeed(None)
dl_state = MagicMock()
dl_state.get_infohash = lambda: b'aaaa'
dl_state.get_status = lambda: DLSTATUS_SEEDING
dl_state.get_download = lambda: fake_download
fake_config = MagicMock()
fake_config.get_hops = lambda: 0
fake_config.get_safe_seeding = lambda: True
fake_download.config = fake_config
return fake_download, dl_state
async def test_load_from_url(file_server, tmpdir):
shutil.copyfile(TORRENT_UBUNTU_FILE, tmpdir / 'ubuntu.torrent')
torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server
torrent_def = await TorrentDef.load_from_url(torrent_url)
assert torrent_def.get_metainfo() == TorrentDef.load(
TORRENT_UBUNTU_FILE).get_metainfo()
assert torrent_def.infohash == TorrentDef.load(
TORRENT_UBUNTU_FILE).infohash
def test_create_invalid_tdef():
"""
Test whether creating invalid TorrentDef objects result in ValueErrors
"""
invalid_metainfo = {}
with pytest.raises(ValueError):
TorrentDef.load_from_memory(bencode(invalid_metainfo))
invalid_metainfo = {b'info': {}}
with pytest.raises(ValueError):
TorrentDef.load_from_memory(bencode(invalid_metainfo))
def test_is_private():
"""
Test whether the private field from an existing torrent is correctly read
"""
privatefn = TESTS_DATA_DIR / "private.torrent"
publicfn = TESTS_DATA_DIR / "bak_single.torrent"
t1 = TorrentDef.load(privatefn)
t2 = TorrentDef.load(publicfn)
assert t1.is_private()
assert not t2.is_private()
def test_tdef(state_dir):
tdef = TorrentDef()
sourcefn = TESTS_DATA_DIR / 'video.avi'
tdef.add_content(sourcefn)
tdef.set_tracker("http://localhost/announce")
torrentfn = state_dir / "gen.torrent"
tdef.save(torrentfn)
return tdef
async def post_commit(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
if channel_id == 0:
for t in self.mds.CollectionNode.commit_all_channels():
self.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t))
else:
coll = self.mds.CollectionNode.get(public_key=channel_pk, id_=channel_id)
if not coll:
return RESTResponse({"success": False}, status=HTTP_NOT_FOUND)
torrent_dict = coll.commit_channel_torrent()
if torrent_dict:
self.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(torrent_dict))
return RESTResponse({"success": True})
async def regenerate_channel_torrent(self, channel_pk, channel_id):
self._logger.info("Regenerating personal channel %s %i",
hexlify(channel_pk), channel_id)
with db_session:
channel = self.mds.ChannelMetadata.get(public_key=channel_pk,
id_=channel_id)
if channel is None:
self._logger.warning(
"Tried to regenerate non-existing channel %s %i",
hexlify(channel_pk), channel_id)
return None
channel_dirname = channel.dirname
for d in self.download_manager.get_downloads_by_name(channel_dirname):
await self.download_manager.remove_download(d, remove_content=True)
with db_session:
channel = self.mds.ChannelMetadata.get_for_update(
public_key=channel_pk, id_=channel_id)
regenerated = channel.consolidate_channel_torrent()
# If the user created their channel, but added no torrents to it,
# the channel torrent will not be created.
if regenerated is None:
return None
tdef = TorrentDef.load_from_dict(regenerated)
self.updated_my_channel(tdef)
return tdef
def test_tdef_init():
"""
Test initializing a TorrentDef object
"""
tdef_params = TorrentDef(
torrent_parameters={'announce': 'http://test.com'})
assert 'announce' in tdef_params.torrent_parameters
def check_watch_folder(self):
if not self.watch_folder.is_dir():
return
# Make sure that we pass a str to os.walk
watch_dir = str(self.watch_folder)
for root, _, files in os.walk(watch_dir):
root = path_util.Path(root)
for name in files:
if not name.endswith(".torrent"):
continue
try:
tdef = TorrentDef.load(root / name)
if not tdef.get_metainfo():
self.cleanup_torrent_file(root, name)
continue
except: # torrent appears to be corrupt
self.cleanup_torrent_file(root, name)
continue
infohash = tdef.get_infohash()
if not self.download_manager.download_exists(infohash):
self._logger.info("Starting download from torrent file %s",
name)
self.download_manager.start_download(torrent_file=root /
name)
def test_restore_torrent_in_channel(metadata_store):
"""
Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it.
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
md = channel_metadata.add_torrent_to_channel(tdef, None)
# Check correct re-add
md.status = TODELETE
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
assert UPDATED == md.status
assert md_updated == md
assert md.has_valid_signature
# Check update of torrent properties from a new tdef
md.status = TODELETE
new_tracker_address = 'http://tribler.org/announce'
tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8')
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
assert md_updated == md
assert md.status == UPDATED
assert md.tracker_info == new_tracker_address
assert md.has_valid_signature
# In addition, check that the trackers table was properly updated
assert len(md.health.trackers) == 2
def add_torrent(self, file, relative_path):
_logger.info(f'Add torrent: {file}')
directory = self.get_directory(relative_path)
decoded_torrent = libtorrent.bdecode(file.read_bytes())
directory.add_torrent_to_channel(TorrentDef(metainfo=decoded_torrent),
None)
def update_trackers(self, infohash, trackers):
""" Update the trackers for a download.
:param infohash: infohash of the torrent that needs to be updated
:param trackers: A list of tracker urls.
"""
download = self.get_download(infohash)
if download:
old_def = download.get_def()
old_trackers = old_def.get_trackers_as_single_tuple()
new_trackers = list(set(trackers) - set(old_trackers))
all_trackers = list(old_trackers) + new_trackers
if new_trackers:
# Add new trackers to the download
download.add_trackers(new_trackers)
# Create a new TorrentDef
if isinstance(old_def, TorrentDefNoMetainfo):
new_def = TorrentDefNoMetainfo(old_def.get_infohash(),
old_def.get_name(),
download.get_magnet_link())
else:
metainfo = old_def.get_metainfo()
if len(all_trackers) > 1:
metainfo["announce-list"] = [all_trackers]
else:
metainfo["announce"] = all_trackers[0]
new_def = TorrentDef.load_from_dict(metainfo)
# Set TorrentDef + checkpoint
download.set_def(new_def)
download.checkpoint()
def on_metadata_received_alert(self, _):
torrent_info = get_info_from_handle(self.handle)
if not torrent_info:
return
metadata = {
b'info': bdecode_compat(torrent_info.metadata()),
b'leechers': 0,
b'seeders': 0
}
trackers = [
tracker['url'].encode('utf-8')
for tracker in self.handle.trackers()
]
if len(trackers) > 1:
metadata[b"announce-list"] = [trackers]
elif trackers:
metadata[b"announce"] = trackers[0]
for peer in self.handle.get_peer_info():
if peer.progress == 1:
metadata[b"seeders"] += 1
else:
metadata[b"leechers"] += 1
try:
self.tdef = TorrentDef.load_from_dict(metadata)
except ValueError as ve:
self._logger.exception(ve)
return
self.set_selected_files()
self.checkpoint()
def add_torrents_from_dir(self, torrents_dir, recursive=False):
torrents_list = []
errors_list = []
def rec_gen(dir_):
for root, _, filenames in os.walk(dir_):
for fn in filenames:
yield Path(root) / fn
filename_generator = rec_gen(
torrents_dir) if recursive else os.listdir(torrents_dir)
# Build list of .torrents to process
torrents_list_generator = (Path(torrents_dir, f)
for f in filename_generator)
torrents_list = [
f for f in torrents_list_generator
if f.is_file() and f.suffix == ".torrent"
]
# 100 is a reasonable chunk size for commits
for chunk in chunks(torrents_list, 100):
for f in chunk:
try:
self.add_torrent_to_channel(TorrentDef.load(f))
except Exception: # pylint: disable=W0703
# Have to use the broad exception clause because Py3 versions of libtorrent
# generate generic Exceptions
errors_list.append(f)
# Optimization to drop excess cache
orm.commit()
return torrents_list, errors_list
def personal_channel(metadata_store):
global update_metainfo
with db_session:
chan = metadata_store.ChannelMetadata.create_channel(
title="my test chan", description="test")
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
chan.add_torrent_to_channel(tdef, None)
update_metainfo = chan.commit_channel_torrent()
return chan
def load_checkpoint(self, filename):
try:
config = DownloadConfig.load(filename)
except Exception:
self._logger.exception("Could not open checkpoint file %s",
filename)
return
metainfo = config.get_metainfo()
if not metainfo:
self._logger.error(
"Could not resume checkpoint %s; metainfo not found", filename)
return
if not isinstance(metainfo, dict):
self._logger.error(
"Could not resume checkpoint %s; metainfo is not dict %s %s",
filename, type(metainfo), repr(metainfo))
return
try:
url = metainfo.get(b'url', None)
url = url.decode('utf-8') if url else url
tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'],
metainfo[b'name'], url) if b'infohash'
in metainfo else TorrentDef.load_from_dict(metainfo))
except (KeyError, ValueError) as e:
self._logger.exception(
"Could not restore tdef from metainfo dict: %s %s ", e,
metainfo)
return
if config.get_bootstrap_download():
# In case the download is marked as bootstrap, remove it if its infohash does not
# match the configured bootstrap infohash
if hexlify(tdef.get_infohash()) != self.bootstrap_infohash:
self.remove_config(tdef.get_infohash())
return
config.state_dir = self.state_dir
if config.get_dest_dir() == '': # removed torrent ignoring
self._logger.info("Removing checkpoint %s destdir is %s", filename,
config.get_dest_dir())
os.remove(filename)
return
try:
if self.download_exists(tdef.get_infohash()):
self._logger.info(
"Not resuming checkpoint because download has already been added"
)
else:
self.start_download(tdef=tdef, config=config)
except Exception:
self._logger.exception(
"Not resume checkpoint due to exception while adding download")
def test_add_torrent_to_channel(metadata_store):
"""
Test adding a torrent to your channel
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'})
assert channel_metadata.contents_list
# Make sure trying to add a duplicate torrent does not result in an error
channel_metadata.add_torrent_to_channel(tdef, None)
def gen_sample_channel(mds):
my_channel = mds.ChannelMetadata.create_channel('test_channel',
'test description')
my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_UBUNTU_FILE),
None)
my_channel.commit_channel_torrent()
t2 = my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_VIDEO_FILE),
None)
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
coll = mds.CollectionNode(origin_id=my_channel.id_,
title='internal collection')
mds.TorrentMetadata.from_dict(
dict(origin_id=coll.id_, **gen_random_entry()))
mds.TorrentMetadata.from_dict(
dict(origin_id=coll.id_, **gen_random_entry()))
my_channel.commit_channel_torrent()
t2.soft_delete()
my_channel.commit_channel_torrent()
# Rename files to stable names
mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob")
torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent")
os.rename(mdblob_name, CHANNEL_METADATA)
os.rename(torrent_name, CHANNEL_TORRENT)
# Update channel
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
my_channel.commit_channel_torrent()
# Rename updated files to stable names
os.rename(mdblob_name, CHANNEL_METADATA_UPDATED)
os.rename(torrent_name, CHANNEL_TORRENT_UPDATED)
def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_disabled=False, hidden=False):
self._logger.debug("Starting download: filename: %s, torrent def: %s", torrent_file, tdef)
# the priority of the parameters is: (1) tdef, (2) torrent_file.
# so if we have tdef, and torrent_file will be ignored, and so on.
if tdef is None:
if torrent_file is None:
raise ValueError("Torrent file must be provided if tdef is not given")
# try to get the torrent from the given torrent file
tdef = TorrentDef.load(torrent_file)
assert tdef is not None, "tdef MUST not be None after loading torrent"
config = config or DownloadConfig()
infohash = tdef.get_infohash()
download = self.get_download(infohash)
if download and infohash not in self.metainfo_requests:
new_trackers = list(set(tdef.get_trackers_as_single_tuple()) -
set(download.get_def().get_trackers_as_single_tuple()))
if new_trackers:
self.update_trackers(tdef.get_infohash(), new_trackers)
return download
# Create the destination directory if it does not exist yet
try:
if not config.get_dest_dir().is_dir():
os.makedirs(config.get_dest_dir())
except OSError:
self._logger.error("Unable to create the download destination directory.")
if config.get_time_added() == 0:
config.set_time_added(int(timemod.time()))
# Create the download
download = Download(tdef=tdef,
config=config,
download_defaults=self.download_defaults,
checkpoint_disabled=checkpoint_disabled,
hidden=hidden or config.get_bootstrap_download(),
notifier=self.notifier,
state_dir=self.state_dir,
download_manager=self,
dummy=self.dummy_mode)
atp = download.get_atp()
# Keep metainfo downloads in self.downloads for now because we will need to remove it later,
# and removing the download at this point will stop us from receiving any further alerts.
if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download:
self.downloads[infohash] = download
if not self.dummy_mode:
self.start_handle(download, atp)
return download
async def test_get_metainfo_with_already_added_torrent(fake_dlmgr):
"""
Testing metainfo fetching for a torrent which is already in session.
"""
sample_torrent = TESTS_DATA_DIR / "bak_single.torrent"
torrent_def = TorrentDef.load(sample_torrent)
download_impl = MagicMock()
download_impl.future_metainfo = succeed(bencode(
torrent_def.get_metainfo()))
download_impl.checkpoint = lambda: succeed(None)
download_impl.stop = lambda: succeed(None)
download_impl.shutdown = lambda: succeed(None)
fake_dlmgr.initialize()
fake_dlmgr.downloads[torrent_def.infohash] = download_impl
assert await fake_dlmgr.get_metainfo(torrent_def.infohash)
def test_consolidate_channel_torrent(torrent_template, metadata_store):
"""
Test completely re-commit your channel
"""
channel = metadata_store.ChannelMetadata.create_channel('test', 'test')
my_dir = Path(metadata_store.ChannelMetadata._channels_dir /
channel.dirname).absolute()
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
# 1st torrent
torrent_entry = channel.add_torrent_to_channel(tdef, None)
channel.commit_channel_torrent()
# 2nd torrent
metadata_store.TorrentMetadata.from_dict(
dict(torrent_template,
public_key=channel.public_key,
origin_id=channel.id_,
status=NEW))
channel.commit_channel_torrent()
# Delete entry
torrent_entry.soft_delete()
channel.commit_channel_torrent()
assert len(channel.contents_list) == 1
assert len(os.listdir(my_dir)) == 3
torrent3 = metadata_store.TorrentMetadata(public_key=channel.public_key,
origin_id=channel.id_,
status=NEW,
infohash=random_infohash())
channel.commit_channel_torrent()
torrent3.soft_delete()
channel.consolidate_channel_torrent()
assert len(os.listdir(my_dir)) == 1
metadata_store.TorrentMetadata.select(
lambda g: g.metadata_type == REGULAR_TORRENT).delete()
channel.local_version = 0
metadata_store.process_channel_dir(my_dir,
channel.public_key,
channel.id_,
skip_personal_metadata_payload=False)
assert len(channel.contents[:]) == 1
async def start_download_from_uri(self, uri, config=None):
scheme = scheme_from_uri(uri)
if scheme in (HTTP_SCHEME, HTTPS_SCHEME):
tdef = await TorrentDef.load_from_url(uri)
return self.start_download(tdef=tdef, config=config)
if scheme == MAGNET_SCHEME:
name, infohash, _ = parse_magnetlink(uri)
if infohash is None:
raise RuntimeError("Missing infohash")
if infohash in self.metainfo_cache:
tdef = TorrentDef.load_from_dict(self.metainfo_cache[infohash]['meta_info'])
else:
tdef = TorrentDefNoMetainfo(infohash, "Unknown name" if name is None else name, url=uri)
return self.start_download(tdef=tdef, config=config)
if scheme == FILE_SCHEME:
file = uri_to_path(uri)
return self.start_download(torrent_file=file, config=config)
raise Exception("invalid uri")
async def test_add_torrent_duplicate(my_channel, rest_api):
"""
Test that adding a duplicate torrent to you channel does not result in an error
"""
with db_session:
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
my_channel.add_torrent_to_channel(tdef, {'description': 'blabla'})
with open(TORRENT_UBUNTU_FILE, "rb") as torrent_file:
base64_content = base64.b64encode(
torrent_file.read()).decode('utf-8')
post_params = {'torrent': base64_content}
await do_request(
rest_api,
f'channels/{hexlify(my_channel.public_key)}/{my_channel.id_}/torrents',
request_type='PUT',
post_data=post_params,
expected_code=200,
)
def test_create_ffa_from_dict(metadata_store):
"""
Test creating a free-for-all torrent entry
"""
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
# Make sure that FFA entry with the infohash that is already known to GigaChannel cannot be created
signed_entry = metadata_store.TorrentMetadata.from_dict(
tdef_to_metadata_dict(tdef))
metadata_store.TorrentMetadata.add_ffa_from_dict(
tdef_to_metadata_dict(tdef))
assert metadata_store.TorrentMetadata.select(
lambda g: g.public_key == EMPTY_BLOB).count() == 0
signed_entry.delete()
# Create FFA entry
metadata_store.TorrentMetadata.add_ffa_from_dict(
tdef_to_metadata_dict(tdef))
assert metadata_store.TorrentMetadata.select(
lambda g: g.public_key == EMPTY_BLOB).count() == 1
def test_delete_torrent_from_channel(metadata_store):
"""
Test deleting a torrent from your channel
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
# Check that nothing is committed when deleting uncommited torrent metadata
torrent = channel_metadata.add_torrent_to_channel(tdef, None)
torrent.soft_delete()
assert not channel_metadata.contents_list
# Check append-only deletion process
torrent = channel_metadata.add_torrent_to_channel(tdef, None)
channel_metadata.commit_channel_torrent()
assert len(channel_metadata.contents_list) == 1
torrent.soft_delete()
channel_metadata.commit_channel_torrent()
assert not channel_metadata.contents_list
def test_commit_channel_torrent(metadata_store):
"""
Test committing a channel torrent
"""
channel = metadata_store.ChannelMetadata.create_channel('test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
channel.add_torrent_to_channel(tdef, None)
# The first run should return the infohash, the second should return None, because nothing was really done
assert channel.commit_channel_torrent()
assert not channel.commit_channel_torrent()
# Test adding flags to channel torrent when adding thumbnail and description
metadata_store.ChannelThumbnail(public_key=channel.public_key,
origin_id=channel.id_,
status=NEW)
metadata_store.ChannelDescription(public_key=channel.public_key,
origin_id=channel.id_,
status=NEW)
assert channel.commit_channel_torrent()
assert channel.reserved_flags == 3
assert not channel.commit_channel_torrent()
async def start_download_from_uri(self, uri, config=None):
if uri.startswith("http"):
tdef = await TorrentDef.load_from_url(uri)
return self.start_download(tdef=tdef, config=config)
if uri.startswith("magnet:"):
name, infohash, _ = parse_magnetlink(uri)
if infohash is None:
raise RuntimeError("Missing infohash")
if infohash in self.metainfo_cache:
tdef = TorrentDef.load_from_dict(
self.metainfo_cache[infohash]['meta_info'])
else:
tdef = TorrentDefNoMetainfo(
infohash,
"Unknown name" if name is None else name,
url=uri)
return self.start_download(tdef=tdef, config=config)
if uri.startswith("file:"):
argument = uri_to_path(uri)
return self.start_download(torrent_file=argument, config=config)
raise Exception("invalid uri")
async def download_channel(self, channel):
"""
Download a channel with a given infohash and title.
:param channel: The channel metadata ORM object.
"""
metainfo = await self.download_manager.get_metainfo(bytes(
channel.infohash),
timeout=60,
hops=0)
if metainfo is None:
# Timeout looking for the channel metainfo. Probably, there are no seeds.
# TODO: count the number of tries we had with the channel, so we can stop trying eventually
return
try:
if metainfo[b'info'][b'name'].decode('utf-8') != channel.dirname:
# Malformed channel
# TODO: stop trying to download this channel until it is updated with a new infohash
return
except (KeyError, TypeError):
return
dcfg = DownloadConfig(state_dir=self.state_dir)
dcfg.set_dest_dir(self.mds.channels_dir)
dcfg.set_channel_download(True)
tdef = TorrentDef(metainfo=metainfo)
download = self.download_manager.start_download(tdef=tdef,
config=dcfg,
hidden=True)
try:
await download.future_finished
except CancelledError:
pass
else:
self.channels_processing_queue[channel.infohash] = (
PROCESS_CHANNEL_DIR, channel)
return download
def test_updated_my_channel(personal_channel, gigachannel_manager, tmpdir):
tdef = TorrentDef.load_from_dict(update_metainfo)
gigachannel_manager.download_manager.start_download = Mock()
gigachannel_manager.download_manager.download_exists = lambda *_: False
gigachannel_manager.updated_my_channel(tdef)
gigachannel_manager.download_manager.start_download.assert_called_once()
def channel_tdef():
return TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' /
'channel_upd.torrent')