async def test_create_torrent(self):
"""
Testing whether the API returns a proper base64 encoded torrent
"""
torrent_path = self.files_path / "video.avi.torrent"
expected_tdef = TorrentDef.load(torrent_path)
export_dir = self.temporary_directory()
post_data = {
"files": [self.files_path / "video.avi",
self.files_path / "video.avi.torrent"],
"description": "Video of my cat",
"trackers": "http://localhost/announce",
"name": "test_torrent",
"export_dir": export_dir
}
response_dict = await self.do_request('createtorrent?download=1', 200, None, 'POST', post_data)
torrent = base64.b64decode(response_dict["torrent"])
tdef = TorrentDef.load_from_memory(torrent)
# Copy expected creation date and created by (Tribler version) from actual result
creation_date = tdef.get_creation_date()
expected_tdef.metainfo[b"creation date"] = creation_date
expected_tdef.metainfo[b"created by"] = tdef.metainfo[b'created by']
self.assertEqual(dir(expected_tdef), dir(tdef))
self.assertTrue((export_dir / "test_torrent.torrent").exists())
def create_fake_download_and_state():
"""
Create a fake download and state which can be passed to the global download callback.
"""
tdef = TorrentDef()
tdef.get_infohash = lambda: b'aaaa'
fake_peer = {'extended_version': 'Tribler', 'id': 'a' * 20, 'dtotal': 10 * 1024 * 1024}
fake_download = Mock()
fake_download.get_def = lambda: tdef
fake_download.get_def().get_name_as_unicode = lambda: "test.iso"
fake_download.get_peerlist = lambda: [fake_peer]
fake_download.hidden = False
fake_download.checkpoint = lambda: succeed(None)
fake_download.stop = lambda: succeed(None)
fake_download.shutdown = lambda: succeed(None)
dl_state = Mock()
dl_state.get_infohash = lambda: b'aaaa'
dl_state.get_status = lambda: DLSTATUS_SEEDING
dl_state.get_download = lambda: fake_download
fake_config = Mock()
fake_config.get_hops = lambda: 0
fake_config.get_safe_seeding = lambda: True
fake_download.config = fake_config
return fake_download, dl_state
def test_get_name_utf8(self):
"""
Check whether we can successfully get the UTF-8 encoded torrent name when using a different encoding
"""
t = TorrentDef()
t.set_name(b'\xA1\xC0')
self.assertEqual(t.get_name_utf8(), u'\xa1\xc0')
def test_get_files_with_length(self):
name_bytes = b'\xe8\xaf\xad\xe8\xa8\x80\xe5\xa4\x84\xe7\x90\x86'
name_unicode = name_bytes.decode()
t = TorrentDef()
t.metainfo = {
b'info': {
b'files': [{
b'path.utf-8': [name_bytes],
b'length': 123
}, {
b'path.utf-8': [b'file.txt'],
b'length': 456
}]
}
}
self.assertEqual(t.get_files_with_length(), [(Path(name_unicode), 123),
(Path('file.txt'), 456)])
t.metainfo = {
b'info': {
b'files': [{
b'path': [name_bytes],
b'length': 123
}, {
b'path': [b'file.txt'],
b'length': 456
}]
}
}
self.assertEqual(t.get_files_with_length(), [(Path(name_unicode), 123),
(Path('file.txt'), 456)])
t.metainfo = {
b'info': {
b'files': [{
b'path': [b'test\xff' + name_bytes],
b'length': 123
}, {
b'path': [b'file.txt'],
b'length': 456
}]
}
}
self.assertEqual(t.get_files_with_length(),
[(Path('test?????????????'), 123),
(Path('file.txt'), 456)])
t.metainfo = {
b'info': {
b'files': [{
b'path.utf-8': [b'test\xff' + name_bytes],
b'length': 123
}, {
b'path': [b'file.txt'],
b'length': 456
}]
}
}
self.assertEqual(t.get_files_with_length(), [(Path('file.txt'), 456)])
def test_get_nr_pieces(self):
"""
Test getting the number of pieces from a TorrentDef
"""
tdef = TorrentDef()
self.assertEqual(tdef.get_nr_pieces(), 0)
tdef.metainfo = {b'info': {b'pieces': b'a' * 40}}
self.assertEqual(tdef.get_nr_pieces(), 2)
async def test_load_from_url(file_server, tmpdir):
shutil.copyfile(TORRENT_UBUNTU_FILE, tmpdir / 'ubuntu.torrent')
torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server
torrent_def = await TorrentDef.load_from_url(torrent_url)
assert torrent_def.get_metainfo() == TorrentDef.load(
TORRENT_UBUNTU_FILE).get_metainfo()
assert torrent_def.infohash == TorrentDef.load(
TORRENT_UBUNTU_FILE).infohash
def test_create_invalid_tdef():
"""
Test whether creating invalid TorrentDef objects result in ValueErrors
"""
invalid_metainfo = {}
with pytest.raises(ValueError):
TorrentDef.load_from_memory(bencode(invalid_metainfo))
invalid_metainfo = {b'info': {}}
with pytest.raises(ValueError):
TorrentDef.load_from_memory(bencode(invalid_metainfo))
def test_is_private():
"""
Test whether the private field from an existing torrent is correctly read
"""
privatefn = TESTS_DATA_DIR / "private.torrent"
publicfn = TESTS_DATA_DIR / "bak_single.torrent"
t1 = TorrentDef.load(privatefn)
t2 = TorrentDef.load(publicfn)
assert t1.is_private()
assert not t2.is_private()
def test_is_private(self):
"""
Test whether the private field from an existing torrent is correctly read
"""
privatefn = TESTS_DATA_DIR / "private.torrent"
publicfn = TESTS_DATA_DIR / "bak_single.torrent"
t1 = TorrentDef.load(privatefn)
t2 = TorrentDef.load(publicfn)
self.assertTrue(t1.is_private())
self.assertFalse(t2.is_private())
async def setup_tunnel_seeder(self, hops):
"""
Setup the seeder.
"""
from tribler_core.session import Session
self.seed_config = self.config.copy()
self.seed_config._state_dir = self.getRootStateDir(2)
self.seed_config.set_libtorrent_enabled(hops == 0)
self.seed_config.set_tunnel_community_socks5_listen_ports(
self.get_ports(5))
if self.session2 is None:
self.session2 = Session(self.seed_config)
self.session2.upgrader_enabled = False
await self.session2.start()
tdef = TorrentDef()
tdef.add_content(TESTS_DATA_DIR / "video.avi")
tdef.set_tracker("http://localhost/announce")
torrentfn = self.session2.config.get_state_dir() / "gen.torrent"
tdef.save(torrent_filepath=torrentfn)
self.seed_tdef = tdef
if hops > 0: # Safe seeding enabled
self.tunnel_community_seeder = await self.load_tunnel_community_in_session(
self.session2, start_lt=True)
self.tunnel_community_seeder.build_tunnels(hops)
else:
await self.sanitize_network(self.session2)
dscfg = DownloadConfig()
dscfg.set_dest_dir(
TESTS_DATA_DIR) # basedir of the file we are seeding
dscfg.set_hops(hops)
d = self.session2.dlmgr.start_download(tdef=tdef, config=dscfg)
d.set_state_callback(self.seeder_state_callback)
def test_tdef(state_dir):
tdef = TorrentDef()
sourcefn = TESTS_DATA_DIR / 'video.avi'
tdef.add_content(sourcefn)
tdef.set_tracker("http://localhost/announce")
torrentfn = state_dir / "gen.torrent"
tdef.save(torrentfn)
return tdef
async def post_commit(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
if channel_id == 0:
for t in self.session.mds.CollectionNode.commit_all_channels():
self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t))
else:
coll = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id)
if not coll:
return RESTResponse({"success": False}, status=HTTP_NOT_FOUND)
torrent_dict = coll.commit_channel_torrent()
if torrent_dict:
self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(torrent_dict))
return RESTResponse({"success": True})
def test_add_content_dir(self):
"""
Test whether adding a single content directory with two files is working correctly
"""
t = TorrentDef()
torrent_dir = TESTS_DATA_DIR / "contentdir"
t.add_content(torrent_dir / "file.txt")
t.add_content(torrent_dir / "otherfile.txt")
t.save()
metainfo = t.get_metainfo()
self.assertEqual(len(metainfo[b'info'][b'files']), 2)
def test_add_content_piece_length(self):
"""
Add a single file with piece length to a TorrentDef
"""
t = TorrentDef()
fn = TESTS_DATA_DIR / VIDEO_FILE_NAME
t.add_content(fn)
t.set_piece_length(2**16)
t.save()
metainfo = t.get_metainfo()
self.assertEqual(metainfo[b'info'][b'piece length'], 2**16)
async def test_load_from_url(self):
# Setup file server to serve torrent file
self.session_base_dir = mkdtemp(suffix="_tribler_test_load_from_url")
files_path = self.session_base_dir / 'http_torrent_files'
os.mkdir(files_path)
shutil.copyfile(TORRENT_UBUNTU_FILE, files_path / 'ubuntu.torrent')
file_server_port = self.get_port()
await self.setUpFileServer(file_server_port, files_path)
torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server_port
torrent_def = await TorrentDef.load_from_url(torrent_url)
self.assertEqual(torrent_def.get_metainfo(),
TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo())
self.assertEqual(torrent_def.infohash,
TorrentDef.load(TORRENT_UBUNTU_FILE).infohash)
def convert_config_to_tribler75(state_dir):
"""
Convert the download config files from Tribler 7.4 to 7.5 format.
"""
for filename in (state_dir / STATEDIR_CHECKPOINT_DIR).glob('*.conf'):
try:
config = DownloadConfig.load(filename)
# Convert resume data
resumedata = config.get_engineresumedata()
if b'mapped_files' in resumedata:
resumedata.pop(b'mapped_files')
config.set_engineresumedata(resumedata)
config.write(str(filename))
# Convert metainfo
metainfo = config.get_metainfo()
if not config.config['download_defaults'].get(
'selected_files') or not metainfo:
continue # no conversion needed/possible, selected files will be reset to their default (i.e., all files)
tdef = TorrentDef.load_from_dict(metainfo)
config.set_selected_files([
tdef.get_index_of_file_in_files(fn) for fn in
config.config['download_defaults'].pop('selected_files')
])
config.write(str(filename))
except ConfigObjParseError:
logger.error("Could not parse %s file so removing it", filename)
os.remove(filename)
def test_tdef_init(self):
"""
Test initializing a TorrentDef object
"""
tdef_params = TorrentDef(
torrent_parameters={'announce': 'http://test.com'})
self.assertIn('announce', tdef_params.torrent_parameters)
def test_tdef_init():
"""
Test initializing a TorrentDef object
"""
tdef_params = TorrentDef(
torrent_parameters={'announce': 'http://test.com'})
assert 'announce' in tdef_params.torrent_parameters
def check_watch_folder(self):
if not self.session.config.get_watch_folder_path().is_dir():
return
# Make sure that we pass a str to os.walk
watch_dir = str(self.session.config.get_watch_folder_path())
for root, _, files in os.walk(watch_dir):
root = path_util.Path(root)
for name in files:
if not name.endswith(".torrent"):
continue
try:
tdef = TorrentDef.load(root / name)
if not tdef.get_metainfo():
self.cleanup_torrent_file(root, name)
continue
except: # torrent appears to be corrupt
self.cleanup_torrent_file(root, name)
continue
infohash = tdef.get_infohash()
if not self.session.dlmgr.download_exists(infohash):
self._logger.info("Starting download from torrent file %s", name)
dl_config = DownloadConfig()
anon_enabled = self.session.config.get_default_anonymity_enabled()
default_num_hops = self.session.config.get_default_number_hops()
dl_config.set_hops(default_num_hops if anon_enabled else 0)
dl_config.set_safe_seeding(self.session.config.get_default_safeseeding_enabled())
dl_config.set_dest_dir(self.session.config.get_default_destination_dir())
self.session.dlmgr.start_download(tdef=tdef, config=dl_config)
def test_restore_torrent_in_channel(self):
"""
Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it.
"""
channel_metadata = self.mds.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
md = channel_metadata.add_torrent_to_channel(tdef, None)
# Check correct re-add
md.status = TODELETE
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
self.assertEqual(UPDATED, md.status)
self.assertEqual(md_updated, md)
self.assertTrue(md.has_valid_signature)
# Check update of torrent properties from a new tdef
md.status = TODELETE
new_tracker_address = u'http://tribler.org/announce'
tdef.torrent_parameters[b'announce'] = new_tracker_address.encode(
'utf-8')
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
self.assertEqual(md_updated, md)
self.assertEqual(md.status, UPDATED)
self.assertEqual(md.tracker_info, new_tracker_address)
self.assertTrue(md.has_valid_signature)
# In addition, check that the trackers table was properly updated
self.assertEqual(len(md.health.trackers), 2)
def test_commit_channel_torrent(self):
channel = self.mds.ChannelMetadata.create_channel('test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
channel.add_torrent_to_channel(tdef, None)
# The first run should return the infohash, the second should return None, because nothing was really done
self.assertTrue(channel.commit_channel_torrent())
self.assertFalse(channel.commit_channel_torrent())
def update_trackers(self, infohash, trackers):
""" Update the trackers for a download.
:param infohash: infohash of the torrent that needs to be updated
:param trackers: A list of tracker urls.
"""
download = self.get_download(infohash)
if download:
old_def = download.get_def()
old_trackers = old_def.get_trackers_as_single_tuple()
new_trackers = list(set(trackers) - set(old_trackers))
all_trackers = list(old_trackers) + new_trackers
if new_trackers:
# Add new trackers to the download
download.add_trackers(new_trackers)
# Create a new TorrentDef
if isinstance(old_def, TorrentDefNoMetainfo):
new_def = TorrentDefNoMetainfo(old_def.get_infohash(),
old_def.get_name(),
download.get_magnet_link())
else:
metainfo = old_def.get_metainfo()
if len(all_trackers) > 1:
metainfo["announce-list"] = [all_trackers]
else:
metainfo["announce"] = all_trackers[0]
new_def = TorrentDef.load_from_dict(metainfo)
# Set TorrentDef + checkpoint
download.set_def(new_def)
download.checkpoint()
def on_metadata_received_alert(self, _):
torrent_info = get_info_from_handle(self.handle)
if not torrent_info:
return
metadata = {
b'info': bdecode_compat(torrent_info.metadata()),
b'leechers': 0,
b'seeders': 0
}
trackers = [
tracker['url'].encode('utf-8')
for tracker in self.handle.trackers()
]
if len(trackers) > 1:
metadata[b"announce-list"] = [trackers]
elif trackers:
metadata[b"announce"] = trackers[0]
for peer in self.handle.get_peer_info():
if peer.progress == 1:
metadata[b"seeders"] += 1
else:
metadata[b"leechers"] += 1
try:
self.tdef = TorrentDef.load_from_dict(metadata)
except ValueError as ve:
self._logger.exception(ve)
return
self.set_selected_files()
self.checkpoint()
def add_torrents_from_dir(self, torrents_dir, recursive=False):
torrents_list = []
errors_list = []
def rec_gen(dir_):
for root, _, filenames in os.walk(dir_):
for fn in filenames:
yield Path(root) / fn
filename_generator = rec_gen(torrents_dir) if recursive else os.listdir(torrents_dir)
# Build list of .torrents to process
torrents_list_generator = (Path(torrents_dir, f) for f in filename_generator)
torrents_list = [f for f in torrents_list_generator if f.is_file() and f.suffix == ".torrent"]
# 100 is a reasonable chunk size for commits
for chunk in chunks(torrents_list, 100):
for f in chunk:
try:
self.add_torrent_to_channel(TorrentDef.load(f))
except DuplicateTorrentFileError:
pass
except Exception: # pylint: disable=W0703
# Have to use the broad exception clause because Py3 versions of libtorrent
# generate generic Exceptions
errors_list.append(f)
# Optimization to drop excess cache
orm.commit()
return torrents_list, errors_list
def add_torrent(self, file, relative_path):
_logger.info(f'Add torrent: {file}')
directory = self.get_directory(relative_path)
decoded_torrent = libtorrent.bdecode(file.read_bytes())
directory.add_torrent_to_channel(TorrentDef(metainfo=decoded_torrent),
None)
def test_restore_torrent_in_channel(metadata_store):
"""
Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it.
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
md = channel_metadata.add_torrent_to_channel(tdef, None)
# Check correct re-add
md.status = TODELETE
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
assert UPDATED == md.status
assert md_updated == md
assert md.has_valid_signature
# Check update of torrent properties from a new tdef
md.status = TODELETE
new_tracker_address = 'http://tribler.org/announce'
tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8')
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
assert md_updated == md
assert md.status == UPDATED
assert md.tracker_info == new_tracker_address
assert md.has_valid_signature
# In addition, check that the trackers table was properly updated
assert len(md.health.trackers) == 2
def test_get_name_as_unicode(self):
name_bytes = b'\xe8\xaf\xad\xe8\xa8\x80\xe5\xa4\x84\xe7\x90\x86'
name_unicode = name_bytes.decode()
t = TorrentDef()
t.metainfo = {b'info': {b'name.utf-8': name_bytes}}
self.assertEqual(t.get_name_as_unicode(), name_unicode)
t.metainfo = {b'info': {b'name': name_bytes}}
self.assertEqual(t.get_name_as_unicode(), name_unicode)
t.metainfo = {b'info': {b'name': b'test\xff' + name_bytes}}
self.assertEqual(t.get_name_as_unicode(), 'test?????????????')
def test_updated_my_channel(enable_chant, personal_channel, channel_manager,
mock_dlmgr, session, tmpdir):
tdef = TorrentDef.load_from_dict(update_metainfo)
session.dlmgr.start_download = Mock()
session.dlmgr.download_exists = lambda *_: False
session.mds.channels_dir = "bla"
session.config.get_state_dir = lambda: Path(tmpdir / "foo")
channel_manager.updated_my_channel(tdef)
session.dlmgr.start_download.assert_called_once()
def personal_channel(session):
global update_metainfo
with db_session:
chan = session.mds.ChannelMetadata.create_channel(title="my test chan",
description="test")
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
chan.add_torrent_to_channel(tdef, None)
update_metainfo = chan.commit_channel_torrent()
return chan
def start_download(self,
torrent_file=None,
tdef=None,
config=None,
checkpoint_disabled=False,
hidden=False):
self._logger.debug("Starting download: filename: %s, torrent def: %s",
torrent_file, tdef)
# the priority of the parameters is: (1) tdef, (2) torrent_file.
# so if we have tdef, and torrent_file will be ignored, and so on.
if tdef is None:
if torrent_file is None:
raise ValueError(
"Torrent file must be provided if tdef is not given")
# try to get the torrent from the given torrent file
tdef = TorrentDef.load(torrent_file)
assert tdef is not None, "tdef MUST not be None after loading torrent"
config = config or DownloadConfig()
infohash = tdef.get_infohash()
download = self.get_download(infohash)
if download and infohash not in self.metainfo_requests:
new_trackers = list(
set(tdef.get_trackers_as_single_tuple()) -
set(download.get_def().get_trackers_as_single_tuple()))
if new_trackers:
self.update_trackers(tdef.get_infohash(), new_trackers)
return download
# Create the destination directory if it does not exist yet
try:
if not config.get_dest_dir().is_dir():
os.makedirs(config.get_dest_dir())
except OSError:
self._logger.error(
"Unable to create the download destination directory.")
if config.get_time_added() == 0:
config.set_time_added(int(timemod.time()))
# Create the download
download = Download(self.tribler_session, tdef, dummy=self.dummy_mode)
atp = download.setup(config,
checkpoint_disabled=checkpoint_disabled,
hidden=hidden or config.get_bootstrap_download())
# Keep metainfo downloads in self.downloads for now because we will need to remove it later,
# and removing the download at this point will stop us from receiving any further alerts.
if infohash not in self.metainfo_requests or self.metainfo_requests[
infohash][0] == download:
self.downloads[infohash] = download
if not self.dummy_mode:
self.start_handle(download, atp)
return download
