async def test_load_from_url(file_server, tmpdir):
shutil.copyfile(TORRENT_UBUNTU_FILE, tmpdir / 'ubuntu.torrent')
torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server
torrent_def = await TorrentDef.load_from_url(torrent_url)
assert torrent_def.get_metainfo() == TorrentDef.load(
TORRENT_UBUNTU_FILE).get_metainfo()
assert torrent_def.infohash == TorrentDef.load(
TORRENT_UBUNTU_FILE).infohash
def test_is_private():
"""
Test whether the private field from an existing torrent is correctly read
"""
privatefn = TESTS_DATA_DIR / "private.torrent"
publicfn = TESTS_DATA_DIR / "bak_single.torrent"
t1 = TorrentDef.load(privatefn)
t2 = TorrentDef.load(publicfn)
assert t1.is_private()
assert not t2.is_private()
def test_restore_torrent_in_channel(metadata_store):
"""
Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it.
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
md = channel_metadata.add_torrent_to_channel(tdef, None)
# Check correct re-add
md.status = TODELETE
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
assert UPDATED == md.status
assert md_updated == md
assert md.has_valid_signature
# Check update of torrent properties from a new tdef
md.status = TODELETE
new_tracker_address = 'http://tribler.org/announce'
tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8')
md_updated = channel_metadata.add_torrent_to_channel(tdef, None)
assert md_updated == md
assert md.status == UPDATED
assert md.tracker_info == new_tracker_address
assert md.has_valid_signature
# In addition, check that the trackers table was properly updated
assert len(md.health.trackers) == 2
def add_torrents_from_dir(self, torrents_dir, recursive=False):
torrents_list = []
errors_list = []
def rec_gen(dir_):
for root, _, filenames in os.walk(dir_):
for fn in filenames:
yield Path(root) / fn
filename_generator = rec_gen(
torrents_dir) if recursive else os.listdir(torrents_dir)
# Build list of .torrents to process
torrents_list_generator = (Path(torrents_dir, f)
for f in filename_generator)
torrents_list = [
f for f in torrents_list_generator
if f.is_file() and f.suffix == ".torrent"
]
# 100 is a reasonable chunk size for commits
for chunk in chunks(torrents_list, 100):
for f in chunk:
try:
self.add_torrent_to_channel(TorrentDef.load(f))
except Exception: # pylint: disable=W0703
# Have to use the broad exception clause because Py3 versions of libtorrent
# generate generic Exceptions
errors_list.append(f)
# Optimization to drop excess cache
orm.commit()
return torrents_list, errors_list
def check_watch_folder(self):
if not self.watch_folder.is_dir():
return
# Make sure that we pass a str to os.walk
watch_dir = str(self.watch_folder)
for root, _, files in os.walk(watch_dir):
root = path_util.Path(root)
for name in files:
if not name.endswith(".torrent"):
continue
try:
tdef = TorrentDef.load(root / name)
if not tdef.get_metainfo():
self.cleanup_torrent_file(root, name)
continue
except: # torrent appears to be corrupt
self.cleanup_torrent_file(root, name)
continue
infohash = tdef.get_infohash()
if not self.download_manager.download_exists(infohash):
self._logger.info("Starting download from torrent file %s",
name)
self.download_manager.start_download(torrent_file=root /
name)
def personal_channel(metadata_store):
global update_metainfo
with db_session:
chan = metadata_store.ChannelMetadata.create_channel(
title="my test chan", description="test")
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
chan.add_torrent_to_channel(tdef, None)
update_metainfo = chan.commit_channel_torrent()
return chan
def test_add_torrent_to_channel(metadata_store):
"""
Test adding a torrent to your channel
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'})
assert channel_metadata.contents_list
# Make sure trying to add a duplicate torrent does not result in an error
channel_metadata.add_torrent_to_channel(tdef, None)
def gen_sample_channel(mds):
my_channel = mds.ChannelMetadata.create_channel('test_channel',
'test description')
my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_UBUNTU_FILE),
None)
my_channel.commit_channel_torrent()
t2 = my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_VIDEO_FILE),
None)
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
coll = mds.CollectionNode(origin_id=my_channel.id_,
title='internal collection')
mds.TorrentMetadata.from_dict(
dict(origin_id=coll.id_, **gen_random_entry()))
mds.TorrentMetadata.from_dict(
dict(origin_id=coll.id_, **gen_random_entry()))
my_channel.commit_channel_torrent()
t2.soft_delete()
my_channel.commit_channel_torrent()
# Rename files to stable names
mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob")
torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent")
os.rename(mdblob_name, CHANNEL_METADATA)
os.rename(torrent_name, CHANNEL_TORRENT)
# Update channel
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
my_channel.commit_channel_torrent()
# Rename updated files to stable names
os.rename(mdblob_name, CHANNEL_METADATA_UPDATED)
os.rename(torrent_name, CHANNEL_TORRENT_UPDATED)
def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_disabled=False, hidden=False):
self._logger.debug("Starting download: filename: %s, torrent def: %s", torrent_file, tdef)
# the priority of the parameters is: (1) tdef, (2) torrent_file.
# so if we have tdef, and torrent_file will be ignored, and so on.
if tdef is None:
if torrent_file is None:
raise ValueError("Torrent file must be provided if tdef is not given")
# try to get the torrent from the given torrent file
tdef = TorrentDef.load(torrent_file)
assert tdef is not None, "tdef MUST not be None after loading torrent"
config = config or DownloadConfig()
infohash = tdef.get_infohash()
download = self.get_download(infohash)
if download and infohash not in self.metainfo_requests:
new_trackers = list(set(tdef.get_trackers_as_single_tuple()) -
set(download.get_def().get_trackers_as_single_tuple()))
if new_trackers:
self.update_trackers(tdef.get_infohash(), new_trackers)
return download
# Create the destination directory if it does not exist yet
try:
if not config.get_dest_dir().is_dir():
os.makedirs(config.get_dest_dir())
except OSError:
self._logger.error("Unable to create the download destination directory.")
if config.get_time_added() == 0:
config.set_time_added(int(timemod.time()))
# Create the download
download = Download(tdef=tdef,
config=config,
download_defaults=self.download_defaults,
checkpoint_disabled=checkpoint_disabled,
hidden=hidden or config.get_bootstrap_download(),
notifier=self.notifier,
state_dir=self.state_dir,
download_manager=self,
dummy=self.dummy_mode)
atp = download.get_atp()
# Keep metainfo downloads in self.downloads for now because we will need to remove it later,
# and removing the download at this point will stop us from receiving any further alerts.
if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download:
self.downloads[infohash] = download
if not self.dummy_mode:
self.start_handle(download, atp)
return download
async def test_get_metainfo_with_already_added_torrent(fake_dlmgr):
"""
Testing metainfo fetching for a torrent which is already in session.
"""
sample_torrent = TESTS_DATA_DIR / "bak_single.torrent"
torrent_def = TorrentDef.load(sample_torrent)
download_impl = MagicMock()
download_impl.future_metainfo = succeed(bencode(
torrent_def.get_metainfo()))
download_impl.checkpoint = lambda: succeed(None)
download_impl.stop = lambda: succeed(None)
download_impl.shutdown = lambda: succeed(None)
fake_dlmgr.initialize()
fake_dlmgr.downloads[torrent_def.infohash] = download_impl
assert await fake_dlmgr.get_metainfo(torrent_def.infohash)
def test_consolidate_channel_torrent(torrent_template, metadata_store):
"""
Test completely re-commit your channel
"""
channel = metadata_store.ChannelMetadata.create_channel('test', 'test')
my_dir = Path(metadata_store.ChannelMetadata._channels_dir /
channel.dirname).absolute()
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
# 1st torrent
torrent_entry = channel.add_torrent_to_channel(tdef, None)
channel.commit_channel_torrent()
# 2nd torrent
metadata_store.TorrentMetadata.from_dict(
dict(torrent_template,
public_key=channel.public_key,
origin_id=channel.id_,
status=NEW))
channel.commit_channel_torrent()
# Delete entry
torrent_entry.soft_delete()
channel.commit_channel_torrent()
assert len(channel.contents_list) == 1
assert len(os.listdir(my_dir)) == 3
torrent3 = metadata_store.TorrentMetadata(public_key=channel.public_key,
origin_id=channel.id_,
status=NEW,
infohash=random_infohash())
channel.commit_channel_torrent()
torrent3.soft_delete()
channel.consolidate_channel_torrent()
assert len(os.listdir(my_dir)) == 1
metadata_store.TorrentMetadata.select(
lambda g: g.metadata_type == REGULAR_TORRENT).delete()
channel.local_version = 0
metadata_store.process_channel_dir(my_dir,
channel.public_key,
channel.id_,
skip_personal_metadata_payload=False)
assert len(channel.contents[:]) == 1
def test_create_ffa_from_dict(metadata_store):
"""
Test creating a free-for-all torrent entry
"""
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
# Make sure that FFA entry with the infohash that is already known to GigaChannel cannot be created
signed_entry = metadata_store.TorrentMetadata.from_dict(
tdef_to_metadata_dict(tdef))
metadata_store.TorrentMetadata.add_ffa_from_dict(
tdef_to_metadata_dict(tdef))
assert metadata_store.TorrentMetadata.select(
lambda g: g.public_key == EMPTY_BLOB).count() == 0
signed_entry.delete()
# Create FFA entry
metadata_store.TorrentMetadata.add_ffa_from_dict(
tdef_to_metadata_dict(tdef))
assert metadata_store.TorrentMetadata.select(
lambda g: g.public_key == EMPTY_BLOB).count() == 1
async def test_add_torrent_duplicate(my_channel, rest_api):
"""
Test that adding a duplicate torrent to you channel does not result in an error
"""
with db_session:
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
my_channel.add_torrent_to_channel(tdef, {'description': 'blabla'})
with open(TORRENT_UBUNTU_FILE, "rb") as torrent_file:
base64_content = base64.b64encode(
torrent_file.read()).decode('utf-8')
post_params = {'torrent': base64_content}
await do_request(
rest_api,
f'channels/{hexlify(my_channel.public_key)}/{my_channel.id_}/torrents',
request_type='PUT',
post_data=post_params,
expected_code=200,
)
def test_delete_torrent_from_channel(metadata_store):
"""
Test deleting a torrent from your channel
"""
channel_metadata = metadata_store.ChannelMetadata.create_channel(
'test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
# Check that nothing is committed when deleting uncommited torrent metadata
torrent = channel_metadata.add_torrent_to_channel(tdef, None)
torrent.soft_delete()
assert not channel_metadata.contents_list
# Check append-only deletion process
torrent = channel_metadata.add_torrent_to_channel(tdef, None)
channel_metadata.commit_channel_torrent()
assert len(channel_metadata.contents_list) == 1
torrent.soft_delete()
channel_metadata.commit_channel_torrent()
assert not channel_metadata.contents_list
def test_commit_channel_torrent(metadata_store):
"""
Test committing a channel torrent
"""
channel = metadata_store.ChannelMetadata.create_channel('test', 'test')
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
channel.add_torrent_to_channel(tdef, None)
# The first run should return the infohash, the second should return None, because nothing was really done
assert channel.commit_channel_torrent()
assert not channel.commit_channel_torrent()
# Test adding flags to channel torrent when adding thumbnail and description
metadata_store.ChannelThumbnail(public_key=channel.public_key,
origin_id=channel.id_,
status=NEW)
metadata_store.ChannelDescription(public_key=channel.public_key,
origin_id=channel.id_,
status=NEW)
assert channel.commit_channel_torrent()
assert channel.reserved_flags == 3
assert not channel.commit_channel_torrent()
def channel_tdef():
return TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' /
'channel_upd.torrent')
def fake_get_metainfo(_, **__):
meta_info = TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo()
return succeed(meta_info)
def test_sanitize_tdef(metadata_store):
tdef = TorrentDef.load(TORRENT_UBUNTU_FILE)
tdef.metainfo["creation date"] = -100000
assert metadata_store.TorrentMetadata.from_dict(
tdef_to_metadata_dict(tdef))
def video_tdef():
return TorrentDef.load(TESTS_DATA_DIR / 'video.avi.torrent')
async def get_torrent_info(self, request):
args = request.query
hops = None
if 'hops' in args:
try:
hops = int(args['hops'])
except ValueError:
return RESTResponse(
{
"error":
f"wrong value of 'hops' parameter: {repr(args['hops'])}"
},
status=HTTP_BAD_REQUEST)
if 'uri' not in args or not args['uri']:
return RESTResponse({"error": "uri parameter missing"},
status=HTTP_BAD_REQUEST)
uri = args['uri']
metainfo = None
if uri.startswith('file:'):
try:
filename = uri_to_path(uri)
tdef = TorrentDef.load(filename)
metainfo = tdef.get_metainfo()
except (TypeError, RuntimeError):
return RESTResponse(
{"error": "error while decoding torrent file"},
status=HTTP_INTERNAL_SERVER_ERROR)
elif uri.startswith('http'):
try:
response = await query_http_uri(uri)
except (ServerConnectionError, ClientResponseError) as e:
return RESTResponse({"error": str(e)},
status=HTTP_INTERNAL_SERVER_ERROR)
if response.startswith(b'magnet'):
_, infohash, _ = parse_magnetlink(response)
if infohash:
metainfo = await self.download_manager.get_metainfo(
infohash, timeout=60, hops=hops, url=response)
else:
metainfo = bdecode_compat(response)
elif uri.startswith('magnet'):
infohash = parse_magnetlink(uri)[1]
if infohash is None:
return RESTResponse({"error": "missing infohash"},
status=HTTP_BAD_REQUEST)
metainfo = await self.download_manager.get_metainfo(infohash,
timeout=60,
hops=hops,
url=uri)
else:
return RESTResponse({"error": "invalid uri"},
status=HTTP_BAD_REQUEST)
if not metainfo:
return RESTResponse({"error": "metainfo error"},
status=HTTP_INTERNAL_SERVER_ERROR)
if not isinstance(metainfo, dict) or b'info' not in metainfo:
self._logger.warning("Received metainfo is not a valid dictionary")
return RESTResponse({"error": "invalid response"},
status=HTTP_INTERNAL_SERVER_ERROR)
# Add the torrent to GigaChannel as a free-for-all entry, so others can search it
self.download_manager.notifier.notify(
NTFY.TORRENT_METADATA_ADDED.value,
tdef_to_metadata_dict(TorrentDef.load_from_dict(metainfo)))
# TODO(Martijn): store the stuff in a database!!!
# TODO(Vadim): this means cache the downloaded torrent in a binary storage, like LevelDB
infohash = hashlib.sha1(lt.bencode(metainfo[b'info'])).digest()
download = self.download_manager.downloads.get(infohash)
metainfo_request = self.download_manager.metainfo_requests.get(
infohash, [None])[0]
download_is_metainfo_request = download == metainfo_request
# Check if the torrent is already in the downloads
encoded_metainfo = deepcopy(metainfo)
# FIXME: json.dumps garbles binary data that is used by the 'pieces' field
# However, this is fine as long as the GUI does not use this field.
encoded_metainfo[b'info'][b'pieces'] = hexlify(
encoded_metainfo[b'info'][b'pieces']).encode('utf-8')
encoded_metainfo = hexlify(
json.dumps(recursive_unicode(encoded_metainfo, ignore_errors=True),
ensure_ascii=False).encode('utf-8'))
return RESTResponse({
"metainfo":
encoded_metainfo,
"download_exists":
download and not download_is_metainfo_request
})