async def test_on_metadata_received_alert(mock_handle, test_download):
"""
Testing whether the right operations happen when we receive metadata
"""
test_future = Future()
mocked_file = Mock()
mocked_file.path = 'test'
test_download.handle.trackers = lambda: []
test_download.handle.get_peer_info = lambda: []
test_download.handle.save_resume_data = lambda: test_future
test_download.handle.rename_file = lambda *_: None
with open(TESTS_DATA_DIR / "bak_single.torrent",
mode='rb') as torrent_file:
encoded_metainfo = torrent_file.read()
decoded_metainfo = bdecode_compat(encoded_metainfo)
get_info_from_handle(test_download.handle).metadata = lambda: bencode(
decoded_metainfo[b'info'])
get_info_from_handle(test_download.handle).files = lambda: [mocked_file]
test_download.checkpoint = lambda: test_future.set_result(None)
test_download.session = MockObject()
test_download.session.torrent_db = None
test_download.handle.save_path = lambda: None
test_download.handle.prioritize_files = lambda _: None
test_download.get_share_mode = lambda: False
test_download.on_metadata_received_alert(None)
await test_future
def on_metadata_received_alert(self, _):
torrent_info = get_info_from_handle(self.handle)
if not torrent_info:
return
metadata = {
b'info': bdecode_compat(torrent_info.metadata()),
b'leechers': 0,
b'seeders': 0
}
trackers = [
tracker['url'].encode('utf-8')
for tracker in self.handle.trackers()
]
if len(trackers) > 1:
metadata[b"announce-list"] = [trackers]
elif trackers:
metadata[b"announce"] = trackers[0]
for peer in self.handle.get_peer_info():
if peer.progress == 1:
metadata[b"seeders"] += 1
else:
metadata[b"leechers"] += 1
try:
self.tdef = TorrentDef.load_from_dict(metadata)
except ValueError as ve:
self._logger.exception(ve)
return
self.set_selected_files()
self.checkpoint()
def _process_scrape_response(self, body):
"""
This function handles the response body of a HTTP tracker,
parsing the results.
"""
# parse the retrieved results
if body is None:
self.failed(msg="no response body")
response_dict = bdecode_compat(body)
if not response_dict:
self.failed(msg="no valid response")
response_list = []
unprocessed_infohash_list = self.infohash_list[:]
if b'files' in response_dict and isinstance(response_dict[b'files'],
dict):
for infohash in response_dict[b'files']:
complete = 0
incomplete = 0
if isinstance(response_dict[b'files'][infohash], dict):
complete = response_dict[b'files'][infohash].get(
b'complete', 0)
incomplete = response_dict[b'files'][infohash].get(
b'incomplete', 0)
# Sow complete as seeders. "complete: number of peers with the entire file, i.e. seeders (integer)"
# - https://wiki.theory.org/BitTorrentSpecification#Tracker_.27scrape.27_Convention
seeders = complete
leechers = incomplete
# Store the information in the dictionary
response_list.append({
'infohash': hexlify(infohash),
'seeders': seeders,
'leechers': leechers
})
# remove this infohash in the infohash list of this session
if infohash in unprocessed_infohash_list:
unprocessed_infohash_list.remove(infohash)
elif b'failure reason' in response_dict:
self._logger.info("%s Failure as reported by tracker [%s]", self,
repr(response_dict[b'failure reason']))
self.failed(msg=repr(response_dict[b'failure reason']))
# handle the infohashes with no result (seeders/leechers = 0/0)
for infohash in unprocessed_infohash_list:
response_list.append({
'infohash': hexlify(infohash),
'seeders': 0,
'leechers': 0
})
self.is_finished = True
return {self.tracker_url: response_list}
def save(self, torrent_filepath=None):
"""
Generate the metainfo and save the torrent file.
:param torrent_filepath: An optional absolute path to where to save the generated .torrent file.
"""
torrent_dict = create_torrent_file(self.files_list, self.torrent_parameters, torrent_filepath=torrent_filepath)
self.metainfo = bdecode_compat(torrent_dict['metainfo'])
self.copy_metainfo_to_torrent_parameters()
self.infohash = torrent_dict['infohash']
def load_from_memory(bencoded_data):
"""
Load some bencoded data into a TorrentDef.
:param bencoded_data: The bencoded data to decode and use as metainfo
"""
metainfo = bdecode_compat(bencoded_data)
# Some versions of libtorrent will not raise an exception when providing invalid data.
# This issue is present in 1.0.8 (included with Tribler 7.3.0), but has been fixed since at least 1.2.1.
if metainfo is None:
raise ValueError("Data is not a bencoded string")
return TorrentDef.load_from_dict(metainfo)
def process_alert(self, alert, hops=0):
alert_type = alert.__class__.__name__
# Periodically, libtorrent will send us a state_update_alert, which contains the torrent status of
# all torrents changed since the last time we received this alert.
if alert_type == 'state_update_alert':
for status in alert.status:
infohash = unhexlify(str(status.info_hash))
if infohash not in self.downloads:
self._logger.debug("Got state_update for unknown torrent %s", hexlify(infohash))
continue
self.downloads[infohash].update_lt_status(status)
infohash = unhexlify(str(alert.handle.info_hash() if hasattr(alert, 'handle') and alert.handle.is_valid()
else getattr(alert, 'info_hash', '')))
download = self.downloads.get(infohash)
if download:
if (download.handle and download.handle.is_valid())\
or (not download.handle and alert_type == 'add_torrent_alert') \
or (download.handle and alert_type == 'torrent_removed_alert'):
download.process_alert(alert, alert_type)
else:
self._logger.debug("Got alert for download without handle %s: %s", hexlify(infohash), alert)
elif infohash:
self._logger.debug("Got alert for unknown download %s: %s", hexlify(infohash), alert)
if alert_type == 'peer_disconnected_alert' and \
self.tribler_session and self.tribler_session.payout_manager:
self.tribler_session.payout_manager.do_payout(alert.pid.to_bytes())
elif alert_type == 'session_stats_alert':
queued_disk_jobs = alert.values['disk.queued_disk_jobs']
queued_write_bytes = alert.values['disk.queued_write_bytes']
num_write_jobs = alert.values['disk.num_write_jobs']
if queued_disk_jobs == queued_write_bytes == num_write_jobs == 0:
self.lt_session_shutdown_ready[hops] = True
if self.session_stats_callback:
self.session_stats_callback(alert)
elif alert_type == "dht_pkt_alert":
# We received a raw DHT message - decode it and check whether it is a BEP33 message.
decoded = bdecode_compat(alert.pkt_buf)
if decoded and b'r' in decoded:
if b'BFsd' in decoded[b'r'] and b'BFpe' in decoded[b'r']:
self.dht_health_manager.received_bloomfilters(decoded[b'r'][b'id'],
bytearray(decoded[b'r'][b'BFsd']),
bytearray(decoded[b'r'][b'BFpe']))
def process_alert(self, alert, hops=0):
alert_type = alert.__class__.__name__
# Periodically, libtorrent will send us a state_update_alert, which contains the torrent status of
# all torrents changed since the last time we received this alert.
if alert_type == 'state_update_alert':
for status in alert.status:
infohash = unhexlify(str(status.info_hash))
if infohash not in self.downloads:
self._logger.debug(
"Got state_update for unknown torrent %s",
hexlify(infohash))
continue
self.downloads[infohash].update_lt_status(status)
infohash = unhexlify(
str(alert.handle.info_hash() if hasattr(alert, 'handle') and alert.
handle.is_valid() else getattr(alert, 'info_hash', '')))
download = self.downloads.get(infohash)
if download:
if (download.handle and download.handle.is_valid())\
or (not download.handle and alert_type == 'add_torrent_alert') \
or (download.handle and alert_type == 'torrent_removed_alert'):
download.process_alert(alert, alert_type)
else:
self._logger.debug(
"Got alert for download without handle %s: %s",
hexlify(infohash), alert)
elif infohash:
self._logger.debug("Got alert for unknown download %s: %s",
hexlify(infohash), alert)
if alert_type == 'listen_succeeded_alert':
# The ``port`` attribute was added in libtorrent 1.1.14.
# Older versions (most notably libtorrent 1.1.13 - the default on Ubuntu 20.04) do not have this attribute.
# We use the now-deprecated ``endpoint`` attribute for these older versions.
self.listen_ports[hops] = getattr(alert, "port", alert.endpoint[1])
elif alert_type == 'peer_disconnected_alert' and \
self.tribler_session and self.tribler_session.payout_manager:
self.tribler_session.payout_manager.do_payout(alert.pid.to_bytes())
elif alert_type == 'session_stats_alert':
queued_disk_jobs = alert.values['disk.queued_disk_jobs']
queued_write_bytes = alert.values['disk.queued_write_bytes']
num_write_jobs = alert.values['disk.num_write_jobs']
if queued_disk_jobs == queued_write_bytes == num_write_jobs == 0:
self.lt_session_shutdown_ready[hops] = True
if self.session_stats_callback:
self.session_stats_callback(alert)
elif alert_type == "dht_pkt_alert":
# Unfortunately, the Python bindings don't have a direction attribute.
# So, we'll have to resort to using the string representation of the alert instead.
incoming = str(alert).startswith('<==')
decoded = bdecode_compat(alert.pkt_buf)
if not decoded:
return
# We are sending a raw DHT message - notify the DHTHealthManager of the outstanding request.
if not incoming and decoded.get(b'y') == b'q' \
and decoded.get(b'q') == b'get_peers' and decoded[b'a'].get(b'scrape') == 1:
self.dht_health_manager.requesting_bloomfilters(
decoded[b't'], decoded[b'a'][b'info_hash'])
# We received a raw DHT message - decode it and check whether it is a BEP33 message.
if incoming and b'r' in decoded and b'BFsd' in decoded[
b'r'] and b'BFpe' in decoded[b'r']:
self.dht_health_manager.received_bloomfilters(
decoded[b't'], bytearray(decoded[b'r'][b'BFsd']),
bytearray(decoded[b'r'][b'BFpe']))
def create_session(self, hops=0, store_listen_port=True):
# Due to a bug in Libtorrent 0.16.18, the outgoing_port and num_outgoing_ports value should be set in
# the settings dictionary
settings = {
'outgoing_port': 0,
'num_outgoing_ports': 1,
'allow_multiple_connections_per_ip': 0
}
# Copy construct so we don't modify the default list
extensions = list(DEFAULT_LT_EXTENSIONS)
# Elric: Strip out the -rcX, -beta, -whatever tail on the version string.
fingerprint = ['TL'] + [
int(x) for x in version_id.split('-')[0].split('.')
] + [0]
if self.dummy_mode:
from unittest.mock import Mock
ltsession = Mock()
ltsession.pop_alerts = lambda: {}
ltsession.listen_port = lambda: 123
ltsession.get_settings = lambda: {"peer_fingerprint": "000"}
else:
ltsession = lt.session(lt.fingerprint(
*fingerprint), flags=0) if hops == 0 else lt.session(flags=0)
if hops == 0:
settings['user_agent'] = 'Tribler/' + version_id
enable_utp = self.tribler_session.config.get_libtorrent_utp()
settings['enable_outgoing_utp'] = enable_utp
settings['enable_incoming_utp'] = enable_utp
if LooseVersion(
self.get_libtorrent_version()) >= LooseVersion("1.1.0"):
settings['prefer_rc4'] = True
settings[
"listen_interfaces"] = "0.0.0.0:%d" % self.tribler_session.config.get_libtorrent_port(
)
else:
pe_settings = lt.pe_settings()
pe_settings.prefer_rc4 = True
ltsession.set_pe_settings(pe_settings)
mid = self.tribler_session.trustchain_keypair.key_to_hash()
settings['peer_fingerprint'] = mid
settings[
'handshake_client_version'] = 'Tribler/' + version_id + '/' + hexlify(
mid)
else:
settings['enable_outgoing_utp'] = True
settings['enable_incoming_utp'] = True
settings['enable_outgoing_tcp'] = False
settings['enable_incoming_tcp'] = False
settings['anonymous_mode'] = True
settings['force_proxy'] = True
if LooseVersion(
self.get_libtorrent_version()) >= LooseVersion("1.1.0"):
settings[
"listen_interfaces"] = "0.0.0.0:%d" % self.tribler_session.config.get_anon_listen_port(
)
# By default block all IPs except 1.1.1.1 (which is used to ensure libtorrent makes a connection to us)
self.update_ip_filter(ltsession, ['1.1.1.1'])
self.set_session_settings(ltsession, settings)
ltsession.set_alert_mask(self.default_alert_mask)
# Load proxy settings
if hops == 0:
proxy_settings = self.tribler_session.config.get_libtorrent_proxy_settings(
)
else:
proxy_settings = list(
self.tribler_session.config.get_anon_proxy_settings())
proxy_host, proxy_ports = proxy_settings[1]
proxy_settings[1] = (proxy_host, proxy_ports[hops - 1])
self.set_proxy_settings(ltsession, *proxy_settings)
for extension in extensions:
ltsession.add_extension(extension)
# Set listen port & start the DHT
if hops == 0:
listen_port = self.tribler_session.config.get_libtorrent_port()
ltsession.listen_on(listen_port, listen_port + 10)
if listen_port != ltsession.listen_port() and store_listen_port:
self.tribler_session.config.set_libtorrent_port_runtime(
ltsession.listen_port())
try:
with open(
self.tribler_session.config.get_state_dir() /
LTSTATE_FILENAME, 'rb') as fp:
lt_state = bdecode_compat(fp.read())
if lt_state is not None:
ltsession.load_state(lt_state)
else:
self._logger.warning(
"the lt.state appears to be corrupt, writing new data on shutdown"
)
except Exception as exc:
self._logger.info(
f"could not load libtorrent state, got exception: {exc!r}. starting from scratch"
)
else:
ltsession.listen_on(
self.tribler_session.config.get_anon_listen_port(),
self.tribler_session.config.get_anon_listen_port() + 20)
settings = {
'upload_rate_limit':
self.tribler_session.config.get_libtorrent_max_upload_rate(),
'download_rate_limit':
self.tribler_session.config.get_libtorrent_max_download_rate()
}
self.set_session_settings(ltsession, settings)
if self.tribler_session.config.get_libtorrent_dht_enabled(
) and not self.dummy_mode:
ltsession.start_dht()
for router in DEFAULT_DHT_ROUTERS:
ltsession.add_dht_router(*router)
ltsession.start_lsd()
self._logger.debug("Started libtorrent session for %d hops on port %d",
hops, ltsession.listen_port())
self.lt_session_shutdown_ready[hops] = False
return ltsession
def get_engineresumedata(self):
return bdecode_compat(
base64.b64decode(
self.config['state']['engineresumedata'].encode('utf-8')))
def get_metainfo(self):
return bdecode_compat(
base64.b64decode(self.config['state']['metainfo'].encode('utf-8')))
def create_session(self, hops=0, store_listen_port=True):
# Due to a bug in Libtorrent 0.16.18, the outgoing_port and num_outgoing_ports value should be set in
# the settings dictionary
self._logger.info('Creating a session')
settings = {
'outgoing_port': 0,
'num_outgoing_ports': 1,
'allow_multiple_connections_per_ip': 0,
'enable_upnp': int(self.config.upnp),
'enable_dht': int(self.config.dht),
'enable_lsd': int(self.config.lsd),
'enable_natpmp': int(self.config.natpmp)
}
# Copy construct so we don't modify the default list
extensions = list(DEFAULT_LT_EXTENSIONS)
self._logger.info(f'Dummy mode: {self.dummy_mode}. Hops: {hops}.')
# Elric: Strip out the -rcX, -beta, -whatever tail on the version string.
fingerprint = ['TL'] + [
int(x) for x in version_id.split('-')[0].split('.')
] + [0]
if self.dummy_mode:
from unittest.mock import Mock
ltsession = Mock()
ltsession.pop_alerts = lambda: {}
ltsession.listen_port = lambda: 123
ltsession.get_settings = lambda: {"peer_fingerprint": "000"}
else:
ltsession = lt.session(lt.fingerprint(
*fingerprint), flags=0) if hops == 0 else lt.session(flags=0)
libtorrent_port = self.config.port or default_network_utils.get_random_free_port(
)
self._libtorrent_port = libtorrent_port
self._logger.info(f'Libtorrent port: {libtorrent_port}')
if hops == 0:
settings['user_agent'] = 'Tribler/' + version_id
enable_utp = self.config.utp
settings['enable_outgoing_utp'] = enable_utp
settings['enable_incoming_utp'] = enable_utp
settings['prefer_rc4'] = True
settings["listen_interfaces"] = "0.0.0.0:%d" % libtorrent_port
settings['peer_fingerprint'] = self.peer_mid
settings[
'handshake_client_version'] = 'Tribler/' + version_id + '/' + hexlify(
self.peer_mid)
else:
settings['enable_outgoing_utp'] = True
settings['enable_incoming_utp'] = True
settings['enable_outgoing_tcp'] = False
settings['enable_incoming_tcp'] = False
settings['anonymous_mode'] = True
settings['force_proxy'] = True
# Anon listen port is never used anywhere, so we let Libtorrent set it
# settings["listen_interfaces"] = "0.0.0.0:%d" % anon_port
# By default block all IPs except 1.1.1.1 (which is used to ensure libtorrent makes a connection to us)
self.update_ip_filter(ltsession, ['1.1.1.1'])
self.set_session_settings(ltsession, settings)
ltsession.set_alert_mask(self.default_alert_mask)
if hops == 0:
proxy_settings = DownloadManager.get_libtorrent_proxy_settings(
self.config)
else:
proxy_settings = [
SOCKS5_PROXY_DEF,
("127.0.0.1", self.socks_listen_ports[hops - 1]), None
]
self.set_proxy_settings(ltsession, *proxy_settings)
for extension in extensions:
ltsession.add_extension(extension)
# Set listen port & start the DHT
if hops == 0:
ltsession.listen_on(libtorrent_port, libtorrent_port + 10)
if libtorrent_port != ltsession.listen_port(
) and store_listen_port:
self.config.port = ltsession.listen_port()
try:
with open(self.state_dir / LTSTATE_FILENAME, 'rb') as fp:
lt_state = bdecode_compat(fp.read())
if lt_state is not None:
ltsession.load_state(lt_state)
else:
self._logger.warning(
"the lt.state appears to be corrupt, writing new data on shutdown"
)
except Exception as exc:
self._logger.info(
f"could not load libtorrent state, got exception: {exc!r}. starting from scratch"
)
else:
# ltsession.listen_on(anon_port, anon_port + 20)
rate = DownloadManager.get_libtorrent_max_upload_rate(self.config)
download_rate = DownloadManager.get_libtorrent_max_download_rate(
self.config)
settings = {
'upload_rate_limit': rate,
'download_rate_limit': download_rate
}
self.set_session_settings(ltsession, settings)
if self.config.dht and not self.dummy_mode:
ltsession.start_dht()
for router in DEFAULT_DHT_ROUTERS:
ltsession.add_dht_router(*router)
ltsession.start_lsd()
self._logger.debug("Started libtorrent session for %d hops on port %d",
hops, ltsession.listen_port())
self.lt_session_shutdown_ready[hops] = False
return ltsession
async def get_torrent_info(self, request):
args = request.query
hops = None
if 'hops' in args:
try:
hops = int(args['hops'])
except ValueError:
return RESTResponse(
{
"error":
f"wrong value of 'hops' parameter: {repr(args['hops'])}"
},
status=HTTP_BAD_REQUEST)
if 'uri' not in args or not args['uri']:
return RESTResponse({"error": "uri parameter missing"},
status=HTTP_BAD_REQUEST)
uri = args['uri']
if uri.startswith('file:'):
try:
filename = url2pathname(uri[5:])
tdef = TorrentDef.load(filename)
metainfo = tdef.get_metainfo()
except (TypeError, RuntimeError):
return RESTResponse(
{"error": "error while decoding torrent file"},
status=HTTP_INTERNAL_SERVER_ERROR)
elif uri.startswith('http'):
try:
async with ClientSession(raise_for_status=True) as session:
response = await session.get(uri)
response = await response.read()
except (ServerConnectionError, ClientResponseError) as e:
return RESTResponse({"error": str(e)},
status=HTTP_INTERNAL_SERVER_ERROR)
if response.startswith(b'magnet'):
_, infohash, _ = parse_magnetlink(response)
if infohash:
metainfo = await self.session.dlmgr.get_metainfo(
infohash, timeout=60, hops=hops, url=response)
else:
metainfo = bdecode_compat(response)
elif uri.startswith('magnet'):
infohash = parse_magnetlink(uri)[1]
if infohash is None:
return RESTResponse({"error": "missing infohash"},
status=HTTP_BAD_REQUEST)
metainfo = await self.session.dlmgr.get_metainfo(infohash,
timeout=60,
hops=hops,
url=uri)
else:
return RESTResponse({"error": "invalid uri"},
status=HTTP_BAD_REQUEST)
if not metainfo:
return RESTResponse({"error": "metainfo error"},
status=HTTP_INTERNAL_SERVER_ERROR)
if not isinstance(metainfo, dict) or b'info' not in metainfo:
self._logger.warning("Received metainfo is not a valid dictionary")
return RESTResponse({"error": "invalid response"},
status=HTTP_INTERNAL_SERVER_ERROR)
# Add the torrent to GigaChannel as a free-for-all entry, so others can search it
self.session.mds.TorrentMetadata.add_ffa_from_dict(
tdef_to_metadata_dict(TorrentDef.load_from_dict(metainfo)))
# TODO(Martijn): store the stuff in a database!!!
# TODO(Vadim): this means cache the downloaded torrent in a binary storage, like LevelDB
infohash = hashlib.sha1(bencode(metainfo[b'info'])).digest()
download = self.session.dlmgr.downloads.get(infohash)
metainfo_request = self.session.dlmgr.metainfo_requests.get(
infohash, [None])[0]
download_is_metainfo_request = download == metainfo_request
# Check if the torrent is already in the downloads
encoded_metainfo = deepcopy(metainfo)
# FIXME: json.dumps garbles binary data that is used by the 'pieces' field
# However, this is fine as long as the GUI does not use this field.
encoded_metainfo[b'info'][b'pieces'] = hexlify(
encoded_metainfo[b'info'][b'pieces']).encode('utf-8')
encoded_metainfo = hexlify(
json.dumps(recursive_unicode(encoded_metainfo, ignore_errors=True),
ensure_ascii=False).encode('utf-8'))
return RESTResponse({
"metainfo":
encoded_metainfo,
"download_exists":
download and not download_is_metainfo_request
})
def test_load_from_dict():
with open(TESTS_DATA_DIR / "bak_single.torrent",
mode='rb') as torrent_file:
encoded_metainfo = torrent_file.read()
assert TorrentDef.load_from_dict(bdecode_compat(encoded_metainfo))
async def create_torrent(self, request):
parameters = await request.json()
params = {}
if 'files' in parameters and parameters['files']:
file_path_list = [
ensure_unicode(f, 'utf-8') for f in parameters['files']
]
else:
return RESTResponse({"error": "files parameter missing"},
status=HTTP_BAD_REQUEST)
if 'description' in parameters and parameters['description']:
params['comment'] = parameters['description']
if 'trackers' in parameters and parameters['trackers']:
tracker_url_list = parameters['trackers']
params['announce'] = tracker_url_list[0]
params['announce-list'] = tracker_url_list
name = 'unknown'
if 'name' in parameters and parameters['name']:
name = parameters['name']
params['name'] = name
export_dir = None
if 'export_dir' in parameters and parameters['export_dir']:
export_dir = Path(parameters['export_dir'])
from tribler_core.version import version_id
params['created by'] = '%s version: %s' % ('Tribler', version_id)
params['nodes'] = False
params['httpseeds'] = False
params['encoding'] = False
params['piece length'] = 0 # auto
try:
result = await self.session.dlmgr.create_torrent_file(
file_path_list, recursive_bytes(params))
except (IOError, UnicodeDecodeError, RuntimeError) as e:
self._logger.exception(e)
return return_handled_exception(request, e)
metainfo_dict = bdecode_compat(result['metainfo'])
if export_dir and export_dir.exists():
save_path = export_dir / ("%s.torrent" % name)
with open(save_path, "wb") as fd:
fd.write(result['metainfo'])
# Download this torrent if specified
if 'download' in request.query and request.query[
'download'] and request.query['download'] == "1":
download_config = DownloadConfig()
download_config.set_dest_dir(result['base_path'] if len(
file_path_list) == 1 else result['base_dir'])
try:
self.session.dlmgr.start_download(
tdef=TorrentDef(metainfo_dict), config=download_config)
except DuplicateDownloadException:
self._logger.warning(
"The created torrent is already being downloaded.")
return RESTResponse(
json.dumps({
"torrent":
base64.b64encode(result['metainfo']).decode('utf-8')
}))
