async def get_channels_peers(self, _):
# Get debug stats for peers serving channels
current_time = time.time()
result = []
mapping = self.gigachannel_community.channels_peers
with db_session:
for id_tuple, peers in mapping._channels_dict.items(): # pylint:disable=W0212
channel_pk, channel_id = id_tuple
chan = self.mds.ChannelMetadata.get(public_key=channel_pk,
id_=channel_id)
peers_list = []
for p in peers:
peers_list.append(
(hexlify(p.mid), int(current_time - p.last_response)))
chan_dict = {
"channel_name": chan.title if chan else None,
"channel_pk": hexlify(channel_pk),
"channel_id": channel_id,
"peers": peers_list,
}
result.append(chan_dict)
return RESTResponse({"channels_list": result})
async def on_remote_select_response(self, peer, response):
request = self.request_cache.get(hexlify(peer.mid), response.id)
if request is None:
return
# Check for limit on the number of packets per request
if request.packets_limit > 1:
request.packets_limit -= 1
else:
self.request_cache.pop(hexlify(peer.mid), response.id)
# We use responses for requests about subscribed channels to bump our local channels ratings
peer_vote = peer if request.request_kwargs.get("subscribed", None) is True else None
result = await self.mds.process_compressed_mdblob_threaded(response.raw_blob, peer_vote_for_channels=peer_vote)
# Maybe move this callback to MetadataStore side?
if self.notifier:
new_channels = [
md.to_simple_dict()
for md, result in result
if md and md.metadata_type == CHANNEL_TORRENT and result == UNKNOWN_CHANNEL and md.origin_id == 0
]
if new_channels:
self.notifier.notify(
NTFY.CHANNEL_DISCOVERED, {"results": new_channels, "uuid": str(CHANNELS_VIEW_UUID)}
)
async def start_handle(self, download, atp):
ltsession = self.get_session(download.config.get_hops())
infohash = download.get_def().get_infohash()
if infohash in self.metainfo_requests and self.metainfo_requests[infohash][0] != download:
self._logger.info("Cancelling metainfo request(s) for infohash:%s", hexlify(infohash))
metainfo_dl, _ = self.metainfo_requests.pop(infohash)
# Leave the checkpoint. Any checkpoint that exists will belong to the download we are currently starting.
await self.remove_download(metainfo_dl, remove_content=True, remove_checkpoint=False)
self.downloads[infohash] = download
known = {unhexlify(str(h.info_hash())): h for h in ltsession.get_torrents()}
existing_handle = known.get(infohash)
if existing_handle:
# Reuse existing handle
self._logger.debug("Reusing handle %s", hexlify(infohash))
download.post_alert('add_torrent_alert', dict(handle=existing_handle))
else:
# Otherwise, add it anew
self._logger.debug("Adding handle %s", hexlify(infohash))
# To prevent flooding the DHT with a short burst of queries and triggering
# flood protection, we postpone adding torrents until we get enough DHT peers.
# The asynchronous wait should be done as close as possible to the actual
# Libtorrent calls, so the higher-level download-adding logic does not block.
# Otherwise, e.g. if added to the Session init sequence, this results in startup
# time increasing by 10-20 seconds.
# See https://github.com/Tribler/tribler/issues/5319
if self.dht_readiness_timeout > 0 and self._dht_ready_task is not None:
try:
await wait_for(shield(self._dht_ready_task), timeout=self.dht_readiness_timeout)
except asyncio.TimeoutError:
self._logger.warning("Timeout waiting for libtorrent DHT getting enough peers")
ltsession.async_add_torrent(encode_atp(atp))
return await download.future_added
async def test_update_multiple_metadata_entries(enable_chant, enable_api,
add_fake_torrents_channels,
session):
"""
Test updating attributes of several metadata entities at once with a PATCH request to REST API
"""
# Test handling the wrong/empty JSON gracefully
await do_request(session,
'metadata',
expected_code=400,
request_type='PATCH',
post_data='abc')
# Test trying update a non-existing entry
await do_request(
session,
'metadata',
post_data=[{
'public_key': hexlify(b'1' * 64),
'id': 111
}],
expected_code=404,
request_type='PATCH',
)
with db_session:
md1 = session.mds.TorrentMetadata(title='old1',
infohash=random_infohash())
md2 = session.mds.ChannelMetadata(title='old2',
infohash=random_infohash(),
subscribed=False)
NEW_NAME1 = "updated1"
NEW_NAME2 = "updated2"
patch_data = [
{
'public_key': hexlify(md1.public_key),
'id': md1.id_,
'title': NEW_NAME1
},
{
'public_key': hexlify(md2.public_key),
'id': md2.id_,
'title': NEW_NAME2,
'subscribed': 1
},
]
await do_request(session,
'metadata',
post_data=patch_data,
expected_code=200,
request_type='PATCH')
with db_session:
entry1 = session.mds.ChannelNode.get(rowid=md1.rowid)
assert NEW_NAME1 == entry1.title
assert UPDATED == entry1.status
entry2 = session.mds.ChannelNode.get(rowid=md2.rowid)
assert NEW_NAME2 == entry2.title
assert UPDATED == entry2.status
assert entry2.subscribed
def test_ffa_serialization(metadata_store):
"""
Test converting free-for-all (unsigned) torrent metadata to payload and back
"""
metadata1 = metadata_store.ChannelNode.from_dict({
"public_key": b"",
"id_": "123"
})
serialized1 = metadata1.serialized()
# Make sure sig is really zeroes
assert hexlify(serialized1).endswith(hexlify(NULL_SIG))
metadata1.delete()
orm.flush()
metadata2 = metadata_store.ChannelNode.from_payload(
ChannelNodePayload.from_signed_blob(serialized1))
serialized2 = metadata2.serialized()
assert serialized1 == serialized2
# Check that it is impossible to create FFA node without specifying id_
with pytest.raises(InvalidChannelNodeException):
metadata_store.ChannelNode.from_dict({"public_key": b""})
# Check that it is impossible to create FFA payload with non-null signature
with pytest.raises(InvalidSignatureException):
ChannelNodePayload(CHANNEL_NODE,
0,
NULL_KEY,
0,
0,
0,
signature=b"123")
# Check that creating a pair of metadata entries do not trigger uniqueness constraints error
metadata_store.ChannelNode.from_dict({"public_key": b"", "id_": "124"})
metadata_store.ChannelNode.from_dict({"public_key": b"", "id_": "125"})
async def test_delete_multiple_metadata_entries(enable_chant, enable_api,
session):
"""
Test deleting multiple entries with JSON REST API
"""
with db_session:
md1 = session.mds.TorrentMetadata(title='old1',
infohash=random_infohash())
md2 = session.mds.TorrentMetadata(title='old2',
infohash=random_infohash())
assert session.mds.ChannelNode.select().count() == 2
patch_data = [
{
'public_key': hexlify(md1.public_key),
'id': md1.id_
},
{
'public_key': hexlify(md2.public_key),
'id': md2.id_
},
]
await do_request(session,
'metadata',
post_data=patch_data,
expected_code=200,
request_type='DELETE')
with db_session:
assert session.mds.ChannelNode.select().count() == 0
async def test_get_channels_peers(rest_api, endpoint, metadata_store,
mock_gigachannel_community): # pylint: disable=W0621, C0321
"""
Test getting debug info about the state of channels to peers mapping
"""
mapping = mock_gigachannel_community.channels_peers = ChannelsPeersMapping(
)
peer_key = default_eccrypto.generate_key("curve25519")
chan_key = default_eccrypto.generate_key("curve25519")
with db_session:
chan = metadata_store.ChannelMetadata(sign_with=chan_key,
name="bla",
infohash=random_infohash())
peer = Peer(peer_key, ("1.2.3.4", 5))
mapping.add(peer, chan.public_key, chan.id_)
result = await do_request(
rest_api,
'remote_query/channels_peers',
request_type="GET",
expected_code=200,
)
first_result = result["channels_list"][0]
assert first_result["channel_name"] == chan.title
assert first_result["channel_pk"] == hexlify(chan.public_key)
assert first_result["channel_id"] == chan.id_
assert first_result["peers"][0][0] == hexlify(peer.mid)
async def remove_download(self,
download,
remove_content=False,
remove_checkpoint=True):
infohash = download.get_def().get_infohash()
handle = download.handle
# Note that the following block of code needs to be able to deal with multiple simultaneous
# calls using the same download object. We need to make sure that we don't return without
# the removal having finished.
if handle:
if handle.is_valid():
download.stream.disable()
self._logger.debug("Removing handle %s", hexlify(infohash))
ltsession = self.get_session(download.config.get_hops())
ltsession.remove_torrent(handle, int(remove_content))
# We need to wait even if the handle is invalid. It's important to synchronize
# here because the upcoming call to shutdown will also cancel future_removed.
await download.future_removed
else:
self._logger.debug(
"Cannot remove handle %s because it does not exists",
hexlify(infohash))
await download.shutdown()
if infohash in self.downloads and self.downloads[infohash] == download:
self.downloads.pop(infohash)
if remove_checkpoint:
self.remove_config(infohash)
else:
self._logger.debug("Cannot remove unknown download")
async def regenerate_channel_torrent(self, channel_pk, channel_id):
self._logger.info("Regenerating personal channel %s %i",
hexlify(channel_pk), channel_id)
with db_session:
channel = self.mds.ChannelMetadata.get(public_key=channel_pk,
id_=channel_id)
if channel is None:
self._logger.warning(
"Tried to regenerate non-existing channel %s %i",
hexlify(channel_pk), channel_id)
return None
channel_dirname = channel.dirname
for d in self.download_manager.get_downloads_by_name(channel_dirname):
await self.download_manager.remove_download(d, remove_content=True)
with db_session:
channel = self.mds.ChannelMetadata.get_for_update(
public_key=channel_pk, id_=channel_id)
regenerated = channel.consolidate_channel_torrent()
# If the user created their channel, but added no torrents to it,
# the channel torrent will not be created.
if regenerated is None:
return None
tdef = TorrentDef.load_from_dict(regenerated)
self.updated_my_channel(tdef)
return tdef
def _process_scrape_response(self, body):
"""
This function handles the response body of a HTTP tracker,
parsing the results.
"""
# parse the retrieved results
if body is None:
self.failed(msg="no response body")
response_dict = bdecode_compat(body)
if not response_dict:
self.failed(msg="no valid response")
response_list = []
unprocessed_infohash_list = self.infohash_list[:]
if b'files' in response_dict and isinstance(response_dict[b'files'],
dict):
for infohash in response_dict[b'files']:
complete = 0
incomplete = 0
if isinstance(response_dict[b'files'][infohash], dict):
complete = response_dict[b'files'][infohash].get(
b'complete', 0)
incomplete = response_dict[b'files'][infohash].get(
b'incomplete', 0)
# Sow complete as seeders. "complete: number of peers with the entire file, i.e. seeders (integer)"
# - https://wiki.theory.org/BitTorrentSpecification#Tracker_.27scrape.27_Convention
seeders = complete
leechers = incomplete
# Store the information in the dictionary
response_list.append({
'infohash': hexlify(infohash),
'seeders': seeders,
'leechers': leechers
})
# remove this infohash in the infohash list of this session
if infohash in unprocessed_infohash_list:
unprocessed_infohash_list.remove(infohash)
elif b'failure reason' in response_dict:
self._logger.info("%s Failure as reported by tracker [%s]", self,
repr(response_dict[b'failure reason']))
self.failed(msg=repr(response_dict[b'failure reason']))
# handle the infohashes with no result (seeders/leechers = 0/0)
for infohash in unprocessed_infohash_list:
response_list.append({
'infohash': hexlify(infohash),
'seeders': 0,
'leechers': 0
})
self.is_finished = True
return {self.tracker_url: response_list}
async def on_remote_select_response(self, peer, response_payload):
"""
Match the the response that we received from the network to a query cache
and process it by adding the corresponding entries to the MetadataStore database.
This processes both direct responses and pushback (updates) responses
"""
self.logger.info(f"Response from {hexlify(peer.mid)}")
# ACHTUNG! the returned request cache can be any one of SelectRequest, PushbackWindow
request = self.request_cache.get(hexlify(peer.mid), response_payload.id)
if request is None:
return
# Remember that at least a single packet was received was received from the queried peer.
if isinstance(request, SelectRequest):
request.peer_responded = True
# Check for limit on the number of packets per request
if request.packets_limit > 1:
request.packets_limit -= 1
else:
self.request_cache.pop(hexlify(peer.mid), response_payload.id)
processing_results = await self.mds.process_compressed_mdblob_threaded(response_payload.raw_blob)
self.logger.info(f"Response result: {processing_results}")
# If we know about updated versions of the received stuff, push the updates back
if isinstance(request, SelectRequest) and self.settings.push_updates_back_enabled:
newer_entities = [r.md_obj for r in processing_results if r.obj_state == ObjState.GOT_NEWER_VERSION]
self.send_db_results(peer, response_payload.id, newer_entities)
if self.settings.channel_query_back_enabled:
for result in processing_results:
# Query back the sender for preview contents for the new channels
# The fact that the object is previously unknown is indicated by process_payload in the
# .obj_state property of returned ProcessingResults objects.
if result.obj_state == ObjState.UNKNOWN_OBJECT and result.md_obj.metadata_type in (
CHANNEL_TORRENT,
COLLECTION_NODE,
):
request_dict = {
"metadata_type": [COLLECTION_NODE, REGULAR_TORRENT],
"channel_pk": hexlify(result.md_obj.public_key),
"origin_id": result.md_obj.id_,
"first": 0,
"last": self.settings.max_channel_query_back,
}
self.send_remote_select(peer=peer, **request_dict)
# Query back for missing dependencies, e.g. thumbnail/description.
# The fact that some dependency is missing is checked by the lower layer during
# the query to process_payload and indicated through .missing_deps property of the
# ProcessingResults objects returned by process_payload.
for dep_query_dict in result.missing_deps:
self.send_remote_select(peer=peer, **dep_query_dict)
if isinstance(request, SelectRequest) and request.processing_callback:
request.processing_callback(request, processing_results)
async def check_torrent_health(self,
infohash,
timeout=20,
scrape_now=False):
"""
Check the health of a torrent with a given infohash.
:param infohash: Torrent infohash.
:param timeout: The timeout to use in the performed requests
:param scrape_now: Flag whether we want to force scraping immediately
"""
tracker_set = []
# We first check whether the torrent is already in the database and checked before
with db_session:
result = self.tribler_session.mds.TorrentState.get(
infohash=database_blob(infohash))
if result:
torrent_id = result.infohash
last_check = result.last_check
time_diff = time.time() - last_check
if time_diff < MIN_TORRENT_CHECK_INTERVAL and not scrape_now:
self._logger.debug(
"time interval too short, not doing torrent health check for %s",
hexlify(infohash))
return {
"db": {
"seeders": result.seeders,
"leechers": result.leechers,
"infohash": hexlify(infohash)
}
}
# get torrent's tracker list from DB
tracker_set = self.get_valid_trackers_of_torrent(torrent_id)
tasks = []
for tracker_url in tracker_set:
session = self._create_session_for_request(tracker_url,
timeout=timeout)
session.add_infohash(infohash)
tasks.append(self.connect_to_tracker(session))
if has_bep33_support():
# Create a (fake) DHT session for the lookup if we have support for BEP33.
session = FakeBep33DHTSession(self.tribler_session, infohash,
timeout)
else:
# Otherwise, fallback on the normal DHT metainfo lookups.
session = FakeDHTSession(self.tribler_session, infohash, timeout)
self._session_list['DHT'].append(session)
tasks.append(self.connect_to_tracker(session))
res = await gather(*tasks, return_exceptions=True)
return self.on_torrent_health_check_completed(infohash, res)
def generate_dht_stats(self):
self.dht_stats = {
"num_tokens": randint(10, 50),
"routing_table_buckets": randint(1, 10),
"num_keys_in_store": randint(100, 500),
"num_store_for_me": {hexlify(os.urandom(20)): randint(1, 8)},
"num_peers_in_store": {},
"node_id": hexlify(os.urandom(20)),
"peer_id": hexlify(os.urandom(20)),
"routing_table_size": randint(10, 50),
}
async def get_view(self, request):
if not self.trust_graph:
self.initialize_graph()
def get_bandwidth_blocks(public_key, limit=5):
return self.trustchain_db.get_latest_blocks(
public_key, limit=limit, block_types=[b'tribler_bandwidth'])
def get_friends(public_key, limit=5):
return self.trustchain_db.get_connected_users(public_key,
limit=limit)
depth = 0
if 'depth' in request.query:
depth = int(request.query['depth'])
# If depth is zero or not provided then fetch all depth levels
fetch_all = depth == 0
try:
if fetch_all:
self.trust_graph.reset(hexlify(self.public_key))
if fetch_all or depth == 1:
self.trust_graph.add_blocks(
get_bandwidth_blocks(self.public_key, limit=100))
if fetch_all or depth == 2:
for friend in get_friends(self.public_key):
self.trust_graph.add_blocks(
get_bandwidth_blocks(unhexlify(friend['public_key']),
limit=10))
if fetch_all or depth == 3:
for friend in get_friends(self.public_key):
self.trust_graph.add_blocks(
get_bandwidth_blocks(unhexlify(friend['public_key'])))
for fof in get_friends(unhexlify(friend['public_key'])):
self.trust_graph.add_blocks(
get_bandwidth_blocks(unhexlify(fof['public_key'])))
if fetch_all or depth == 4:
for user_block in self.trustchain_db.get_users():
self.trust_graph.add_blocks(
get_bandwidth_blocks(
unhexlify(user_block['public_key'])))
except TrustGraphException as tgex:
self.logger.warning(tgex)
graph_data = self.trust_graph.compute_node_graph()
return RESTResponse({
'root_public_key': hexlify(self.public_key),
'graph': graph_data,
'bootstrap': self.get_bootstrap_info(),
'num_tx': len(graph_data['edge']),
'depth': depth,
})
def update_peer(self, mid, infohash, balance):
"""
Update a peer with a specific mid for a specific infohash.
"""
self.logger.debug("Updating peer with mid %s and ih %s (balance: %d)",
hexlify(mid), hexlify(infohash), balance)
if mid not in self.tribler_peers:
self.tribler_peers[mid] = {}
self.tribler_peers[mid][infohash] = balance
def setup(self, config=None, hidden=False, checkpoint_disabled=False):
"""
Create a Download object. Used internally by Session.
@param config DownloadConfig or None (in which case a new DownloadConfig() is created
:returns a Deferred to which a callback can be added which returns the result of network_create_engine_wrapper.
"""
self.hidden = hidden
self.checkpoint_disabled = checkpoint_disabled or self.dummy
self.config = config or DownloadConfig(
state_dir=self.session.config.get_state_dir())
self._logger.debug("Setup: %s", hexlify(self.tdef.get_infohash()))
self.checkpoint()
atp = {
"save_path":
path_util.normpath(get_default_dest_dir() /
self.config.get_dest_dir()),
"storage_mode":
lt.storage_mode_t.storage_mode_sparse,
"flags":
lt.add_torrent_params_flags_t.flag_paused
| lt.add_torrent_params_flags_t.flag_duplicate_is_error
| lt.add_torrent_params_flags_t.flag_update_subscribe
}
if self.config.get_share_mode():
atp["flags"] = atp[
"flags"] | lt.add_torrent_params_flags_t.flag_share_mode
if self.config.get_upload_mode():
atp["flags"] = atp[
"flags"] | lt.add_torrent_params_flags_t.flag_upload_mode
resume_data = self.config.get_engineresumedata()
if not isinstance(self.tdef, TorrentDefNoMetainfo):
metainfo = self.tdef.get_metainfo()
torrentinfo = lt.torrent_info(metainfo)
atp["ti"] = torrentinfo
if resume_data and isinstance(resume_data, dict):
# Rewrite save_path as a global path, if it is given as a relative path
if b"save_path" in resume_data and not path_util.isabs(
ensure_unicode(resume_data[b"save_path"], 'utf8')):
resume_data[
b"save_path"] = self.state_dir / ensure_unicode(
resume_data[b"save_path"], 'utf8')
atp["resume_data"] = lt.bencode(resume_data)
else:
atp["url"] = self.tdef.get_url(
) or "magnet:?xt=urn:btih:" + hexlify(self.tdef.get_infohash())
atp["name"] = self.tdef.get_name_as_unicode()
return atp
async def _on_remote_select_basic(self, peer, request_payload, force_eva_response=False):
try:
db_results = await self.process_rpc_query(request_payload.json)
# When we send our response to a host, we open a window of opportunity
# for it to push back updates
if db_results and not self.request_cache.has(hexlify(peer.mid), request_payload.id):
self.request_cache.add(PushbackWindow(self.request_cache, hexlify(peer.mid), request_payload.id))
self.send_db_results(peer, request_payload.id, db_results, force_eva_response)
except (OperationalError, TypeError, ValueError) as error:
self.logger.error(f"Remote select. The error occurred: {error}")
async def test_remote_select_torrents(self):
"""
Test dropping packets that go over the response limit for a remote select.
"""
peer = self.nodes[0].my_peer
mds0 = self.nodes[0].overlay.mds
mds1 = self.nodes[1].overlay.mds
with db_session:
chan = mds0.ChannelMetadata.create_channel(random_string(100), "")
torrent_infohash = random_infohash()
torrent = mds0.TorrentMetadata(origin_id=chan.id_,
infohash=torrent_infohash,
title='title1')
torrent.sign()
processing_results = []
def callback(request, results): # pylint: disable=unused-argument
processing_results.extend(results)
self.nodes[1].overlay.send_remote_select(
peer,
metadata_type=REGULAR_TORRENT,
infohash=hexlify(torrent_infohash),
processing_callback=callback)
await self.deliver_messages()
assert len(processing_results) == 1
obj = processing_results[0].md_obj
assert isinstance(obj, mds1.TorrentMetadata)
assert obj.title == 'title1'
assert obj.health.seeders == 0
with db_session:
torrent = mds0.TorrentMetadata.get(infohash=torrent_infohash)
torrent.timestamp += 1
torrent.title = 'title2'
torrent.sign()
processing_results = []
self.nodes[1].overlay.send_remote_select(
peer,
metadata_type=REGULAR_TORRENT,
infohash=hexlify(torrent_infohash),
processing_callback=callback)
await self.deliver_messages()
assert len(processing_results) == 1
obj = processing_results[0].md_obj
assert isinstance(obj, mds1.TorrentMetadata)
assert obj.health.seeders == 0
def on_torrent_health_check_completed(self, infohash, result):
final_response = {}
if not result or not isinstance(result, list):
self._logger.info("Received invalid torrent checker result")
self.tribler_session.notifier.notify(
NTFY.CHANNEL_ENTITY_UPDATED, {
"infohash": hexlify(infohash),
"num_seeders": 0,
"num_leechers": 0,
"last_tracker_check": int(time.time()),
"health": "updated"
})
return final_response
torrent_update_dict = {
'infohash': infohash,
'seeders': 0,
'leechers': 0,
'last_check': int(time.time())
}
for response in reversed(result):
if isinstance(response, Exception):
final_response[response.tracker_url] = {'error': str(response)}
continue
elif response is None:
self._logger.warning("Torrent health response is none!")
continue
response_keys = list(response.keys())
final_response[response_keys[0]] = response[response_keys[0]][0]
s = response[response_keys[0]][0]['seeders']
l = response[response_keys[0]][0]['leechers']
# More leeches is better, because undefined peers are marked as leeches in DHT
if s > torrent_update_dict['seeders'] or \
(s == torrent_update_dict['seeders'] and l > torrent_update_dict['leechers']):
torrent_update_dict['seeders'] = s
torrent_update_dict['leechers'] = l
self._update_torrent_result(torrent_update_dict)
self.update_torrents_checked(torrent_update_dict)
# TODO: DRY! Stop doing lots of formats, just make REST endpoint automatically encode binary data to hex!
self.tribler_session.notifier.notify(
NTFY.CHANNEL_ENTITY_UPDATED, {
"infohash": hexlify(infohash),
"num_seeders": torrent_update_dict["seeders"],
"num_leechers": torrent_update_dict["leechers"],
"last_tracker_check": torrent_update_dict["last_check"],
"health": "updated"
})
return final_response
async def on_remote_select_response(self, peer, response_payload):
"""
Match the the response that we received from the network to a query cache
and process it by adding the corresponding entries to the MetadataStore database.
This processes both direct responses and pushback (updates) responses
"""
self.logger.info(f"Response from {hexlify(peer.mid)}")
# ACHTUNG! the returned request cache can be either a SelectRequest or PushbackWindow
request = self.request_cache.get(hexlify(peer.mid),
response_payload.id)
if request is None:
return
# Check for limit on the number of packets per request
if request.packets_limit > 1:
request.packets_limit -= 1
else:
self.request_cache.pop(hexlify(peer.mid), response_payload.id)
processing_results = await self.mds.process_compressed_mdblob_threaded(
response_payload.raw_blob)
self.logger.info(f"Response result: {processing_results}")
# If we now about updated versions of the received stuff, push the updates back
if isinstance(
request,
SelectRequest) and self.settings.push_updates_back_enabled:
newer_entities = [
md for md, result in processing_results
if result == GOT_NEWER_VERSION
]
self.send_db_results(peer, response_payload.id, newer_entities)
# Query back the sender for preview contents for the new channels
# TODO: maybe transform this into a processing_callback?
if self.settings.channel_query_back_enabled:
new_channels = [
md for md, result in processing_results
if result in (UNKNOWN_CHANNEL, UNKNOWN_COLLECTION)
]
for channel in new_channels:
request_dict = {
"channel_pk": hexlify(channel.public_key),
"origin_id": channel.id_,
"first": 0,
"last": self.settings.max_channel_query_back,
}
self.send_remote_select(peer=peer, **request_dict)
if isinstance(request, SelectRequest) and request.processing_callback:
request.processing_callback(request, processing_results)
def commit_channel_torrent(self,
new_start_timestamp=None,
commit_list=None):
"""
Collect new/uncommitted and marked for deletion metadata entries, commit them to a channel torrent and
remove the obsolete entries if the commit succeeds.
:param new_start_timestamp: change the start_timestamp of the committed channel entry to this value
:param commit_list: the list of ORM objects to commit into this channel torrent
:return The new infohash, should be used to update the downloads
"""
md_list = commit_list or self.get_contents_to_commit()
if not md_list:
return None
try:
update_dict, torrent = self.update_channel_torrent(md_list)
except OSError:
self._logger.error(
"Error during channel torrent commit, not going to garbage collect the channel. Channel %s",
hexlify(self.public_key),
)
return None
if new_start_timestamp:
update_dict['start_timestamp'] = new_start_timestamp
# Update channel infohash, etc
for attr, val in update_dict.items():
setattr(self, attr, val)
self.local_version = self.timestamp
self.sign()
# Change the statuses of committed entries and clean up obsolete TODELETE entries
for g in md_list:
if g.status in [NEW, UPDATED]:
g.status = COMMITTED
elif g.status == TODELETE:
g.delete()
# Write the channel mdblob to disk
self.status = COMMITTED # pylint: disable=W0201
self.to_file(self._channels_dir / (self.dirname + BLOB_EXTENSION))
self._logger.info(
"Channel %s committed with %i new entries. New version is %i",
hexlify(self.public_key),
len(md_list),
update_dict['timestamp'],
)
return torrent
async def test_check_torrent_health(self):
"""
Test the endpoint to fetch the health of a chant-managed, infohash-only torrent
"""
infohash = b'a' * 20
tracker_url = 'udp://localhost:%s/announce' % self.udp_port
self.udp_tracker.tracker_info.add_info_about_infohash(
infohash, 12, 11, 1)
with db_session:
tracker_state = self.session.mds.TrackerState(url=tracker_url)
torrent_state = self.session.mds.TorrentState(
trackers=tracker_state, infohash=infohash)
self.session.mds.TorrentMetadata(infohash=infohash,
title='ubuntu-torrent.iso',
size=42,
tracker_info=tracker_url,
health=torrent_state)
url = 'metadata/torrents/%s/health?timeout=%s&refresh=1' % (
hexlify(infohash), TORRENT_CHECK_TIMEOUT)
# Initialize the torrent checker
self.session.torrent_checker = TorrentChecker(self.session)
await self.session.torrent_checker.initialize()
# Add mock DHT response - we both need to account for the case when BEP33 is used and the old lookup method
self.session.dlmgr = MockObject()
self.session.dlmgr.get_metainfo = lambda _, **__: succeed(None)
self.session.dlmgr.dht_health_manager = MockObject()
dht_health_dict = {
"infohash": hexlify(infohash),
"seeders": 1,
"leechers": 2
}
self.session.dlmgr.dht_health_manager.get_health = lambda *_, **__: succeed(
{"DHT": [dht_health_dict]})
self.session.dlmgr.get_channel_downloads = lambda: []
# Left for compatibility with other tests in this object
await self.udp_tracker.start()
json_response = await self.do_request(url)
self.assertIn("health", json_response)
self.assertIn("udp://localhost:%s" % self.udp_port,
json_response['health'])
if has_bep33_support():
self.assertIn("DHT", json_response['health'])
json_response = await self.do_request(url + '&nowait=1')
self.assertDictEqual(json_response, {'checking': '1'})
def process_alert(self, alert, hops=0):
alert_type = alert.__class__.__name__
# Periodically, libtorrent will send us a state_update_alert, which contains the torrent status of
# all torrents changed since the last time we received this alert.
if alert_type == 'state_update_alert':
for status in alert.status:
infohash = unhexlify(str(status.info_hash))
if infohash not in self.downloads:
self._logger.debug("Got state_update for unknown torrent %s", hexlify(infohash))
continue
self.downloads[infohash].update_lt_status(status)
infohash = unhexlify(str(alert.handle.info_hash() if hasattr(alert, 'handle') and alert.handle.is_valid()
else getattr(alert, 'info_hash', '')))
download = self.downloads.get(infohash)
if download:
if (download.handle and download.handle.is_valid())\
or (not download.handle and alert_type == 'add_torrent_alert') \
or (download.handle and alert_type == 'torrent_removed_alert'):
download.process_alert(alert, alert_type)
else:
self._logger.debug("Got alert for download without handle %s: %s", hexlify(infohash), alert)
elif infohash:
self._logger.debug("Got alert for unknown download %s: %s", hexlify(infohash), alert)
if alert_type == 'peer_disconnected_alert' and \
self.tribler_session and self.tribler_session.payout_manager:
self.tribler_session.payout_manager.do_payout(alert.pid.to_bytes())
elif alert_type == 'session_stats_alert':
queued_disk_jobs = alert.values['disk.queued_disk_jobs']
queued_write_bytes = alert.values['disk.queued_write_bytes']
num_write_jobs = alert.values['disk.num_write_jobs']
if queued_disk_jobs == queued_write_bytes == num_write_jobs == 0:
self.lt_session_shutdown_ready[hops] = True
if self.session_stats_callback:
self.session_stats_callback(alert)
elif alert_type == "dht_pkt_alert":
# We received a raw DHT message - decode it and check whether it is a BEP33 message.
decoded = bdecode_compat(alert.pkt_buf)
if decoded and b'r' in decoded:
if b'BFsd' in decoded[b'r'] and b'BFpe' in decoded[b'r']:
self.dht_health_manager.received_bloomfilters(decoded[b'r'][b'id'],
bytearray(decoded[b'r'][b'BFsd']),
bytearray(decoded[b'r'][b'BFpe']))
def remove_config(self, infohash):
if infohash not in self.downloads:
try:
basename = hexlify(infohash) + '.conf'
filename = self.get_checkpoint_dir() / basename
self._logger.debug("Removing download checkpoint %s", filename)
if os.access(filename, os.F_OK):
os.remove(filename)
except:
# Show must go on
self._logger.exception("Could not remove state")
else:
self._logger.warning(
"Download is back, restarted? Cancelling removal! %s",
hexlify(infohash))
async def test_get_entry_not_found(enable_chant, enable_api, session):
"""
Test trying to get a non-existing entry with the REST API GET request
"""
await do_request(session,
'metadata/%s/%i' % (hexlify(b"0" * 64), 123),
expected_code=404)
async def test_update_entry(enable_chant, enable_api, session):
"""
Test updating a metadata entry with REST API
"""
new_title = 'bla2'
new_tags = "Compressed"
with db_session:
chan = session.mds.ChannelMetadata.create_channel(title="bla")
chan.status = COMMITTED
patch_params = {'title': new_title, 'tags': new_tags}
result = await do_request(
session,
'metadata/%s/%i' % (hexlify(chan.public_key), chan.id_),
request_type='PATCH',
post_data=patch_params,
expected_code=200,
)
assert new_title == result['name']
assert new_tags == result['category']
with db_session:
chan = session.mds.ChannelMetadata.get_my_channels().first()
assert chan.status == UPDATED
assert chan.tags == new_tags
assert chan.title == new_title
async def check_and_regen_personal_channels(self):
# Test if our channels are there, but we don't share these because Tribler was closed unexpectedly
try:
with db_session:
for channel in self.session.mds.ChannelMetadata.get_my_channels(
).where(lambda g: g.status == COMMITTED):
channel_download = self.session.dlmgr.get_download(
bytes(channel.infohash))
if channel_download is None:
self._logger.warning(
"Torrent for personal channel %s %i does not exist.",
hexlify(channel.public_key),
channel.id_,
)
self.regenerate_channel_torrent(
channel.public_key, channel.id_)
else:
self.register_task(
f"Check personal channel {hexlify(channel.public_key), channel.id_}",
self.check_and_regen_personal_channel_torrent,
channel.public_key,
channel.id_,
channel_download,
)
except Exception:
self._logger.exception(
"Error when tried to resume personal channel seeding on GigaChannel Manager startup"
)
def on_save_resume_data_alert(self, alert):
"""
Callback for the alert that contains the resume data of a specific download.
This resume data will be written to a file on disk.
"""
self._logger.debug(f'On save resume data alert: {alert}')
if self.checkpoint_disabled:
return
resume_data = alert.resume_data
# Make save_path relative if the torrent is saved in the Tribler state directory
if self.state_dir and b'save_path' in resume_data:
save_path = Path(resume_data[b'save_path'].decode('utf8'))
if save_path.exists():
resume_data[b'save_path'] = str(
save_path.normalize_to(self.state_dir))
metainfo = {
'infohash': self.tdef.get_infohash(),
'name': self.tdef.get_name_as_unicode(),
'url': self.tdef.get_url()
} if isinstance(self.tdef,
TorrentDefNoMetainfo) else self.tdef.get_metainfo()
self.config.set_metainfo(metainfo)
self.config.set_engineresumedata(resume_data)
# Save it to file
basename = hexlify(resume_data[b'info-hash']) + '.conf'
filename = self.dlmgr.get_checkpoint_dir() / basename
self.config.config['download_defaults'][
'name'] = self.tdef.get_name_as_unicode(
) # store name (for debugging)
self.config.write(str(filename))
self._logger.debug('Saving download config to file %s', filename)
async def process_torrents_health(self, peer, torrent_healths):
infohashes_to_resolve = []
with db_session:
for infohash, seeders, leechers, last_check in torrent_healths:
torrent_state = self.mds.TorrentState.get(infohash=infohash)
if torrent_state and last_check > torrent_state.last_check:
# Replace current information
torrent_state.seeders = seeders
torrent_state.leechers = leechers
torrent_state.last_check = last_check
self.logger.info(
f"{hexlify(infohash)} updated ({seeders},{leechers})")
elif not torrent_state:
self.mds.TorrentState(infohash=infohash,
seeders=seeders,
leechers=leechers,
last_check=last_check)
self.logger.info(
f"{hexlify(infohash)} added ({seeders},{leechers})")
infohashes_to_resolve.append(infohash)
for infohash in infohashes_to_resolve:
# Get a single result per infohash to avoid duplicates
self.send_remote_select(peer=peer,
infohash=hexlify(infohash),
last=1)
async def get_metainfo(self, infohash, timeout=30, hops=None, url=None):
"""
Lookup metainfo for a given infohash. The mechanism works by joining the swarm for the infohash connecting
to a few peers, and downloading the metadata for the torrent.
:param infohash: The (binary) infohash to lookup metainfo for.
:param timeout: A timeout in seconds.
:param hops: the number of tunnel hops to use for this lookup. If None, use config default.
:param url: Optional URL. Can contain trackers info, etc.
:return: The metainfo
"""
infohash_hex = hexlify(infohash)
if infohash in self.metainfo_cache:
self._logger.info('Returning metainfo from cache for %s',
infohash_hex)
return self.metainfo_cache[infohash]['meta_info']
self._logger.info('Trying to fetch metainfo for %s', infohash_hex)
if infohash in self.metainfo_requests:
download = self.metainfo_requests[infohash][0]
self.metainfo_requests[infohash][1] += 1
elif infohash in self.downloads:
download = self.downloads[infohash]
else:
tdef = TorrentDefNoMetainfo(infohash, 'metainfo request', url=url)
dcfg = DownloadConfig()
dcfg.set_hops(hops or self.download_defaults.number_hops)
dcfg.set_upload_mode(
True
) # Upload mode should prevent libtorrent from creating files
dcfg.set_dest_dir(self.metadata_tmpdir)
try:
download = self.start_download(tdef=tdef,
config=dcfg,
hidden=True,
checkpoint_disabled=True)
except TypeError:
return
self.metainfo_requests[infohash] = [download, 1]
try:
metainfo = download.tdef.get_metainfo() or await wait_for(
shield(download.future_metainfo), timeout)
self._logger.info('Successfully retrieved metainfo for %s',
infohash_hex)
self.metainfo_cache[infohash] = {
'time': timemod.time(),
'meta_info': metainfo
}
except (CancelledError, asyncio.TimeoutError):
metainfo = None
self._logger.info('Failed to retrieve metainfo for %s',
infohash_hex)
if infohash in self.metainfo_requests:
self.metainfo_requests[infohash][1] -= 1
if self.metainfo_requests[infohash][1] <= 0:
await self.remove_download(download, remove_content=True)
self.metainfo_requests.pop(infohash, None)
return metainfo
