class BandwidthEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing requests for bandwidth accounting data.
"""
def setup_routes(self) -> None:
self.app.add_routes([web.get('/statistics', self.get_statistics)])
self.app.add_routes([web.get('/history', self.get_history)])
@docs(tags=["Bandwidth"],
summary="Return statistics about the bandwidth community.",
responses={
200: {
"schema":
schema(
BandwidthStatisticsResponse={
'statistics':
schema(
BandwidthStatistics={
'id': String,
'num_peers_helped': Integer,
'num_peers_helped_by': Integer,
'total_taken': Integer,
'total_given': Integer
})
})
}
})
async def get_statistics(self, request) -> RESTResponse:
if not self.session.bandwidth_community:
return RESTResponse({"error": "Bandwidth community not found"},
status=HTTP_NOT_FOUND)
return RESTResponse(
{'statistics': self.session.bandwidth_community.get_statistics()})
@docs(tags=["Bandwidth"],
summary="Return a list of the balance history.",
responses={
200: {
"schema":
schema(
BandwidthHistoryResponse={
"history": [
schema(BandwidthHistoryItem={
"timestamp": Integer,
"balance": Integer
})
]
})
}
})
async def get_history(self, request) -> RESTResponse:
if not self.session.bandwidth_community:
return RESTResponse({"error": "Bandwidth community not found"},
status=HTTP_NOT_FOUND)
return RESTResponse({
'history':
self.session.bandwidth_community.database.get_history()
})
class ShutdownEndpoint(RESTEndpoint):
"""
With this endpoint you can shutdown Tribler.
"""
def __init__(self, shutdown_callback):
super().__init__()
self.shutdown_callback = shutdown_callback
def setup_routes(self):
self.app.add_routes([web.put('', self.shutdown)])
@docs(
tags=["General"],
summary="Shutdown Tribler.",
responses={
200: {
"schema": schema(TriblerShutdownResponse={
'shutdown': Boolean
})
}
}
)
async def shutdown(self, request):
self.shutdown_callback()
return RESTResponse({"shutdown": True})
class StateEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing all requests regarding the state of Tribler.
"""
def __init__(self, session):
super().__init__(session)
self.tribler_state = STATE_STARTING
self.last_exception = None
self.sentry_event = None
self.session.notifier.add_observer(NTFY.UPGRADER_STARTED,
self.on_tribler_upgrade_started)
self.session.notifier.add_observer(NTFY.UPGRADER_DONE,
self.on_tribler_upgrade_finished)
self.session.notifier.add_observer(NTFY.TRIBLER_STARTED,
self.on_tribler_started)
def setup_routes(self):
self.app.add_routes([web.get('', self.get_state)])
def on_tribler_upgrade_started(self, *_):
self.tribler_state = STATE_UPGRADING
def on_tribler_upgrade_finished(self, *_):
self.tribler_state = STATE_STARTING
def on_tribler_started(self, *_):
self.tribler_state = STATE_STARTED
def on_tribler_exception(self, exception_text, sentry_event):
self.tribler_state = STATE_EXCEPTION
self.last_exception = exception_text
self.sentry_event = sentry_event
@docs(
tags=["General"],
summary="Return the current state of the Tribler core.",
responses={
200: {
"schema":
schema(
TriblerStateResponse={
'state':
(String,
'One of three stats: STARTING, UPGRADING, STARTED, EXCEPTION'
),
'last_exception':
String,
'readable_state':
String
})
}
})
async def get_state(self, request):
return RESTResponse({
"state": self.tribler_state,
"last_exception": self.last_exception,
"readable_state": self.session.readable_status
})
class UpgraderEndpoint(RESTEndpoint):
"""
With this endpoint you can control DB upgrade process of Tribler.
"""
def setup_routes(self):
self.app.add_routes([web.post('', self.skip_upgrade)])
@docs(tags=["Upgrader"],
summary="Skip the DB upgrade process, if it is running.",
responses={
200: {
"schema":
schema(UpgraderResponse={'skip_db_upgrade': Boolean}),
'examples': {
"skip_db_upgrade": True
}
},
HTTP_NOT_FOUND: {
'schema': HandledErrorSchema
},
HTTP_BAD_REQUEST: {
'schema': HandledErrorSchema
}
})
@json_schema(
schema(
UpgraderRequest={
'skip_db_upgrade': (
Boolean, 'Whether to skip the DB upgrade process or not'),
}))
async def skip_upgrade(self, request):
parameters = await request.json()
if SKIP_DB_UPGRADE_STR not in parameters:
return RESTResponse({"error": "attribute to change is missing"},
status=HTTP_BAD_REQUEST)
elif not self.session.upgrader:
return RESTResponse({"error": "upgrader is not running"},
status=HTTP_NOT_FOUND)
if self.session.upgrader and parameters[SKIP_DB_UPGRADE_STR]:
self.session.upgrader.skip()
return RESTResponse({SKIP_DB_UPGRADE_STR: True})
class RemoteQueryEndpoint(MetadataEndpointBase):
"""
This endpoint is responsible for searching in channels and torrents present in the local Tribler database.
It also fires a remote search in the IPv8 channel community.
"""
def setup_routes(self):
self.app.add_routes([web.put('', self.create_remote_search_request)])
def sanitize_parameters(self, parameters):
sanitized = super(RemoteQueryEndpoint, self).sanitize_parameters(parameters)
sanitized.update({'uuid': parameters['uuid'], 'channel_pk': unhexlify(parameters.get('channel_pk', ""))})
return sanitized
@docs(
tags=['Metadata'],
summary="Perform a search for a given query.",
responses={
200: {
'schema': schema(RemoteSearchResponse={
'success': Boolean
})
}
}
)
@querystring_schema(RemoteQueryParameters)
async def create_remote_search_request(self, request):
# Query remote results from the GigaChannel Community v1.0.
# v1.0 does not support searching for text limited by public key.
# GigaChannel v1.0 search community sends requests for channel contents by putting channel's public key
# into the text filter field. To communicate with older clients we have to shape the request accordingly.
# Results are returned over the Events endpoint.
try:
sanitized = self.sanitize_parameters(request.query)
if sanitized["txt_filter"] and sanitized["channel_pk"]:
return RESTResponse({"error": "Remote search by text and pk is not supported"}, status=HTTP_BAD_REQUEST)
except (ValueError, KeyError) as e:
return RESTResponse({"error": "Error processing request parameters: %s" % e}, status=HTTP_BAD_REQUEST)
self.session.gigachannel_community.send_search_request(
sanitized['txt_filter'] or ('"%s"*' % hexlify(sanitized['channel_pk'])),
metadata_type=sanitized.get('metadata_type'),
sort_by=sanitized['sort_by'],
sort_asc=sanitized['sort_desc'],
hide_xxx=sanitized['hide_xxx'],
uuid=sanitized['uuid'],
)
return RESTResponse({"success": True})
class ShutdownEndpoint(RESTEndpoint):
"""
With this endpoint you can shutdown Tribler.
"""
def __init__(self, session):
super(ShutdownEndpoint, self).__init__(session)
self.process_checker = ProcessChecker()
def setup_routes(self):
self.app.add_routes([web.put('', self.shutdown)])
@docs(
tags=["General"],
summary="Shutdown Tribler.",
responses={
200: {
"schema": schema(TriblerShutdownResponse={
'shutdown': Boolean
})
}
}
)
async def shutdown(self, request):
async def shutdown():
try:
keep_loop_running = await self.session.shutdown()
except Exception as e:
self._logger.error(e)
keep_loop_running = False
self.process_checker.remove_lock_file()
# Flush the logs to the file before exiting
for handler in logging.getLogger().handlers:
handler.flush()
if not keep_loop_running:
get_event_loop().stop()
ensure_future(shutdown())
return RESTResponse({"shutdown": True})
class RemoteQueryEndpoint(MetadataEndpointBase):
"""
This endpoint fires a remote search in the IPv8 GigaChannel Community.
"""
def setup_routes(self):
self.app.add_routes([web.put('', self.create_remote_search_request)])
def sanitize_parameters(self, parameters):
sanitized = super().sanitize_parameters(parameters)
# Convert frozenset to string
if "metadata_type" in sanitized:
sanitized["metadata_type"] = [str(mt) for mt in sanitized["metadata_type"] if mt]
if "channel_pk" in parameters:
sanitized["channel_pk"] = parameters["channel_pk"]
if "origin_id" in parameters:
sanitized["origin_id"] = parameters["origin_id"]
return sanitized
@docs(
tags=['Metadata'],
summary="Perform a search for a given query.",
responses={200: {'schema': schema(RemoteSearchResponse={'request_uuid': String()})}},
)
@querystring_schema(RemoteQueryParameters)
async def create_remote_search_request(self, request):
# Query remote results from the GigaChannel Community.
# Results are returned over the Events endpoint.
try:
sanitized = self.sanitize_parameters(request.query)
except (ValueError, KeyError) as e:
return RESTResponse({"error": f"Error processing request parameters: {e}"}, status=HTTP_BAD_REQUEST)
request_uuid = self.session.gigachannel_community.send_search_request(**sanitized)
return RESTResponse({"request_uuid": str(request_uuid)})
class SettingsEndpoint(RESTEndpoint):
"""
This endpoint is reponsible for handing all requests regarding settings and configuration.
"""
def __init__(self,
tribler_config: TriblerConfig,
download_manager: DownloadManager = None):
super().__init__()
self.tribler_config = tribler_config
self.download_manager = download_manager
def setup_routes(self):
self.app.add_routes([
web.get('', self.get_settings),
web.post('', self.update_settings)
])
@docs(
tags=["General"],
summary="Return all the session settings that can be found in Tribler.",
responses={200: {
"schema": schema(GetTriblerSettingsResponse={})
}},
description=
"This endpoint returns all the session settings that can be found in Tribler.\n\n It also returns "
"the runtime-determined ports")
async def get_settings(self, request):
self._logger.info(f'Get settings. Request: {request}')
return RESTResponse({
"settings": self.tribler_config.dict(),
"ports": list(default_network_utils.ports_in_use)
})
@docs(tags=["General"],
summary="Update Tribler settings.",
responses={
200: {
"schema":
schema(UpdateTriblerSettingsResponse={'modified': Boolean})
}
})
@json_schema(schema(UpdateTriblerSettingsRequest={}))
async def update_settings(self, request):
settings_dict = await request.json()
await self.parse_settings_dict(settings_dict)
self.tribler_config.write()
return RESTResponse({"modified": True})
async def parse_setting(self, section, option, value):
"""
Set a specific Tribler setting. Throw a ValueError if this setting is not available.
"""
# if section in self.config.config and option in self.config.config[section]:
self.tribler_config.__getattribute__(section).__setattr__(
option, value)
# else:
# raise ValueError(f"Section {section} with option {option} does not exist")
# Perform some actions when specific keys are set
if section == "libtorrent" and (option == "max_download_rate"
or option == "max_upload_rate"):
if self.download_manager:
self.download_manager.update_max_rates_from_config()
async def parse_settings_dict(self, settings_dict, depth=1, root_key=None):
"""
Parse the settings dictionary.
"""
for key, value in settings_dict.items():
if isinstance(value, dict):
await self.parse_settings_dict(value,
depth=depth + 1,
root_key=key)
else:
await self.parse_setting(root_key, key, value)
class EventsEndpoint(RESTEndpoint, TaskManager):
"""
Important events in Tribler are returned over the events endpoint. This connection is held open. Each event is
pushed over this endpoint in the form of a JSON dictionary. Each JSON dictionary contains a type field that
indicates the type of the event. Individual events are separated by a newline character.
"""
def __init__(self, session):
RESTEndpoint.__init__(self, session)
TaskManager.__init__(self)
self.events_responses = []
self.app.on_shutdown.append(self.on_shutdown)
# We need to know that Tribler completed its startup sequence
self.tribler_started = False
self.session.notifier.add_observer(NTFY.TRIBLER_STARTED, self._tribler_started)
for event_type, event_lambda in reactions_dict.items():
self.session.notifier.add_observer(event_type,
lambda *args, el=event_lambda, et=event_type:
self.write_data({"type": et.value, "event": el(*args)}))
def on_circuit_removed(circuit, *args):
if isinstance(circuit, Circuit):
event = {
"circuit_id": circuit.circuit_id,
"bytes_up": circuit.bytes_up,
"bytes_down": circuit.bytes_down,
"uptime": time.time() - circuit.creation_time
}
self.write_data({"type": NTFY.TUNNEL_REMOVE.value, "event": event})
# Tribler tunnel circuit has been removed
self.session.notifier.add_observer(NTFY.TUNNEL_REMOVE, on_circuit_removed)
async def on_shutdown(self, _):
await self.shutdown_task_manager()
def _tribler_started(self):
self.tribler_started = True
def setup_routes(self):
self.app.add_routes([web.get('', self.get_events)])
@task
async def write_data(self, message):
"""
Write data over the event socket if it's open.
"""
if not self.events_responses:
return
try:
message = json.dumps(message)
except UnicodeDecodeError:
# The message contains invalid characters; fix them
self._logger.error("Event contains non-unicode characters, fixing")
message = json.dumps(fix_unicode_dict(message))
message_bytes = b'data: ' + message.encode('utf-8') + b'\n\n'
for request in self.events_responses:
await request.write(message_bytes)
# An exception has occurred in Tribler. The event includes a readable string of the error.
def on_tribler_exception(self, exception_text):
self.write_data({"type": NTFY.TRIBLER_EXCEPTION.value, "event": {"text": exception_text}})
@docs(
tags=["General"],
summary="Open an EventStream for receiving Tribler events.",
responses={
200: {
"schema": schema(EventsResponse={'type': String,
'event': Dict})
}
}
)
async def get_events(self, request):
"""
.. http:get:: /events
A GET request to this endpoint will open the event connection.
**Example request**:
.. sourcecode:: none
curl -X GET http://localhost:8085/events
"""
# Setting content-type to text/event-stream to ensure browsers will handle the content properly
response = RESTStreamResponse(status=200,
reason='OK',
headers={'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'})
await response.prepare(request)
# FIXME: Proper start check!
await response.write(b'data: ' + json.dumps({"type": NTFY.EVENTS_START.value,
"event": {"tribler_started": self.tribler_started,
"version": version_id}}).encode('utf-8') + b'\n\n')
self.events_responses.append(response)
try:
while True:
await self.register_anonymous_task('event_sleep', lambda: None, delay=3600)
except CancelledError:
self.events_responses.remove(response)
return response
class EventsEndpoint(RESTEndpoint, TaskManager):
"""
Important events in Tribler are returned over the events endpoint. This connection is held open. Each event is
pushed over this endpoint in the form of a JSON dictionary. Each JSON dictionary contains a type field that
indicates the type of the event. Individual events are separated by a newline character.
"""
def __init__(self, notifier: Notifier, public_key: str = None):
RESTEndpoint.__init__(self)
TaskManager.__init__(self)
self.events_responses: List[RESTStreamResponse] = []
self.app.on_shutdown.append(self.on_shutdown)
self.notifier = None
self.undelivered_error: Optional[dict] = None
self.connect_notifier(notifier)
self.public_key = public_key
def connect_notifier(self, notifier: Notifier):
self.notifier = notifier
for event_type, event_lambda in reactions_dict.items():
self.notifier.add_observer(event_type,
lambda *args, el=event_lambda, et=event_type:
self.write_data({"type": et.value, "event": el(*args)}))
def on_circuit_removed(circuit, *args):
if isinstance(circuit, Circuit):
event = {
"circuit_id": circuit.circuit_id,
"bytes_up": circuit.bytes_up,
"bytes_down": circuit.bytes_down,
"uptime": time.time() - circuit.creation_time
}
self.write_data({"type": NTFY.TUNNEL_REMOVE.value, "event": event})
# Tribler tunnel circuit has been removed
self.notifier.add_observer(NTFY.TUNNEL_REMOVE, on_circuit_removed)
async def on_shutdown(self, _):
await self.shutdown_task_manager()
def setup_routes(self):
self.app.add_routes([web.get('', self.get_events)])
def initial_message(self) -> dict:
return {
"type": NTFY.EVENTS_START.value,
"event": {"public_key": self.public_key, "version": version_id}
}
def error_message(self, reported_error: ReportedError) -> dict:
return {
"type": NTFY.TRIBLER_EXCEPTION.value,
"event": asdict(reported_error),
}
def encode_message(self, message: dict) -> bytes:
try:
message = json.dumps(message)
except UnicodeDecodeError:
# The message contains invalid characters; fix them
self._logger.error("Event contains non-unicode characters, fixing")
message = json.dumps(fix_unicode_dict(message))
return b'data: ' + message.encode('utf-8') + b'\n\n'
def has_connection_to_gui(self):
return bool(self.events_responses)
@task
async def write_data(self, message):
"""
Write data over the event socket if it's open.
"""
if not self.has_connection_to_gui():
return
message_bytes = self.encode_message(message)
for response in self.events_responses:
await response.write(message_bytes)
# An exception has occurred in Tribler. The event includes a readable
# string of the error and a Sentry event.
def on_tribler_exception(self, reported_error: ReportedError):
message = self.error_message(reported_error)
if self.has_connection_to_gui():
self.write_data(message)
elif not self.undelivered_error:
# If there are several undelivered errors, we store the first error as more important and skip other
self.undelivered_error = message
@docs(
tags=["General"],
summary="Open an EventStream for receiving Tribler events.",
responses={
200: {
"schema": schema(EventsResponse={'type': String,
'event': Dict})
}
}
)
async def get_events(self, request):
"""
.. http:get:: /events
A GET request to this endpoint will open the event connection.
**Example request**:
.. sourcecode:: none
curl -X GET http://localhost:52194/events
"""
# Setting content-type to text/event-stream to ensure browsers will handle the content properly
response = RESTStreamResponse(status=200,
reason='OK',
headers={'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'})
await response.prepare(request)
await response.write(self.encode_message(self.initial_message()))
if self.undelivered_error:
error = self.undelivered_error
self.undelivered_error = None
await response.write(self.encode_message(error))
self.events_responses.append(response)
try:
while True:
await self.register_anonymous_task('event_sleep', lambda: None, delay=3600)
except CancelledError:
self.events_responses.remove(response)
return response
class StatisticsEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing requests regarding statistics in Tribler.
"""
def setup_routes(self):
self.app.add_routes([
web.get('/tribler', self.get_tribler_stats),
web.get('/ipv8', self.get_ipv8_stats)
])
@docs(tags=["General"],
summary="Return general statistics of Tribler.",
responses={
200: {
"schema":
schema(
TriblerStatisticsResponse={
'statistics':
schema(
TriblerStatistics={
'num_channels':
Integer,
'database_size':
Integer,
'torrent_queue_stats': [
schema(
TorrentQueueStats={
'failed': Integer,
'total': Integer,
'type': String,
'pending': Integer,
'success': Integer
})
],
})
})
}
})
async def get_tribler_stats(self, request):
return RESTResponse(
{'tribler_statistics': self.session.get_tribler_statistics()})
@docs(tags=["General"],
summary="Return general statistics of IPv8.",
responses={
200: {
"schema":
schema(
IPv8StatisticsResponse={
'statistics':
schema(IPv8Statistics={
'total_up': Integer,
'total_down': Integer
})
})
}
})
async def get_ipv8_stats(self, request):
"""
.. sourcecode:: javascript
{
"ipv8_statistics": {
"total_up": 3424324,
"total_down": 859484
}
}
"""
return RESTResponse(
{'ipv8_statistics': self.session.get_ipv8_statistics()})
class SearchEndpoint(MetadataEndpointBase):
"""
This endpoint is responsible for searching in channels and torrents present in the local Tribler database.
"""
def setup_routes(self):
self.app.add_routes([
web.get('', self.search),
web.get('/completions', self.completions)
])
@staticmethod
def get_uuid(parameters):
return parameters['uuid'] if 'uuid' in parameters else None
@docs(tags=['Metadata'],
summary="Perform a search for a given query.",
responses={
200: {
'schema':
schema(
SearchResponse={
'torrents': [
schema(
Torrent={
'commit_status': Integer,
'num_leechers': Integer,
'date': Integer,
'relevance_score': Integer,
'id': Integer,
'size': Integer,
'category': String,
'public_key': String,
'name': String,
'last_tracker_check': Integer,
'infohash': String,
'num_seeders': Integer,
'type': String,
})
],
'chant_dirty':
Boolean
})
}
})
@querystring_schema(MetadataParameters)
async def search(self, request):
try:
sanitized = self.sanitize_parameters(request.query)
except (ValueError, KeyError):
return RESTResponse(
{"error": "Error processing request parameters"},
status=HTTP_BAD_REQUEST)
if not sanitized["txt_filter"]:
return RESTResponse({"error": "Filter parameter missing"},
status=HTTP_BAD_REQUEST)
include_total = request.query.get('include_total', '')
def search_db():
with db_session:
pony_query = self.session.mds.MetadataNode.get_entries(
**sanitized)
total = self.session.mds.MetadataNode.get_total_count(
**sanitized) if include_total else None
search_results = [r.to_simple_dict() for r in pony_query]
self.session.mds._db.disconnect(
) # DB must be disconnected explicitly if run on a thread
return search_results, total
try:
search_results, total = await asyncio.get_event_loop(
).run_in_executor(None, search_db)
except Exception as e:
self._logger.error("Error while performing DB search: %s", e)
return RESTResponse(status=HTTP_BAD_REQUEST)
response_dict = {
"results": search_results,
"first": sanitized["first"],
"last": sanitized["last"],
"sort_by": sanitized["sort_by"],
"sort_desc": sanitized["sort_desc"],
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(tags=['Metadata'],
summary="Return auto-completion suggestions for a given query.",
parameters=[{
'in': 'query',
'name': 'q',
'description': 'Search query',
'type': 'string',
'required': True
}],
responses={
200: {
'schema': schema(CompletionsResponse={
'completions': [String],
}),
'examples': {
'completions': ['pioneer one', 'pioneer movie']
}
}
})
async def completions(self, request):
args = request.query
if 'q' not in args:
return RESTResponse({"error": "query parameter missing"},
status=HTTP_BAD_REQUEST)
keywords = args['q'].strip().lower()
# TODO: add XXX filtering for completion terms
results = self.session.mds.TorrentMetadata.get_auto_complete_terms(
keywords, max_terms=5)
return RESTResponse({"completions": results})
class RemoteQueryEndpoint(MetadataEndpointBase):
"""
This endpoint fires a remote search in the IPv8 GigaChannel Community.
"""
def __init__(self, gigachannel_community: GigaChannelCommunity, *args,
**kwargs):
MetadataEndpointBase.__init__(self, *args, **kwargs)
self.gigachannel_community = gigachannel_community
def setup_routes(self):
self.app.add_routes([web.put('', self.create_remote_search_request)])
self.app.add_routes(
[web.get('/channels_peers', self.get_channels_peers)])
def sanitize_parameters(self, parameters):
sanitized = super().sanitize_parameters(parameters)
if "channel_pk" in parameters:
sanitized["channel_pk"] = unhexlify(parameters["channel_pk"])
if "origin_id" in parameters:
sanitized["origin_id"] = int(parameters["origin_id"])
return sanitized
@docs(
tags=['Metadata'],
summary="Perform a search for a given query.",
responses={
200: {
'schema':
schema(RemoteSearchResponse={'request_uuid': String()})
}
},
)
@querystring_schema(RemoteQueryParameters)
async def create_remote_search_request(self, request):
self._logger.info('Create remote search request')
# Query remote results from the GigaChannel Community.
# Results are returned over the Events endpoint.
try:
sanitized = self.sanitize_parameters(request.query)
except (ValueError, KeyError) as e:
return RESTResponse(
{"error": f"Error processing request parameters: {e}"},
status=HTTP_BAD_REQUEST)
self._logger.info(f'Parameters: {sanitized}')
request_uuid, peers_list = self.gigachannel_community.send_search_request(
**sanitized)
peers_mid_list = [hexlify(p.mid) for p in peers_list]
return RESTResponse({
"request_uuid": str(request_uuid),
"peers": peers_mid_list
})
async def get_channels_peers(self, _):
# Get debug stats for peers serving channels
current_time = time.time()
result = []
mapping = self.gigachannel_community.channels_peers
with db_session:
for id_tuple, peers in mapping._channels_dict.items(): # pylint:disable=W0212
channel_pk, channel_id = id_tuple
chan = self.mds.ChannelMetadata.get(public_key=channel_pk,
id_=channel_id)
peers_list = []
for p in peers:
peers_list.append(
(hexlify(p.mid), int(current_time - p.last_response)))
chan_dict = {
"channel_name": chan.title if chan else None,
"channel_pk": hexlify(channel_pk),
"channel_id": channel_id,
"peers": peers_list,
}
result.append(chan_dict)
return RESTResponse({"channels_list": result})
class TrustchainEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing requests for trustchain data.
"""
def setup_routes(self):
self.app.add_routes([
web.get('/statistics', self.get_statistics),
web.get('/bootstrap', self.bootstrap)
])
@docs(tags=["TrustChain"],
summary="Return statistics about the trustchain community.",
responses={
200: {
"schema":
schema(
TrustchainStatisticsResponse={
'statistics':
schema(
TrustchainStatistics={
'id': String,
'peers_that_pk_helped': Integer,
'peers_that_helped_pk': Integer,
'total_blocks': Integer,
'total_down': Integer,
'total_up': Integer
})
})
}
})
async def get_statistics(self, request):
if 'MB' not in self.session.wallets:
return RESTResponse({"error": "TrustChain community not found"},
status=HTTP_NOT_FOUND)
return RESTResponse({
'statistics':
recursive_unicode(self.session.wallets['MB'].get_statistics())
})
@docs(
tags=["TrustChain"],
summary=
"Generate a new identity and transfers bandwidth tokens to it..",
parameters=[{
'in': 'query',
'name': 'amount',
'description':
'Specifies how much tokens need to be emptied into the new identity',
'type': 'integer',
'required': True
}],
responses={
200: {
"schema":
schema(
TrustchainBootstrapResponse={
'private_key':
String,
'transaction':
schema(BootstrapTransaction={
'down': Integer,
'up': Integer
}),
'block':
schema(BootstrapBlock={
'block_hash': String,
'sequence_number': String
})
})
}
})
async def bootstrap(self, request):
if 'MB' not in self.session.wallets:
return RESTResponse({"error": "bandwidth wallet not found"},
status=HTTP_NOT_FOUND)
bandwidth_wallet = self.session.wallets['MB']
available_tokens = bandwidth_wallet.get_bandwidth_tokens()
args = request.query
if 'amount' in args:
try:
amount = int(args['amount'])
except ValueError:
return RESTResponse(
{"error": "Provided token amount is not a number"},
status=HTTP_BAD_REQUEST)
if amount <= 0:
return RESTResponse(
{"error": "Provided token amount is zero or negative"},
status=HTTP_BAD_REQUEST)
else:
amount = available_tokens
if amount <= 0 or amount > available_tokens:
return RESTResponse(
{"error": "Not enough bandwidth tokens available"},
status=HTTP_BAD_REQUEST)
result = bandwidth_wallet.bootstrap_new_identity(amount)
result['private_key'] = result['private_key'].decode('utf-8')
result['block']['block_hash'] = result['block']['block_hash'].decode(
'utf-8')
return RESTResponse(result)
class TorrentInfoEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing all requests regarding torrent info in Tribler.
"""
def setup_routes(self):
self.app.add_routes([web.get('', self.get_torrent_info)])
@docs(
tags=["Libtorrent"],
summary="Return metainfo from a torrent found at a provided URI.",
parameters=[{
'in':
'query',
'name':
'torrent',
'description':
'URI for which to return torrent information. This URI can either represent '
'a file location, a magnet link or a HTTP(S) url.',
'type':
'string',
'required':
True
}],
responses={
200: {
'description':
'Return a hex-encoded json-encoded string with torrent metainfo',
"schema": schema(GetMetainfoResponse={'metainfo': String})
}
})
async def get_torrent_info(self, request):
args = request.query
hops = None
if 'hops' in args:
try:
hops = int(args['hops'])
except ValueError:
return RESTResponse(
{
"error":
f"wrong value of 'hops' parameter: {repr(args['hops'])}"
},
status=HTTP_BAD_REQUEST)
if 'uri' not in args or not args['uri']:
return RESTResponse({"error": "uri parameter missing"},
status=HTTP_BAD_REQUEST)
uri = args['uri']
if uri.startswith('file:'):
try:
filename = url2pathname(uri[5:])
tdef = TorrentDef.load(filename)
metainfo = tdef.get_metainfo()
except (TypeError, RuntimeError):
return RESTResponse(
{"error": "error while decoding torrent file"},
status=HTTP_INTERNAL_SERVER_ERROR)
elif uri.startswith('http'):
try:
async with ClientSession(raise_for_status=True) as session:
response = await session.get(uri)
response = await response.read()
except (ServerConnectionError, ClientResponseError) as e:
return RESTResponse({"error": str(e)},
status=HTTP_INTERNAL_SERVER_ERROR)
if response.startswith(b'magnet'):
_, infohash, _ = parse_magnetlink(response)
if infohash:
metainfo = await self.session.dlmgr.get_metainfo(
infohash, timeout=60, hops=hops, url=response)
else:
metainfo = bdecode_compat(response)
elif uri.startswith('magnet'):
infohash = parse_magnetlink(uri)[1]
if infohash is None:
return RESTResponse({"error": "missing infohash"},
status=HTTP_BAD_REQUEST)
metainfo = await self.session.dlmgr.get_metainfo(infohash,
timeout=60,
hops=hops,
url=uri)
else:
return RESTResponse({"error": "invalid uri"},
status=HTTP_BAD_REQUEST)
if not metainfo:
return RESTResponse({"error": "metainfo error"},
status=HTTP_INTERNAL_SERVER_ERROR)
if not isinstance(metainfo, dict) or b'info' not in metainfo:
self._logger.warning("Received metainfo is not a valid dictionary")
return RESTResponse({"error": "invalid response"},
status=HTTP_INTERNAL_SERVER_ERROR)
# Add the torrent to GigaChannel as a free-for-all entry, so others can search it
self.session.mds.TorrentMetadata.add_ffa_from_dict(
tdef_to_metadata_dict(TorrentDef.load_from_dict(metainfo)))
# TODO(Martijn): store the stuff in a database!!!
# TODO(Vadim): this means cache the downloaded torrent in a binary storage, like LevelDB
infohash = hashlib.sha1(bencode(metainfo[b'info'])).digest()
download = self.session.dlmgr.downloads.get(infohash)
metainfo_request = self.session.dlmgr.metainfo_requests.get(
infohash, [None])[0]
download_is_metainfo_request = download == metainfo_request
# Check if the torrent is already in the downloads
encoded_metainfo = deepcopy(metainfo)
# FIXME: json.dumps garbles binary data that is used by the 'pieces' field
# However, this is fine as long as the GUI does not use this field.
encoded_metainfo[b'info'][b'pieces'] = hexlify(
encoded_metainfo[b'info'][b'pieces']).encode('utf-8')
encoded_metainfo = hexlify(
json.dumps(recursive_unicode(encoded_metainfo, ignore_errors=True),
ensure_ascii=False).encode('utf-8'))
return RESTResponse({
"metainfo":
encoded_metainfo,
"download_exists":
download and not download_is_metainfo_request
})
class TrustViewEndpoint(RESTEndpoint):
def __init__(self, bandwidth_db: BandwidthDatabase):
super().__init__()
self.bandwidth_db = bandwidth_db
def setup_routes(self):
self.app.add_routes([web.get('', self.get_view)])
@cached_property
def trust_graph(self) -> TrustGraph:
trust_graph = TrustGraph(self.bandwidth_db.my_pub_key,
self.bandwidth_db)
trust_graph.compose_graph_data()
return trust_graph
@docs(tags=["TrustGraph"],
summary="Return the trust graph.",
parameters=[],
responses={
200: {
"schema":
schema(
GraphResponse={
'root_public_key':
String,
'graph':
schema(
Graph={
'node':
schema(
Node={
'id': Integer,
'key': String,
'pos': [Float],
'sequence_number': Integer,
'total_up': Integer,
'total_down': Integer
}),
'edge':
List(List(Integer))
}),
'bootstrap':
schema(
Bootstrap={
'download': Integer,
'upload': Integer,
'progress': Float
}),
'num_tx':
Integer
})
}
})
async def get_view(self, request):
refresh_graph = int(request.query.get('refresh', '0'))
if refresh_graph:
self.trust_graph.compose_graph_data()
graph_data = self.trust_graph.compute_node_graph()
return RESTResponse({
'root_public_key':
hexlify(self.bandwidth_db.my_pub_key),
'graph':
graph_data,
'bootstrap':
0,
'num_tx':
len(graph_data['edge'])
})
class StatisticsEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing requests regarding statistics in Tribler.
"""
def __init__(self,
ipv8: IPv8 = None,
metadata_store: MetadataStore = None):
super().__init__()
self.mds = metadata_store
self.ipv8 = ipv8
def setup_routes(self):
self.app.add_routes([
web.get('/tribler', self.get_tribler_stats),
web.get('/ipv8', self.get_ipv8_stats)
])
@docs(tags=["General"],
summary="Return general statistics of Tribler.",
responses={
200: {
"schema":
schema(
TriblerStatisticsResponse={
'statistics':
schema(
TriblerStatistics={
'num_channels':
Integer,
'database_size':
Integer,
'torrent_queue_stats': [
schema(
TorrentQueueStats={
'failed': Integer,
'total': Integer,
'type': String,
'pending': Integer,
'success': Integer
})
],
})
})
}
})
async def get_tribler_stats(self, request):
stats_dict = {}
if self.mds:
db_size = self.mds.get_db_file_size()
stats_dict = {
"db_size": db_size,
"num_channels": self.mds.get_num_channels(),
"num_torrents": self.mds.get_num_torrents()
}
return RESTResponse({'tribler_statistics': stats_dict})
@docs(tags=["General"],
summary="Return general statistics of IPv8.",
responses={
200: {
"schema":
schema(
IPv8StatisticsResponse={
'statistics':
schema(IPv8Statistics={
'total_up': Integer,
'total_down': Integer
})
})
}
})
async def get_ipv8_stats(self, request):
"""
.. sourcecode:: javascript
{
"ipv8_statistics": {
"total_up": 3424324,
"total_down": 859484
}
}
"""
stats_dict = {}
if self.ipv8:
stats_dict = {
"total_up": self.ipv8.endpoint.bytes_up,
"total_down": self.ipv8.endpoint.bytes_down,
# "session_uptime": time.time() - self.ipv8_start_time
}
return RESTResponse({'ipv8_statistics': stats_dict})
class LibTorrentEndpoint(RESTEndpoint):
"""
Endpoint for getting information about libtorrent sessions and settings.
"""
def __init__(self, download_manager: DownloadManager):
super().__init__()
self.download_manager = download_manager
def setup_routes(self):
self.app.add_routes([web.get('/settings', self.get_libtorrent_settings),
web.get('/session', self.get_libtorrent_session_info)])
@docs(
tags=["Libtorrent"],
summary="Return Libtorrent session settings.",
parameters=[{
'in': 'query',
'name': 'hop',
'description': 'The hop count of the session for which to return settings',
'type': 'string',
'required': False
}],
responses={
200: {
'description': 'Return a dictonary with key-value pairs from the Libtorrent session settings',
"schema": schema(LibtorrentSessionResponse={'hop': Integer,
'settings': schema(LibtorrentSettings={})})
}
}
)
async def get_libtorrent_settings(self, request):
args = request.query
hop = 0
if 'hop' in args and args['hop']:
hop = int(args['hop'])
if hop not in self.download_manager.ltsessions:
return RESTResponse({'hop': hop, "settings": {}})
lt_session = self.download_manager.ltsessions[hop]
if hop == 0:
lt_settings = self.download_manager.get_session_settings(lt_session)
lt_settings['peer_fingerprint'] = hexlify(lt_settings['peer_fingerprint'])
else:
lt_settings = lt_session.get_settings()
return RESTResponse({'hop': hop, "settings": lt_settings})
@docs(
tags=["Libtorrent"],
summary="Return Libtorrent session information.",
parameters=[{
'in': 'query',
'name': 'hop',
'description': 'The hop count of the session for which to return information',
'type': 'string',
'required': False
}],
responses={
200: {
'description': 'Return a dictonary with key-value pairs from the Libtorrent session information',
"schema": schema(LibtorrentinfoResponse={'hop': Integer,
'settings': schema(LibtorrentInfo={})})
}
}
)
async def get_libtorrent_session_info(self, request):
session_stats = Future()
def on_session_stats_alert_received(alert):
if not session_stats.done():
session_stats.set_result(alert.values)
args = request.query
hop = 0
if 'hop' in args and args['hop']:
hop = int(args['hop'])
if hop not in self.download_manager.ltsessions or \
not hasattr(self.download_manager.ltsessions[hop], "post_session_stats"):
return RESTResponse({'hop': hop, 'session': {}})
self.download_manager.session_stats_callback = on_session_stats_alert_received
self.download_manager.ltsessions[hop].post_session_stats()
stats = await session_stats
return RESTResponse({'hop': hop, 'session': stats})
class SettingsEndpoint(RESTEndpoint):
"""
This endpoint is reponsible for handing all requests regarding settings and configuration.
"""
def setup_routes(self):
self.app.add_routes([
web.get('', self.get_settings),
web.post('', self.update_settings)
])
@docs(
tags=["General"],
summary="Return all the session settings that can be found in Tribler.",
responses={200: {
"schema": schema(GetTriblerSettingsResponse={})
}},
description=
"This endpoint returns all the session settings that can be found in Tribler.\n\n It also returns "
"the runtime-determined ports, i.e. the ports for the SOCKS5 servers. Please note that a port "
"with a value of -1 in the settings means that the port is randomly assigned at startup."
)
async def get_settings(self, request):
return RESTResponse({
"settings": self.session.config.config,
"ports": self.session.config.selected_ports
})
@docs(tags=["General"],
summary="Update Tribler settings.",
responses={
200: {
"schema":
schema(UpdateTriblerSettingsResponse={'modified': Boolean})
}
})
@json_schema(schema(UpdateTriblerSettingsRequest={}))
async def update_settings(self, request):
settings_dict = await request.json()
await self.parse_settings_dict(settings_dict)
self.session.config.write()
return RESTResponse({"modified": True})
async def parse_setting(self, section, option, value):
"""
Set a specific Tribler setting. Throw a ValueError if this setting is not available.
"""
if section in self.session.config.config and option in self.session.config.config[
section]:
self.session.config.config[section][option] = value
else:
raise ValueError(
f"Section {section} with option {option} does not exist")
# Perform some actions when specific keys are set
if section == "libtorrent" and (option == "max_download_rate"
or option == "max_upload_rate"):
self.session.dlmgr.update_max_rates_from_config()
async def parse_settings_dict(self, settings_dict, depth=1, root_key=None):
"""
Parse the settings dictionary.
"""
for key, value in settings_dict.items():
if isinstance(value, dict):
await self.parse_settings_dict(value,
depth=depth + 1,
root_key=key)
else:
await self.parse_setting(root_key, key, value)
class TrustViewEndpoint(RESTEndpoint):
def __init__(self, session):
super().__init__(session)
self.logger = logging.getLogger(self.__class__.__name__)
self.bandwidth_db = None
self.trust_graph = None
self.public_key = None
def setup_routes(self):
self.app.add_routes([web.get('', self.get_view)])
def initialize_graph(self):
if self.session.bandwidth_community:
self.bandwidth_db = self.session.bandwidth_community.database
self.public_key = self.session.bandwidth_community.my_pk
self.trust_graph = TrustGraph(self.public_key, self.bandwidth_db)
# Start bootstrap download if not already done
if not self.session.bootstrap:
self.session.start_bootstrap_download()
@docs(tags=["TrustGraph"],
summary="Return the trust graph.",
parameters=[],
responses={
200: {
"schema":
schema(
GraphResponse={
'root_public_key':
String,
'graph':
schema(
Graph={
'node':
schema(
Node={
'id': Integer,
'key': String,
'pos': [Float],
'sequence_number': Integer,
'total_up': Integer,
'total_down': Integer
}),
'edge':
List(List(Integer))
}),
'bootstrap':
schema(
Bootstrap={
'download': Integer,
'upload': Integer,
'progress': Float
}),
'num_tx':
Integer
})
}
})
async def get_view(self, request):
if not self.trust_graph:
self.initialize_graph()
self.trust_graph.compose_graph_data()
refresh_graph = int(request.query.get('refresh', '0'))
if refresh_graph:
self.trust_graph.compose_graph_data()
graph_data = self.trust_graph.compute_node_graph()
return RESTResponse({
'root_public_key': hexlify(self.public_key),
'graph': graph_data,
'bootstrap': self.get_bootstrap_info(),
'num_tx': len(graph_data['edge'])
})
def get_bootstrap_info(self):
if self.session.bootstrap.download and self.session.bootstrap.download.get_state(
):
state = self.session.bootstrap.download.get_state()
return {
'download': state.get_total_transferred(DOWNLOAD),
'upload': state.get_total_transferred(UPLOAD),
'progress': state.get_progress(),
}
return {'download': 0, 'upload': 0, 'progress': 0}
class ChannelsEndpoint(ChannelsEndpointBase):
def setup_routes(self):
self.app.add_routes([
web.get('', self.get_channels),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}',
self.get_channel_contents),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/description',
self.get_channel_description),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/description',
self.put_channel_description),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/thumbnail',
self.get_channel_thumbnail),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/thumbnail',
self.put_channel_thumbnail),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/copy',
self.copy_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/channels',
self.create_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/collections',
self.create_collection),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/torrents',
self.add_torrent_to_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/commit',
self.post_commit),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/commit',
self.is_channel_dirty),
web.get('/popular_torrents', self.get_popular_torrents_channel),
])
def add_download_progress_to_metadata_list(self, contents_list):
for torrent in contents_list:
if torrent['type'] == REGULAR_TORRENT:
dl = self.session.dlmgr.get_download(
unhexlify(torrent['infohash']))
if dl is not None and dl.tdef.infohash not in self.session.dlmgr.metainfo_requests:
torrent['progress'] = dl.get_state().get_progress()
def get_channel_from_request(self, request):
channel_pk = (self.session.mds.my_key.pub().key_to_bin()[10:]
if request.match_info['channel_pk'] == 'mychannel' else
unhexlify(request.match_info['channel_pk']))
channel_id = int(request.match_info['channel_id'])
return channel_pk, channel_id
@docs(
tags=['Metadata'],
summary='Get a list of all channels known to the system.',
responses={
200: {
'schema':
schema(
GetChannelsResponse={
'results': [ChannelSchema],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
})
}
},
)
async def get_channels(self, request):
sanitized = self.sanitize_parameters(request.query)
sanitized[
'subscribed'] = None if 'subscribed' not in request.query else bool(
int(request.query['subscribed']))
include_total = request.query.get('include_total', '')
sanitized.update({"origin_id": 0})
sanitized['metadata_type'] = CHANNEL_TORRENT
with db_session:
channels = self.session.mds.get_entries(**sanitized)
total = self.session.mds.get_total_count(
**sanitized) if include_total else None
channels_list = []
for channel in channels:
channel_dict = channel.to_simple_dict()
# Add progress info for those channels that are still being processed
if channel.subscribed:
if channel_dict["state"] == CHANNEL_STATE.UPDATING.value:
try:
progress = self.session.mds.compute_channel_update_progress(
channel)
channel_dict["progress"] = progress
except (ZeroDivisionError, FileNotFoundError) as e:
self._logger.error(
"Error %s when calculating channel update progress. Channel data: %s-%i %i/%i",
e,
hexlify(channel.public_key),
channel.id_,
channel.start_timestamp,
channel.local_version,
)
elif channel_dict[
"state"] == CHANNEL_STATE.METAINFO_LOOKUP.value:
if not self.session.dlmgr.metainfo_requests.get(
bytes(channel.infohash)
) and self.session.dlmgr.download_exists(
bytes(channel.infohash)):
channel_dict[
"state"] = CHANNEL_STATE.DOWNLOADING.value
channels_list.append(channel_dict)
response_dict = {
"results": channels_list,
"first": sanitized["first"],
"last": sanitized["last"],
"sort_by": sanitized["sort_by"],
"sort_desc": int(sanitized["sort_desc"]),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary=
'Get a list of the channel\'s contents (torrents/channels/etc.).',
responses={
200: {
'schema':
schema(
GetChannelContentsResponse={
'results': [Dict()],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
})
}
},
)
async def get_channel_contents(self, request):
sanitized = self.sanitize_parameters(request.query)
include_total = request.query.get('include_total', '')
channel_pk, channel_id = self.get_channel_from_request(request)
sanitized.update({"channel_pk": channel_pk, "origin_id": channel_id})
remote = sanitized.pop("remote", None)
total = None
remote_failed = False
if remote:
try:
contents_list = await self.session.gigachannel_community.remote_select_channel_contents(
**sanitized)
except (RequestTimeoutException, NoChannelSourcesException,
CancelledError):
remote_failed = True
if not remote or remote_failed:
with db_session:
contents = self.session.mds.get_entries(**sanitized)
contents_list = [c.to_simple_dict() for c in contents]
total = self.session.mds.get_total_count(
**sanitized) if include_total else None
self.add_download_progress_to_metadata_list(contents_list)
response_dict = {
"results": contents_list,
"first": sanitized['first'],
"last": sanitized['last'],
"sort_by": sanitized['sort_by'],
"sort_desc": int(sanitized['sort_desc']),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
async def get_channel_description(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel_description = self.session.mds.ChannelDescription.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id).first()
response_dict = loads(channel_description.json_text) if (
channel_description is not None) else {}
return RESTResponse(response_dict)
async def put_channel_description(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
updated_json_text = dumps(
{"description_text": request_parsed["description_text"]})
with db_session:
channel_description = self.session.mds.ChannelDescription.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id).first()
if channel_description is not None:
channel_description.update_properties(
{"json_text": updated_json_text})
else:
channel_description = self.session.mds.ChannelDescription(
public_key=channel_pk,
origin_id=channel_id,
json_text=updated_json_text,
status=NEW)
return RESTResponse(loads(channel_description.json_text))
async def get_channel_thumbnail(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
obj = self.session.mds.ChannelThumbnail.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id).first()
return web.Response(
body=obj.binary_data,
content_type=obj.data_type) if obj else web.Response(status=400)
async def put_channel_thumbnail(self, request):
content_type = request.headers["Content-Type"]
post_body = await request.read()
channel_pk, channel_id = self.get_channel_from_request(request)
obj_properties = {"binary_data": post_body, "data_type": content_type}
with db_session:
obj = self.session.mds.ChannelThumbnail.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id, ).first()
if obj is not None:
obj.update_properties(obj_properties)
else:
self.session.mds.ChannelThumbnail(public_key=channel_pk,
origin_id=channel_id,
status=NEW,
**obj_properties)
return web.Response(status=201)
@docs(
tags=['Metadata'],
summary='Create a copy of an entry/entries from another channel.',
parameters=[{
'in': 'body',
'name': 'entries',
'description': 'List of entries to copy',
'example': [{
'public_key': '1234567890',
'id': 123
}],
'required': True,
}],
responses={
200: {
'description': 'Returns a list of copied content'
},
HTTP_NOT_FOUND: {
'schema': HandledErrorSchema,
'example': {
"error": "Target channel not found"
}
},
HTTP_BAD_REQUEST: {
'schema': HandledErrorSchema,
'example': {
"error": "Source entry not found"
}
},
},
)
async def copy_channel(self, request):
with db_session:
channel_pk, channel_id = self.get_channel_from_request(request)
personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub(
).key_to_bin()[10:]
# TODO: better error handling
target_collection = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
try:
request_parsed = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON"},
status=HTTP_BAD_REQUEST)
if not target_collection and not personal_root:
return RESTResponse({"error": "Target channel not found"},
status=HTTP_NOT_FOUND)
results_list = []
for entry in request_parsed:
public_key, id_ = database_blob(unhexlify(
entry["public_key"])), entry["id"]
source = self.session.mds.ChannelNode.get(
public_key=public_key, id_=id_)
if not source:
return RESTResponse({"error": "Source entry not found"},
status=HTTP_BAD_REQUEST)
# We must upgrade Collections to Channels when moving them to root channel, and, vice-versa,
# downgrade Channels to Collections when moving them into existing channels
if isinstance(source, self.session.mds.CollectionNode):
src_dict = source.to_dict()
if channel_id == 0:
rslt = self.session.mds.ChannelMetadata.create_channel(
title=source.title)
else:
dst_dict = {'origin_id': channel_id, "status": NEW}
for k in self.session.mds.CollectionNode.nonpersonal_attributes:
dst_dict[k] = src_dict[k]
dst_dict.pop("metadata_type")
rslt = self.session.mds.CollectionNode(**dst_dict)
for child in source.actual_contents:
child.make_copy(rslt.id_)
else:
rslt = source.make_copy(channel_id)
results_list.append(rslt.to_simple_dict())
return RESTResponse(results_list)
@docs(
tags=['Metadata'],
summary='Create a new channel entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created channel',
'schema': schema(CreateChannelResponse={'results': [Dict()]}),
}
},
)
async def create_channel(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
channel_name = request_parsed.get("name", "New channel")
md = self.session.mds.ChannelMetadata.create_channel(
channel_name, origin_id=channel_id)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Create a new collection entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created collection',
'schema':
schema(CreateCollectionResponse={'results': [Dict()]}),
}
},
)
async def create_collection(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
collection_name = request_parsed.get("name", "New collection")
md = self.session.mds.CollectionNode(origin_id=channel_id,
title=collection_name,
status=NEW)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Add a torrent file to your own channel.',
responses={
200: {
'schema':
schema(
AddTorrentToChannelResponse={
'added': (
Integer,
'Number of torrent that were added to the channel')
})
},
HTTP_NOT_FOUND: {
'schema': HandledErrorSchema,
'example': {
"error": "Unknown channel"
}
},
HTTP_BAD_REQUEST: {
'schema': HandledErrorSchema,
'example': {
"error": "unknown uri type"
}
},
},
)
@json_schema(
schema(
AddTorrentToChannelRequest={
'torrent': (String, 'Base64-encoded torrent file'),
'uri': (String, 'Add a torrent from a magnet link or URL'),
'torrents_dir': (
String, 'Add all .torrent files from a chosen directory'),
'recursive':
(Boolean,
'Toggle recursive scanning of the chosen directory for .torrent files'
),
'description': (String, 'Description for the torrent'),
}))
async def add_torrent_to_channel(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
if not channel:
return RESTResponse({"error": "Unknown channel"},
status=HTTP_NOT_FOUND)
parameters = await request.json()
extra_info = {}
if parameters.get('description', None):
extra_info = {'description': parameters['description']}
# First, check whether we did upload a magnet link or URL
if parameters.get('uri', None):
uri = parameters['uri']
if uri.startswith("http:") or uri.startswith("https:"):
async with ClientSession() as session:
response = await session.get(uri)
data = await response.read()
tdef = TorrentDef.load_from_memory(data)
elif uri.startswith("magnet:"):
_, xt, _ = parse_magnetlink(uri)
if (xt and is_infohash(codecs.encode(xt, 'hex')) and
(self.session.mds.torrent_exists_in_personal_channel(xt)
or channel.copy_torrent_from_infohash(xt))):
return RESTResponse({"added": 1})
meta_info = await self.session.dlmgr.get_metainfo(xt,
timeout=30,
url=uri)
if not meta_info:
raise RuntimeError("Metainfo timeout")
tdef = TorrentDef.load_from_dict(meta_info)
else:
return RESTResponse({"error": "unknown uri type"},
status=HTTP_BAD_REQUEST)
added = 0
if tdef:
channel.add_torrent_to_channel(tdef, extra_info)
added = 1
return RESTResponse({"added": added})
torrents_dir = None
if parameters.get('torrents_dir', None):
torrents_dir = parameters['torrents_dir']
if not path_util.isabs(torrents_dir):
return RESTResponse(
{"error": "the torrents_dir should point to a directory"},
status=HTTP_BAD_REQUEST)
recursive = False
if parameters.get('recursive'):
recursive = parameters['recursive']
if not torrents_dir:
return RESTResponse(
{
"error":
"the torrents_dir parameter should be provided when the recursive parameter is set"
},
status=HTTP_BAD_REQUEST,
)
if torrents_dir:
torrents_list, errors_list = channel.add_torrents_from_dir(
torrents_dir, recursive)
return RESTResponse({
"added": len(torrents_list),
"errors": errors_list
})
if not parameters.get('torrent', None):
return RESTResponse({"error": "torrent parameter missing"},
status=HTTP_BAD_REQUEST)
# Try to parse the torrent data
# Any errors will be handled by the error_middleware
torrent = base64.b64decode(parameters['torrent'])
torrent_def = TorrentDef.load_from_memory(torrent)
channel.add_torrent_to_channel(torrent_def, extra_info)
return RESTResponse({"added": 1})
@docs(
tags=['Metadata'],
summary='Commit a channel.',
responses={
200: {
'schema': schema(CommitResponse={'success': Boolean()})
}
},
)
async def post_commit(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
if channel_id == 0:
for t in self.session.mds.CollectionNode.commit_all_channels():
self.session.gigachannel_manager.updated_my_channel(
TorrentDef.load_from_dict(t))
else:
coll = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
if not coll:
return RESTResponse({"success": False},
status=HTTP_NOT_FOUND)
torrent_dict = coll.commit_channel_torrent()
if torrent_dict:
self.session.gigachannel_manager.updated_my_channel(
TorrentDef.load_from_dict(torrent_dict))
return RESTResponse({"success": True})
@docs(
tags=['Metadata'],
summary='Check if a channel has uncommitted changes.',
responses={
200: {
'schema': schema(IsChannelDirtyResponse={'dirty': Boolean()})
}
},
)
async def is_channel_dirty(self, request):
channel_pk, _ = self.get_channel_from_request(request)
with db_session:
dirty = self.session.mds.MetadataNode.exists(
lambda g: g.public_key == database_blob(
channel_pk) and g.status in DIRTY_STATUSES)
return RESTResponse({"dirty": dirty})
@docs(
tags=['Metadata'],
summary=
'Get the list of most popular torrents. Functions as a pseudo-channel.',
responses={
200: {
'schema':
schema(GetChannelContentsResponse={
'results': [Dict()],
'first': Integer(),
'last': Integer(),
})
}
},
)
async def get_popular_torrents_channel(self, request):
sanitized = self.sanitize_parameters(request.query)
sanitized["metadata_type"] = REGULAR_TORRENT
sanitized["popular"] = True
with db_session:
contents = self.session.mds.get_entries(**sanitized)
contents_list = [c.to_simple_dict() for c in contents]
self.add_download_progress_to_metadata_list(contents_list)
response_dict = {
"results": contents_list,
"first": sanitized['first'],
"last": sanitized['last'],
}
return RESTResponse(response_dict)
class CreateTorrentEndpoint(RESTEndpoint):
"""
Create a torrent file from local files.
See: http://www.bittorrent.org/beps/bep_0012.html
"""
def setup_routes(self):
self.app.add_routes([web.post('', self.create_torrent)])
@docs(
tags=["Libtorrent"],
summary=
"Create a torrent from local files and return it in base64 encoding.",
parameters=[{
'in': 'query',
'name': 'download',
'description':
'Flag indicating whether or not to start downloading',
'type': 'boolean',
'required': False
}],
responses={
200: {
"schema":
schema(CreateTorrentResponse={
'torrent': 'base64 encoded torrent file'
}),
"examples": {
'Success': {
"success": True
}
}
},
HTTP_BAD_REQUEST: {
"schema": HandledErrorSchema,
"examples": {
"Error": {
"error": "files parameter missing"
}
}
}
})
@json_schema(
schema(
CreateTorrentRequest={
'files': [String],
'name': String,
'description': String,
'trackers': [String],
'export_dir': String
}))
async def create_torrent(self, request):
parameters = await request.json()
params = {}
if 'files' in parameters and parameters['files']:
file_path_list = [
ensure_unicode(f, 'utf-8') for f in parameters['files']
]
else:
return RESTResponse({"error": "files parameter missing"},
status=HTTP_BAD_REQUEST)
if 'description' in parameters and parameters['description']:
params['comment'] = parameters['description']
if 'trackers' in parameters and parameters['trackers']:
tracker_url_list = parameters['trackers']
params['announce'] = tracker_url_list[0]
params['announce-list'] = tracker_url_list
name = 'unknown'
if 'name' in parameters and parameters['name']:
name = parameters['name']
params['name'] = name
export_dir = None
if 'export_dir' in parameters and parameters['export_dir']:
export_dir = Path(parameters['export_dir'])
from tribler_core.version import version_id
params['created by'] = '%s version: %s' % ('Tribler', version_id)
params['nodes'] = False
params['httpseeds'] = False
params['encoding'] = False
params['piece length'] = 0 # auto
try:
result = await self.session.dlmgr.create_torrent_file(
file_path_list, recursive_bytes(params))
except (IOError, UnicodeDecodeError, RuntimeError) as e:
self._logger.exception(e)
return return_handled_exception(request, e)
metainfo_dict = bdecode_compat(result['metainfo'])
if export_dir and export_dir.exists():
save_path = export_dir / ("%s.torrent" % name)
with open(save_path, "wb") as fd:
fd.write(result['metainfo'])
# Download this torrent if specified
if 'download' in request.query and request.query[
'download'] and request.query['download'] == "1":
download_config = DownloadConfig()
download_config.set_dest_dir(result['base_path'] if len(
file_path_list) == 1 else result['base_dir'])
try:
self.session.dlmgr.start_download(
tdef=TorrentDef(metainfo_dict), config=download_config)
except DuplicateDownloadException:
self._logger.warning(
"The created torrent is already being downloaded.")
return RESTResponse(
json.dumps({
"torrent":
base64.b64encode(result['metainfo']).decode('utf-8')
}))
class SearchEndpoint(MetadataEndpointBase):
"""
This endpoint is responsible for searching in channels and torrents present in the local Tribler database.
"""
def setup_routes(self):
self.app.add_routes([
web.get('', self.search),
web.get('/completions', self.completions)
])
@staticmethod
def get_uuid(parameters):
return parameters['uuid'] if 'uuid' in parameters else None
@classmethod
def sanitize_parameters(cls, parameters):
sanitized = super().sanitize_parameters(parameters)
if "max_rowid" in parameters:
sanitized["max_rowid"] = int(parameters["max_rowid"])
return sanitized
@docs(
tags=['Metadata'],
summary="Perform a search for a given query.",
responses={
200: {
'schema':
schema(
SearchResponse={
'torrents': [
schema(
Torrent={
'commit_status': Integer,
'num_leechers': Integer,
'date': Integer,
'relevance_score': Integer,
'id': Integer,
'size': Integer,
'category': String,
'public_key': String,
'name': String,
'last_tracker_check': Integer,
'infohash': String,
'num_seeders': Integer,
'type': String,
})
],
'chant_dirty':
Boolean,
})
}
},
)
@querystring_schema(MetadataParameters)
async def search(self, request):
try:
sanitized = self.sanitize_parameters(request.query)
except (ValueError, KeyError):
return RESTResponse(
{"error": "Error processing request parameters"},
status=HTTP_BAD_REQUEST)
if not sanitized["txt_filter"]:
return RESTResponse({"error": "Filter parameter missing"},
status=HTTP_BAD_REQUEST)
include_total = request.query.get('include_total', '')
mds: MetadataStore = self.session.mds
def search_db():
with db_session:
pony_query = mds.get_entries(**sanitized)
search_results = [r.to_simple_dict() for r in pony_query]
if include_total:
total = mds.get_total_count(**sanitized)
max_rowid = mds.get_max_rowid()
else:
total = max_rowid = None
return search_results, total, max_rowid
try:
search_results, total, max_rowid = await mds.run_threaded(search_db
)
except Exception as e: # pylint: disable=broad-except; # pragma: no cover
self._logger.error("Error while performing DB search: %s: %s",
type(e).__name__, e)
return RESTResponse(status=HTTP_BAD_REQUEST)
response_dict = {
"results": search_results,
"first": sanitized["first"],
"last": sanitized["last"],
"sort_by": sanitized["sort_by"],
"sort_desc": sanitized["sort_desc"],
}
if include_total:
response_dict.update(total=total, max_rowid=max_rowid)
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary="Return auto-completion suggestions for a given query.",
parameters=[{
'in': 'query',
'name': 'q',
'description': 'Search query',
'type': 'string',
'required': True
}],
responses={
200: {
'schema': schema(CompletionsResponse={
'completions': [String],
}),
'examples': {
'completions': ['pioneer one', 'pioneer movie']
},
}
},
)
async def completions(self, request):
args = request.query
if 'q' not in args:
return RESTResponse({"error": "query parameter missing"},
status=HTTP_BAD_REQUEST)
keywords = args['q'].strip().lower()
# TODO: add XXX filtering for completion terms
results = self.session.mds.get_auto_complete_terms(keywords,
max_terms=5)
return RESTResponse({"completions": results})
class TagsEndpoint(RESTEndpoint):
"""
Top-level endpoint for tags.
"""
def __init__(self, db: TagDatabase, community: TagCommunity):
super().__init__()
self.db: TagDatabase = db
self.community: TagCommunity = community
@staticmethod
def validate_infohash(infohash: str) -> Tuple[bool, Optional[RESTResponse]]:
try:
infohash = unhexlify(infohash)
if len(infohash) != 20:
return False, RESTResponse({"error": "Invalid infohash"}, status=HTTP_BAD_REQUEST)
except binascii.Error:
return False, RESTResponse({"error": "Invalid infohash"}, status=HTTP_BAD_REQUEST)
return True, None
def setup_routes(self):
self.app.add_routes(
[
web.patch('/{infohash}', self.update_tags_entries),
web.get('/{infohash}/suggestions', self.get_suggestions),
]
)
@docs(
tags=["General"],
summary="Update a particular torrent with tags.",
responses={
200: {
"schema": schema(UpdateTagsResponse={'success': Boolean()})
},
HTTP_BAD_REQUEST: {
"schema": HandledErrorSchema, 'example': {"error": "Invalid tag length"}},
},
description="This endpoint updates a particular torrent with the provided tags."
)
async def update_tags_entries(self, request):
params = await request.json()
infohash = request.match_info["infohash"]
ih_valid, error_response = TagsEndpoint.validate_infohash(infohash)
if not ih_valid:
return error_response
tags = {tag.lower() for tag in params["tags"]}
# Validate whether the size of the tag is within the allowed range
for tag in tags:
if len(tag) < MIN_TAG_LENGTH or len(tag) > MAX_TAG_LENGTH:
return RESTResponse({"error": "Invalid tag length"}, status=HTTP_BAD_REQUEST)
self.modify_tags(unhexlify(infohash), tags)
return RESTResponse({"success": True})
@db_session
def modify_tags(self, infohash: bytes, new_tags: Set[str]):
"""
Modify the tags of a particular content item.
"""
if not self.community:
return
# First, get the current tags and compute the diff between the old and new tags
old_tags = set(self.db.get_tags(infohash))
added_tags = new_tags - old_tags
removed_tags = old_tags - new_tags
# Create individual tag operations for the added/removed tags
public_key = self.community.tags_key.pub().key_to_bin()
for tag in added_tags.union(removed_tags):
type_of_operation = TagOperationEnum.ADD if tag in added_tags else TagOperationEnum.REMOVE
operation = TagOperation(infohash=infohash, operation=type_of_operation, clock=0,
creator_public_key=public_key, tag=tag)
operation.clock = self.db.get_clock(operation) + 1
signature = self.community.sign(operation)
self.db.add_tag_operation(operation, signature, is_local_peer=True)
@docs(
tags=["General"],
summary="Get tag suggestions for a torrent with a particular infohash.",
responses={
200: {
"schema": schema(SuggestedTagsResponse={'suggestions': List(String)})
},
HTTP_BAD_REQUEST: {
"schema": HandledErrorSchema, 'example': {"error": "Invalid infohash"}},
},
description="This endpoint updates a particular torrent with the provided tags."
)
async def get_suggestions(self, request):
"""
Get suggestions for a particular tag.
"""
infohash = request.match_info["infohash"]
ih_valid, error_response = TagsEndpoint.validate_infohash(infohash)
if not ih_valid:
return error_response
with db_session:
suggestions = self.db.get_suggestions(unhexlify(infohash))
return RESTResponse({"suggestions": suggestions})
class TrustViewEndpoint(RESTEndpoint):
def __init__(self, session):
super(TrustViewEndpoint, self).__init__(session)
self.logger = logging.getLogger(self.__class__.__name__)
self.trustchain_db = None
self.trust_graph = None
self.public_key = None
def setup_routes(self):
self.app.add_routes([web.get('', self.get_view)])
def initialize_graph(self):
if self.session.trustchain_community:
self.trustchain_db = self.session.trustchain_community.persistence
self.public_key = self.session.trustchain_community.my_peer.public_key.key_to_bin(
)
self.trust_graph = TrustGraph(hexlify(self.public_key))
# Start bootstrap download if not already done
if not self.session.bootstrap:
self.session.start_bootstrap_download()
@docs(tags=["TrustChain"],
summary="Return the trust graph.",
parameters=[{
'in': 'query',
'name': 'depth',
'description': 'Depth level (0 = all depths)',
'enum': [0, 1, 2, 3, 4],
'type': 'integer',
'required': False
}],
responses={
200: {
"schema":
schema(
GraphResponse={
'root_public_key':
String,
'graph':
schema(
Graph={
'node':
schema(
Node={
'id': Integer,
'key': String,
'pos': [Float],
'sequence_number': Integer,
'total_up': Integer,
'total_down': Integer
}),
'edge':
List(List(Integer))
}),
'bootstrap':
schema(
Bootstrap={
'download': Integer,
'upload': Integer,
'progress': Float
}),
'num_tx':
Integer,
'depth':
Integer
})
}
})
async def get_view(self, request):
if not self.trust_graph:
self.initialize_graph()
def get_bandwidth_blocks(public_key, limit=5):
return self.trustchain_db.get_latest_blocks(
public_key, limit=limit, block_types=[b'tribler_bandwidth'])
def get_friends(public_key, limit=5):
return self.trustchain_db.get_connected_users(public_key,
limit=limit)
depth = 0
if 'depth' in request.query:
depth = int(request.query['depth'])
# If depth is zero or not provided then fetch all depth levels
fetch_all = depth == 0
try:
if fetch_all:
self.trust_graph.reset(hexlify(self.public_key))
if fetch_all or depth == 1:
self.trust_graph.add_blocks(
get_bandwidth_blocks(self.public_key, limit=100))
if fetch_all or depth == 2:
for friend in get_friends(self.public_key):
self.trust_graph.add_blocks(
get_bandwidth_blocks(unhexlify(friend['public_key']),
limit=10))
if fetch_all or depth == 3:
for friend in get_friends(self.public_key):
self.trust_graph.add_blocks(
get_bandwidth_blocks(unhexlify(friend['public_key'])))
for fof in get_friends(unhexlify(friend['public_key'])):
self.trust_graph.add_blocks(
get_bandwidth_blocks(unhexlify(fof['public_key'])))
if fetch_all or depth == 4:
for user_block in self.trustchain_db.get_users():
self.trust_graph.add_blocks(
get_bandwidth_blocks(
unhexlify(user_block['public_key'])))
except TrustGraphException as tgex:
self.logger.warning(tgex)
graph_data = self.trust_graph.compute_node_graph()
return RESTResponse({
'root_public_key': hexlify(self.public_key),
'graph': graph_data,
'bootstrap': self.get_bootstrap_info(),
'num_tx': len(graph_data['edge']),
'depth': depth,
})
def get_bootstrap_info(self):
if self.session.bootstrap.download and self.session.bootstrap.download.get_state(
):
state = self.session.bootstrap.download.get_state()
return {
'download': state.get_total_transferred(DOWNLOAD),
'upload': state.get_total_transferred(UPLOAD),
'progress': state.get_progress(),
}
return {'download': 0, 'upload': 0, 'progress': 0}
class DebugEndpoint(RESTEndpoint):
"""
This endpoint is responsible for handing requests regarding debug information in Tribler.
"""
def __init__(self,
state_dir: Path,
log_dir: Path,
tunnel_community: TunnelCommunity = None,
resource_monitor: ResourceMonitor = None):
super().__init__()
self.state_dir = state_dir
self.log_dir = log_dir
self.tunnel_community = tunnel_community
self.resource_monitor = resource_monitor
def setup_routes(self):
self.app.add_routes([
web.get('/circuits/slots', self.get_circuit_slots),
web.get('/open_files', self.get_open_files),
web.get('/open_sockets', self.get_open_sockets),
web.get('/threads', self.get_threads),
web.get('/cpu/history', self.get_cpu_history),
web.get('/memory/history', self.get_memory_history),
web.get('/log', self.get_log),
web.get('/profiler', self.get_profiler_state),
web.put('/profiler', self.start_profiler),
web.delete('/profiler', self.stop_profiler)
])
if HAS_MELIAE:
self.app.add_routes(
[web.get('/memory/dump', self.get_memory_dump)])
@docs(tags=['Debug'],
summary="Return information about the slots in the tunnel overlay.",
responses={
200: {
'schema':
schema(
CircuitSlotsResponse={
'slots': [
schema(CircuitSlot={
'random': Integer,
'competing': Integer
})
]
})
}
})
async def get_circuit_slots(self, request):
return RESTResponse({
"slots": {
"random": self.tunnel_community.random_slots,
"competing": self.tunnel_community.competing_slots
}
})
@docs(tags=['Debug'],
summary="Return information about files opened by Tribler.",
responses={
200: {
'schema':
schema(
OpenFilesResponse={
'open_files':
[schema(OpenFile={
'path': String,
'fd': Integer
})]
})
}
})
async def get_open_files(self, request):
my_process = psutil.Process()
return RESTResponse({
"open_files": [{
"path": open_file.path,
"fd": open_file.fd
} for open_file in my_process.open_files()]
})
@docs(tags=['Debug'],
summary="Return information about open sockets.",
responses={
200: {
'schema':
schema(
OpenSocketsResponse={
'open_sockets': [
schema(
OpenSocket={
'family': Integer,
'status': String,
'laddr': String,
'raddr': String,
'type': Integer
})
]
})
}
})
async def get_open_sockets(self, request):
my_process = psutil.Process()
sockets = []
for open_socket in my_process.connections():
sockets.append({
"family":
open_socket.family,
"status":
open_socket.status,
"laddr":
("%s:%d" % open_socket.laddr) if open_socket.laddr else "-",
"raddr":
("%s:%d" % open_socket.raddr) if open_socket.raddr else "-",
"type":
open_socket.type
})
return RESTResponse({"open_sockets": sockets})
@docs(tags=['Debug'],
summary="Return information about running threads.",
responses={
200: {
'schema':
schema(
ThreadsResponse={
'threads': [
schema(
Thread={
'thread_id': Integer,
'thread_name': String,
'frames': [String]
})
]
})
}
})
async def get_threads(self, request):
watchdog = WatchDog()
return RESTResponse({"threads": watchdog.get_threads_info()})
@docs(tags=['Debug'],
summary="Return information about CPU usage history.",
responses={
200: {
'schema':
schema(
CPUHistoryResponse={
'cpu_history':
[schema(CPUHistory={
'time': Integer,
'cpu': Float
})]
})
}
})
async def get_cpu_history(self, request):
history = self.resource_monitor.get_cpu_history_dict(
) if self.resource_monitor else {}
return RESTResponse({"cpu_history": history})
@docs(tags=['Debug'],
summary="Return information about memory usage history.",
responses={
200: {
'schema':
schema(
MemoryHistoryResponse={
'memory_history': [
schema(MemoryHistory={
'time': Integer,
'mem': Integer
})
]
})
}
})
async def get_memory_history(self, request):
history = self.resource_monitor.get_memory_history_dict(
) if self.resource_monitor else {}
return RESTResponse({"memory_history": history})
@docs(tags=['Debug'],
summary="Return a Meliae-compatible dump of the memory contents.",
responses={
200: {
'description': 'The content of the memory dump file'
}
})
async def get_memory_dump(self, request):
if sys.platform == "win32":
# On Windows meliae (especially older versions) segfault on writing to file
dump_buffer = MemoryDumpBuffer()
try:
scanner.dump_all_objects(dump_buffer)
except OverflowError as e:
# https://bugs.launchpad.net/meliae/+bug/569947
logging.error(
"meliae dump failed (your version may be too old): %s",
str(e))
content = dump_buffer.getvalue()
dump_buffer.close()
else:
# On other platforms, simply writing to file is much faster
dump_file_path = self.state_dir / 'memory_dump.json'
scanner.dump_all_objects(dump_file_path)
with open(dump_file_path) as dump_file:
content = dump_file.read()
date_str = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
return RESTResponse(
content,
headers={
'Content-Type', 'application/json', 'Content-Disposition',
'attachment; filename=tribler_memory_dump_%s.json' % date_str
})
@docs(
tags=['Debug'],
summary="Return content of core or gui log file & max_lines requested.",
parameters=[{
'in': 'query',
'name': 'process',
'description': 'Specifies which log to return',
'enum': ['core', 'gui'],
'type': 'string',
'required': False
}, {
'in': 'query',
'name': 'max_lines',
'description':
'Maximum number of lines to return from the log file',
'type': 'integer',
'required': False
}],
responses={
200: {
'schema':
schema(LogFileResponse={
'max_lines': Integer,
'content': String
})
}
})
async def get_log(self, request):
# First, flush all the logs to make sure it is written to file
for handler in logging.getLogger().handlers:
handler.flush()
# Default response
response = {'content': '', 'max_lines': 0}
# Get the location of log file
param_process = request.query.get('process', 'core')
log_name = f'tribler-{param_process}-info.log'
log_file_name = self.log_dir / log_name
# If the log file is not present in the versioned state directory, try root state directory location
if not log_file_name.exists():
log_file_name = get_root_state_directory() / log_name
# If the log file is still not found, maybe it is not created yet, then return the default response
if not log_file_name.exists():
return RESTResponse(response)
# If the log file exists and return last requested 'max_lines' of log
try:
max_lines = int(request.query['max_lines'])
with log_file_name.open(mode='r') as log_file:
response['content'] = self.tail(log_file, max_lines)
response['max_lines'] = max_lines
except ValueError:
with log_file_name.open(mode='r') as log_file:
response['content'] = self.tail(log_file,
100) # default 100 lines
response['max_lines'] = 0
return RESTResponse(response)
def tail(self, file_handler, lines=1):
"""Tail a file and get X lines from the end"""
# place holder for the lines found
lines_found = []
byte_buffer = 1024
# block counter will be multiplied by buffer
# to get the block size from the end
block_counter = -1
# loop until we find X lines
while len(lines_found) < lines:
try:
file_handler.seek(block_counter * byte_buffer, os.SEEK_END)
except OSError: # either file is too small, or too many lines requested
file_handler.seek(0)
lines_found = file_handler.readlines()
break
lines_found = file_handler.readlines()
# we found enough lines, get out
if len(lines_found) > lines:
break
# decrement the block counter to get the
# next X bytes
block_counter -= 1
return ''.join(lines_found[-lines:])
@docs(
tags=['Debug'],
summary="Return information about the state of the profiler.",
responses={
200: {
'schema':
schema(
ProfilerStateResponse={
'state': (String,
'State of the profiler (STARTED or STOPPED)')
})
}
})
async def get_profiler_state(self, _):
if self.resource_monitor is None:
return RESTResponse(status=404)
state = "STARTED" if self.resource_monitor.profiler.is_running(
) else "STOPPED"
return RESTResponse({"state": state})
@docs(tags=['Debug'],
summary="Start the profiler.",
responses={
200: {
'schema': schema(StartProfilerResponse={'success': Boolean})
}
})
async def start_profiler(self, _):
self.resource_monitor.profiler.start()
return RESTResponse({"success": True})
@docs(tags=['Debug'],
summary="Stop the profiler.",
responses={
200: {
'schema': schema(StopProfilerResponse={'success': Boolean})
}
})
async def stop_profiler(self, _):
file_path = self.resource_monitor.profiler.stop()
return RESTResponse({"success": True, "profiler_file": str(file_path)})
class MetadataEndpoint(MetadataEndpointBase, UpdateEntryMixin):
"""
This is the top-level endpoint class that serves other endpoints.
# /metadata
# /channels
# /torrents
# /<public_key>
"""
def setup_routes(self):
self.app.add_routes(
[
web.patch('', self.update_channel_entries),
web.delete('', self.delete_channel_entries),
web.get('/torrents/{infohash}/health', self.get_torrent_health),
web.patch(r'/{public_key:\w*}/{id:\w*}', self.update_channel_entry),
web.get(r'/{public_key:\w*}/{id:\w*}', self.get_channel_entries),
]
)
@docs(
tags=['Metadata'],
summary='Update channel entries.',
parameters=[
{
'in': 'body',
'name': 'entries',
'description': 'List of entries to update',
'example': [{'public_key': '1234567890', 'id': 123, 'property_to_update': 'new_value'}],
'required': True,
}
],
responses={
200: {'description': 'Returns a list of updated entries'},
HTTP_NOT_FOUND: {'schema': HandledErrorSchema},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema},
},
)
async def update_channel_entries(self, request):
try:
request_parsed = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON"}, status=HTTP_BAD_REQUEST)
results_list = []
for entry in request_parsed:
public_key = database_blob(unhexlify(entry.pop("public_key")))
id_ = entry.pop("id")
error, result = self.update_entry(public_key, id_, entry)
# TODO: handle the results for a list that contains some errors in a smarter way
if error:
return RESTResponse(result, status=error)
results_list.append(result)
return RESTResponse(results_list)
@docs(
tags=['Metadata'],
summary='Delete channel entries.',
parameters=[
{
'in': 'body',
'name': 'entries',
'description': 'List of entries to delete',
'example': [{'public_key': '1234567890', 'id': 123}],
'required': True,
}
],
responses={
200: {'description': 'Returns a list of deleted entries'},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema},
},
)
async def delete_channel_entries(self, request):
with db_session:
request_parsed = await request.json()
results_list = []
for entry in request_parsed:
public_key = database_blob(unhexlify(entry.pop("public_key")))
id_ = entry.pop("id")
entry = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_)
if not entry:
return RESTResponse({"error": "Entry %i not found" % id_}, status=HTTP_BAD_REQUEST)
entry.delete()
result = {"public_key": hexlify(public_key), "id": id_, "state": "Deleted"}
results_list.append(result)
return RESTResponse(results_list)
@docs(
tags=['Metadata'],
summary='Update a single channel entry.',
responses={
200: {'description': 'The updated entry'},
HTTP_NOT_FOUND: {'schema': HandledErrorSchema},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema},
},
)
async def update_channel_entry(self, request):
# TODO: unify checks for parts of the path, i.e. proper hex for public key, etc.
try:
parameters = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON input data"}, status=HTTP_BAD_REQUEST)
public_key = unhexlify(request.match_info['public_key'])
id_ = request.match_info['id']
error, result = self.update_entry(public_key, id_, parameters)
return RESTResponse(result, status=error or 200)
@docs(
tags=['Metadata'],
summary='Get channel entries.',
responses={200: {'description': 'Returns a list of entries'}, HTTP_NOT_FOUND: {'schema': HandledErrorSchema}},
)
async def get_channel_entries(self, request):
public_key = unhexlify(request.match_info['public_key'])
id_ = request.match_info['id']
with db_session:
entry = self.session.mds.ChannelNode.get(public_key=database_blob(public_key), id_=id_)
if entry:
# TODO: handle costly attributes in a more graceful and generic way for all types of metadata
entry_dict = entry.to_simple_dict(include_trackers=isinstance(entry, self.session.mds.TorrentMetadata))
else:
return RESTResponse({"error": "entry not found in database"}, status=HTTP_NOT_FOUND)
return RESTResponse(entry_dict)
@docs(
tags=["Metadata"],
summary="Fetch the swarm health of a specific torrent.",
parameters=[
{
'in': 'path',
'name': 'infohash',
'description': 'Infohash of the download to remove',
'type': 'string',
'required': True,
},
{
'in': 'query',
'name': 'timeout',
'description': 'Timeout to be used in the connections to the trackers',
'type': 'integer',
'default': 20,
'required': False,
},
{
'in': 'query',
'name': 'refresh',
'description': 'Whether or not to force a health recheck. Settings this to 0 means that the '
'health of a torrent will not be checked again if it was recently checked.',
'type': 'integer',
'enum': [0, 1],
'required': False,
},
{
'in': 'query',
'name': 'nowait',
'description': 'Whether or not to return immediately. If enabled, results '
'will be passed through to the events endpoint.',
'type': 'integer',
'enum': [0, 1],
'required': False,
},
],
responses={
200: {
'schema': schema(
HealthCheckResponse={
'tracker': schema(
HealthCheck={'seeders': Integer, 'leechers': Integer, 'infohash': String, 'error': String}
)
}
),
'examples': [
{
"health": {
"http://mytracker.com:80/announce": {
"seeders": 43,
"leechers": 20,
"infohash": "97d2d8f5d37e56cfaeaae151d55f05b077074779",
},
"http://nonexistingtracker.com:80/announce": {"error": "timeout"},
}
},
{'checking': 1},
],
}
},
)
async def get_torrent_health(self, request):
timeout = request.query.get('timeout')
if not timeout:
timeout = TORRENT_CHECK_TIMEOUT
elif timeout.isdigit():
timeout = int(timeout)
else:
return RESTResponse({"error": f"Error processing timeout parameter '{timeout}'"}, status=HTTP_BAD_REQUEST)
refresh = request.query.get('refresh', '0') == '1'
nowait = request.query.get('nowait', '0') == '1'
infohash = unhexlify(request.match_info['infohash'])
result_future = self.session.torrent_checker.check_torrent_health(infohash, timeout=timeout, scrape_now=refresh)
# Return immediately. Used by GUI to schedule health updates through the EventsEndpoint
if nowait:
return RESTResponse({'checking': '1'})
# Errors will be handled by error_middleware
result = await result_future
return RESTResponse({'health': result})
class ChannelsEndpoint(ChannelsEndpointBase):
def setup_routes(self):
self.app.add_routes(
[
web.get('', self.get_channels),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}', self.get_channel_contents),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/copy', self.copy_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/channels', self.create_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/collections', self.create_collection),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/torrents', self.add_torrent_to_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.post_commit),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.is_channel_dirty),
]
)
def get_channel_from_request(self, request):
channel_pk = (
self.session.mds.my_key.pub().key_to_bin()[10:]
if request.match_info['channel_pk'] == 'mychannel'
else unhexlify(request.match_info['channel_pk'])
)
channel_id = int(request.match_info['channel_id'])
return channel_pk, channel_id
@docs(
tags=['Metadata'],
summary='Get a list of all channels known to the system.',
responses={
200: {
'schema': schema(
GetChannelsResponse={
'results': [ChannelSchema],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
}
)
}
},
)
# TODO: DRY it with SpecificChannel endpoint?
async def get_channels(self, request):
sanitized = self.sanitize_parameters(request.query)
sanitized['subscribed'] = None if 'subscribed' not in request.query else bool(int(request.query['subscribed']))
include_total = request.query.get('include_total', '')
sanitized.update({"origin_id": 0})
with db_session:
channels = self.session.mds.ChannelMetadata.get_entries(**sanitized)
total = self.session.mds.ChannelMetadata.get_total_count(**sanitized) if include_total else None
channels_list = [channel.to_simple_dict() for channel in channels]
response_dict = {
"results": channels_list,
"first": sanitized["first"],
"last": sanitized["last"],
"sort_by": sanitized["sort_by"],
"sort_desc": int(sanitized["sort_desc"]),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary='Get a list of the channel\'s contents (torrents/channels/etc.).',
responses={
200: {
'schema': schema(
GetChannelContentsResponse={
'results': [Dict()],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
}
)
}
},
)
async def get_channel_contents(self, request):
sanitized = self.sanitize_parameters(request.query)
include_total = request.query.get('include_total', '')
channel_pk, channel_id = self.get_channel_from_request(request)
sanitized.update({"channel_pk": channel_pk, "origin_id": channel_id})
with db_session:
contents = self.session.mds.MetadataNode.get_entries(**sanitized)
contents_list = [c.to_simple_dict() for c in contents]
total = self.session.mds.MetadataNode.get_total_count(**sanitized) if include_total else None
response_dict = {
"results": contents_list,
"first": sanitized['first'],
"last": sanitized['last'],
"sort_by": sanitized['sort_by'],
"sort_desc": int(sanitized['sort_desc']),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary='Create a copy of an entry/entries from another channel.',
parameters=[
{
'in': 'body',
'name': 'entries',
'description': 'List of entries to copy',
'example': [{'public_key': '1234567890', 'id': 123}],
'required': True,
}
],
responses={
200: {'description': 'Returns a list of copied content'},
HTTP_NOT_FOUND: {'schema': HandledErrorSchema, 'example': {"error": "Target channel not found"}},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema, 'example': {"error": "Source entry not found"}},
},
)
async def copy_channel(self, request):
with db_session:
channel_pk, channel_id = self.get_channel_from_request(request)
personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub().key_to_bin()[10:]
# TODO: better error handling
target_collection = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id
)
try:
request_parsed = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON"}, status=HTTP_BAD_REQUEST)
if not target_collection and not personal_root:
return RESTResponse({"error": "Target channel not found"}, status=HTTP_NOT_FOUND)
results_list = []
for entry in request_parsed:
public_key, id_ = database_blob(unhexlify(entry["public_key"])), entry["id"]
source = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_)
if not source:
return RESTResponse({"error": "Source entry not found"}, status=HTTP_BAD_REQUEST)
# We must upgrade Collections to Channels when moving them to root channel, and, vice-versa,
# downgrade Channels to Collections when moving them into existing channels
if isinstance(source, self.session.mds.CollectionNode):
src_dict = source.to_dict()
if channel_id == 0:
rslt = self.session.mds.ChannelMetadata.create_channel(title=source.title)
else:
dst_dict = {'origin_id': channel_id, "status": NEW}
for k in self.session.mds.CollectionNode.nonpersonal_attributes:
dst_dict[k] = src_dict[k]
dst_dict.pop("metadata_type")
rslt = self.session.mds.CollectionNode(**dst_dict)
for child in source.actual_contents:
child.make_copy(rslt.id_)
else:
rslt = source.make_copy(channel_id)
results_list.append(rslt.to_simple_dict())
return RESTResponse(results_list)
@docs(
tags=['Metadata'],
summary='Create a new channel entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created channel',
'schema': schema(CreateChannelResponse={'results': [Dict()]}),
}
},
)
async def create_channel(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
channel_name = request_parsed.get("name", "New channel")
md = self.session.mds.ChannelMetadata.create_channel(channel_name, origin_id=channel_id)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Create a new collection entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created collection',
'schema': schema(CreateCollectionResponse={'results': [Dict()]}),
}
},
)
async def create_collection(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
collection_name = request_parsed.get("name", "New collection")
md = self.session.mds.CollectionNode(origin_id=channel_id, title=collection_name, status=NEW)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Add a torrent file to your own channel.',
responses={
200: {
'schema': schema(
AddTorrentToChannelResponse={'added': (Integer, 'Number of torrent that were added to the channel')}
)
},
HTTP_NOT_FOUND: {'schema': HandledErrorSchema, 'example': {"error": "Unknown channel"}},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema, 'example': {"error": "unknown uri type"}},
},
)
@json_schema(
schema(
AddTorrentToChannelRequest={
'torrent': (String, 'Base64-encoded torrent file'),
'uri': (String, 'Add a torrent from a magnet link or URL'),
'torrents_dir': (String, 'Add all .torrent files from a chosen directory'),
'recursive': (Boolean, 'Toggle recursive scanning of the chosen directory for .torrent files'),
'description': (String, 'Description for the torrent'),
'filesize': (Integer, "Filesize of the torrent file, this parameter is used for "
"skipping metadata check when uri is a magnet link"),
}
)
)
async def add_torrent_to_channel(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id)
if not channel:
return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND)
parameters = await request.json()
extra_info = {}
if parameters.get('description', None):
extra_info = {'description': parameters['description']}
# First, check whether we did upload a magnet link or URL
if parameters.get('uri', None):
uri = parameters['uri']
if uri.startswith("http:") or uri.startswith("https:"):
async with ClientSession() as session:
response = await session.get(uri)
data = await response.read()
tdef = TorrentDef.load_from_memory(data)
elif uri.startswith("magnet:"):
_, xt, _ = parse_magnetlink(uri)
if (
xt
and is_infohash(codecs.encode(xt, 'hex'))
and (channel.torrent_exists(xt) or channel.copy_torrent_from_infohash(xt))
):
return RESTResponse({"added": 1})
filesize = parameters.get("filesize")
if filesize and not (isinstance(filesize, int) or int is None):
return RESTResponse({"error": "filesize must be an integer"},
status=HTTP_BAD_REQUEST,)
if filesize:
dn, xt, _ = parse_magnetlink(uri)
tdef = TorrentDefNoMetainfo(xt, dn, uri, filesize)
else:
meta_info = await self.session.dlmgr.get_metainfo(xt, timeout=30, url=uri)
if not meta_info:
raise RuntimeError("Metainfo timeout")
tdef = TorrentDef.load_from_dict(meta_info)
else:
return RESTResponse({"error": "unknown uri type"}, status=HTTP_BAD_REQUEST)
added = 0
if tdef:
channel.add_torrent_to_channel(tdef, extra_info)
added = 1
return RESTResponse({"added": added})
torrents_dir = None
if parameters.get('torrents_dir', None):
torrents_dir = parameters['torrents_dir']
if not path_util.isabs(torrents_dir):
return RESTResponse({"error": "the torrents_dir should point to a directory"}, status=HTTP_BAD_REQUEST)
recursive = False
if parameters.get('recursive'):
recursive = parameters['recursive']
if not torrents_dir:
return RESTResponse(
{"error": "the torrents_dir parameter should be provided when the recursive parameter is set"},
status=HTTP_BAD_REQUEST,
)
if torrents_dir:
torrents_list, errors_list = channel.add_torrents_from_dir(torrents_dir, recursive)
return RESTResponse({"added": len(torrents_list), "errors": errors_list})
if not parameters.get('torrent', None):
return RESTResponse({"error": "torrent parameter missing"}, status=HTTP_BAD_REQUEST)
# Try to parse the torrent data
# Any errors will be handled by the error_middleware
torrent = base64.b64decode(parameters['torrent'])
torrent_def = TorrentDef.load_from_memory(torrent)
channel.add_torrent_to_channel(torrent_def, extra_info)
return RESTResponse({"added": 1})
@docs(
tags=['Metadata'],
summary='Commit a channel.',
responses={200: {'schema': schema(CommitResponse={'success': Boolean()})}},
)
async def post_commit(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
if channel_id == 0:
for t in self.session.mds.CollectionNode.commit_all_channels():
self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t))
else:
coll = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id)
if not coll:
return RESTResponse({"success": False}, status=HTTP_NOT_FOUND)
torrent_dict = coll.commit_channel_torrent()
if torrent_dict:
self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(torrent_dict))
return RESTResponse({"success": True})
@docs(
tags=['Metadata'],
summary='Check if a channel has uncommitted changes.',
responses={200: {'schema': schema(IsChannelDirtyResponse={'dirty': Boolean()})}},
)
async def is_channel_dirty(self, request):
channel_pk, _ = self.get_channel_from_request(request)
with db_session:
dirty = self.session.mds.MetadataNode.exists(
lambda g: g.public_key == database_blob(channel_pk) and g.status in DIRTY_STATUSES
)
return RESTResponse({"dirty": dirty})
class DownloadsEndpoint(RESTEndpoint):
"""
This endpoint is responsible for all requests regarding downloads. Examples include getting all downloads,
starting, pausing and stopping downloads.
"""
def __init__(self,
download_manager,
metadata_store=None,
tunnel_community=None):
super().__init__()
self.download_manager = download_manager
self.mds = metadata_store
self.tunnel_community = tunnel_community
self.app.on_shutdown.append(self.on_shutdown)
async def on_shutdown(self, _):
pass
def setup_routes(self):
self.app.add_routes([
web.get('', self.get_downloads),
web.put('', self.add_download),
web.delete('/{infohash}', self.delete_download),
web.patch('/{infohash}', self.update_download),
web.get('/{infohash}/torrent', self.get_torrent),
web.get('/{infohash}/files', self.get_files),
web.get('/{infohash}/stream/{fileindex}',
self.stream,
allow_head=False)
])
@staticmethod
def return_404(request, message="this download does not exist"):
"""
Returns a 404 response code if your channel has not been created.
"""
return RESTResponse({"error": message}, status=HTTP_NOT_FOUND)
@staticmethod
def create_dconfig_from_params(parameters):
"""
Create a download configuration based on some given parameters. Possible parameters are:
- anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download
- safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on)
- destination: the destination path of the torrent (where it is saved on disk)
"""
download_config = DownloadConfig()
anon_hops = parameters.get('anon_hops', 0)
safe_seeding = bool(parameters.get('safe_seeding', 0))
if anon_hops > 0 and not safe_seeding:
return None, "Cannot set anonymous download without safe seeding enabled"
if anon_hops > 0:
download_config.set_hops(anon_hops)
if safe_seeding:
download_config.set_safe_seeding(True)
if 'destination' in parameters:
download_config.set_dest_dir(parameters['destination'])
if 'selected_files' in parameters:
download_config.set_selected_files(parameters['selected_files'])
return download_config, None
@staticmethod
def get_files_info_json(download):
"""
Return file information as JSON from a specified download.
"""
files_json = []
files_completion = {
name: progress
for name, progress in download.get_state().get_files_completion()
}
selected_files = download.config.get_selected_files()
file_index = 0
for fn, size in download.get_def().get_files_with_length():
files_json.append({
"index":
file_index,
# We always return files in Posix format to make GUI independent of Core and simplify testing
"name":
str(PurePosixPath(fn)),
"size":
size,
"included": (file_index in selected_files
or not selected_files),
"progress":
files_completion.get(fn, 0.0)
})
file_index += 1
return files_json
@docs(
tags=["Libtorrent"],
summary="Return all downloads, both active and inactive",
parameters=[{
'in': 'query',
'name': 'get_peers',
'description': 'Flag indicating whether or not to include peers',
'type': 'boolean',
'required': False
}, {
'in': 'query',
'name': 'get_pieces',
'description': 'Flag indicating whether or not to include pieces',
'type': 'boolean',
'required': False
}, {
'in': 'query',
'name': 'get_files',
'description': 'Flag indicating whether or not to include files',
'type': 'boolean',
'required': False
}],
responses={
200: {
"schema":
schema(
DownloadsResponse={
'downloads':
schema(
Download={
'name': String,
'progress': Float,
'infohash': String,
'speed_down': Float,
'speed_up': Float,
'status': String,
'size': Integer,
'eta': Integer,
'num_peers': Integer,
'num_seeds': Integer,
'total_up': Integer,
'total_down': Integer,
'ratio': Float,
'files': String,
'trackers': String,
'hops': Integer,
'anon_download': Boolean,
'safe_seeding': Boolean,
'max_upload_speed': Integer,
'max_download_speed': Integer,
'destination': String,
'availability': Float,
'peers': String,
'total_pieces': Integer,
'vod_mode': Boolean,
'vod_prebuffering_progress': Float,
'vod_prebuffering_progress_consec': Float,
'error': String,
'time_added': Integer
})
}),
}
},
description=
"This endpoint returns all downloads in Tribler, both active and inactive. The progress "
"is a number ranging from 0 to 1, indicating the progress of the specific state (downloading, "
"checking etc). The download speeds have the unit bytes/sec. The size of the torrent is given "
"in bytes. The estimated time assumed is given in seconds.\n\n"
"Detailed information about peers and pieces is only requested when the get_peers and/or "
"get_pieces flag is set. Note that setting this flag has a negative impact on performance "
"and should only be used in situations where this data is required. ")
async def get_downloads(self, request):
get_peers = request.query.get('get_peers', '0') == '1'
get_pieces = request.query.get('get_pieces', '0') == '1'
get_files = request.query.get('get_files', '0') == '1'
downloads_json = []
downloads = self.download_manager.get_downloads()
for download in downloads:
if download.hidden and not download.config.get_channel_download():
# We still want to send channel downloads since they are displayed in the GUI
continue
state = download.get_state()
tdef = download.get_def()
# Create tracker information of the download
tracker_info = []
for url, url_info in download.get_tracker_status().items():
tracker_info.append({
"url": url,
"peers": url_info[0],
"status": url_info[1]
})
num_seeds, num_peers = state.get_num_seeds_peers()
num_connected_seeds, num_connected_peers = download.get_num_connected_seeds_peers(
)
if download.config.get_channel_download():
download_name = self.mds.ChannelMetadata.get_channel_name_cached(
tdef.get_name_utf8(), tdef.get_infohash())
elif self.mds is None:
download_name = tdef.get_name_utf8()
else:
download_name = self.mds.TorrentMetadata.get_torrent_title(tdef.get_infohash()) or \
tdef.get_name_utf8()
download_status = get_extended_status(
self.tunnel_community, download
) if self.tunnel_community else download.get_state().get_status()
download_json = {
"name":
download_name,
"progress":
state.get_progress(),
"infohash":
hexlify(tdef.get_infohash()),
"speed_down":
state.get_current_payload_speed(DOWNLOAD),
"speed_up":
state.get_current_payload_speed(UPLOAD),
"status":
dlstatus_strings[download_status],
"size":
tdef.get_length(),
"eta":
state.get_eta(),
"num_peers":
num_peers,
"num_seeds":
num_seeds,
"num_connected_peers":
num_connected_peers,
"num_connected_seeds":
num_connected_seeds,
"total_up":
state.get_total_transferred(UPLOAD),
"total_down":
state.get_total_transferred(DOWNLOAD),
"ratio":
state.get_seeding_ratio(),
"trackers":
tracker_info,
"hops":
download.config.get_hops(),
"anon_download":
download.get_anon_mode(),
"safe_seeding":
download.config.get_safe_seeding(),
# Maximum upload/download rates are set for entire sessions
"max_upload_speed":
DownloadManager.get_libtorrent_max_upload_rate(
self.download_manager.config),
"max_download_speed":
DownloadManager.get_libtorrent_max_download_rate(
self.download_manager.config),
"destination":
str(download.config.get_dest_dir()),
"availability":
state.get_availability(),
"total_pieces":
tdef.get_nr_pieces(),
"vod_mode":
download.stream and download.stream.enabled,
"error":
repr(state.get_error()) if state.get_error() else "",
"time_added":
download.config.get_time_added(),
"channel_download":
download.config.get_channel_download()
}
if download.stream:
download_json.update({
"vod_prebuffering_progress":
download.stream.prebuffprogress,
"vod_prebuffering_progress_consec":
download.stream.prebuffprogress_consec,
"vod_header_progress":
download.stream.headerprogress,
"vod_footer_progress":
download.stream.footerprogress,
})
# Add peers information if requested
if get_peers:
peer_list = state.get_peerlist()
for peer_info in peer_list: # Remove have field since it is very large to transmit.
del peer_info['have']
if 'extended_version' in peer_info:
peer_info[
'extended_version'] = _safe_extended_peer_info(
peer_info['extended_version'])
# Does this peer represent a hidden services circuit?
if peer_info.get(
'port'
) == CIRCUIT_ID_PORT and self.tunnel_community:
tc = self.tunnel_community
circuit_id = tc.ip_to_circuit_id(peer_info['ip'])
circuit = tc.circuits.get(circuit_id, None)
if circuit:
peer_info['circuit'] = circuit_id
download_json["peers"] = peer_list
# Add piece information if requested
if get_pieces:
download_json["pieces"] = download.get_pieces_base64().decode(
'utf-8')
# Add files if requested
if get_files:
download_json["files"] = self.get_files_info_json(download)
downloads_json.append(download_json)
return RESTResponse({"downloads": downloads_json})
@docs(
tags=["Libtorrent"],
summary="Start a download from a provided URI.",
parameters=[{
'in': 'query',
'name': 'get_peers',
'description': 'Flag indicating whether or not to include peers',
'type': 'boolean',
'required': False
}, {
'in': 'query',
'name': 'get_pieces',
'description': 'Flag indicating whether or not to include pieces',
'type': 'boolean',
'required': False
}, {
'in': 'query',
'name': 'get_files',
'description': 'Flag indicating whether or not to include files',
'type': 'boolean',
'required': False
}],
responses={
200: {
"schema":
schema(AddDownloadResponse={
"started": Boolean,
"infohash": String
}),
'examples': {
"started": True,
"infohash": "4344503b7e797ebf31582327a5baae35b11bda01"
}
}
},
)
@json_schema(
schema(
AddDownloadRequest={
'anon_hops':
(Integer,
'Number of hops for the anonymous download. No hops is equivalent to a plain download'
),
'safe_seeding':
(Boolean,
'Whether the seeding of the download should be anonymous or not'
),
'destination': (
String, 'the download destination path of the torrent'),
'uri*':
(String,
'The URI of the torrent file that should be downloaded. This URI can either represent a file '
'location, a magnet link or a HTTP(S) url.'),
}))
async def add_download(self, request):
parameters = await request.json()
if not parameters.get('uri'):
return RESTResponse({"error": "uri parameter missing"},
status=HTTP_BAD_REQUEST)
download_config, error = DownloadsEndpoint.create_dconfig_from_params(
parameters)
if error:
return RESTResponse({"error": error}, status=HTTP_BAD_REQUEST)
try:
download = await self.download_manager.start_download_from_uri(
parameters['uri'], config=download_config)
except Exception as e:
return RESTResponse({"error": str(e)},
status=HTTP_INTERNAL_SERVER_ERROR)
return RESTResponse({
"started":
True,
"infohash":
hexlify(download.get_def().get_infohash())
})
@docs(
tags=["Libtorrent"],
summary="Remove a specific download.",
parameters=[{
'in': 'path',
'name': 'infohash',
'description': 'Infohash of the download to remove',
'type': 'string',
'required': True
}],
responses={
200: {
"schema":
schema(DeleteDownloadResponse={
"removed": Boolean,
"infohash": String
}),
'examples': {
"removed": True,
"infohash": "4344503b7e797ebf31582327a5baae35b11bda01"
}
}
},
)
@json_schema(
schema(
RemoveDownloadRequest={
'remove_data': (
Boolean, 'Whether or not to remove the associated data'),
}))
async def delete_download(self, request):
parameters = await request.json()
if 'remove_data' not in parameters:
return RESTResponse({"error": "remove_data parameter missing"},
status=HTTP_BAD_REQUEST)
infohash = unhexlify(request.match_info['infohash'])
download = self.download_manager.get_download(infohash)
if not download:
return DownloadsEndpoint.return_404(request)
try:
await self.download_manager.remove_download(
download, remove_content=parameters['remove_data'])
except Exception as e:
self._logger.exception(e)
return return_handled_exception(request, e)
return RESTResponse({
"removed":
True,
"infohash":
hexlify(download.get_def().get_infohash())
})
async def vod_response(self, download, parameters, request, vod_mode):
modified = False
if vod_mode:
file_index = parameters.get("fileindex")
if file_index is None:
return RESTResponse(
{"error": "fileindex is necessary to enable vod_mode"},
status=HTTP_BAD_REQUEST)
if download.stream is None:
download.add_stream()
if not download.stream.enabled or download.stream.fileindex != file_index:
await wait_for(
download.stream.enable(file_index, request.http_range.start
or 0), 10)
await download.stream.updateprios()
modified = True
elif not vod_mode and download.stream is not None and download.stream.enabled:
download.stream.disable()
modified = True
return RESTResponse({
"vod_prebuffering_progress":
download.stream.prebuffprogress,
"vod_prebuffering_progress_consec":
download.stream.prebuffprogress_consec,
"vod_header_progress":
download.stream.headerprogress,
"vod_footer_progress":
download.stream.footerprogress,
"vod_mode":
download.stream.enabled,
"infohash":
hexlify(download.get_def().get_infohash()),
"modified":
modified,
})
@docs(
tags=["Libtorrent"],
summary="Update a specific download.",
parameters=[{
'in': 'path',
'name': 'infohash',
'description': 'Infohash of the download to update',
'type': 'string',
'required': True
}],
responses={
200: {
"schema":
schema(UpdateDownloadResponse={
"modified": Boolean,
"infohash": String
}),
'examples': {
"modified": True,
"infohash": "4344503b7e797ebf31582327a5baae35b11bda01"
}
}
},
)
@json_schema(
schema(
UpdateDownloadRequest={
'state':
(String,
'State parameter to be passed to modify the state of the download (resume/stop/recheck)'
),
'selected_files': (
List(Integer),
'File indexes to be included in the download'),
'anon_hops':
(Integer,
'The anonymity of a download can be changed at runtime by passing the anon_hops '
'parameter, however, this must be the only parameter in this request.'
)
}))
async def update_download(self, request):
infohash = unhexlify(request.match_info['infohash'])
download = self.download_manager.get_download(infohash)
if not download:
return DownloadsEndpoint.return_404(request)
parameters = await request.json()
vod_mode = parameters.get("vod_mode")
if vod_mode is not None:
if not isinstance(vod_mode, bool):
return RESTResponse({"error": "vod_mode must be bool flag"},
status=HTTP_BAD_REQUEST)
return await self.vod_response(download, parameters, request,
vod_mode)
if len(parameters) > 1 and 'anon_hops' in parameters:
return RESTResponse(
{
"error":
"anon_hops must be the only parameter in this request"
},
status=HTTP_BAD_REQUEST)
elif 'anon_hops' in parameters:
anon_hops = int(parameters['anon_hops'])
try:
await self.download_manager.update_hops(download, anon_hops)
except Exception as e:
self._logger.exception(e)
return return_handled_exception(request, e)
return RESTResponse({
"modified":
True,
"infohash":
hexlify(download.get_def().get_infohash())
})
if 'selected_files' in parameters:
selected_files_list = parameters['selected_files']
num_files = len(download.tdef.get_files())
if not all(
[0 <= index < num_files for index in selected_files_list]):
return RESTResponse({"error": "index out of range"},
status=HTTP_BAD_REQUEST)
download.set_selected_files(selected_files_list)
if parameters.get('state'):
state = parameters['state']
if state == "resume":
download.resume()
elif state == "stop":
await download.stop(user_stopped=True)
elif state == "recheck":
download.force_recheck()
elif state == "move_storage":
dest_dir = Path(parameters['dest_dir'])
if not dest_dir.exists():
return RESTResponse({
"error":
f"Target directory ({dest_dir}) does not exist"
})
download.move_storage(dest_dir)
download.checkpoint()
else:
return RESTResponse({"error": "unknown state parameter"},
status=HTTP_BAD_REQUEST)
return RESTResponse({
"modified":
True,
"infohash":
hexlify(download.get_def().get_infohash())
})
@docs(tags=["Libtorrent"],
summary=
"Return the .torrent file associated with the specified download.",
parameters=[{
'in': 'path',
'name': 'infohash',
'description':
'Infohash of the download from which to get the .torrent file',
'type': 'string',
'required': True
}],
responses={200: {
'description': 'The torrent'
}})
async def get_torrent(self, request):
infohash = unhexlify(request.match_info['infohash'])
download = self.download_manager.get_download(infohash)
if not download:
return DownloadsEndpoint.return_404(request)
torrent = download.get_torrent_data()
if not torrent:
return DownloadsEndpoint.return_404(request)
return RESTResponse(lt.bencode(torrent),
headers={
'content-type':
'application/x-bittorrent',
'Content-Disposition':
'attachment; filename=%s.torrent' %
hexlify(infohash).encode('utf-8')
})
@docs(tags=["Libtorrent"],
summary="Return file information of a specific download.",
parameters=[{
'in': 'path',
'name': 'infohash',
'description':
'Infohash of the download to from which to get file information',
'type': 'string',
'required': True
}],
responses={
200: {
"schema":
schema(
GetFilesResponse={
"files": [
schema(
File={
'index': Integer,
'name': String,
'size': Integer,
'included': Boolean,
'progress': Float
})
]
})
}
})
async def get_files(self, request):
infohash = unhexlify(request.match_info['infohash'])
download = self.download_manager.get_download(infohash)
if not download:
return DownloadsEndpoint.return_404(request)
return RESTResponse({"files": self.get_files_info_json(download)})
@docs(tags=["Libtorrent"],
summary="Stream the contents of a file that is being downloaded.",
parameters=[{
'in': 'path',
'name': 'infohash',
'description': 'Infohash of the download to stream',
'type': 'string',
'required': True
}, {
'in': 'path',
'name': 'fileindex',
'description': 'The fileindex to stream',
'type': 'string',
'required': True
}],
responses={206: {
'description': 'Contents of the stream'
}})
async def stream(self, request):
infohash = unhexlify(request.match_info['infohash'])
download = self.download_manager.get_download(infohash)
if not download:
return DownloadsEndpoint.return_404(request)
file_index = int(request.match_info['fileindex'])
http_range = request.http_range
start = http_range.start or 0
if download.stream is None:
download.add_stream()
await wait_for(
download.stream.enable(file_index, None if start > 0 else 0), 10)
stop = download.stream.filesize if http_range.stop is None else min(
http_range.stop, download.stream.filesize)
if not start < stop or not 0 <= start < download.stream.filesize or not 0 < stop <= download.stream.filesize:
return RESTResponse('Requested Range Not Satisfiable', status=416)
response = RESTStreamResponse(
status=206,
reason='OK',
headers={
'Accept-Ranges': 'bytes',
'Content-Type': 'application/octet-stream',
'Content-Length': f'{stop - start}',
'Content-Range': f'{start}-{stop}/{download.stream.filesize}'
})
response.force_close()
with suppress(CancelledError, ConnectionResetError):
async with StreamChunk(download.stream, start) as chunk:
await response.prepare(request)
bytes_todo = stop - start
bytes_done = 0
self._logger.info('Got range request for %s-%s (%s bytes)',
start, stop, bytes_todo)
while not request.transport.is_closing():
if chunk.seekpos >= download.stream.filesize:
break
data = await chunk.read()
try:
if len(data) == 0:
break
if bytes_done + len(data) > bytes_todo:
# if we have more data than we need
endlen = bytes_todo - bytes_done
if endlen != 0:
await wait_for(response.write(data[:endlen]),
STREAM_PAUSE_TIME)
bytes_done += endlen
break
await wait_for(response.write(data), STREAM_PAUSE_TIME)
bytes_done += len(data)
if chunk.resume():
self._logger.debug(
"Stream %s-%s is resumed, starting sequential buffer",
start, stop)
except AsyncTimeoutError:
# This means that stream writer has a full buffer, in practice means that
# the client keeps the conenction but sets the window size to 0. In this case
# there is no need to keep sequenial buffer if there are other chunks waiting for prios
if chunk.pause():
self._logger.debug(
"Stream %s-%s is paused, stopping sequential buffer",
start, stop)
return response
class UnhandledErrorSchema(Schema):
error = schema(DetailedErrorSchema={'handled': Boolean, 'code': Integer, 'message': String})
