class PopularityCommunity(PubSubCommunity):
"""
Community for disseminating the content across the network. Follows publish-subscribe model.
"""
MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b8104002703819200040504278d20d6776ce7081ad57d99fe066bb2a93" \
"ce7cc92405a534ef7175bab702be557d8c7d3b725ea0eb09c686e798f6c7ad85e8781a4c3b20e54c15ede38077c" \
"8f5c801b71d13105f261da7ddcaa94ae14bd177bf1a05a66f595b9bb99117d11f73b4c8d3dcdcdc2b3f838b8ba3" \
"5a9f600d2c543e8b3ba646083307b917bbbccfc53fc5ab6ded90b711d7eeda46f5f"
master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex'))
def __init__(self, *args, **kwargs):
self.torrent_db = kwargs.pop('torrent_db', None)
self.channel_db = kwargs.pop('channel_db', None)
self.trustchain = kwargs.pop('trustchain_community', None)
self.tribler_session = kwargs.pop('session', None)
super(PopularityCommunity, self).__init__(*args, **kwargs)
self.content_repository = ContentRepository(self.torrent_db, self.channel_db)
self.decode_map.update({
chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response,
chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response,
chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request,
chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response,
chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request,
chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response
})
self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX'))
@inlineCallbacks
def unload(self):
self.content_repository.cleanup()
self.content_repository = None
yield super(PopularityCommunity, self).unload()
def on_subscribe(self, source_address, data):
auth, _, _ = self._ez_unpack_auth(ContentSubscription, data)
peer = self.get_peer_from_auth(auth, source_address)
subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data)
# Publish the latest torrents to the subscriber
if subscribed:
self.publish_latest_torrents(peer=peer)
def on_torrent_health_response(self, source_address, data):
"""
Message handler for torrent health response. Torrent health response is part of periodic update message from
the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply
dropped. In other case, a decision to accept or reject the message is made based on freshness of the message
and the trustscore (check update_torrent in ContentRepository for the implementation).
"""
self.logger.debug("Got torrent health response from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.publishers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
infohash = payload.infohash
if not self.content_repository.has_torrent(infohash):
self.send_torrent_info_request(infohash, peer=peer)
peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0
self.content_repository.update_torrent_health(payload, peer_trust)
def on_channel_health_response(self, source_address, data):
"""
Message handler for channel health response. Currently, not sure how to handle it.
"""
def on_torrent_info_request(self, source_address, data):
"""
Message handler for torrent info request.
"""
self.logger.debug("Got torrent info request from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentInfoRequestPayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.subscribers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
self.send_torrent_info_response(payload.infohash, peer=peer)
def on_torrent_info_response(self, source_address, data):
"""
Message handler for torrent info response.
"""
self.logger.debug("Got torrent info response from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentInfoResponsePayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.publishers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
self.content_repository.update_torrent_info(payload)
def on_content_info_request(self, source_address, data):
auth, _, payload = self._ez_unpack_auth(ContentInfoRequest, data)
peer = self.get_peer_from_auth(auth, source_address)
if payload.content_type == SEARCH_TORRENT_REQUEST:
db_results = self.content_repository.search_torrent(payload.query_list)
self.send_content_info_response(peer, payload.identifier, SEARCH_TORRENT_RESPONSE, db_results)
def on_content_info_response(self, source_address, data):
_, _, payload = self._ez_unpack_auth(ContentInfoResponse, data)
identifier = int(payload.identifier)
if not self.request_cache.has(u'request', identifier):
return
cache = self.request_cache.get(u'request', identifier)
if payload.content_type == SEARCH_TORRENT_RESPONSE:
self.process_torrent_search_response(cache.query, payload)
if not payload.pagination.more:
cache = self.request_cache.pop(u'request', identifier)
cache.finish()
def process_torrent_search_response(self, query, payload):
item_format = SearchResponseItemPayload.format_list
response, _ = self.serializer.unpack_multiple_as_list(item_format, payload.response)
# Decode the category string to list
for response_item in response:
response_item[4] = decode_values(response_item[4])
self.content_repository.update_from_torrent_search_results(response)
result_dict = dict()
result_dict['keywords'] = query
result_dict['results'] = response
result_dict['candidate'] = None
if self.tribler_session:
self.tribler_session.notifier.notify(SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None,
result_dict)
# MESSAGE SENDING FUNCTIONS
def send_torrent_health_response(self, payload, peer=None):
"""
Method to send torrent health response. This message is sent to all the subscribers by default but if a
peer is specified then only that peer receives this message.
"""
if peer and peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload)
self.broadcast_message(packet, peer=peer)
def send_channel_health_response(self, payload, peer=None):
"""
Method to send channel health response. This message is sent to all the subscribers by default but if a
peer is specified then only that peer receives this message.
"""
if peer and peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
packet = self.create_message_packet(MSG_CHANNEL_HEALTH_RESPONSE, payload)
self.broadcast_message(packet, peer=peer)
def send_torrent_info_request(self, infohash, peer):
"""
Method to request information about a torrent with given infohash to a peer.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
info_request = TorrentInfoRequestPayload(infohash)
packet = self.create_message_packet(MSG_TORRENT_INFO_REQUEST, info_request)
self.broadcast_message(packet, peer=peer)
def send_torrent_info_response(self, infohash, peer):
"""
Method to send information about a torrent with given infohash to the requesting peer.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
db_torrent = self.content_repository.get_torrent(infohash)
info_response = TorrentInfoResponsePayload(infohash, db_torrent['name'], db_torrent['length'],
db_torrent['creation_date'], db_torrent['num_files'],
db_torrent['comment'])
packet = self.create_message_packet(MSG_TORRENT_INFO_RESPONSE, info_response)
self.broadcast_message(packet, peer=peer)
def send_content_info_request(self, content_type, request_list, limit=25, peer=None):
"""
Sends the generic content request of given content_type.
:param content_type: request content type
:param request_list: List<string> request queries
:param limit: Number of expected responses
:param peer: Peer to send this request to
:return a Deferred that fires when we receive the content
:rtype Deferred
"""
cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list))
self.logger.debug("Sending search request query:%s, identifier:%s", request_list, cache.number)
content_request = ContentInfoRequest(cache.number, content_type, request_list, limit)
packet = self.create_message_packet(MSG_CONTENT_INFO_REQUEST, content_request)
if peer:
self.broadcast_message(packet, peer=peer)
else:
for connected_peer in self.get_peers():
self.broadcast_message(packet, peer=connected_peer)
return cache.deferred
def send_content_info_response(self, peer, identifier, content_type, response_list):
"""
Sends the generic content info response with payload response list.
:param peer: Receiving peer
:param identifier: Request identifier
:param content_type: Message content type
:param response_list: Content response
"""
num_results = len(response_list)
current_index = 0
page_num = 1
while current_index < num_results:
serialized_results, current_index, page_size = self.pack_sized(response_list, MAX_PACKET_PAYLOAD_SIZE,
start_index=current_index)
if not serialized_results:
self.logger.info("Item too big probably to fit into package. Skipping it")
current_index += 1
else:
pagination = Pagination(page_num, page_size, num_results, more=current_index == num_results)
response_payload = ContentInfoResponse(identifier, content_type, serialized_results, pagination)
packet = self.create_message_packet(MSG_CONTENT_INFO_RESPONSE, response_payload)
self.broadcast_message(packet, peer=peer)
def send_torrent_search_request(self, query):
"""
Sends torrent search query as a content info request with content_type as SEARCH_TORRENT_REQUEST.
"""
self.send_content_info_request(SEARCH_TORRENT_REQUEST, query)
def send_channel_search_request(self, query):
"""
Sends channel search query to All Channel 2.0 to get a list of channels.
"""
# TODO: Not implemented yet. Waiting for All Channel 2.0
# CONTENT REPOSITORY STUFFS
def publish_next_content(self):
"""
Publishes the next content from the queue to the subscribers.
Does nothing if there are none subscribers.
Only Torrent health response is published at the moment.
"""
self.logger.info("Content to publish: %d", self.content_repository.count_content())
if not self.subscribers:
self.logger.info("No subscribers found. Not publishing anything")
return
content_type, content = self.content_repository.pop_content()
if content_type is None:
self.logger.error(ERROR_NO_CONTENT)
return
self.logger.info("Publishing content[type:%d]", content_type)
if content_type == TYPE_TORRENT_HEALTH:
infohash, seeders, leechers, timestamp = content
payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp)
self.send_torrent_health_response(payload)
def publish_latest_torrents(self, peer):
"""
Publishes the latest torrents in local database to the given peer.
"""
torrents = self.content_repository.get_top_torrents()
self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer)
for torrent in torrents:
infohash, seeders, leechers, timestamp = torrent[:4]
payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp)
self.send_torrent_health_response(payload, peer=peer)
def queue_content(self, content_type, content):
"""
Basically adds a given content to the queue of content repository.
"""
self.content_repository.add_content(content_type, content)
class PopularityCommunity(PubSubCommunity):
"""
Community for disseminating the content across the network. Follows publish-subscribe model.
"""
MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b810400270381920004073beff7002b6a9fc2824a3b1bbb1c4fc32546" \
"261e3ef7537874560346c5fdc0c17fe654f67d23b08cbb44141879f79a7a4c8deddf9beb4fbc7a0f02ee1586cc" \
"ebedb623eeef51710108d702f9250361c071482e83c0a4a86c8f45a0b13a19ef83eacb6267b4bfccf220ae5f6d" \
"1db7125ea1d10da3744b65679828f23376e28b76ab33132b7fa984a77f159dba7351a7"
master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex'))
def __init__(self, *args, **kwargs):
self.torrent_db = kwargs.pop('torrent_db', None)
self.channel_db = kwargs.pop('channel_db', None)
self.trustchain = kwargs.pop('trustchain_community', None)
self.tribler_session = kwargs.pop('session', None)
super(PopularityCommunity, self).__init__(*args, **kwargs)
self.content_repository = ContentRepository(self.torrent_db, self.channel_db)
self.decode_map.update({
chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response,
chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response,
chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request,
chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response,
chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request,
chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response
})
self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX'))
@inlineCallbacks
def unload(self):
self.content_repository.cleanup()
self.content_repository = None
yield super(PopularityCommunity, self).unload()
def on_subscribe(self, source_address, data):
auth, _, _ = self._ez_unpack_auth(ContentSubscription, data)
peer = self.get_peer_from_auth(auth, source_address)
subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data)
# Publish the latest torrents to the subscriber
if subscribed:
self.publish_latest_torrents(peer=peer)
def on_torrent_health_response(self, source_address, data):
"""
Message handler for torrent health response. Torrent health response is part of periodic update message from
the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply
dropped. In other case, a decision to accept or reject the message is made based on freshness of the message
and the trustscore (check update_torrent in ContentRepository for the implementation).
"""
self.logger.info("Got torrent health response from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.publishers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
infohash = payload.infohash
if not self.content_repository.has_torrent(infohash):
self.send_torrent_info_request(infohash, peer=peer)
peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0
self.content_repository.update_torrent_health(payload, peer_trust)
def on_channel_health_response(self, source_address, data):
"""
Message handler for channel health response. Currently, not sure how to handle it.
"""
def on_torrent_info_request(self, source_address, data):
"""
Message handler for torrent info request.
"""
self.logger.info("Got torrent info request from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentInfoRequestPayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.subscribers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
self.send_torrent_info_response(payload.infohash, peer=peer)
def on_torrent_info_response(self, source_address, data):
"""
Message handler for torrent info response.
"""
self.logger.info("Got torrent info response from %s", source_address)
auth, _, payload = self._ez_unpack_auth(TorrentInfoResponsePayload, data)
peer = self.get_peer_from_auth(auth, source_address)
if peer not in self.publishers:
self.logger.error(ERROR_UNKNOWN_RESPONSE)
return
self.content_repository.update_torrent_info(payload)
def on_content_info_request(self, source_address, data):
auth, _, payload = self._ez_unpack_auth(ContentInfoRequest, data)
peer = self.get_peer_from_auth(auth, source_address)
if payload.content_type == SEARCH_TORRENT_REQUEST:
db_results = self.content_repository.search_torrent(payload.query_list)
self.send_content_info_response(peer, payload.identifier, SEARCH_TORRENT_RESPONSE, db_results)
def on_content_info_response(self, source_address, data):
_, _, payload = self._ez_unpack_auth(ContentInfoResponse, data)
identifier = int(payload.identifier)
if not self.request_cache.has(u'request', identifier):
return
cache = self.request_cache.get(u'request', identifier)
if payload.content_type == SEARCH_TORRENT_RESPONSE:
self.process_torrent_search_response(cache.query, payload)
if not payload.pagination.more:
cache = self.request_cache.pop(u'request', identifier)
cache.finish()
def process_torrent_search_response(self, query, payload):
item_format = SearchResponseItemPayload.format_list
response, _ = self.serializer.unpack_multiple_as_list(item_format, payload.response)
# Decode the category string to list
for response_item in response:
response_item[4] = decode_values(response_item[4])
self.content_repository.update_from_torrent_search_results(response)
result_dict = dict()
result_dict['keywords'] = query
result_dict['results'] = response
result_dict['candidate'] = None
if self.tribler_session:
self.tribler_session.notifier.notify(SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None,
result_dict)
# MESSAGE SENDING FUNCTIONS
def send_torrent_health_response(self, payload, peer=None):
"""
Method to send torrent health response. This message is sent to all the subscribers by default but if a
peer is specified then only that peer receives this message.
"""
if peer and peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload)
self.broadcast_message(packet, peer=peer)
def send_channel_health_response(self, payload, peer=None):
"""
Method to send channel health response. This message is sent to all the subscribers by default but if a
peer is specified then only that peer receives this message.
"""
if peer and peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
packet = self.create_message_packet(MSG_CHANNEL_HEALTH_RESPONSE, payload)
self.broadcast_message(packet, peer=peer)
def send_torrent_info_request(self, infohash, peer):
"""
Method to request information about a torrent with given infohash to a peer.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
info_request = TorrentInfoRequestPayload(infohash)
packet = self.create_message_packet(MSG_TORRENT_INFO_REQUEST, info_request)
self.broadcast_message(packet, peer=peer)
def send_torrent_info_response(self, infohash, peer):
"""
Method to send information about a torrent with given infohash to the requesting peer.
"""
if peer not in self.get_peers():
self.logger.error(ERROR_UNKNOWN_PEER)
return
db_torrent = self.content_repository.get_torrent(infohash)
info_response = TorrentInfoResponsePayload(infohash, db_torrent['name'], db_torrent['length'],
db_torrent['creation_date'], db_torrent['num_files'],
db_torrent['comment'])
packet = self.create_message_packet(MSG_TORRENT_INFO_RESPONSE, info_response)
self.broadcast_message(packet, peer=peer)
def send_content_info_request(self, content_type, request_list, limit=25, peer=None):
"""
Sends the generic content request of given content_type.
:param content_type: request content type
:param request_list: List<string> request queries
:param limit: Number of expected responses
:param peer: Peer to send this request to
:return a Deferred that fires when we receive the content
:rtype Deferred
"""
cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list))
self.logger.info("Sending search request query:%s, identifier:%s", request_list, cache.number)
content_request = ContentInfoRequest(cache.number, content_type, request_list, limit)
packet = self.create_message_packet(MSG_CONTENT_INFO_REQUEST, content_request)
if peer:
self.broadcast_message(packet, peer=peer)
else:
for connected_peer in self.get_peers():
self.broadcast_message(packet, peer=connected_peer)
return cache.deferred
def send_content_info_response(self, peer, identifier, content_type, response_list):
"""
Sends the generic content info response with payload response list.
:param peer: Receiving peer
:param identifier: Request identifier
:param content_type: Message content type
:param response_list: Content response
"""
num_results = len(response_list)
current_index = 0
page_num = 1
while current_index < num_results:
serialized_results, current_index, page_size = self.pack_sized(response_list, MAX_PACKET_PAYLOAD_SIZE,
start_index=current_index)
if not serialized_results:
self.logger.info("Item too big probably to fit into package. Skipping it")
current_index += 1
else:
pagination = Pagination(page_num, page_size, num_results, more=current_index == num_results)
response_payload = ContentInfoResponse(identifier, content_type, serialized_results, pagination)
packet = self.create_message_packet(MSG_CONTENT_INFO_RESPONSE, response_payload)
self.broadcast_message(packet, peer=peer)
def send_torrent_search_request(self, query):
"""
Sends torrent search query as a content info request with content_type as SEARCH_TORRENT_REQUEST.
"""
self.send_content_info_request(SEARCH_TORRENT_REQUEST, query)
def send_channel_search_request(self, query):
"""
Sends channel search query to All Channel 2.0 to get a list of channels.
"""
# TODO: Not implemented yet. Waiting for All Channel 2.0
# CONTENT REPOSITORY STUFFS
def publish_next_content(self):
"""
Publishes the next content from the queue to the subscribers.
Does nothing if there are none subscribers.
Only Torrent health response is published at the moment.
"""
self.logger.info("Content to publish: %d", self.content_repository.count_content())
if not self.subscribers:
self.logger.info("No subscribers found. Not publishing anything")
return
content_type, content = self.content_repository.pop_content()
if content_type is None:
self.logger.error(ERROR_NO_CONTENT)
return
self.logger.info("Publishing content[type:%d]", content_type)
if content_type == TYPE_TORRENT_HEALTH:
infohash, seeders, leechers, timestamp = content
payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp)
self.send_torrent_health_response(payload)
def publish_latest_torrents(self, peer):
"""
Publishes the latest torrents in local database to the given peer.
"""
torrents = self.content_repository.get_top_torrents()
self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer)
for torrent in torrents:
infohash, seeders, leechers, timestamp = torrent[:4]
payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp)
self.send_torrent_health_response(payload, peer=peer)
def queue_content(self, content_type, content):
"""
Basically adds a given content to the queue of content repository.
"""
self.content_repository.add_content(content_type, content)
class TestContentRepository(unittest.TestCase):
def setUp(self):
torrent_db = MockObject()
channel_db = MockObject()
self.content_repository = ContentRepository(torrent_db, channel_db)
def test_add_content(self):
"""
Test adding and removing content works as expected.
"""
# Initial content queue is zero
self.assertEqual(self.content_repository.count_content(), 0,
"No item expected in queue initially")
# Add a sample content and check the size
sample_content = ('a' * 20, 6, 3, 123456789)
sample_content_type = 1
self.content_repository.add_content(sample_content_type,
sample_content)
self.assertEqual(self.content_repository.count_content(), 1,
"One item expected in queue")
# Pop an item
(content_type, content) = self.content_repository.pop_content()
self.assertEqual(content_type, sample_content_type,
"Content type should be equal")
self.assertEqual(content, sample_content, "Content should be equal")
# Check size again
self.assertEqual(self.content_repository.count_content(), 0,
"No item expected in queue")
def test_get_top_torrents(self):
"""
Test if content repository returns expected top torrents.
"""
def get_fake_torrents(limit):
return [[chr(x) * 20, x, 0, 1525704192] for x in range(limit)]
self.content_repository.torrent_db.getRecentlyCheckedTorrents = get_fake_torrents
limit = 10
self.assertEqual(self.content_repository.get_top_torrents(limit=limit),
get_fake_torrents(limit))
def test_update_torrent_health(self):
"""
Tests update torrent health.
"""
def update_torrent(repo, _):
repo.update_torrent_called = True
# Assume a fake torrent response
fake_torrent_health_payload = TorrentHealthPayload(
'a' * 20, 10, 4, time.time())
self.content_repository.torrent_db = MockObject()
self.content_repository.torrent_db.updateTorrent = lambda infohash, *args, **kw: \
update_torrent(self.content_repository, infohash)
# If torrent does not exist in the database, then it should be added to the database
self.content_repository.has_torrent = lambda infohash: False
self.content_repository.update_torrent_health(
fake_torrent_health_payload, peer_trust=0)
self.assertTrue(self.content_repository.update_torrent_called)
def test_update_torrent_with_higher_trust(self):
"""
Scenario: The database torrent has still fresh last_check_time and you receive a new response from
peer with trust > 1.
Expect: Torrent in database is updated.
"""
# last_check_time for existing torrent in database
db_last_time_check = time.time() - 10
# Peer trust, higher than 1 in this scenario
peer_trust = 10
# Database record is expected to be updated
self.assertTrue(
self.try_torrent_update_with_options(db_last_time_check,
peer_trust))
def test_update_torrent_with_stale_check_time(self):
"""
Scenario: The database torrent has stale last_check_time and you receive a new response from
peer with no previous trust.
Expect: Torrent in database is still updated.
"""
# last_check_time for existing torrent in database
db_last_time_check = time.time() - DEFAULT_FRESHNESS_LIMIT
# Peer trust, higher than 1 in this scenario
peer_trust = 0
# Database record is expected to be updated
self.assertTrue(
self.try_torrent_update_with_options(db_last_time_check,
peer_trust))
def try_torrent_update_with_options(self, db_last_check_time, peer_trust):
"""
Tries updating torrent considering the given last check time of existing torrent and a new response
obtained from a peer with given peer_trust value.
"""
sample_infohash, seeders, leechers, timestamp = 'a' * 20, 10, 5, db_last_check_time
sample_payload = TorrentHealthPayload(sample_infohash, seeders,
leechers, timestamp)
def update_torrent(content_repo, _):
content_repo.update_torrent_called = True
def get_torrent(infohash):
return {
'infohash': infohash,
'num_seeders': seeders,
'num_leechers': leechers,
'last_tracker_check': timestamp
}
self.content_repository.torrent_db.getTorrent = lambda infohash, **kw: get_torrent(
infohash)
self.content_repository.torrent_db.hasTorrent = lambda infohash: infohash == sample_infohash
self.content_repository.torrent_db.updateTorrent = \
lambda infohash, *args, **kw: update_torrent(self.content_repository, infohash)
self.content_repository.update_torrent_called = False
self.content_repository.update_torrent_health(sample_payload,
peer_trust=peer_trust)
return self.content_repository.update_torrent_called
def test_update_torrent_info(self):
""" Test updating torrent info """
self.content_repository.called_update_torrent = False
def fake_update_torrent(ref):
ref.called_update_torrent = True
self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: \
fake_update_torrent(self.content_repository)
self.content_repository.has_torrent = lambda infohash: False
torrent_info_response = MockObject()
torrent_info_response.infohash = 'a' * 20
torrent_info_response.name = 'ubuntu'
torrent_info_response.length = 123
torrent_info_response.creation_date = 123123123
torrent_info_response.num_files = 2
torrent_info_response.comment = 'Ubuntu ISO'
self.content_repository.update_torrent_info(torrent_info_response)
self.assertTrue(self.content_repository.called_update_torrent)
def test_update_conflicting_torrent_info(self):
""" Test updating torrent info response with existing record in the database."""
torrent_info_response = MockObject()
torrent_info_response.infohash = 'a' * 20
torrent_info_response.name = 'ubuntu'
torrent_info_response.length = 123
torrent_info_response.creation_date = 123123123
torrent_info_response.num_files = 2
torrent_info_response.comment = 'Ubuntu ISO'
self.content_repository.called_update_torrent = False
def fake_update_torrent(ref):
ref.called_update_torrent = True
def fake_get_torrent(infohash, name):
torrent = {'infohash': infohash, 'name': name}
return torrent
self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent(
self.content_repository)
self.content_repository.has_torrent = lambda infohash: True
self.content_repository.get_torrent = lambda infohash: fake_get_torrent(
infohash, torrent_info_response.name)
self.content_repository.update_torrent_info(torrent_info_response)
self.assertFalse(self.content_repository.called_update_torrent)
def test_search_torrent(self):
""" Test torrent search """
def random_string(size=6,
chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def random_infohash():
return ''.join(
random.choice('0123456789abcdef') for _ in range(20))
sample_torrents = []
for _ in range(10):
infohash = random_infohash()
name = random_string()
length = random.randint(1000, 9999)
num_files = random.randint(1, 10)
category_list = ['video', 'audio']
creation_date = random.randint(1000000, 111111111)
seeders = random.randint(10, 200)
leechers = random.randint(5, 1000)
cid = random_string(size=20)
sample_torrents.append([
infohash, name, length, num_files, category_list,
creation_date, seeders, leechers, cid
])
def fake_torrentdb_search_names(_):
return sample_torrents
self.content_repository.torrent_db.searchNames = lambda query, **kw: fake_torrentdb_search_names(
query)
search_query = "Ubuntu"
search_results = self.content_repository.search_torrent(search_query)
for index in range(10):
db_torrent = sample_torrents[index]
search_result = search_results[index]
self.assertEqual(db_torrent[0], search_result.infohash)
self.assertEqual(db_torrent[1], search_result.name)
self.assertEqual(db_torrent[2], search_result.length)
self.assertEqual(db_torrent[3], search_result.num_files)
self.assertEqual(db_torrent[6], search_result.seeders)
self.assertEqual(db_torrent[7], search_result.leechers)
def test_search_channel(self):
""" Test channel search """
def random_string(size=6,
chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
sample_channels = []
for index in range(10):
dbid = index
cid = random_string(size=20)
name = random_string()
description = random_string(20)
nr_torrents = random.randint(1, 10)
nr_favorite = random.randint(1, 10)
nr_spam = random.randint(1, 10)
my_vote = 1
modified = random.randint(1, 10000000)
relevance_score = 0.0
sample_channels.append([
dbid, cid, name, description, nr_torrents, nr_favorite,
nr_spam, my_vote, modified, relevance_score
])
def fake_torrentdb_search_channels(_):
return sample_channels
self.content_repository.channel_db.search_in_local_channels_db = lambda query, **kw: \
fake_torrentdb_search_channels(query)
search_query = "Ubuntu"
search_results = self.content_repository.search_channels(search_query)
for index in range(10):
db_channel = sample_channels[index]
search_result = search_results[index]
self.assertEqual(db_channel[0], search_result.id)
self.assertEqual(db_channel[1], search_result.cid)
self.assertEqual(db_channel[2], search_result.name)
self.assertEqual(db_channel[3], search_result.description)
self.assertEqual(db_channel[4], search_result.nr_torrents)
self.assertEqual(db_channel[5], search_result.nr_favorite)
self.assertEqual(db_channel[6], search_result.nr_spam)
self.assertEqual(db_channel[8], search_result.modified)
def test_update_torrent_from_search_results(self):
""" Tests updating database from the search results """
def random_string(size=6,
chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def random_infohash():
return ''.join(
random.choice('0123456789abcdef') for _ in range(20))
search_results = dict()
for _ in range(10):
infohash = random_infohash()
name = random_string()
length = random.randint(1000, 9999)
num_files = random.randint(1, 10)
category_list = ['video', 'audio']
creation_date = random.randint(1000000, 111111111)
seeders = random.randint(10, 200)
leechers = random.randint(5, 1000)
cid = random_string(size=20)
search_results[infohash] = [
infohash, name, length, num_files, category_list,
creation_date, seeders, leechers, cid
]
def get_torrent(torrent_as_list):
return {
'infohash': torrent_as_list[0],
'name': torrent_as_list[1],
'length': torrent_as_list[2],
'num_files': torrent_as_list[3],
'category_list': torrent_as_list[4],
'creation_date': torrent_as_list[5],
'seeders': torrent_as_list[6],
'leechers': torrent_as_list[7],
'cid': torrent_as_list[8]
}
def fake_update_torrent(ref):
ref.called_update_torrent = True
def fake_add_or_get_torrent_id(ref):
ref.called_add_or_get_torrent_id = True
self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent(
self.content_repository)
self.content_repository.torrent_db.addOrGetTorrentID = lambda infohash: fake_add_or_get_torrent_id(
self.content_repository)
# Case 1: Assume torrent does not exist in the database
self.content_repository.has_torrent = lambda infohash: False
self.content_repository.get_torrent = lambda infohash: None
self.content_repository.called_update_torrent = False
self.content_repository.update_from_torrent_search_results(
search_results.values())
self.assertTrue(self.content_repository.called_update_torrent)
self.assertTrue(self.content_repository.called_add_or_get_torrent_id)
# Case 2: Torrent already exist in the database
self.content_repository.has_torrent = lambda infohash: infohash in search_results
self.content_repository.get_torrent = lambda infohash: get_torrent(
search_results[infohash])
self.content_repository.called_update_torrent = False
self.content_repository.called_add_or_get_torrent_id = False
self.content_repository.update_from_torrent_search_results(
search_results.values())
self.assertFalse(self.content_repository.called_update_torrent)
self.assertFalse(self.content_repository.called_add_or_get_torrent_id)
class TestContentRepository(unittest.TestCase):
def setUp(self):
torrent_db = MockObject()
channel_db = MockObject()
self.content_repository = ContentRepository(torrent_db, channel_db)
def test_add_content(self):
"""
Test adding and removing content works as expected.
"""
# Initial content queue is zero
self.assertEqual(self.content_repository.count_content(), 0, "No item expected in queue initially")
# Add a sample content and check the size
sample_content = ('a' * 20, 6, 3, 123456789)
sample_content_type = 1
self.content_repository.add_content(sample_content_type, sample_content)
self.assertEqual(self.content_repository.count_content(), 1, "One item expected in queue")
# Pop an item
(content_type, content) = self.content_repository.pop_content()
self.assertEqual(content_type, sample_content_type, "Content type should be equal")
self.assertEqual(content, sample_content, "Content should be equal")
# Check size again
self.assertEqual(self.content_repository.count_content(), 0, "No item expected in queue")
def test_get_top_torrents(self):
"""
Test if content repository returns expected top torrents.
"""
def get_fake_torrents(limit):
return [[chr(x) * 20, x, 0, 1525704192] for x in range(limit)]
self.content_repository.torrent_db.getRecentlyCheckedTorrents = get_fake_torrents
limit = 10
self.assertEqual(self.content_repository.get_top_torrents(limit=limit), get_fake_torrents(limit))
def test_update_torrent_health(self):
"""
Tests update torrent health.
"""
def update_torrent(repo, _):
repo.update_torrent_called = True
# Assume a fake torrent response
fake_torrent_health_payload = TorrentHealthPayload('a' * 20, 10, 4, time.time())
self.content_repository.torrent_db = MockObject()
self.content_repository.torrent_db.updateTorrent = lambda infohash, *args, **kw: \
update_torrent(self.content_repository, infohash)
# If torrent does not exist in the database, then it should be added to the database
self.content_repository.has_torrent = lambda infohash: False
self.content_repository.update_torrent_health(fake_torrent_health_payload, peer_trust=0)
self.assertTrue(self.content_repository.update_torrent_called)
def test_update_torrent_with_higher_trust(self):
"""
Scenario: The database torrent has still fresh last_check_time and you receive a new response from
peer with trust > 1.
Expect: Torrent in database is updated.
"""
# last_check_time for existing torrent in database
db_last_time_check = time.time() - 10
# Peer trust, higher than 1 in this scenario
peer_trust = 10
# Database record is expected to be updated
self.assertTrue(self.try_torrent_update_with_options(db_last_time_check, peer_trust))
def test_update_torrent_with_stale_check_time(self):
"""
Scenario: The database torrent has stale last_check_time and you receive a new response from
peer with no previous trust.
Expect: Torrent in database is still updated.
"""
# last_check_time for existing torrent in database
db_last_time_check = time.time() - DEFAULT_FRESHNESS_LIMIT
# Peer trust, higher than 1 in this scenario
peer_trust = 0
# Database record is expected to be updated
self.assertTrue(self.try_torrent_update_with_options(db_last_time_check, peer_trust))
def try_torrent_update_with_options(self, db_last_check_time, peer_trust):
"""
Tries updating torrent considering the given last check time of existing torrent and a new response
obtained from a peer with given peer_trust value.
"""
sample_infohash, seeders, leechers, timestamp = 'a' * 20, 10, 5, db_last_check_time
sample_payload = TorrentHealthPayload(sample_infohash, seeders, leechers, timestamp)
def update_torrent(content_repo, _):
content_repo.update_torrent_called = True
def get_torrent(infohash):
return {'infohash': infohash, 'num_seeders': seeders,
'num_leechers': leechers, 'last_tracker_check': timestamp}
self.content_repository.torrent_db.getTorrent = lambda infohash, **kw: get_torrent(infohash)
self.content_repository.torrent_db.hasTorrent = lambda infohash: infohash == sample_infohash
self.content_repository.torrent_db.updateTorrent = \
lambda infohash, *args, **kw: update_torrent(self.content_repository, infohash)
self.content_repository.update_torrent_called = False
self.content_repository.update_torrent_health(sample_payload, peer_trust=peer_trust)
return self.content_repository.update_torrent_called
def test_update_torrent_info(self):
""" Test updating torrent info """
self.content_repository.called_update_torrent = False
def fake_update_torrent(ref):
ref.called_update_torrent = True
self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: \
fake_update_torrent(self.content_repository)
self.content_repository.has_torrent = lambda infohash: False
torrent_info_response = MockObject()
torrent_info_response.infohash = 'a' * 20
torrent_info_response.name = 'ubuntu'
torrent_info_response.length = 123
torrent_info_response.creation_date = 123123123
torrent_info_response.num_files = 2
torrent_info_response.comment = 'Ubuntu ISO'
self.content_repository.update_torrent_info(torrent_info_response)
self.assertTrue(self.content_repository.called_update_torrent)
def test_update_conflicting_torrent_info(self):
""" Test updating torrent info response with existing record in the database."""
torrent_info_response = MockObject()
torrent_info_response.infohash = 'a' * 20
torrent_info_response.name = 'ubuntu'
torrent_info_response.length = 123
torrent_info_response.creation_date = 123123123
torrent_info_response.num_files = 2
torrent_info_response.comment = 'Ubuntu ISO'
self.content_repository.called_update_torrent = False
def fake_update_torrent(ref):
ref.called_update_torrent = True
def fake_get_torrent(infohash, name):
torrent = {'infohash': infohash, 'name': name}
return torrent
self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent(
self.content_repository)
self.content_repository.has_torrent = lambda infohash: True
self.content_repository.get_torrent = lambda infohash: fake_get_torrent(infohash, torrent_info_response.name)
self.content_repository.update_torrent_info(torrent_info_response)
self.assertFalse(self.content_repository.called_update_torrent)
def test_search_torrent(self):
""" Test torrent search """
def random_string(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def random_infohash():
return ''.join(random.choice('0123456789abcdef') for _ in range(20))
sample_torrents = []
for _ in range(10):
infohash = random_infohash()
name = random_string()
length = random.randint(1000, 9999)
num_files = random.randint(1, 10)
category_list = ['video', 'audio']
creation_date = random.randint(1000000, 111111111)
seeders = random.randint(10, 200)
leechers = random.randint(5, 1000)
cid = random_string(size=20)
sample_torrents.append([infohash, name, length, num_files, category_list, creation_date, seeders,
leechers, cid])
def fake_torrentdb_search_names(_):
return sample_torrents
self.content_repository.torrent_db.searchNames = lambda query, **kw: fake_torrentdb_search_names(query)
search_query = "Ubuntu"
search_results = self.content_repository.search_torrent(search_query)
for index in range(10):
db_torrent = sample_torrents[index]
search_result = search_results[index]
self.assertEqual(db_torrent[0], search_result.infohash)
self.assertEqual(db_torrent[1], search_result.name)
self.assertEqual(db_torrent[2], search_result.length)
self.assertEqual(db_torrent[3], search_result.num_files)
self.assertEqual(db_torrent[6], search_result.seeders)
self.assertEqual(db_torrent[7], search_result.leechers)
def test_search_channel(self):
""" Test channel search """
def random_string(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
sample_channels = []
for index in range(10):
dbid = index
cid = random_string(size=20)
name = random_string()
description = random_string(20)
nr_torrents = random.randint(1, 10)
nr_favorite = random.randint(1, 10)
nr_spam = random.randint(1, 10)
my_vote = 1
modified = random.randint(1, 10000000)
relevance_score = 0.0
sample_channels.append([dbid, cid, name, description, nr_torrents, nr_favorite, nr_spam, my_vote,
modified, relevance_score])
def fake_torrentdb_search_channels(_):
return sample_channels
self.content_repository.channel_db.search_in_local_channels_db = lambda query, **kw: \
fake_torrentdb_search_channels(query)
search_query = "Ubuntu"
search_results = self.content_repository.search_channels(search_query)
for index in range(10):
db_channel = sample_channels[index]
search_result = search_results[index]
self.assertEqual(db_channel[0], search_result.id)
self.assertEqual(db_channel[1], search_result.cid)
self.assertEqual(db_channel[2], search_result.name)
self.assertEqual(db_channel[3], search_result.description)
self.assertEqual(db_channel[4], search_result.nr_torrents)
self.assertEqual(db_channel[5], search_result.nr_favorite)
self.assertEqual(db_channel[6], search_result.nr_spam)
self.assertEqual(db_channel[8], search_result.modified)
def test_update_torrent_from_search_results(self):
""" Tests updating database from the search results """
def random_string(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def random_infohash():
return ''.join(random.choice('0123456789abcdef') for _ in range(20))
search_results = dict()
for _ in range(10):
infohash = random_infohash()
name = random_string()
length = random.randint(1000, 9999)
num_files = random.randint(1, 10)
category_list = ['video', 'audio']
creation_date = random.randint(1000000, 111111111)
seeders = random.randint(10, 200)
leechers = random.randint(5, 1000)
cid = random_string(size=20)
search_results[infohash] = [infohash, name, length, num_files, category_list, creation_date,
seeders, leechers, cid]
def get_torrent(torrent_as_list):
return {'infohash': torrent_as_list[0],
'name': torrent_as_list[1],
'length': torrent_as_list[2],
'num_files': torrent_as_list[3],
'category_list': torrent_as_list[4],
'creation_date': torrent_as_list[5],
'seeders': torrent_as_list[6],
'leechers': torrent_as_list[7],
'cid': torrent_as_list[8]}
def fake_update_torrent(ref):
ref.called_update_torrent = True
def fake_add_or_get_torrent_id(ref):
ref.called_add_or_get_torrent_id = True
self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent(
self.content_repository)
self.content_repository.torrent_db.addOrGetTorrentID = lambda infohash: fake_add_or_get_torrent_id(
self.content_repository)
# Case 1: Assume torrent does not exist in the database
self.content_repository.has_torrent = lambda infohash: False
self.content_repository.get_torrent = lambda infohash: None
self.content_repository.torrent_db._db = MockObject()
self.content_repository.torrent_db._db.commit_now = lambda x=None: None
self.content_repository.called_update_torrent = False
self.content_repository.update_from_torrent_search_results(search_results.values())
self.assertTrue(self.content_repository.called_update_torrent)
self.assertTrue(self.content_repository.called_add_or_get_torrent_id)
# Case 2: Torrent already exist in the database
self.content_repository.has_torrent = lambda infohash: infohash in search_results
self.content_repository.get_torrent = lambda infohash: get_torrent(search_results[infohash])
self.content_repository.called_update_torrent = False
self.content_repository.called_add_or_get_torrent_id = False
self.content_repository.update_from_torrent_search_results(search_results.values())
self.assertFalse(self.content_repository.called_update_torrent)
self.assertFalse(self.content_repository.called_add_or_get_torrent_id)