def __init__(self, *args, **kwargs): self.torrent_db = kwargs.pop('torrent_db', None) self.channel_db = kwargs.pop('channel_db', None) self.trustchain = kwargs.pop('trustchain_community', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.torrent_db, self.channel_db) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response, chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response, chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request, chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response, chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request, chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response }) self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX'))
class TestContentRepositoryWithRealDatabase(TestBase): """ Tests content repository with real database. """ def setUp(self): super(TestContentRepositoryWithRealDatabase, self).setUp() session_base_dir = self.temporary_directory() tar = tarfile.open(os.path.join(TESTS_DATA_DIR, 'bak_new_tribler.sdb.tar.gz'), 'r|gz') tar.extractall(session_base_dir) db_path = os.path.join(session_base_dir, 'bak_new_tribler.sdb') self.sqlitedb = SQLiteCacheDB(db_path, busytimeout=BUSYTIMEOUT) session = MockObject() session.sqlite_db = self.sqlitedb session.notifier = MockObject() self.torrent_db = TorrentDBHandler(session) channel_db = MockObject() self.content_repository = ContentRepository(self.torrent_db, channel_db) def tearDown(self): self.torrent_db.close() self.sqlitedb.close() super(TestContentRepositoryWithRealDatabase, self).tearDown() def test_update_db_from_search_results(self): """ Test if database is properly updated with the search results. Should not raise any UnicodeDecodeError. """ # Add a torrent infohash before updating from search results infohash = unhexlify('ed81da94d21ad1b305133f2726cdaec5a57fed98') self.content_repository.torrent_db.addOrGetTorrentID(infohash) # Sample search results name = 'Puppy.Linux.manual.301.espa\xc3\xb1ol.pdf' length = random.randint(1000, 9999) num_files = random.randint(1, 10) category_list = ['other'] creation_date = random.randint(1000000, 111111111) seeders = random.randint(10, 200) leechers = random.randint(5, 1000) cid = None search_results = [[infohash, name, length, num_files, category_list, creation_date, seeders, leechers, cid]] # Update from search results self.content_repository.update_from_torrent_search_results(search_results) # Check if database has correct results torrent_info = self.content_repository.get_torrent(infohash) expected_name = u'Puppy.Linux.manual.301.espa\xc3\xb1ol.pdf' self.assertEqual(expected_name, torrent_info['name']) self.assertEqual(seeders, torrent_info['num_seeders']) self.assertEqual(leechers, torrent_info['num_leechers']) self.assertEqual(creation_date, torrent_info['creation_date']) self.assertEqual(num_files, torrent_info['num_files']) self.assertEqual(length, torrent_info['length'])
def setUp(self): yield super(TestContentRepository, self).setUp() self.my_key = default_eccrypto.generate_key(u"curve25519") mds = MetadataStore(':memory:', self.session_base_dir, self.my_key) self.content_repository = ContentRepository(mds) # Add some content to the metadata database with db_session: mds.ChannelMetadata.create_channel('test', 'test') for torrent_ind in xrange(5): torrent = mds.TorrentMetadata(title='torrent%d' % torrent_ind, infohash=('%d' % torrent_ind) * 20) torrent.health.seeders = torrent_ind + 1
def __init__(self, *args, **kwargs): self.metadata_store = kwargs.pop('metadata_store', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.metadata_store) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response }) self.logger.info('Popular Community initialized (peer mid %s)', hexlify(self.my_peer.mid))
def setUp(self): super(TestContentRepositoryWithRealDatabase, self).setUp() session_base_dir = self.temporary_directory() tar = tarfile.open(os.path.join(TESTS_DATA_DIR, 'bak_new_tribler.sdb.tar.gz'), 'r|gz') tar.extractall(session_base_dir) db_path = os.path.join(session_base_dir, 'bak_new_tribler.sdb') self.sqlitedb = SQLiteCacheDB(db_path, busytimeout=BUSYTIMEOUT) session = MockObject() session.sqlite_db = self.sqlitedb session.notifier = MockObject() self.torrent_db = TorrentDBHandler(session) channel_db = MockObject() self.content_repository = ContentRepository(self.torrent_db, channel_db)
def __init__(self, *args, **kwargs): self.torrent_db = kwargs.pop('torrent_db', None) self.channel_db = kwargs.pop('channel_db', None) self.trustchain = kwargs.pop('trustchain_community', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.torrent_db, self.channel_db) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response, chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response, chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request, chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response, chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request, chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response }) self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX'))
class PopularityCommunity(PubSubCommunity): """ Community for disseminating the content across the network. Follows publish-subscribe model. """ MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b8104002703819200040504278d20d6776ce7081ad57d99fe066bb2a93" \ "ce7cc92405a534ef7175bab702be557d8c7d3b725ea0eb09c686e798f6c7ad85e8781a4c3b20e54c15ede38077c" \ "8f5c801b71d13105f261da7ddcaa94ae14bd177bf1a05a66f595b9bb99117d11f73b4c8d3dcdcdc2b3f838b8ba3" \ "5a9f600d2c543e8b3ba646083307b917bbbccfc53fc5ab6ded90b711d7eeda46f5f" master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex')) def __init__(self, *args, **kwargs): self.torrent_db = kwargs.pop('torrent_db', None) self.channel_db = kwargs.pop('channel_db', None) self.trustchain = kwargs.pop('trustchain_community', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.torrent_db, self.channel_db) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response, chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response, chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request, chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response, chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request, chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response }) self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX')) @inlineCallbacks def unload(self): self.content_repository.cleanup() self.content_repository = None yield super(PopularityCommunity, self).unload() def on_subscribe(self, source_address, data): auth, _, _ = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data) # Publish the latest torrents to the subscriber if subscribed: self.publish_latest_torrents(peer=peer) def on_torrent_health_response(self, source_address, data): """ Message handler for torrent health response. Torrent health response is part of periodic update message from the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply dropped. In other case, a decision to accept or reject the message is made based on freshness of the message and the trustscore (check update_torrent in ContentRepository for the implementation). """ self.logger.debug("Got torrent health response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return infohash = payload.infohash if not self.content_repository.has_torrent(infohash): self.send_torrent_info_request(infohash, peer=peer) peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0 self.content_repository.update_torrent_health(payload, peer_trust) def on_channel_health_response(self, source_address, data): """ Message handler for channel health response. Currently, not sure how to handle it. """ def on_torrent_info_request(self, source_address, data): """ Message handler for torrent info request. """ self.logger.debug("Got torrent info request from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentInfoRequestPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.subscribers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return self.send_torrent_info_response(payload.infohash, peer=peer) def on_torrent_info_response(self, source_address, data): """ Message handler for torrent info response. """ self.logger.debug("Got torrent info response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentInfoResponsePayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return self.content_repository.update_torrent_info(payload) def on_content_info_request(self, source_address, data): auth, _, payload = self._ez_unpack_auth(ContentInfoRequest, data) peer = self.get_peer_from_auth(auth, source_address) if payload.content_type == SEARCH_TORRENT_REQUEST: db_results = self.content_repository.search_torrent(payload.query_list) self.send_content_info_response(peer, payload.identifier, SEARCH_TORRENT_RESPONSE, db_results) def on_content_info_response(self, source_address, data): _, _, payload = self._ez_unpack_auth(ContentInfoResponse, data) identifier = int(payload.identifier) if not self.request_cache.has(u'request', identifier): return cache = self.request_cache.get(u'request', identifier) if payload.content_type == SEARCH_TORRENT_RESPONSE: self.process_torrent_search_response(cache.query, payload) if not payload.pagination.more: cache = self.request_cache.pop(u'request', identifier) cache.finish() def process_torrent_search_response(self, query, payload): item_format = SearchResponseItemPayload.format_list response, _ = self.serializer.unpack_multiple_as_list(item_format, payload.response) # Decode the category string to list for response_item in response: response_item[4] = decode_values(response_item[4]) self.content_repository.update_from_torrent_search_results(response) result_dict = dict() result_dict['keywords'] = query result_dict['results'] = response result_dict['candidate'] = None if self.tribler_session: self.tribler_session.notifier.notify(SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None, result_dict) # MESSAGE SENDING FUNCTIONS def send_torrent_health_response(self, payload, peer=None): """ Method to send torrent health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) def send_channel_health_response(self, payload, peer=None): """ Method to send channel health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_CHANNEL_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) def send_torrent_info_request(self, infohash, peer): """ Method to request information about a torrent with given infohash to a peer. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return info_request = TorrentInfoRequestPayload(infohash) packet = self.create_message_packet(MSG_TORRENT_INFO_REQUEST, info_request) self.broadcast_message(packet, peer=peer) def send_torrent_info_response(self, infohash, peer): """ Method to send information about a torrent with given infohash to the requesting peer. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return db_torrent = self.content_repository.get_torrent(infohash) info_response = TorrentInfoResponsePayload(infohash, db_torrent['name'], db_torrent['length'], db_torrent['creation_date'], db_torrent['num_files'], db_torrent['comment']) packet = self.create_message_packet(MSG_TORRENT_INFO_RESPONSE, info_response) self.broadcast_message(packet, peer=peer) def send_content_info_request(self, content_type, request_list, limit=25, peer=None): """ Sends the generic content request of given content_type. :param content_type: request content type :param request_list: List<string> request queries :param limit: Number of expected responses :param peer: Peer to send this request to :return a Deferred that fires when we receive the content :rtype Deferred """ cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list)) self.logger.debug("Sending search request query:%s, identifier:%s", request_list, cache.number) content_request = ContentInfoRequest(cache.number, content_type, request_list, limit) packet = self.create_message_packet(MSG_CONTENT_INFO_REQUEST, content_request) if peer: self.broadcast_message(packet, peer=peer) else: for connected_peer in self.get_peers(): self.broadcast_message(packet, peer=connected_peer) return cache.deferred def send_content_info_response(self, peer, identifier, content_type, response_list): """ Sends the generic content info response with payload response list. :param peer: Receiving peer :param identifier: Request identifier :param content_type: Message content type :param response_list: Content response """ num_results = len(response_list) current_index = 0 page_num = 1 while current_index < num_results: serialized_results, current_index, page_size = self.pack_sized(response_list, MAX_PACKET_PAYLOAD_SIZE, start_index=current_index) if not serialized_results: self.logger.info("Item too big probably to fit into package. Skipping it") current_index += 1 else: pagination = Pagination(page_num, page_size, num_results, more=current_index == num_results) response_payload = ContentInfoResponse(identifier, content_type, serialized_results, pagination) packet = self.create_message_packet(MSG_CONTENT_INFO_RESPONSE, response_payload) self.broadcast_message(packet, peer=peer) def send_torrent_search_request(self, query): """ Sends torrent search query as a content info request with content_type as SEARCH_TORRENT_REQUEST. """ self.send_content_info_request(SEARCH_TORRENT_REQUEST, query) def send_channel_search_request(self, query): """ Sends channel search query to All Channel 2.0 to get a list of channels. """ # TODO: Not implemented yet. Waiting for All Channel 2.0 # CONTENT REPOSITORY STUFFS def publish_next_content(self): """ Publishes the next content from the queue to the subscribers. Does nothing if there are none subscribers. Only Torrent health response is published at the moment. """ self.logger.info("Content to publish: %d", self.content_repository.count_content()) if not self.subscribers: self.logger.info("No subscribers found. Not publishing anything") return content_type, content = self.content_repository.pop_content() if content_type is None: self.logger.error(ERROR_NO_CONTENT) return self.logger.info("Publishing content[type:%d]", content_type) if content_type == TYPE_TORRENT_HEALTH: infohash, seeders, leechers, timestamp = content payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp) self.send_torrent_health_response(payload) def publish_latest_torrents(self, peer): """ Publishes the latest torrents in local database to the given peer. """ torrents = self.content_repository.get_top_torrents() self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer) for torrent in torrents: infohash, seeders, leechers, timestamp = torrent[:4] payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp) self.send_torrent_health_response(payload, peer=peer) def queue_content(self, content_type, content): """ Basically adds a given content to the queue of content repository. """ self.content_repository.add_content(content_type, content)
def setUp(self): torrent_db = MockObject() channel_db = MockObject() self.content_repository = ContentRepository(torrent_db, channel_db)
class TestContentRepository(unittest.TestCase): def setUp(self): torrent_db = MockObject() channel_db = MockObject() self.content_repository = ContentRepository(torrent_db, channel_db) def test_add_content(self): """ Test adding and removing content works as expected. """ # Initial content queue is zero self.assertEqual(self.content_repository.count_content(), 0, "No item expected in queue initially") # Add a sample content and check the size sample_content = ('a' * 20, 6, 3, 123456789) sample_content_type = 1 self.content_repository.add_content(sample_content_type, sample_content) self.assertEqual(self.content_repository.count_content(), 1, "One item expected in queue") # Pop an item (content_type, content) = self.content_repository.pop_content() self.assertEqual(content_type, sample_content_type, "Content type should be equal") self.assertEqual(content, sample_content, "Content should be equal") # Check size again self.assertEqual(self.content_repository.count_content(), 0, "No item expected in queue") def test_get_top_torrents(self): """ Test if content repository returns expected top torrents. """ def get_fake_torrents(limit): return [[chr(x) * 20, x, 0, 1525704192] for x in range(limit)] self.content_repository.torrent_db.getRecentlyCheckedTorrents = get_fake_torrents limit = 10 self.assertEqual(self.content_repository.get_top_torrents(limit=limit), get_fake_torrents(limit)) def test_update_torrent_health(self): """ Tests update torrent health. """ def update_torrent(repo, _): repo.update_torrent_called = True # Assume a fake torrent response fake_torrent_health_payload = TorrentHealthPayload( 'a' * 20, 10, 4, time.time()) self.content_repository.torrent_db = MockObject() self.content_repository.torrent_db.updateTorrent = lambda infohash, *args, **kw: \ update_torrent(self.content_repository, infohash) # If torrent does not exist in the database, then it should be added to the database self.content_repository.has_torrent = lambda infohash: False self.content_repository.update_torrent_health( fake_torrent_health_payload, peer_trust=0) self.assertTrue(self.content_repository.update_torrent_called) def test_update_torrent_with_higher_trust(self): """ Scenario: The database torrent has still fresh last_check_time and you receive a new response from peer with trust > 1. Expect: Torrent in database is updated. """ # last_check_time for existing torrent in database db_last_time_check = time.time() - 10 # Peer trust, higher than 1 in this scenario peer_trust = 10 # Database record is expected to be updated self.assertTrue( self.try_torrent_update_with_options(db_last_time_check, peer_trust)) def test_update_torrent_with_stale_check_time(self): """ Scenario: The database torrent has stale last_check_time and you receive a new response from peer with no previous trust. Expect: Torrent in database is still updated. """ # last_check_time for existing torrent in database db_last_time_check = time.time() - DEFAULT_FRESHNESS_LIMIT # Peer trust, higher than 1 in this scenario peer_trust = 0 # Database record is expected to be updated self.assertTrue( self.try_torrent_update_with_options(db_last_time_check, peer_trust)) def try_torrent_update_with_options(self, db_last_check_time, peer_trust): """ Tries updating torrent considering the given last check time of existing torrent and a new response obtained from a peer with given peer_trust value. """ sample_infohash, seeders, leechers, timestamp = 'a' * 20, 10, 5, db_last_check_time sample_payload = TorrentHealthPayload(sample_infohash, seeders, leechers, timestamp) def update_torrent(content_repo, _): content_repo.update_torrent_called = True def get_torrent(infohash): return { 'infohash': infohash, 'num_seeders': seeders, 'num_leechers': leechers, 'last_tracker_check': timestamp } self.content_repository.torrent_db.getTorrent = lambda infohash, **kw: get_torrent( infohash) self.content_repository.torrent_db.hasTorrent = lambda infohash: infohash == sample_infohash self.content_repository.torrent_db.updateTorrent = \ lambda infohash, *args, **kw: update_torrent(self.content_repository, infohash) self.content_repository.update_torrent_called = False self.content_repository.update_torrent_health(sample_payload, peer_trust=peer_trust) return self.content_repository.update_torrent_called def test_update_torrent_info(self): """ Test updating torrent info """ self.content_repository.called_update_torrent = False def fake_update_torrent(ref): ref.called_update_torrent = True self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: \ fake_update_torrent(self.content_repository) self.content_repository.has_torrent = lambda infohash: False torrent_info_response = MockObject() torrent_info_response.infohash = 'a' * 20 torrent_info_response.name = 'ubuntu' torrent_info_response.length = 123 torrent_info_response.creation_date = 123123123 torrent_info_response.num_files = 2 torrent_info_response.comment = 'Ubuntu ISO' self.content_repository.update_torrent_info(torrent_info_response) self.assertTrue(self.content_repository.called_update_torrent) def test_update_conflicting_torrent_info(self): """ Test updating torrent info response with existing record in the database.""" torrent_info_response = MockObject() torrent_info_response.infohash = 'a' * 20 torrent_info_response.name = 'ubuntu' torrent_info_response.length = 123 torrent_info_response.creation_date = 123123123 torrent_info_response.num_files = 2 torrent_info_response.comment = 'Ubuntu ISO' self.content_repository.called_update_torrent = False def fake_update_torrent(ref): ref.called_update_torrent = True def fake_get_torrent(infohash, name): torrent = {'infohash': infohash, 'name': name} return torrent self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent( self.content_repository) self.content_repository.has_torrent = lambda infohash: True self.content_repository.get_torrent = lambda infohash: fake_get_torrent( infohash, torrent_info_response.name) self.content_repository.update_torrent_info(torrent_info_response) self.assertFalse(self.content_repository.called_update_torrent) def test_search_torrent(self): """ Test torrent search """ def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) def random_infohash(): return ''.join( random.choice('0123456789abcdef') for _ in range(20)) sample_torrents = [] for _ in range(10): infohash = random_infohash() name = random_string() length = random.randint(1000, 9999) num_files = random.randint(1, 10) category_list = ['video', 'audio'] creation_date = random.randint(1000000, 111111111) seeders = random.randint(10, 200) leechers = random.randint(5, 1000) cid = random_string(size=20) sample_torrents.append([ infohash, name, length, num_files, category_list, creation_date, seeders, leechers, cid ]) def fake_torrentdb_search_names(_): return sample_torrents self.content_repository.torrent_db.searchNames = lambda query, **kw: fake_torrentdb_search_names( query) search_query = "Ubuntu" search_results = self.content_repository.search_torrent(search_query) for index in range(10): db_torrent = sample_torrents[index] search_result = search_results[index] self.assertEqual(db_torrent[0], search_result.infohash) self.assertEqual(db_torrent[1], search_result.name) self.assertEqual(db_torrent[2], search_result.length) self.assertEqual(db_torrent[3], search_result.num_files) self.assertEqual(db_torrent[6], search_result.seeders) self.assertEqual(db_torrent[7], search_result.leechers) def test_search_channel(self): """ Test channel search """ def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) sample_channels = [] for index in range(10): dbid = index cid = random_string(size=20) name = random_string() description = random_string(20) nr_torrents = random.randint(1, 10) nr_favorite = random.randint(1, 10) nr_spam = random.randint(1, 10) my_vote = 1 modified = random.randint(1, 10000000) relevance_score = 0.0 sample_channels.append([ dbid, cid, name, description, nr_torrents, nr_favorite, nr_spam, my_vote, modified, relevance_score ]) def fake_torrentdb_search_channels(_): return sample_channels self.content_repository.channel_db.search_in_local_channels_db = lambda query, **kw: \ fake_torrentdb_search_channels(query) search_query = "Ubuntu" search_results = self.content_repository.search_channels(search_query) for index in range(10): db_channel = sample_channels[index] search_result = search_results[index] self.assertEqual(db_channel[0], search_result.id) self.assertEqual(db_channel[1], search_result.cid) self.assertEqual(db_channel[2], search_result.name) self.assertEqual(db_channel[3], search_result.description) self.assertEqual(db_channel[4], search_result.nr_torrents) self.assertEqual(db_channel[5], search_result.nr_favorite) self.assertEqual(db_channel[6], search_result.nr_spam) self.assertEqual(db_channel[8], search_result.modified) def test_update_torrent_from_search_results(self): """ Tests updating database from the search results """ def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) def random_infohash(): return ''.join( random.choice('0123456789abcdef') for _ in range(20)) search_results = dict() for _ in range(10): infohash = random_infohash() name = random_string() length = random.randint(1000, 9999) num_files = random.randint(1, 10) category_list = ['video', 'audio'] creation_date = random.randint(1000000, 111111111) seeders = random.randint(10, 200) leechers = random.randint(5, 1000) cid = random_string(size=20) search_results[infohash] = [ infohash, name, length, num_files, category_list, creation_date, seeders, leechers, cid ] def get_torrent(torrent_as_list): return { 'infohash': torrent_as_list[0], 'name': torrent_as_list[1], 'length': torrent_as_list[2], 'num_files': torrent_as_list[3], 'category_list': torrent_as_list[4], 'creation_date': torrent_as_list[5], 'seeders': torrent_as_list[6], 'leechers': torrent_as_list[7], 'cid': torrent_as_list[8] } def fake_update_torrent(ref): ref.called_update_torrent = True def fake_add_or_get_torrent_id(ref): ref.called_add_or_get_torrent_id = True self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent( self.content_repository) self.content_repository.torrent_db.addOrGetTorrentID = lambda infohash: fake_add_or_get_torrent_id( self.content_repository) # Case 1: Assume torrent does not exist in the database self.content_repository.has_torrent = lambda infohash: False self.content_repository.get_torrent = lambda infohash: None self.content_repository.called_update_torrent = False self.content_repository.update_from_torrent_search_results( search_results.values()) self.assertTrue(self.content_repository.called_update_torrent) self.assertTrue(self.content_repository.called_add_or_get_torrent_id) # Case 2: Torrent already exist in the database self.content_repository.has_torrent = lambda infohash: infohash in search_results self.content_repository.get_torrent = lambda infohash: get_torrent( search_results[infohash]) self.content_repository.called_update_torrent = False self.content_repository.called_add_or_get_torrent_id = False self.content_repository.update_from_torrent_search_results( search_results.values()) self.assertFalse(self.content_repository.called_update_torrent) self.assertFalse(self.content_repository.called_add_or_get_torrent_id)
class PopularityCommunity(PubSubCommunity): """ Community for disseminating the content across the network. Follows publish-subscribe model. """ MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b810400270381920004073beff7002b6a9fc2824a3b1bbb1c4fc32546" \ "261e3ef7537874560346c5fdc0c17fe654f67d23b08cbb44141879f79a7a4c8deddf9beb4fbc7a0f02ee1586cc" \ "ebedb623eeef51710108d702f9250361c071482e83c0a4a86c8f45a0b13a19ef83eacb6267b4bfccf220ae5f6d" \ "1db7125ea1d10da3744b65679828f23376e28b76ab33132b7fa984a77f159dba7351a7" master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex')) def __init__(self, *args, **kwargs): self.torrent_db = kwargs.pop('torrent_db', None) self.channel_db = kwargs.pop('channel_db', None) self.trustchain = kwargs.pop('trustchain_community', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.torrent_db, self.channel_db) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response, chr(MSG_CHANNEL_HEALTH_RESPONSE): self.on_channel_health_response, chr(MSG_TORRENT_INFO_REQUEST): self.on_torrent_info_request, chr(MSG_TORRENT_INFO_RESPONSE): self.on_torrent_info_response, chr(MSG_CONTENT_INFO_REQUEST): self.on_content_info_request, chr(MSG_CONTENT_INFO_RESPONSE): self.on_content_info_response }) self.logger.info('Popular Community initialized (peer mid %s)', self.my_peer.mid.encode('HEX')) @inlineCallbacks def unload(self): self.content_repository.cleanup() self.content_repository = None yield super(PopularityCommunity, self).unload() def on_subscribe(self, source_address, data): auth, _, _ = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data) # Publish the latest torrents to the subscriber if subscribed: self.publish_latest_torrents(peer=peer) def on_torrent_health_response(self, source_address, data): """ Message handler for torrent health response. Torrent health response is part of periodic update message from the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply dropped. In other case, a decision to accept or reject the message is made based on freshness of the message and the trustscore (check update_torrent in ContentRepository for the implementation). """ self.logger.info("Got torrent health response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return infohash = payload.infohash if not self.content_repository.has_torrent(infohash): self.send_torrent_info_request(infohash, peer=peer) peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0 self.content_repository.update_torrent_health(payload, peer_trust) def on_channel_health_response(self, source_address, data): """ Message handler for channel health response. Currently, not sure how to handle it. """ def on_torrent_info_request(self, source_address, data): """ Message handler for torrent info request. """ self.logger.info("Got torrent info request from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentInfoRequestPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.subscribers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return self.send_torrent_info_response(payload.infohash, peer=peer) def on_torrent_info_response(self, source_address, data): """ Message handler for torrent info response. """ self.logger.info("Got torrent info response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentInfoResponsePayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return self.content_repository.update_torrent_info(payload) def on_content_info_request(self, source_address, data): auth, _, payload = self._ez_unpack_auth(ContentInfoRequest, data) peer = self.get_peer_from_auth(auth, source_address) if payload.content_type == SEARCH_TORRENT_REQUEST: db_results = self.content_repository.search_torrent(payload.query_list) self.send_content_info_response(peer, payload.identifier, SEARCH_TORRENT_RESPONSE, db_results) def on_content_info_response(self, source_address, data): _, _, payload = self._ez_unpack_auth(ContentInfoResponse, data) identifier = int(payload.identifier) if not self.request_cache.has(u'request', identifier): return cache = self.request_cache.get(u'request', identifier) if payload.content_type == SEARCH_TORRENT_RESPONSE: self.process_torrent_search_response(cache.query, payload) if not payload.pagination.more: cache = self.request_cache.pop(u'request', identifier) cache.finish() def process_torrent_search_response(self, query, payload): item_format = SearchResponseItemPayload.format_list response, _ = self.serializer.unpack_multiple_as_list(item_format, payload.response) # Decode the category string to list for response_item in response: response_item[4] = decode_values(response_item[4]) self.content_repository.update_from_torrent_search_results(response) result_dict = dict() result_dict['keywords'] = query result_dict['results'] = response result_dict['candidate'] = None if self.tribler_session: self.tribler_session.notifier.notify(SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None, result_dict) # MESSAGE SENDING FUNCTIONS def send_torrent_health_response(self, payload, peer=None): """ Method to send torrent health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) def send_channel_health_response(self, payload, peer=None): """ Method to send channel health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_CHANNEL_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) def send_torrent_info_request(self, infohash, peer): """ Method to request information about a torrent with given infohash to a peer. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return info_request = TorrentInfoRequestPayload(infohash) packet = self.create_message_packet(MSG_TORRENT_INFO_REQUEST, info_request) self.broadcast_message(packet, peer=peer) def send_torrent_info_response(self, infohash, peer): """ Method to send information about a torrent with given infohash to the requesting peer. """ if peer not in self.get_peers(): self.logger.error(ERROR_UNKNOWN_PEER) return db_torrent = self.content_repository.get_torrent(infohash) info_response = TorrentInfoResponsePayload(infohash, db_torrent['name'], db_torrent['length'], db_torrent['creation_date'], db_torrent['num_files'], db_torrent['comment']) packet = self.create_message_packet(MSG_TORRENT_INFO_RESPONSE, info_response) self.broadcast_message(packet, peer=peer) def send_content_info_request(self, content_type, request_list, limit=25, peer=None): """ Sends the generic content request of given content_type. :param content_type: request content type :param request_list: List<string> request queries :param limit: Number of expected responses :param peer: Peer to send this request to :return a Deferred that fires when we receive the content :rtype Deferred """ cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list)) self.logger.info("Sending search request query:%s, identifier:%s", request_list, cache.number) content_request = ContentInfoRequest(cache.number, content_type, request_list, limit) packet = self.create_message_packet(MSG_CONTENT_INFO_REQUEST, content_request) if peer: self.broadcast_message(packet, peer=peer) else: for connected_peer in self.get_peers(): self.broadcast_message(packet, peer=connected_peer) return cache.deferred def send_content_info_response(self, peer, identifier, content_type, response_list): """ Sends the generic content info response with payload response list. :param peer: Receiving peer :param identifier: Request identifier :param content_type: Message content type :param response_list: Content response """ num_results = len(response_list) current_index = 0 page_num = 1 while current_index < num_results: serialized_results, current_index, page_size = self.pack_sized(response_list, MAX_PACKET_PAYLOAD_SIZE, start_index=current_index) if not serialized_results: self.logger.info("Item too big probably to fit into package. Skipping it") current_index += 1 else: pagination = Pagination(page_num, page_size, num_results, more=current_index == num_results) response_payload = ContentInfoResponse(identifier, content_type, serialized_results, pagination) packet = self.create_message_packet(MSG_CONTENT_INFO_RESPONSE, response_payload) self.broadcast_message(packet, peer=peer) def send_torrent_search_request(self, query): """ Sends torrent search query as a content info request with content_type as SEARCH_TORRENT_REQUEST. """ self.send_content_info_request(SEARCH_TORRENT_REQUEST, query) def send_channel_search_request(self, query): """ Sends channel search query to All Channel 2.0 to get a list of channels. """ # TODO: Not implemented yet. Waiting for All Channel 2.0 # CONTENT REPOSITORY STUFFS def publish_next_content(self): """ Publishes the next content from the queue to the subscribers. Does nothing if there are none subscribers. Only Torrent health response is published at the moment. """ self.logger.info("Content to publish: %d", self.content_repository.count_content()) if not self.subscribers: self.logger.info("No subscribers found. Not publishing anything") return content_type, content = self.content_repository.pop_content() if content_type is None: self.logger.error(ERROR_NO_CONTENT) return self.logger.info("Publishing content[type:%d]", content_type) if content_type == TYPE_TORRENT_HEALTH: infohash, seeders, leechers, timestamp = content payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp) self.send_torrent_health_response(payload) def publish_latest_torrents(self, peer): """ Publishes the latest torrents in local database to the given peer. """ torrents = self.content_repository.get_top_torrents() self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer) for torrent in torrents: infohash, seeders, leechers, timestamp = torrent[:4] payload = TorrentHealthPayload(infohash, seeders, leechers, timestamp) self.send_torrent_health_response(payload, peer=peer) def queue_content(self, content_type, content): """ Basically adds a given content to the queue of content repository. """ self.content_repository.add_content(content_type, content)
class PopularityCommunity(PubSubCommunity): """ Community for disseminating the content across the network. Follows publish-subscribe model. """ MASTER_PUBLIC_KEY = ("3081a7301006072a8648ce3d020106052b8104002703819200040504278d20d6776ce7081ad57d99fe066bb2a93" "ce7cc92405a534ef7175bab702be557d8c7d3b725ea0eb09c686e798f6c7ad85e8781a4c3b20e54c15ede38077c" "8f5c801b71d13105f261da7ddcaa94ae14bd177bf1a05a66f595b9bb99117d11f73b4c8d3dcdcdc2b3f838b8ba3" "5a9f600d2c543e8b3ba646083307b917bbbccfc53fc5ab6ded90b711d7eeda46f5f") master_peer = Peer(unhexlify(MASTER_PUBLIC_KEY)) def __init__(self, *args, **kwargs): self.metadata_store = kwargs.pop('metadata_store', None) self.tribler_session = kwargs.pop('session', None) super(PopularityCommunity, self).__init__(*args, **kwargs) self.content_repository = ContentRepository(self.metadata_store) self.decode_map.update({ chr(MSG_TORRENT_HEALTH_RESPONSE): self.on_torrent_health_response }) self.logger.info('Popular Community initialized (peer mid %s)', hexlify(self.my_peer.mid)) @inlineCallbacks def unload(self): self.content_repository.cleanup() self.content_repository = None yield super(PopularityCommunity, self).unload() def on_subscribe(self, source_address, data): auth, _, _ = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) subscribed = super(PopularityCommunity, self).on_subscribe(source_address, data) # Publish the latest torrents to the subscriber if subscribed: self.publish_latest_torrents(peer=peer) def on_torrent_health_response(self, source_address, data): """ Message handler for torrent health response. Torrent health response is part of periodic update message from the publisher. If the message was from an unknown publisher then we are not interested in it and it is simply dropped. In other case, a decision to accept or reject the message is made based on freshness of the message and the trustscore (check update_torrent in ContentRepository for the implementation). """ self.logger.debug("Got torrent health response from %s", source_address) auth, _, payload = self._ez_unpack_auth(TorrentHealthPayload, data) peer = self.get_peer_from_auth(auth, source_address) if peer not in self.publishers: self.logger.error(ERROR_UNKNOWN_RESPONSE) return infohash = payload.infohash if not self.content_repository.has_torrent(infohash): # TODO(Martijn): we should probably try to fetch the torrent info from the other peer return peer_trust = self.trustchain.get_trust(peer) if self.trustchain else 0 self.content_repository.update_torrent_health(payload, peer_trust) # MESSAGE SENDING FUNCTIONS def send_torrent_health_response(self, payload, peer=None): """ Method to send torrent health response. This message is sent to all the subscribers by default but if a peer is specified then only that peer receives this message. """ if peer and peer not in self.get_peers(): self.logger.debug(ERROR_UNKNOWN_PEER) return packet = self.create_message_packet(MSG_TORRENT_HEALTH_RESPONSE, payload) self.broadcast_message(packet, peer=peer) # CONTENT REPOSITORY STUFFS def publish_next_content(self): """ Publishes the next content from the queue to the subscribers. Does nothing if there are no subscribers. Only Torrent health response is published at the moment. """ self.logger.info("Content to publish: %d", self.content_repository.queue_length()) if not self.subscribers: self.logger.info("No subscribers found. Not publishing anything") return content = self.content_repository.pop_content() if content: infohash, seeders, leechers, timestamp = content payload = TorrentHealthPayload(bytes(infohash), seeders, leechers, timestamp) self.send_torrent_health_response(payload) def publish_latest_torrents(self, peer): """ Publishes the latest torrents in local database to the given peer. """ with db_session: torrents = self.content_repository.get_top_torrents() self.logger.info("Publishing %d torrents to peer %s", len(torrents), peer) to_send = [TorrentHealthPayload(bytes(torrent.infohash), torrent.health.seeders, torrent.health.leechers, torrent.health.last_check) for torrent in torrents] for payload in to_send: self.send_torrent_health_response(payload, peer=peer) def queue_content(self, content): """ Basically adds a given content to the queue of content repository. """ self.content_repository.add_content_to_queue(content)
def setUp(self): torrent_db = MockObject() channel_db = MockObject() self.content_repository = ContentRepository(torrent_db, channel_db)
class TestContentRepository(unittest.TestCase): def setUp(self): torrent_db = MockObject() channel_db = MockObject() self.content_repository = ContentRepository(torrent_db, channel_db) def test_add_content(self): """ Test adding and removing content works as expected. """ # Initial content queue is zero self.assertEqual(self.content_repository.count_content(), 0, "No item expected in queue initially") # Add a sample content and check the size sample_content = ('a' * 20, 6, 3, 123456789) sample_content_type = 1 self.content_repository.add_content(sample_content_type, sample_content) self.assertEqual(self.content_repository.count_content(), 1, "One item expected in queue") # Pop an item (content_type, content) = self.content_repository.pop_content() self.assertEqual(content_type, sample_content_type, "Content type should be equal") self.assertEqual(content, sample_content, "Content should be equal") # Check size again self.assertEqual(self.content_repository.count_content(), 0, "No item expected in queue") def test_get_top_torrents(self): """ Test if content repository returns expected top torrents. """ def get_fake_torrents(limit): return [[chr(x) * 20, x, 0, 1525704192] for x in range(limit)] self.content_repository.torrent_db.getRecentlyCheckedTorrents = get_fake_torrents limit = 10 self.assertEqual(self.content_repository.get_top_torrents(limit=limit), get_fake_torrents(limit)) def test_update_torrent_health(self): """ Tests update torrent health. """ def update_torrent(repo, _): repo.update_torrent_called = True # Assume a fake torrent response fake_torrent_health_payload = TorrentHealthPayload('a' * 20, 10, 4, time.time()) self.content_repository.torrent_db = MockObject() self.content_repository.torrent_db.updateTorrent = lambda infohash, *args, **kw: \ update_torrent(self.content_repository, infohash) # If torrent does not exist in the database, then it should be added to the database self.content_repository.has_torrent = lambda infohash: False self.content_repository.update_torrent_health(fake_torrent_health_payload, peer_trust=0) self.assertTrue(self.content_repository.update_torrent_called) def test_update_torrent_with_higher_trust(self): """ Scenario: The database torrent has still fresh last_check_time and you receive a new response from peer with trust > 1. Expect: Torrent in database is updated. """ # last_check_time for existing torrent in database db_last_time_check = time.time() - 10 # Peer trust, higher than 1 in this scenario peer_trust = 10 # Database record is expected to be updated self.assertTrue(self.try_torrent_update_with_options(db_last_time_check, peer_trust)) def test_update_torrent_with_stale_check_time(self): """ Scenario: The database torrent has stale last_check_time and you receive a new response from peer with no previous trust. Expect: Torrent in database is still updated. """ # last_check_time for existing torrent in database db_last_time_check = time.time() - DEFAULT_FRESHNESS_LIMIT # Peer trust, higher than 1 in this scenario peer_trust = 0 # Database record is expected to be updated self.assertTrue(self.try_torrent_update_with_options(db_last_time_check, peer_trust)) def try_torrent_update_with_options(self, db_last_check_time, peer_trust): """ Tries updating torrent considering the given last check time of existing torrent and a new response obtained from a peer with given peer_trust value. """ sample_infohash, seeders, leechers, timestamp = 'a' * 20, 10, 5, db_last_check_time sample_payload = TorrentHealthPayload(sample_infohash, seeders, leechers, timestamp) def update_torrent(content_repo, _): content_repo.update_torrent_called = True def get_torrent(infohash): return {'infohash': infohash, 'num_seeders': seeders, 'num_leechers': leechers, 'last_tracker_check': timestamp} self.content_repository.torrent_db.getTorrent = lambda infohash, **kw: get_torrent(infohash) self.content_repository.torrent_db.hasTorrent = lambda infohash: infohash == sample_infohash self.content_repository.torrent_db.updateTorrent = \ lambda infohash, *args, **kw: update_torrent(self.content_repository, infohash) self.content_repository.update_torrent_called = False self.content_repository.update_torrent_health(sample_payload, peer_trust=peer_trust) return self.content_repository.update_torrent_called def test_update_torrent_info(self): """ Test updating torrent info """ self.content_repository.called_update_torrent = False def fake_update_torrent(ref): ref.called_update_torrent = True self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: \ fake_update_torrent(self.content_repository) self.content_repository.has_torrent = lambda infohash: False torrent_info_response = MockObject() torrent_info_response.infohash = 'a' * 20 torrent_info_response.name = 'ubuntu' torrent_info_response.length = 123 torrent_info_response.creation_date = 123123123 torrent_info_response.num_files = 2 torrent_info_response.comment = 'Ubuntu ISO' self.content_repository.update_torrent_info(torrent_info_response) self.assertTrue(self.content_repository.called_update_torrent) def test_update_conflicting_torrent_info(self): """ Test updating torrent info response with existing record in the database.""" torrent_info_response = MockObject() torrent_info_response.infohash = 'a' * 20 torrent_info_response.name = 'ubuntu' torrent_info_response.length = 123 torrent_info_response.creation_date = 123123123 torrent_info_response.num_files = 2 torrent_info_response.comment = 'Ubuntu ISO' self.content_repository.called_update_torrent = False def fake_update_torrent(ref): ref.called_update_torrent = True def fake_get_torrent(infohash, name): torrent = {'infohash': infohash, 'name': name} return torrent self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent( self.content_repository) self.content_repository.has_torrent = lambda infohash: True self.content_repository.get_torrent = lambda infohash: fake_get_torrent(infohash, torrent_info_response.name) self.content_repository.update_torrent_info(torrent_info_response) self.assertFalse(self.content_repository.called_update_torrent) def test_search_torrent(self): """ Test torrent search """ def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) def random_infohash(): return ''.join(random.choice('0123456789abcdef') for _ in range(20)) sample_torrents = [] for _ in range(10): infohash = random_infohash() name = random_string() length = random.randint(1000, 9999) num_files = random.randint(1, 10) category_list = ['video', 'audio'] creation_date = random.randint(1000000, 111111111) seeders = random.randint(10, 200) leechers = random.randint(5, 1000) cid = random_string(size=20) sample_torrents.append([infohash, name, length, num_files, category_list, creation_date, seeders, leechers, cid]) def fake_torrentdb_search_names(_): return sample_torrents self.content_repository.torrent_db.searchNames = lambda query, **kw: fake_torrentdb_search_names(query) search_query = "Ubuntu" search_results = self.content_repository.search_torrent(search_query) for index in range(10): db_torrent = sample_torrents[index] search_result = search_results[index] self.assertEqual(db_torrent[0], search_result.infohash) self.assertEqual(db_torrent[1], search_result.name) self.assertEqual(db_torrent[2], search_result.length) self.assertEqual(db_torrent[3], search_result.num_files) self.assertEqual(db_torrent[6], search_result.seeders) self.assertEqual(db_torrent[7], search_result.leechers) def test_search_channel(self): """ Test channel search """ def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) sample_channels = [] for index in range(10): dbid = index cid = random_string(size=20) name = random_string() description = random_string(20) nr_torrents = random.randint(1, 10) nr_favorite = random.randint(1, 10) nr_spam = random.randint(1, 10) my_vote = 1 modified = random.randint(1, 10000000) relevance_score = 0.0 sample_channels.append([dbid, cid, name, description, nr_torrents, nr_favorite, nr_spam, my_vote, modified, relevance_score]) def fake_torrentdb_search_channels(_): return sample_channels self.content_repository.channel_db.search_in_local_channels_db = lambda query, **kw: \ fake_torrentdb_search_channels(query) search_query = "Ubuntu" search_results = self.content_repository.search_channels(search_query) for index in range(10): db_channel = sample_channels[index] search_result = search_results[index] self.assertEqual(db_channel[0], search_result.id) self.assertEqual(db_channel[1], search_result.cid) self.assertEqual(db_channel[2], search_result.name) self.assertEqual(db_channel[3], search_result.description) self.assertEqual(db_channel[4], search_result.nr_torrents) self.assertEqual(db_channel[5], search_result.nr_favorite) self.assertEqual(db_channel[6], search_result.nr_spam) self.assertEqual(db_channel[8], search_result.modified) def test_update_torrent_from_search_results(self): """ Tests updating database from the search results """ def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) def random_infohash(): return ''.join(random.choice('0123456789abcdef') for _ in range(20)) search_results = dict() for _ in range(10): infohash = random_infohash() name = random_string() length = random.randint(1000, 9999) num_files = random.randint(1, 10) category_list = ['video', 'audio'] creation_date = random.randint(1000000, 111111111) seeders = random.randint(10, 200) leechers = random.randint(5, 1000) cid = random_string(size=20) search_results[infohash] = [infohash, name, length, num_files, category_list, creation_date, seeders, leechers, cid] def get_torrent(torrent_as_list): return {'infohash': torrent_as_list[0], 'name': torrent_as_list[1], 'length': torrent_as_list[2], 'num_files': torrent_as_list[3], 'category_list': torrent_as_list[4], 'creation_date': torrent_as_list[5], 'seeders': torrent_as_list[6], 'leechers': torrent_as_list[7], 'cid': torrent_as_list[8]} def fake_update_torrent(ref): ref.called_update_torrent = True def fake_add_or_get_torrent_id(ref): ref.called_add_or_get_torrent_id = True self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent( self.content_repository) self.content_repository.torrent_db.addOrGetTorrentID = lambda infohash: fake_add_or_get_torrent_id( self.content_repository) # Case 1: Assume torrent does not exist in the database self.content_repository.has_torrent = lambda infohash: False self.content_repository.get_torrent = lambda infohash: None self.content_repository.torrent_db._db = MockObject() self.content_repository.torrent_db._db.commit_now = lambda x=None: None self.content_repository.called_update_torrent = False self.content_repository.update_from_torrent_search_results(search_results.values()) self.assertTrue(self.content_repository.called_update_torrent) self.assertTrue(self.content_repository.called_add_or_get_torrent_id) # Case 2: Torrent already exist in the database self.content_repository.has_torrent = lambda infohash: infohash in search_results self.content_repository.get_torrent = lambda infohash: get_torrent(search_results[infohash]) self.content_repository.called_update_torrent = False self.content_repository.called_add_or_get_torrent_id = False self.content_repository.update_from_torrent_search_results(search_results.values()) self.assertFalse(self.content_repository.called_update_torrent) self.assertFalse(self.content_repository.called_add_or_get_torrent_id)
class TestContentRepository(TriblerCoreTest): @inlineCallbacks def setUp(self): yield super(TestContentRepository, self).setUp() self.my_key = default_eccrypto.generate_key(u"curve25519") mds = MetadataStore(':memory:', self.session_base_dir, self.my_key) self.content_repository = ContentRepository(mds) # Add some content to the metadata database with db_session: mds.ChannelMetadata.create_channel('test', 'test') for torrent_ind in xrange(5): torrent = mds.TorrentMetadata(title='torrent%d' % torrent_ind, infohash=('%d' % torrent_ind) * 20) torrent.health.seeders = torrent_ind + 1 def test_has_get_torrent(self): """ Test fetching a torrent from the metadata store """ self.assertFalse(self.content_repository.get_torrent('9' * 20)) self.assertTrue(self.content_repository.get_torrent('0' * 20)) self.assertFalse(self.content_repository.has_torrent('9' * 20)) self.assertTrue(self.content_repository.has_torrent('0' * 20)) self.assertFalse(self.content_repository.get_torrent('\x89' * 20)) @db_session def test_get_top_torrents(self): """ Test fetching the top torrents from the metadata store """ torrents = self.content_repository.get_top_torrents() self.assertEqual(len(torrents), 5) self.assertEqual(torrents[0].health.seeders, 5) self.assertEqual(len(self.content_repository.get_top_torrents(limit=1)), 1) def test_add_content(self): """ Test adding and removing content works as expected. """ # Initial content queue is zero self.assertEqual(self.content_repository.queue_length(), 0, "No item expected in queue initially") # Add a sample content and check the size torrent = self.content_repository.get_torrent('0' * 20) self.content_repository.add_content_to_queue(torrent) self.assertEqual(self.content_repository.queue_length(), 1, "One item expected in queue") # Pop an item content = self.content_repository.pop_content() self.assertEqual(content, torrent, "Content should be equal") # Check size again self.assertEqual(self.content_repository.queue_length(), 0, "No item expected in queue") def test_update_torrent_health(self): """ Tests update torrent health. """ fake_torrent_health_payload = TorrentHealthPayload('0' * 20, 10, 4, time.time()) self.content_repository.update_torrent_health(fake_torrent_health_payload, peer_trust=0) with db_session: torrent = self.content_repository.get_torrent('0' * 20) self.assertEqual(torrent.health.seeders, 10) self.assertEqual(torrent.health.leechers, 4)