def test_squash_mdblobs_multiple_chunks(metadata_store): r = random.Random(123) md_list = [ metadata_store.TorrentMetadata( title=''.join( r.choice(string.ascii_uppercase + string.digits) for _ in range(20)), infohash=database_blob(random_infohash()), torrent_date=datetime.utcfromtimestamp(100), ) for _ in range(0, 10) ] # Test splitting into multiple chunks chunk, index = entries_to_chunk(md_list, chunk_size=900) chunk2, _ = entries_to_chunk(md_list, chunk_size=900, start_index=index) dict_list = [d.to_dict()["signature"] for d in md_list] for d in md_list: d.delete() assert dict_list[:index] == [ d.md_obj.to_dict()["signature"] for d in metadata_store.process_compressed_mdblob( chunk, skip_personal_metadata_payload=False) ] assert dict_list[index:] == [ d.md_obj.to_dict()["signature"] for d in metadata_store.process_compressed_mdblob( chunk2, skip_personal_metadata_payload=False) ]
def test_squash_mdblobs_multiple_chunks(self): with db_session: md_list = [ self.mds.TorrentMetadata( title=''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(20)), infohash=database_blob(random_infohash()), ) for _ in range(0, 10) ] # Test splitting into multiple chunks chunk, index = entries_to_chunk(md_list, chunk_size=900) chunk2, _ = entries_to_chunk(md_list, chunk_size=900, start_index=index) dict_list = [d.to_dict()["signature"] for d in md_list] for d in md_list: d.delete() self.assertListEqual( dict_list[:index], [ d[0].to_dict()["signature"] for d in self.mds.process_compressed_mdblob( chunk, skip_personal_metadata_payload=False) ], ) self.assertListEqual( dict_list[index:], [ d[0].to_dict()["signature"] for d in self.mds.process_compressed_mdblob( chunk2, skip_personal_metadata_payload=False) ], )
def _prepare_gossip_blob_cache(self): # Choose some random entries and try to pack them into maximum_payload_size bytes with db_session: # Generate and cache the gossip blob for the personal channel personal_channels = list( self.metadata_store.ChannelMetadata.get_my_channels().where( lambda g: g.num_entries > 0).random(1)) personal_channel = personal_channels[ 0] if personal_channels else None md_list = ( [personal_channel] + list(personal_channel.get_random_contents(max_entries - 1)) if personal_channel else None) self.gossip_blob_personal_channel = ( entries_to_chunk(md_list, maximum_payload_size)[0] if md_list and len(md_list) > 1 else None) # Generate and cache the gossip blob for a subscribed channel # TODO: when the health table will be there, send popular torrents instead channel_l = list( self.metadata_store.ChannelMetadata.get_random_channels( 1, only_subscribed=True, only_downloaded=True)) md_list = channel_l + list(channel_l[0].get_random_contents( max_entries - 1)) if channel_l else None self.gossip_blob = entries_to_chunk( md_list, maximum_payload_size)[0] if md_list else None self.metadata_store.disconnect_thread()
def test_mdblob_dont_fit_exception(metadata_store): with pytest.raises(Exception): md_list = [ metadata_store.TorrentMetadata(title='test' + str(x), infohash=random_infohash()) for x in range(0, 1) ] entries_to_chunk(md_list, chunk_size=1)
def send_db_results(self, peer, request_payload_id, db_results): index = 0 while index < len(db_results): data, index = entries_to_chunk(db_results, self.settings.maximum_payload_size, start_index=index) self.ez_send(peer, SelectResponsePayload(request_payload_id, data))
def send_db_results(self, peer, request_payload_id, db_results, force_eva_response=False): # Special case of empty results list - sending empty lz4 archive if len(db_results) == 0: self.ez_send( peer, SelectResponsePayload(request_payload_id, LZ4_EMPTY_ARCHIVE)) return index = 0 while index < len(db_results): transfer_size = (self.eva_protocol.binary_size_limit if force_eva_response else self.settings.maximum_payload_size) data, index = entries_to_chunk(db_results, transfer_size, start_index=index) payload = SelectResponsePayload(request_payload_id, data) if force_eva_response or (len(data) > self.settings.maximum_payload_size): self.eva_send_binary(peer, struct.pack('>i', request_payload_id), self.ezr_pack(payload.msg_id, payload)) else: self.ez_send(peer, payload)
def gen_have_newer_results_blob(md_list): with db_session: reply_list = [ md for md, result in md_list if (md and (md.metadata_type == CHANNEL_TORRENT)) and ( result == GOT_NEWER_VERSION) ] return entries_to_chunk( reply_list, maximum_payload_size)[0] if reply_list else None
def _get_search_results(): with db_session: db_results = self.metadata_store.MetadataNode.get_entries( **request_dict) result = entries_to_chunk( db_results[:max_entries], maximum_payload_size)[0] if db_results else None self.metadata_store.disconnect_thread() return result
async def on_remote_select(self, peer, request): request_sanitized = sanitize_query(json.loads(request.json), self.settings.max_response_size) db_results = await self.mds.MetadataNode.get_entries_threaded(**request_sanitized) if not db_results: return index = 0 while index < len(db_results): data, index = entries_to_chunk(db_results, self.settings.maximum_payload_size, start_index=index) self.ez_send(peer, SelectResponsePayload(request.id, data))
def test_data_dont_fit_in_mdblob(metadata_store): import random as rng # pylint: disable=import-outside-toplevel rng.seed(123) md_list = [ metadata_store.TorrentMetadata( title='test' + str(x), infohash=random_infohash(rng), id_=rng.randint(0, 100000000), timestamp=rng.randint(0, 100000000), ) for x in range(0, 1) ] chunk, index = entries_to_chunk(md_list, chunk_size=1) assert index == 1 assert len(chunk) == 206 # Test corner case of empty list and/or too big index with pytest.raises(Exception): entries_to_chunk(md_list, chunk_size=1000, start_index=1000) with pytest.raises(Exception): entries_to_chunk([], chunk_size=1)
def test_squash_mdblobs(metadata_store): chunk_size = metadata_store.ChannelMetadata._CHUNK_SIZE_LIMIT md_list = [ metadata_store.TorrentMetadata( title=''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20)), infohash=database_blob(random_infohash()), torrent_date=datetime.utcfromtimestamp(100), ) for _ in range(0, 10) ] chunk, _ = entries_to_chunk(md_list, chunk_size=chunk_size) dict_list = [d.to_dict()["signature"] for d in md_list] for d in md_list: d.delete() assert dict_list == [ d[0].to_dict()["signature"] for d in metadata_store.process_compressed_mdblob(chunk, skip_personal_metadata_payload=False) ]
def test_squash_mdblobs(self): chunk_size = self.mds.ChannelMetadata._CHUNK_SIZE_LIMIT md_list = [ self.mds.TorrentMetadata( title=''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(20)), infohash=database_blob(random_infohash()), ) for _ in range(0, 10) ] chunk, _ = entries_to_chunk(md_list, chunk_size=chunk_size) dict_list = [d.to_dict()["signature"] for d in md_list] for d in md_list: d.delete() self.assertListEqual( dict_list, [ d[0].to_dict()["signature"] for d in self.mds.process_compressed_mdblob( chunk, skip_personal_metadata_payload=False) ], )