def test_skip_processing_of_received_personal_channel_torrents(metadata_store): """ Test that personal torrent is ignored by default when processing the torrent metadata payload """ channel = metadata_store.ChannelMetadata.create_channel('testchan') torrent_md = metadata_store.TorrentMetadata(origin_id=channel.id_, title='test', status=NEW, infohash=random_infohash()) channel.commit_channel_torrent() torrent_md.delete() channel_dir = Path( metadata_store.ChannelMetadata._channels_dir) / channel.dirname assert os.listdir(Path.fix_win_long_file(channel_dir)) # By default, personal channel torrent metadata processing is skipped so there should be no torrents # added to the channel channel.local_version = 0 metadata_store.process_channel_dir(channel_dir, channel.public_key, channel.id_) assert not channel.contents # Enable processing of personal channel torrent metadata channel.local_version = 0 metadata_store.process_channel_dir(channel_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False) assert len(channel.contents) == 1
def get_base_path(): """ Get absolute path to resource, works for dev and for PyInstaller """ try: # PyInstaller creates a temp folder and stores path in _MEIPASS base_path = Path(sys._MEIPASS) except Exception: base_path = Path(tribler_core.__file__).parent fixed_filename = Path.fix_win_long_file(base_path) return Path(fixed_filename)
def process_mdblob_file(self, filepath, **kwargs): """ Process a file with metadata in a channel directory. :param filepath: The path to the file :param skip_personal_metadata_payload: if this is set to True, personal torrent metadata payload received through gossip will be ignored. The default value is True. :param external_thread: indicate to the lower lever that we're running in the backround thread, to possibly pace down the upload process :return: a list of tuples of (<metadata or payload>, <action type>) """ path = Path.fix_win_long_file(filepath) with open(path, 'rb') as f: serialized_data = f.read() if path.endswith('.lz4'): return self.process_compressed_mdblob(serialized_data, **kwargs) return self.process_squashed_mdblob(serialized_data, **kwargs)
def consolidate_channel_torrent(self): """ Delete the channel dir contents and create it anew. Use it to consolidate fragmented channel torrent directories. :param key: The public/private key, used to sign the data """ # Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms # Cleanup entries marked for deletion db.CollectionNode.collapse_deleted_subtrees() # Note: It should be possible to stop alling get_contents_to_commit here commit_queue = self.get_contents_to_commit() for entry in commit_queue: if entry.status == TODELETE: entry.delete() folder = Path(self._channels_dir) / self.dirname # We check if we need to re-create the channel dir in case it was deleted for some reason if not folder.is_dir(): os.makedirs(folder) for filename in os.listdir(folder): file_path = folder / filename # We only remove mdblobs and leave the rest as it is if filename.endswith(BLOB_EXTENSION) or filename.endswith( BLOB_EXTENSION + '.lz4'): os.unlink(Path.fix_win_long_file(file_path)) # Channel should get a new starting timestamp and its contents should get higher timestamps start_timestamp = clock.tick() def update_timestamps_recursive(node): if issubclass(type(node), db.CollectionNode): for child in node.contents: update_timestamps_recursive(child) if node.status in [COMMITTED, UPDATED, NEW]: node.status = UPDATED node.timestamp = clock.tick() node.sign() update_timestamps_recursive(self) return self.commit_channel_torrent( new_start_timestamp=start_timestamp)
def update_channel_torrent(self, metadata_list): """ Channel torrents are append-only to support seeding the old versions from the same dir and avoid updating already downloaded blobs. :param metadata_list: The list of metadata entries to add to the torrent dir. ACHTUNG: TODELETE entries _MUST_ be sorted to the end of the list to prevent channel corruption! :return The newly create channel torrent infohash, final timestamp for the channel and torrent date """ # As a workaround for delete entries not having a timestamp in the DB, delete entries should # be placed after create/modify entries: # | create/modify entries | delete entries | <- final timestamp # Create dir for the metadata files channel_dir = Path(self._channels_dir / self.dirname).absolute() if not channel_dir.is_dir(): os.makedirs(Path.fix_win_long_file(channel_dir)) existing_contents = sorted(channel_dir.iterdir()) last_existing_blob_number = get_mdblob_sequence_number(existing_contents[-1]) if existing_contents else None index = 0 while index < len(metadata_list): # Squash several serialized and signed metadata entries into a single file data, index = entries_to_chunk(metadata_list, self._CHUNK_SIZE_LIMIT, start_index=index) # Blobs ending with TODELETE entries increase the final timestamp as a workaround for delete commands # possessing no timestamp. if metadata_list[index - 1].status == TODELETE: blob_timestamp = clock.tick() else: blob_timestamp = metadata_list[index - 1].timestamp # The final file in the sequence should get a timestamp that is higher than the timestamp of # the last channel contents entry. This final timestamp then should be returned to the calling function # to be assigned to the corresponding channel entry. # Otherwise, the local channel version will never become equal to its timestamp. if index >= len(metadata_list): blob_timestamp = clock.tick() # Check that the mdblob we're going to create has a greater timestamp than the existing ones assert last_existing_blob_number is None or (blob_timestamp > last_existing_blob_number) blob_filename = Path(channel_dir, str(blob_timestamp).zfill(12) + BLOB_EXTENSION + '.lz4') assert not blob_filename.exists() # Never ever write over existing files. blob_filename.write_bytes(data) last_existing_blob_number = blob_timestamp with db_session: thumb_exists = db.ChannelThumbnail.exists( lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE ) descr_exists = db.ChannelDescription.exists( lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE ) flags = CHANNEL_THUMBNAIL_FLAG * (int(thumb_exists)) + CHANNEL_DESCRIPTION_FLAG * (int(descr_exists)) # Note: the timestamp can end up messed in case of an error # Make torrent out of dir with metadata files torrent, infohash = create_torrent_from_dir(channel_dir, self._channels_dir / (self.dirname + ".torrent")) torrent_date = datetime.utcfromtimestamp(torrent[b'creation date']) return { "infohash": infohash, "timestamp": last_existing_blob_number, "torrent_date": torrent_date, "reserved_flags": flags, }, torrent
def write(self, filename): self.config.filename = Path.fix_win_long_file(filename) self.config.write()
def load(config_path=None): return DownloadConfig( ConfigObj(infile=Path.fix_win_long_file(config_path), file_error=True, configspec=str(CONFIG_SPEC_PATH), default_encoding='utf-8'))
def to_delete_file(self, filename): with open(Path.fix_win_long_file(filename), 'wb') as output_file: output_file.write(self.serialized_delete())
def to_file(self, filename, key=None): with open(Path.fix_win_long_file(filename), 'wb') as output_file: output_file.write(self.serialized(key))