def test_delete_noncompliant_state(self): STATE_DIR = TESTS_DATA_DIR / 'noncompliant_state_dir' tmpdir = self.temporary_directory() / STATE_DIR.name shutil.copytree(str_path(STATE_DIR), str_path(tmpdir)) cleanup_noncompliant_channel_torrents(tmpdir) # Check cleanup of the channels dir dir_listing = list((tmpdir / "channels").iterdir()) self.assertEqual(3, len(dir_listing)) for f in (tmpdir / "channels").iterdir(): self.assertEqual(CHANNEL_DIR_NAME_LENGTH, len(f.stem)) # Check cleanup of torrent state dir checkpoints_dir = tmpdir / "dlcheckpoints" dir_listing = os.listdir(checkpoints_dir) self.assertEqual(1, len(dir_listing)) file_path = checkpoints_dir / dir_listing[0] pstate = CallbackConfigParser() pstate.read_file(file_path) self.assertEqual(CHANNEL_DIR_NAME_LENGTH, len(pstate.get('state', 'metainfo')['info']['name']))
def process_mdblob_file(self, filepath, **kwargs): """ Process a file with metadata in a channel directory. :param filepath: The path to the file :param skip_personal_metadata_payload: if this is set to True, personal torrent metadata payload received through gossip will be ignored. The default value is True. :param external_thread: indicate to the lower lever that we're running in the backround thread, to possibly pace down the upload process :return: a list of tuples of (<metadata or payload>, <action type>) """ with open(str_path(filepath), 'rb') as f: serialized_data = f.read() if str(filepath).endswith('.lz4'): return self.process_compressed_mdblob(serialized_data, **kwargs) return self.process_squashed_mdblob(serialized_data, **kwargs)
def consolidate_channel_torrent(self): """ Delete the channel dir contents and create it anew. Use it to consolidate fragmented channel torrent directories. :param key: The public/private key, used to sign the data """ # Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms # Cleanup entries marked for deletion db.CollectionNode.collapse_deleted_subtrees() # Note: It should be possible to stop alling get_contents_to_commit here commit_queue = self.get_contents_to_commit() for entry in commit_queue: if entry.status == TODELETE: entry.delete() folder = Path(self._channels_dir) / self.dirname # We check if we need to re-create the channel dir in case it was deleted for some reason if not folder.is_dir(): os.makedirs(folder) for filename in os.listdir(folder): file_path = folder / filename # We only remove mdblobs and leave the rest as it is if filename.endswith(BLOB_EXTENSION) or filename.endswith( BLOB_EXTENSION + '.lz4'): os.unlink(str_path(file_path)) # Channel should get a new starting timestamp and its contents should get higher timestamps start_timestamp = clock.tick() def update_timestamps_recursive(node): if issubclass(type(node), db.CollectionNode): for child in node.contents: update_timestamps_recursive(child) if node.status in [COMMITTED, UPDATED, NEW]: node.status = UPDATED node.timestamp = clock.tick() node.sign() update_timestamps_recursive(self) return self.commit_channel_torrent( new_start_timestamp=start_timestamp)
def update_channel_torrent(self, metadata_list): """ Channel torrents are append-only to support seeding the old versions from the same dir and avoid updating already downloaded blobs. :param metadata_list: The list of metadata entries to add to the torrent dir. ACHTUNG: TODELETE entries _MUST_ be sorted to the end of the list to prevent channel corruption! :return The newly create channel torrent infohash, final timestamp for the channel and torrent date """ # As a workaround for delete entries not having a timestamp in the DB, delete entries should # be placed after create/modify entries: # | create/modify entries | delete entries | <- final timestamp # Create dir for the metadata files channel_dir = path_util.abspath(self._channels_dir / self.dirname) if not channel_dir.is_dir(): os.makedirs(str_path(channel_dir)) existing_contents = sorted(channel_dir.iterdir()) last_existing_blob_number = get_mdblob_sequence_number( existing_contents[-1]) if existing_contents else None index = 0 while index < len(metadata_list): # Squash several serialized and signed metadata entries into a single file data, index = entries_to_chunk(metadata_list, self._CHUNK_SIZE_LIMIT, start_index=index) # Blobs ending with TODELETE entries increase the final timestamp as a workaround for delete commands # possessing no timestamp. if metadata_list[index - 1].status == TODELETE: blob_timestamp = clock.tick() else: blob_timestamp = metadata_list[index - 1].timestamp # The final file in the sequence should get a timestamp that is higher than the timestamp of # the last channel contents entry. This final timestamp then should be returned to the calling function # to be assigned to the corresponding channel entry. # Otherwise, the local channel version will never become equal to its timestamp. if index >= len(metadata_list): blob_timestamp = clock.tick() # Check that the mdblob we're going to create has a greater timestamp than the existing ones assert last_existing_blob_number is None or ( blob_timestamp > last_existing_blob_number) blob_filename = Path( channel_dir, str(blob_timestamp).zfill(12) + BLOB_EXTENSION + '.lz4') assert not blob_filename.exists( ) # Never ever write over existing files. blob_filename.write_bytes(data) last_existing_blob_number = blob_timestamp with db_session: thumb_exists = db.ChannelThumbnail.exists( lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE) descr_exists = db.ChannelDescription.exists( lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE) flags = CHANNEL_THUMBNAIL_FLAG * ( int(thumb_exists)) + CHANNEL_DESCRIPTION_FLAG * ( int(descr_exists)) # Note: the timestamp can end up messed in case of an error # Make torrent out of dir with metadata files torrent, infohash = create_torrent_from_dir( channel_dir, self._channels_dir / (self.dirname + ".torrent")) torrent_date = datetime.utcfromtimestamp(torrent[b'creation date']) return { "infohash": infohash, "timestamp": last_existing_blob_number, "torrent_date": torrent_date, "reserved_flags": flags, }, torrent
def write(self, filename): self.config.filename = str_path(filename) self.config.write()
def load(config_path=None): return DownloadConfig( ConfigObj(infile=str_path(config_path), file_error=True, configspec=str(CONFIG_SPEC_PATH), default_encoding='utf-8'))
def to_delete_file(self, filename): with open(str_path(filename), 'wb') as output_file: output_file.write(self.serialized_delete())
def to_file(self, filename, key=None): with open(str_path(filename), 'wb') as output_file: output_file.write(self.serialized(key))
async def convert_personal_channel(self): with db_session: # Reflect conversion state v = self.mds.MiscData.get_for_update( name=CONVERSION_FROM_72_PERSONAL) if v: if v.value == CONVERSION_STARTED: # Just drop the entries from the previous try my_channels = self.mds.ChannelMetadata.get_my_channels() for my_channel in my_channels: my_channel.contents.delete(bulk=True) my_channel.delete() else: # Something is wrong, this should never happen raise Exception( "Previous conversion resulted in invalid state") else: self.mds.MiscData(name=CONVERSION_FROM_72_PERSONAL, value=CONVERSION_STARTED) my_channels_count = self.mds.ChannelMetadata.get_my_channels( ).count() # Make sure every precondition is met if self.personal_channel_id and not my_channels_count: total_to_convert = self.get_personal_channel_torrents_count() with db_session: my_channel = self.mds.ChannelMetadata.create_channel( title=self.personal_channel_title, description='') def get_old_stuff(batch_size, offset): return self.get_old_torrents(personal_channel_only=True, sign=True, batch_size=batch_size, offset=offset) def add_to_pony(t): return self.mds.TorrentMetadata(origin_id=my_channel.id_, **t) await self.convert_async( add_to_pony, get_old_stuff, total_to_convert, offset=0, message="Converting personal channel torrents.") with db_session: my_channel = self.mds.ChannelMetadata.get_my_channels().first() folder = Path(my_channel._channels_dir) / my_channel.dirname # We check if we need to re-create the channel dir in case it was deleted for some reason if not folder.is_dir(): os.makedirs(str_path(folder)) for filename in os.listdir(folder): file_path = folder / filename # We only remove mdblobs and leave the rest as it is if filename.endswith(BLOB_EXTENSION) or filename.endswith( BLOB_EXTENSION + '.lz4'): os.unlink(str_path(file_path)) my_channel.commit_channel_torrent() with db_session: v = self.mds.MiscData.get_for_update( name=CONVERSION_FROM_72_PERSONAL) v.value = CONVERSION_FINISHED