def prepare_commit_queue_for_channel(commit_queue): """ This routine prepares the raw commit queue for commit by updating the elements' properties and re-signing them. Also, it removes the channel entry itself from the queue [:-1], because its meaningless to put it in the blobs, as it must be updated with the new infohash after commit. :param commit_queue: :return: """ for node in commit_queue: # Avoid updating entries that must be deleted: # soft delete payloads require signatures of unmodified entries if issubclass(type(node), db.CollectionNode) and node.status != TODELETE: # Update recursive count of actual non-collection contents node.num_entries = select( # For each subnode, if it is a collection, add the count of its contents to the recursive sum. # Otherwise, add just 1 to the sum (to count the subnode itself). (g.num_entries if g.metadata_type == COLLECTION_NODE else 1) for g in node.actual_contents ).sum() node.timestamp = clock.tick() node.sign() # This perverted comparator lambda is necessary to ensure that delete entries are always # sorted to the end of the list, as required by the channel serialization routine. return sorted(commit_queue[:-1], key=lambda x: int(x.status == TODELETE) - 1 / x.timestamp)
def add_torrent_to_channel(self, tdef, extra_info=None): """ Add a torrent to your channel. :param tdef: The torrent definition file of the torrent to add :param extra_info: Optional extra info to add to the torrent """ new_entry_dict = dict(tdef_to_metadata_dict(tdef), status=NEW) if extra_info: new_entry_dict['tags'] = extra_info.get('description', '') # See if the torrent is already in the channel old_torrent = db.TorrentMetadata.get(public_key=self.public_key, infohash=tdef.get_infohash()) if old_torrent: # If it is there, check if we were going to delete it if old_torrent.status == TODELETE: new_timestamp = clock.tick() old_torrent.set(timestamp=new_timestamp, origin_id=self.id_, **new_entry_dict) old_torrent.sign() # As we really don't know what status this torrent had _before_ it got its TODELETE status, # we _must_ set its status to UPDATED, for safety old_torrent.status = UPDATED torrent_metadata = old_torrent else: raise DuplicateTorrentFileError() else: torrent_metadata = db.TorrentMetadata.from_dict(dict(origin_id=self.id_, **new_entry_dict)) return torrent_metadata
def update_timestamps_recursive(node): if issubclass(type(node), db.CollectionNode): for child in node.contents: update_timestamps_recursive(child) if node.status in [COMMITTED, UPDATED, NEW]: node.status = UPDATED node.timestamp = clock.tick() node.sign()
def update_properties(self, update_dict): signed_attribute_changed = False for k, value in update_dict.items(): if getattr(self, k) != value: setattr(self, k, value) signed_attribute_changed = signed_attribute_changed or (k in self.payload_arguments) if signed_attribute_changed: if self.status != NEW: self.status = UPDATED self.timestamp = clock.tick() self.sign() return self
def update_properties(self, update_dict): signed_attribute_changed = False for k, value in update_dict.items(): if getattr(self, k) != value: setattr(self, k, value) signed_attribute_changed = signed_attribute_changed or (k in self.payload_arguments) if signed_attribute_changed: if self.status != NEW: self.status = UPDATED # ACHTUNG! When using the key argument, the thing will still use _local_ timestamp counter! self.timestamp = clock.tick() self.sign() return self
def consolidate_channel_torrent(self): """ Delete the channel dir contents and create it anew. Use it to consolidate fragmented channel torrent directories. :param key: The public/private key, used to sign the data """ # Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms # Cleanup entries marked for deletion db.CollectionNode.collapse_deleted_subtrees() # Note: It should be possible to stop alling get_contents_to_commit here commit_queue = self.get_contents_to_commit() for entry in commit_queue: if entry.status == TODELETE: entry.delete() folder = Path(self._channels_dir) / self.dirname # We check if we need to re-create the channel dir in case it was deleted for some reason if not folder.is_dir(): os.makedirs(folder) for filename in os.listdir(folder): file_path = folder / filename # We only remove mdblobs and leave the rest as it is if filename.endswith(BLOB_EXTENSION) or filename.endswith( BLOB_EXTENSION + '.lz4'): os.unlink(str_path(file_path)) # Channel should get a new starting timestamp and its contents should get higher timestamps start_timestamp = clock.tick() def update_timestamps_recursive(node): if issubclass(type(node), db.CollectionNode): for child in node.contents: update_timestamps_recursive(child) if node.status in [COMMITTED, UPDATED, NEW]: node.status = UPDATED node.timestamp = clock.tick() node.sign() update_timestamps_recursive(self) return self.commit_channel_torrent( new_start_timestamp=start_timestamp)
def update_channel_torrent(self, metadata_list): """ Channel torrents are append-only to support seeding the old versions from the same dir and avoid updating already downloaded blobs. :param metadata_list: The list of metadata entries to add to the torrent dir. ACHTUNG: TODELETE entries _MUST_ be sorted to the end of the list to prevent channel corruption! :return The newly create channel torrent infohash, final timestamp for the channel and torrent date """ # As a workaround for delete entries not having a timestamp in the DB, delete entries should # be placed after create/modify entries: # | create/modify entries | delete entries | <- final timestamp # Create dir for the metadata files channel_dir = path_util.abspath(self._channels_dir / self.dirname) if not channel_dir.is_dir(): os.makedirs(str_path(channel_dir)) existing_contents = sorted(channel_dir.iterdir()) last_existing_blob_number = get_mdblob_sequence_number( existing_contents[-1]) if existing_contents else None index = 0 while index < len(metadata_list): # Squash several serialized and signed metadata entries into a single file data, index = entries_to_chunk(metadata_list, self._CHUNK_SIZE_LIMIT, start_index=index) # Blobs ending with TODELETE entries increase the final timestamp as a workaround for delete commands # possessing no timestamp. if metadata_list[index - 1].status == TODELETE: blob_timestamp = clock.tick() else: blob_timestamp = metadata_list[index - 1].timestamp # The final file in the sequence should get a timestamp that is higher than the timestamp of # the last channel contents entry. This final timestamp then should be returned to the calling function # to be assigned to the corresponding channel entry. # Otherwise, the local channel version will never become equal to its timestamp. if index >= len(metadata_list): blob_timestamp = clock.tick() # Check that the mdblob we're going to create has a greater timestamp than the existing ones assert last_existing_blob_number is None or ( blob_timestamp > last_existing_blob_number) blob_filename = Path( channel_dir, str(blob_timestamp).zfill(12) + BLOB_EXTENSION + '.lz4') assert not blob_filename.exists( ) # Never ever write over existing files. blob_filename.write_bytes(data) last_existing_blob_number = blob_timestamp with db_session: thumb_exists = db.ChannelThumbnail.exists( lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE) descr_exists = db.ChannelDescription.exists( lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE) flags = CHANNEL_THUMBNAIL_FLAG * ( int(thumb_exists)) + CHANNEL_DESCRIPTION_FLAG * ( int(descr_exists)) # Note: the timestamp can end up messed in case of an error # Make torrent out of dir with metadata files torrent, infohash = create_torrent_from_dir( channel_dir, self._channels_dir / (self.dirname + ".torrent")) torrent_date = datetime.utcfromtimestamp(torrent[b'creation date']) return { "infohash": infohash, "timestamp": last_existing_blob_number, "torrent_date": torrent_date, "reserved_flags": flags, }, torrent
def __init__(self, *args, **kwargs): """ Initialize a metadata object. All this dance is required to ensure that the signature is there and it is correct. """ skip_key_check = False # FIXME: refactor this method by moving different ways to create an entry into separate methods # Process special keyworded arguments # "sign_with" argument given, sign with it private_key_override = None if "sign_with" in kwargs: kwargs["public_key"] = database_blob( kwargs["sign_with"].pub().key_to_bin()[10:]) private_key_override = kwargs.pop("sign_with") # Free-for-all entries require special treatment if "public_key" in kwargs and kwargs["public_key"] == b"": # We have to give the entry an unique sig to honor the DB constraints. We use the entry's id_ # as the sig to keep it unique and short. The uniqueness is guaranteed by DB as it already # imposes uniqueness constraints on the id_+public_key combination. if "id_" in kwargs: kwargs["signature"] = None skip_key_check = True else: # Trying to create an FFA entry without specifying the id_ should be considered an error, # because assigning id_ automatically by clock breaks anonymity. # FFA entries should be "timeless" and anonymous. raise InvalidChannelNodeException( "Attempted to create %s free-for-all (unsigned) object without specifying id_ : " % str(self.__class__.__name__)) # For putting legacy/test stuff in skip_key_check = kwargs.pop("skip_key_check", skip_key_check) if "timestamp" not in kwargs: kwargs["timestamp"] = clock.tick() if "id_" not in kwargs: kwargs["id_"] = int(random.getrandbits(63)) if not private_key_override and not skip_key_check: # No key/signature given, sign with our own key. if ("signature" not in kwargs) and ( ("public_key" not in kwargs) or (kwargs["public_key"] == database_blob( self._my_key.pub().key_to_bin()[10:]))): private_key_override = self._my_key # Key/signature given, check them for correctness elif ("public_key" in kwargs) and ("signature" in kwargs): try: self._payload_class(**kwargs) except InvalidSignatureException: raise InvalidSignatureException( f"Attempted to create {str(self.__class__.__name__)} object with invalid signature/PK: " + (hexlify(kwargs["signature"]) if "signature" in kwargs else "empty signature ") + " / " + (hexlify(kwargs["public_key"]) if "public_key" in kwargs else " empty PK")) if private_key_override: # Get default values for Pony class attributes. We have to do it manually because we need # to know the payload signature *before* creating the object. kwargs = generate_dict_from_pony_args( self.__class__, skip_list=["signature", "public_key"], **kwargs) payload = self._payload_class(**dict( kwargs, public_key=private_key_override.pub().key_to_bin()[10:], key=private_key_override, metadata_type=self.metadata_type, )) kwargs["public_key"] = payload.public_key kwargs["signature"] = payload.signature super().__init__(*args, **kwargs)