def prepare_commit_queue_for_channel(commit_queue):
"""
This routine prepares the raw commit queue for commit by updating the elements' properties and
re-signing them. Also, it removes the channel entry itself from the queue [:-1], because its
meaningless to put it in the blobs, as it must be updated with the new infohash after commit.
:param commit_queue:
:return:
"""
for node in commit_queue:
# Avoid updating entries that must be deleted:
# soft delete payloads require signatures of unmodified entries
if issubclass(type(node), db.CollectionNode) and node.status != TODELETE:
# Update recursive count of actual non-collection contents
node.num_entries = select(
# For each subnode, if it is a collection, add the count of its contents to the recursive sum.
# Otherwise, add just 1 to the sum (to count the subnode itself).
(g.num_entries if g.metadata_type == COLLECTION_NODE else 1)
for g in node.actual_contents
).sum()
node.timestamp = clock.tick()
node.sign()
# This perverted comparator lambda is necessary to ensure that delete entries are always
# sorted to the end of the list, as required by the channel serialization routine.
return sorted(commit_queue[:-1], key=lambda x: int(x.status == TODELETE) - 1 / x.timestamp)
def add_torrent_to_channel(self, tdef, extra_info=None):
"""
Add a torrent to your channel.
:param tdef: The torrent definition file of the torrent to add
:param extra_info: Optional extra info to add to the torrent
"""
new_entry_dict = dict(tdef_to_metadata_dict(tdef), status=NEW)
if extra_info:
new_entry_dict['tags'] = extra_info.get('description', '')
# See if the torrent is already in the channel
old_torrent = db.TorrentMetadata.get(public_key=self.public_key, infohash=tdef.get_infohash())
if old_torrent:
# If it is there, check if we were going to delete it
if old_torrent.status == TODELETE:
new_timestamp = clock.tick()
old_torrent.set(timestamp=new_timestamp, origin_id=self.id_, **new_entry_dict)
old_torrent.sign()
# As we really don't know what status this torrent had _before_ it got its TODELETE status,
# we _must_ set its status to UPDATED, for safety
old_torrent.status = UPDATED
torrent_metadata = old_torrent
else:
raise DuplicateTorrentFileError()
else:
torrent_metadata = db.TorrentMetadata.from_dict(dict(origin_id=self.id_, **new_entry_dict))
return torrent_metadata
def update_timestamps_recursive(node):
if issubclass(type(node), db.CollectionNode):
for child in node.contents:
update_timestamps_recursive(child)
if node.status in [COMMITTED, UPDATED, NEW]:
node.status = UPDATED
node.timestamp = clock.tick()
node.sign()
def update_properties(self, update_dict):
signed_attribute_changed = False
for k, value in update_dict.items():
if getattr(self, k) != value:
setattr(self, k, value)
signed_attribute_changed = signed_attribute_changed or (k in self.payload_arguments)
if signed_attribute_changed:
if self.status != NEW:
self.status = UPDATED
self.timestamp = clock.tick()
self.sign()
return self
def update_properties(self, update_dict):
signed_attribute_changed = False
for k, value in update_dict.items():
if getattr(self, k) != value:
setattr(self, k, value)
signed_attribute_changed = signed_attribute_changed or (k in self.payload_arguments)
if signed_attribute_changed:
if self.status != NEW:
self.status = UPDATED
# ACHTUNG! When using the key argument, the thing will still use _local_ timestamp counter!
self.timestamp = clock.tick()
self.sign()
return self
def consolidate_channel_torrent(self):
"""
Delete the channel dir contents and create it anew.
Use it to consolidate fragmented channel torrent directories.
:param key: The public/private key, used to sign the data
"""
# Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms
# Cleanup entries marked for deletion
db.CollectionNode.collapse_deleted_subtrees()
# Note: It should be possible to stop alling get_contents_to_commit here
commit_queue = self.get_contents_to_commit()
for entry in commit_queue:
if entry.status == TODELETE:
entry.delete()
folder = Path(self._channels_dir) / self.dirname
# We check if we need to re-create the channel dir in case it was deleted for some reason
if not folder.is_dir():
os.makedirs(folder)
for filename in os.listdir(folder):
file_path = folder / filename
# We only remove mdblobs and leave the rest as it is
if filename.endswith(BLOB_EXTENSION) or filename.endswith(
BLOB_EXTENSION + '.lz4'):
os.unlink(str_path(file_path))
# Channel should get a new starting timestamp and its contents should get higher timestamps
start_timestamp = clock.tick()
def update_timestamps_recursive(node):
if issubclass(type(node), db.CollectionNode):
for child in node.contents:
update_timestamps_recursive(child)
if node.status in [COMMITTED, UPDATED, NEW]:
node.status = UPDATED
node.timestamp = clock.tick()
node.sign()
update_timestamps_recursive(self)
return self.commit_channel_torrent(
new_start_timestamp=start_timestamp)
def update_channel_torrent(self, metadata_list):
"""
Channel torrents are append-only to support seeding the old versions
from the same dir and avoid updating already downloaded blobs.
:param metadata_list: The list of metadata entries to add to the torrent dir.
ACHTUNG: TODELETE entries _MUST_ be sorted to the end of the list to prevent channel corruption!
:return The newly create channel torrent infohash, final timestamp for the channel and torrent date
"""
# As a workaround for delete entries not having a timestamp in the DB, delete entries should
# be placed after create/modify entries:
# | create/modify entries | delete entries | <- final timestamp
# Create dir for the metadata files
channel_dir = path_util.abspath(self._channels_dir / self.dirname)
if not channel_dir.is_dir():
os.makedirs(str_path(channel_dir))
existing_contents = sorted(channel_dir.iterdir())
last_existing_blob_number = get_mdblob_sequence_number(
existing_contents[-1]) if existing_contents else None
index = 0
while index < len(metadata_list):
# Squash several serialized and signed metadata entries into a single file
data, index = entries_to_chunk(metadata_list,
self._CHUNK_SIZE_LIMIT,
start_index=index)
# Blobs ending with TODELETE entries increase the final timestamp as a workaround for delete commands
# possessing no timestamp.
if metadata_list[index - 1].status == TODELETE:
blob_timestamp = clock.tick()
else:
blob_timestamp = metadata_list[index - 1].timestamp
# The final file in the sequence should get a timestamp that is higher than the timestamp of
# the last channel contents entry. This final timestamp then should be returned to the calling function
# to be assigned to the corresponding channel entry.
# Otherwise, the local channel version will never become equal to its timestamp.
if index >= len(metadata_list):
blob_timestamp = clock.tick()
# Check that the mdblob we're going to create has a greater timestamp than the existing ones
assert last_existing_blob_number is None or (
blob_timestamp > last_existing_blob_number)
blob_filename = Path(
channel_dir,
str(blob_timestamp).zfill(12) + BLOB_EXTENSION + '.lz4')
assert not blob_filename.exists(
) # Never ever write over existing files.
blob_filename.write_bytes(data)
last_existing_blob_number = blob_timestamp
with db_session:
thumb_exists = db.ChannelThumbnail.exists(
lambda g: g.public_key == self.public_key and g.origin_id
== self.id_ and g.status != TODELETE)
descr_exists = db.ChannelDescription.exists(
lambda g: g.public_key == self.public_key and g.origin_id
== self.id_ and g.status != TODELETE)
flags = CHANNEL_THUMBNAIL_FLAG * (
int(thumb_exists)) + CHANNEL_DESCRIPTION_FLAG * (
int(descr_exists))
# Note: the timestamp can end up messed in case of an error
# Make torrent out of dir with metadata files
torrent, infohash = create_torrent_from_dir(
channel_dir, self._channels_dir / (self.dirname + ".torrent"))
torrent_date = datetime.utcfromtimestamp(torrent[b'creation date'])
return {
"infohash": infohash,
"timestamp": last_existing_blob_number,
"torrent_date": torrent_date,
"reserved_flags": flags,
}, torrent
def __init__(self, *args, **kwargs):
"""
Initialize a metadata object.
All this dance is required to ensure that the signature is there and it is correct.
"""
skip_key_check = False
# FIXME: refactor this method by moving different ways to create an entry into separate methods
# Process special keyworded arguments
# "sign_with" argument given, sign with it
private_key_override = None
if "sign_with" in kwargs:
kwargs["public_key"] = database_blob(
kwargs["sign_with"].pub().key_to_bin()[10:])
private_key_override = kwargs.pop("sign_with")
# Free-for-all entries require special treatment
if "public_key" in kwargs and kwargs["public_key"] == b"":
# We have to give the entry an unique sig to honor the DB constraints. We use the entry's id_
# as the sig to keep it unique and short. The uniqueness is guaranteed by DB as it already
# imposes uniqueness constraints on the id_+public_key combination.
if "id_" in kwargs:
kwargs["signature"] = None
skip_key_check = True
else:
# Trying to create an FFA entry without specifying the id_ should be considered an error,
# because assigning id_ automatically by clock breaks anonymity.
# FFA entries should be "timeless" and anonymous.
raise InvalidChannelNodeException(
"Attempted to create %s free-for-all (unsigned) object without specifying id_ : "
% str(self.__class__.__name__))
# For putting legacy/test stuff in
skip_key_check = kwargs.pop("skip_key_check", skip_key_check)
if "timestamp" not in kwargs:
kwargs["timestamp"] = clock.tick()
if "id_" not in kwargs:
kwargs["id_"] = int(random.getrandbits(63))
if not private_key_override and not skip_key_check:
# No key/signature given, sign with our own key.
if ("signature" not in kwargs) and (
("public_key" not in kwargs) or
(kwargs["public_key"] == database_blob(
self._my_key.pub().key_to_bin()[10:]))):
private_key_override = self._my_key
# Key/signature given, check them for correctness
elif ("public_key" in kwargs) and ("signature" in kwargs):
try:
self._payload_class(**kwargs)
except InvalidSignatureException:
raise InvalidSignatureException(
f"Attempted to create {str(self.__class__.__name__)} object with invalid signature/PK: "
+ (hexlify(kwargs["signature"]) if "signature" in
kwargs else "empty signature ") + " / " +
(hexlify(kwargs["public_key"]) if "public_key" in
kwargs else " empty PK"))
if private_key_override:
# Get default values for Pony class attributes. We have to do it manually because we need
# to know the payload signature *before* creating the object.
kwargs = generate_dict_from_pony_args(
self.__class__,
skip_list=["signature", "public_key"],
**kwargs)
payload = self._payload_class(**dict(
kwargs,
public_key=private_key_override.pub().key_to_bin()[10:],
key=private_key_override,
metadata_type=self.metadata_type,
))
kwargs["public_key"] = payload.public_key
kwargs["signature"] = payload.signature
super().__init__(*args, **kwargs)
