def test_skip_processing_of_received_personal_channel_torrents(metadata_store):
"""
Test that personal torrent is ignored by default when processing the torrent metadata payload
"""
channel = metadata_store.ChannelMetadata.create_channel('testchan')
torrent_md = metadata_store.TorrentMetadata(origin_id=channel.id_,
title='test',
status=NEW,
infohash=random_infohash())
channel.commit_channel_torrent()
torrent_md.delete()
channel_dir = Path(
metadata_store.ChannelMetadata._channels_dir) / channel.dirname
assert os.listdir(Path.fix_win_long_file(channel_dir))
# By default, personal channel torrent metadata processing is skipped so there should be no torrents
# added to the channel
channel.local_version = 0
metadata_store.process_channel_dir(channel_dir, channel.public_key,
channel.id_)
assert not channel.contents
# Enable processing of personal channel torrent metadata
channel.local_version = 0
metadata_store.process_channel_dir(channel_dir,
channel.public_key,
channel.id_,
skip_personal_metadata_payload=False)
assert len(channel.contents) == 1
def get_base_path():
""" Get absolute path to resource, works for dev and for PyInstaller """
try:
# PyInstaller creates a temp folder and stores path in _MEIPASS
base_path = Path(sys._MEIPASS)
except Exception:
base_path = Path(tribler_core.__file__).parent
fixed_filename = Path.fix_win_long_file(base_path)
return Path(fixed_filename)
def process_mdblob_file(self, filepath, **kwargs):
"""
Process a file with metadata in a channel directory.
:param filepath: The path to the file
:param skip_personal_metadata_payload: if this is set to True, personal torrent metadata payload received
through gossip will be ignored. The default value is True.
:param external_thread: indicate to the lower lever that we're running in the backround thread,
to possibly pace down the upload process
:return: a list of tuples of (<metadata or payload>, <action type>)
"""
path = Path.fix_win_long_file(filepath)
with open(path, 'rb') as f:
serialized_data = f.read()
if path.endswith('.lz4'):
return self.process_compressed_mdblob(serialized_data, **kwargs)
return self.process_squashed_mdblob(serialized_data, **kwargs)
def consolidate_channel_torrent(self):
"""
Delete the channel dir contents and create it anew.
Use it to consolidate fragmented channel torrent directories.
:param key: The public/private key, used to sign the data
"""
# Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms
# Cleanup entries marked for deletion
db.CollectionNode.collapse_deleted_subtrees()
# Note: It should be possible to stop alling get_contents_to_commit here
commit_queue = self.get_contents_to_commit()
for entry in commit_queue:
if entry.status == TODELETE:
entry.delete()
folder = Path(self._channels_dir) / self.dirname
# We check if we need to re-create the channel dir in case it was deleted for some reason
if not folder.is_dir():
os.makedirs(folder)
for filename in os.listdir(folder):
file_path = folder / filename
# We only remove mdblobs and leave the rest as it is
if filename.endswith(BLOB_EXTENSION) or filename.endswith(
BLOB_EXTENSION + '.lz4'):
os.unlink(Path.fix_win_long_file(file_path))
# Channel should get a new starting timestamp and its contents should get higher timestamps
start_timestamp = clock.tick()
def update_timestamps_recursive(node):
if issubclass(type(node), db.CollectionNode):
for child in node.contents:
update_timestamps_recursive(child)
if node.status in [COMMITTED, UPDATED, NEW]:
node.status = UPDATED
node.timestamp = clock.tick()
node.sign()
update_timestamps_recursive(self)
return self.commit_channel_torrent(
new_start_timestamp=start_timestamp)
def update_channel_torrent(self, metadata_list):
"""
Channel torrents are append-only to support seeding the old versions
from the same dir and avoid updating already downloaded blobs.
:param metadata_list: The list of metadata entries to add to the torrent dir.
ACHTUNG: TODELETE entries _MUST_ be sorted to the end of the list to prevent channel corruption!
:return The newly create channel torrent infohash, final timestamp for the channel and torrent date
"""
# As a workaround for delete entries not having a timestamp in the DB, delete entries should
# be placed after create/modify entries:
# | create/modify entries | delete entries | <- final timestamp
# Create dir for the metadata files
channel_dir = Path(self._channels_dir / self.dirname).absolute()
if not channel_dir.is_dir():
os.makedirs(Path.fix_win_long_file(channel_dir))
existing_contents = sorted(channel_dir.iterdir())
last_existing_blob_number = get_mdblob_sequence_number(existing_contents[-1]) if existing_contents else None
index = 0
while index < len(metadata_list):
# Squash several serialized and signed metadata entries into a single file
data, index = entries_to_chunk(metadata_list, self._CHUNK_SIZE_LIMIT, start_index=index)
# Blobs ending with TODELETE entries increase the final timestamp as a workaround for delete commands
# possessing no timestamp.
if metadata_list[index - 1].status == TODELETE:
blob_timestamp = clock.tick()
else:
blob_timestamp = metadata_list[index - 1].timestamp
# The final file in the sequence should get a timestamp that is higher than the timestamp of
# the last channel contents entry. This final timestamp then should be returned to the calling function
# to be assigned to the corresponding channel entry.
# Otherwise, the local channel version will never become equal to its timestamp.
if index >= len(metadata_list):
blob_timestamp = clock.tick()
# Check that the mdblob we're going to create has a greater timestamp than the existing ones
assert last_existing_blob_number is None or (blob_timestamp > last_existing_blob_number)
blob_filename = Path(channel_dir, str(blob_timestamp).zfill(12) + BLOB_EXTENSION + '.lz4')
assert not blob_filename.exists() # Never ever write over existing files.
blob_filename.write_bytes(data)
last_existing_blob_number = blob_timestamp
with db_session:
thumb_exists = db.ChannelThumbnail.exists(
lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE
)
descr_exists = db.ChannelDescription.exists(
lambda g: g.public_key == self.public_key and g.origin_id == self.id_ and g.status != TODELETE
)
flags = CHANNEL_THUMBNAIL_FLAG * (int(thumb_exists)) + CHANNEL_DESCRIPTION_FLAG * (int(descr_exists))
# Note: the timestamp can end up messed in case of an error
# Make torrent out of dir with metadata files
torrent, infohash = create_torrent_from_dir(channel_dir, self._channels_dir / (self.dirname + ".torrent"))
torrent_date = datetime.utcfromtimestamp(torrent[b'creation date'])
return {
"infohash": infohash,
"timestamp": last_existing_blob_number,
"torrent_date": torrent_date,
"reserved_flags": flags,
}, torrent
def write(self, filename):
self.config.filename = Path.fix_win_long_file(filename)
self.config.write()
def load(config_path=None):
return DownloadConfig(
ConfigObj(infile=Path.fix_win_long_file(config_path),
file_error=True,
configspec=str(CONFIG_SPEC_PATH),
default_encoding='utf-8'))
def to_delete_file(self, filename):
with open(Path.fix_win_long_file(filename), 'wb') as output_file:
output_file.write(self.serialized_delete())
def to_file(self, filename, key=None):
with open(Path.fix_win_long_file(filename), 'wb') as output_file:
output_file.write(self.serialized(key))