def test_delete_noncompliant_state(self):
STATE_DIR = TESTS_DATA_DIR / 'noncompliant_state_dir'
tmpdir = self.temporary_directory() / STATE_DIR.name
shutil.copytree(str_path(STATE_DIR), str_path(tmpdir))
cleanup_noncompliant_channel_torrents(tmpdir)
# Check cleanup of the channels dir
dir_listing = list((tmpdir / "channels").iterdir())
self.assertEqual(3, len(dir_listing))
for f in (tmpdir / "channels").iterdir():
self.assertEqual(CHANNEL_DIR_NAME_LENGTH, len(f.stem))
# Check cleanup of torrent state dir
checkpoints_dir = tmpdir / "dlcheckpoints"
dir_listing = os.listdir(checkpoints_dir)
self.assertEqual(1, len(dir_listing))
file_path = checkpoints_dir / dir_listing[0]
pstate = CallbackConfigParser()
pstate.read_file(file_path)
self.assertEqual(CHANNEL_DIR_NAME_LENGTH,
len(pstate.get('state', 'metainfo')['info']['name']))
def process_mdblob_file(self, filepath, **kwargs):
"""
Process a file with metadata in a channel directory.
:param filepath: The path to the file
:param skip_personal_metadata_payload: if this is set to True, personal torrent metadata payload received
through gossip will be ignored. The default value is True.
:param external_thread: indicate to the lower lever that we're running in the backround thread,
to possibly pace down the upload process
:return: a list of tuples of (<metadata or payload>, <action type>)
"""
with open(str_path(filepath), 'rb') as f:
serialized_data = f.read()
if str(filepath).endswith('.lz4'):
return self.process_compressed_mdblob(serialized_data, **kwargs)
return self.process_squashed_mdblob(serialized_data, **kwargs)
def consolidate_channel_torrent(self):
"""
Delete the channel dir contents and create it anew.
Use it to consolidate fragmented channel torrent directories.
:param key: The public/private key, used to sign the data
"""
# Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms
# Cleanup entries marked for deletion
db.CollectionNode.collapse_deleted_subtrees()
# Note: It should be possible to stop alling get_contents_to_commit here
commit_queue = self.get_contents_to_commit()
for entry in commit_queue:
if entry.status == TODELETE:
entry.delete()
folder = Path(self._channels_dir) / self.dirname
# We check if we need to re-create the channel dir in case it was deleted for some reason
if not folder.is_dir():
os.makedirs(folder)
for filename in os.listdir(folder):
file_path = folder / filename
# We only remove mdblobs and leave the rest as it is
if filename.endswith(BLOB_EXTENSION) or filename.endswith(
BLOB_EXTENSION + '.lz4'):
os.unlink(str_path(file_path))
# Channel should get a new starting timestamp and its contents should get higher timestamps
start_timestamp = clock.tick()
def update_timestamps_recursive(node):
if issubclass(type(node), db.CollectionNode):
for child in node.contents:
update_timestamps_recursive(child)
if node.status in [COMMITTED, UPDATED, NEW]:
node.status = UPDATED
node.timestamp = clock.tick()
node.sign()
update_timestamps_recursive(self)
return self.commit_channel_torrent(
new_start_timestamp=start_timestamp)
def update_channel_torrent(self, metadata_list):
"""
Channel torrents are append-only to support seeding the old versions
from the same dir and avoid updating already downloaded blobs.
:param metadata_list: The list of metadata entries to add to the torrent dir.
ACHTUNG: TODELETE entries _MUST_ be sorted to the end of the list to prevent channel corruption!
:return The newly create channel torrent infohash, final timestamp for the channel and torrent date
"""
# As a workaround for delete entries not having a timestamp in the DB, delete entries should
# be placed after create/modify entries:
# | create/modify entries | delete entries | <- final timestamp
# Create dir for the metadata files
channel_dir = path_util.abspath(self._channels_dir / self.dirname)
if not channel_dir.is_dir():
os.makedirs(str_path(channel_dir))
existing_contents = sorted(channel_dir.iterdir())
last_existing_blob_number = get_mdblob_sequence_number(
existing_contents[-1]) if existing_contents else None
index = 0
while index < len(metadata_list):
# Squash several serialized and signed metadata entries into a single file
data, index = entries_to_chunk(metadata_list,
self._CHUNK_SIZE_LIMIT,
start_index=index)
# Blobs ending with TODELETE entries increase the final timestamp as a workaround for delete commands
# possessing no timestamp.
if metadata_list[index - 1].status == TODELETE:
blob_timestamp = clock.tick()
else:
blob_timestamp = metadata_list[index - 1].timestamp
# The final file in the sequence should get a timestamp that is higher than the timestamp of
# the last channel contents entry. This final timestamp then should be returned to the calling function
# to be assigned to the corresponding channel entry.
# Otherwise, the local channel version will never become equal to its timestamp.
if index >= len(metadata_list):
blob_timestamp = clock.tick()
# Check that the mdblob we're going to create has a greater timestamp than the existing ones
assert last_existing_blob_number is None or (
blob_timestamp > last_existing_blob_number)
blob_filename = Path(
channel_dir,
str(blob_timestamp).zfill(12) + BLOB_EXTENSION + '.lz4')
assert not blob_filename.exists(
) # Never ever write over existing files.
blob_filename.write_bytes(data)
last_existing_blob_number = blob_timestamp
with db_session:
thumb_exists = db.ChannelThumbnail.exists(
lambda g: g.public_key == self.public_key and g.origin_id
== self.id_ and g.status != TODELETE)
descr_exists = db.ChannelDescription.exists(
lambda g: g.public_key == self.public_key and g.origin_id
== self.id_ and g.status != TODELETE)
flags = CHANNEL_THUMBNAIL_FLAG * (
int(thumb_exists)) + CHANNEL_DESCRIPTION_FLAG * (
int(descr_exists))
# Note: the timestamp can end up messed in case of an error
# Make torrent out of dir with metadata files
torrent, infohash = create_torrent_from_dir(
channel_dir, self._channels_dir / (self.dirname + ".torrent"))
torrent_date = datetime.utcfromtimestamp(torrent[b'creation date'])
return {
"infohash": infohash,
"timestamp": last_existing_blob_number,
"torrent_date": torrent_date,
"reserved_flags": flags,
}, torrent
def write(self, filename):
self.config.filename = str_path(filename)
self.config.write()
def load(config_path=None):
return DownloadConfig(
ConfigObj(infile=str_path(config_path),
file_error=True,
configspec=str(CONFIG_SPEC_PATH),
default_encoding='utf-8'))
def to_delete_file(self, filename):
with open(str_path(filename), 'wb') as output_file:
output_file.write(self.serialized_delete())
def to_file(self, filename, key=None):
with open(str_path(filename), 'wb') as output_file:
output_file.write(self.serialized(key))
async def convert_personal_channel(self):
with db_session:
# Reflect conversion state
v = self.mds.MiscData.get_for_update(
name=CONVERSION_FROM_72_PERSONAL)
if v:
if v.value == CONVERSION_STARTED:
# Just drop the entries from the previous try
my_channels = self.mds.ChannelMetadata.get_my_channels()
for my_channel in my_channels:
my_channel.contents.delete(bulk=True)
my_channel.delete()
else:
# Something is wrong, this should never happen
raise Exception(
"Previous conversion resulted in invalid state")
else:
self.mds.MiscData(name=CONVERSION_FROM_72_PERSONAL,
value=CONVERSION_STARTED)
my_channels_count = self.mds.ChannelMetadata.get_my_channels(
).count()
# Make sure every precondition is met
if self.personal_channel_id and not my_channels_count:
total_to_convert = self.get_personal_channel_torrents_count()
with db_session:
my_channel = self.mds.ChannelMetadata.create_channel(
title=self.personal_channel_title, description='')
def get_old_stuff(batch_size, offset):
return self.get_old_torrents(personal_channel_only=True,
sign=True,
batch_size=batch_size,
offset=offset)
def add_to_pony(t):
return self.mds.TorrentMetadata(origin_id=my_channel.id_, **t)
await self.convert_async(
add_to_pony,
get_old_stuff,
total_to_convert,
offset=0,
message="Converting personal channel torrents.")
with db_session:
my_channel = self.mds.ChannelMetadata.get_my_channels().first()
folder = Path(my_channel._channels_dir) / my_channel.dirname
# We check if we need to re-create the channel dir in case it was deleted for some reason
if not folder.is_dir():
os.makedirs(str_path(folder))
for filename in os.listdir(folder):
file_path = folder / filename
# We only remove mdblobs and leave the rest as it is
if filename.endswith(BLOB_EXTENSION) or filename.endswith(
BLOB_EXTENSION + '.lz4'):
os.unlink(str_path(file_path))
my_channel.commit_channel_torrent()
with db_session:
v = self.mds.MiscData.get_for_update(
name=CONVERSION_FROM_72_PERSONAL)
v.value = CONVERSION_FINISHED
