def metadata_store(tmp_path):
mds = MetadataStore(db_filename=tmp_path / 'test.db',
channels_dir=tmp_path / 'channels',
my_key=TEST_PERSONAL_KEY,
disable_sync=True)
yield mds
mds.shutdown()
def test_upgrade_pony_8to10(upgrader, channels_dir, mds_path,
trustchain_keypair): # pylint: disable=W0621
_copy('pony_v8.db', mds_path)
upgrader.upgrade_pony_db_8to10()
mds = MetadataStore(mds_path,
channels_dir,
trustchain_keypair,
check_tables=False,
db_version=10)
with db_session:
assert mds.get_value("db_version") == '10'
assert mds.ChannelNode.select().count() == 23
mds.shutdown()
async def test_upgrade_pony_10to11(upgrader, channels_dir, mds_path,
trustchain_keypair):
_copy('pony_v10.db', mds_path)
upgrader.upgrade_pony_db_10to11()
mds = MetadataStore(mds_path,
channels_dir,
trustchain_keypair,
check_tables=False,
db_version=11)
with db_session:
# pylint: disable=protected-access
assert upgrader.column_exists_in_table(mds._db, 'TorrentState',
'self_checked')
assert mds.get_value("db_version") == '11'
mds.shutdown()
def upgrade_pony_db_12to13(self):
"""
Upgrade GigaChannel DB from version 12 (7.9.x) to version 13 (7.11.x).
Version 12 adds index for TorrentState.last_check attribute.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
if database_path.exists():
mds = MetadataStore(database_path,
self.channels_dir,
self.trustchain_keypair,
disable_sync=True,
check_tables=False,
db_version=12)
self.do_upgrade_pony_db_12to13(mds)
mds.shutdown()
def do_recreate_indexes(self, mds: MetadataStore, base_duration):
index_total = None
index_num = 0
t0 = t1 = now()
# SQLite callback handler to update progress bar during index creation
def index_callback_handler():
try:
t2 = now()
index_percentage = calc_progress(t2 - t1, base_duration / 8.0)
total_percentage = (index_num * 100.0 + index_percentage) / index_total
self.notification_callback(f"recreating indexes\n"
f"{total_percentage:.2f}% done")
except Exception as e:
self._logger.error(f"Error in SQLite callback handler: {type(e).__name__}:{str(e)}")
self.shutting_down = True
# Recreate table indexes
with db_session(ddl=True):
connection = mds._db.get_connection()
try:
db_objects = mds.get_objects_to_create()
index_total = len(db_objects)
for i, obj in enumerate(db_objects):
index_num = i
t1 = now()
connection.set_progress_handler(index_callback_handler, 5000)
obj.create(mds._db.schema.provider, connection)
duration = now() - t1
self._logger.info(f"Upgrade: created {obj.name} in {duration:.2f} seconds")
finally:
connection.set_progress_handler(None, 0)
duration = now() - t0
self._logger.info(f'Recreated all indexes in {duration:.2f} seconds')
t1 = now()
# SQLite callback handler to update progress bar during FTS index creation
def fts_callback_handler():
try:
t2 = now()
self.notification_callback("adding full text search index...\n"
f"{calc_progress(t2 - t1, base_duration):.2f}% done")
except Exception as e:
self._logger.error(f"Error in SQLite callback handler: {type(e).__name__}:{str(e)}")
self.shutting_down = True
# Create FTS index
with db_session(ddl=True):
mds.create_fts_triggers()
connection = mds._db.get_connection()
connection.set_progress_handler(fts_callback_handler, 5000)
try:
t = now()
mds.fill_fts_index()
duration = now() - t
self._logger.info(f'Upgrade: fill FTS in {duration:.2f} seconds')
finally:
connection.set_progress_handler(None, 0)
mds.shutdown()
def upgrade_pony_db_10to11(self):
"""
Upgrade GigaChannel DB from version 10 (7.6.x) to version 11 (7.7.x).
Version 11 adds a `self_checked` field to TorrentState table if it
already does not exist.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
if not database_path.exists():
return
mds = MetadataStore(database_path,
self.channels_dir,
self.trustchain_keypair,
disable_sync=True,
check_tables=False,
db_version=10)
self.do_upgrade_pony_db_10to11(mds)
mds.shutdown()
def upgrade_pony_db_11to12(self):
"""
Upgrade GigaChannel DB from version 11 (7.8.x) to version 12 (7.9.x).
Version 12 adds a `json_text`, `binary_data` and `data_type` fields
to TorrentState table if it already does not exist.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
if not database_path.exists():
return
mds = MetadataStore(database_path,
self.channels_dir,
self.trustchain_keypair,
disable_sync=True,
check_tables=False,
db_version=11)
self.do_upgrade_pony_db_11to12(mds)
mds.shutdown()
def upgrade_pony_db_13to14(self):
mds_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
tagdb_path = self.state_dir / STATEDIR_DB_DIR / 'tags.db'
mds = MetadataStore(mds_path,
self.channels_dir,
self.trustchain_keypair,
disable_sync=True,
check_tables=False,
db_version=13) if mds_path.exists() else None
tag_db = TagDatabase(
str(tagdb_path), create_tables=False,
check_tables=False) if tagdb_path.exists() else None
self.do_upgrade_pony_db_13to14(mds, tag_db)
if mds:
mds.shutdown()
if tag_db:
tag_db.shutdown()
def upgrade_pony_db_8to10(self):
"""
Upgrade GigaChannel DB from version 8 (7.5.x) to version 10 (7.6.x).
This will recreate the database anew, which can take quite some time.
"""
database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
if not database_path.exists() or get_db_version(database_path) >= 10:
# Either no old db exists, or the old db version is up to date - nothing to do
return
# Otherwise, start upgrading
self.update_status("STARTING")
tmp_database_path = database_path.parent / 'metadata_upgraded.db'
# Clean the previous temp database
tmp_database_path.unlink(missing_ok=True)
# Create the new database
mds = MetadataStore(tmp_database_path,
None,
self.trustchain_keypair,
disable_sync=True,
db_version=10)
with db_session(ddl=True):
mds.drop_indexes()
mds.drop_fts_triggers()
mds.shutdown()
self._pony2pony = PonyToPonyMigration(
database_path,
tmp_database_path,
self.update_status,
logger=self._logger,
shutdown_set_callback=self.interrupt_upgrade_event)
duration_base = self._pony2pony.do_migration()
self._pony2pony.recreate_indexes(mds, duration_base)
# Remove the old DB
database_path.unlink(missing_ok=True)
if not self._pony2pony.shutting_down:
# Move the upgraded db in its place
tmp_database_path.rename(database_path)
else:
# The upgrade process was either skipped or interrupted. Delete the temp upgrade DB.
tmp_database_path.unlink(missing_ok=True)
self.update_status("FINISHED")
title='internal collection')
mds.TorrentMetadata.from_dict(
dict(origin_id=coll.id_, **gen_random_entry()))
mds.TorrentMetadata.from_dict(
dict(origin_id=coll.id_, **gen_random_entry()))
my_channel.commit_channel_torrent()
t2.soft_delete()
my_channel.commit_channel_torrent()
# Rename files to stable names
mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob")
torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent")
os.rename(mdblob_name, CHANNEL_METADATA)
os.rename(torrent_name, CHANNEL_TORRENT)
# Update channel
mds.TorrentMetadata.from_dict(
dict(origin_id=my_channel.id_, **gen_random_entry()))
my_channel.commit_channel_torrent()
# Rename updated files to stable names
os.rename(mdblob_name, CHANNEL_METADATA_UPDATED)
os.rename(torrent_name, CHANNEL_TORRENT_UPDATED)
if __name__ == "__main__":
mds = MetadataStore(MEMORY_DB, SAMPLE_DIR, my_key)
gen_sample_channel(mds)