def metadata_store(tmp_path): mds = MetadataStore(db_filename=tmp_path / 'test.db', channels_dir=tmp_path / 'channels', my_key=TEST_PERSONAL_KEY, disable_sync=True) yield mds mds.shutdown()
def test_upgrade_pony_8to10(upgrader, channels_dir, mds_path, trustchain_keypair): # pylint: disable=W0621 _copy('pony_v8.db', mds_path) upgrader.upgrade_pony_db_8to10() mds = MetadataStore(mds_path, channels_dir, trustchain_keypair, check_tables=False, db_version=10) with db_session: assert mds.get_value("db_version") == '10' assert mds.ChannelNode.select().count() == 23 mds.shutdown()
async def test_upgrade_pony_10to11(upgrader, channels_dir, mds_path, trustchain_keypair): _copy('pony_v10.db', mds_path) upgrader.upgrade_pony_db_10to11() mds = MetadataStore(mds_path, channels_dir, trustchain_keypair, check_tables=False, db_version=11) with db_session: # pylint: disable=protected-access assert upgrader.column_exists_in_table(mds._db, 'TorrentState', 'self_checked') assert mds.get_value("db_version") == '11' mds.shutdown()
def upgrade_pony_db_12to13(self): """ Upgrade GigaChannel DB from version 12 (7.9.x) to version 13 (7.11.x). Version 12 adds index for TorrentState.last_check attribute. """ # We have to create the Metadata Store object because Session-managed Store has not been started yet database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db' if database_path.exists(): mds = MetadataStore(database_path, self.channels_dir, self.trustchain_keypair, disable_sync=True, check_tables=False, db_version=12) self.do_upgrade_pony_db_12to13(mds) mds.shutdown()
def do_recreate_indexes(self, mds: MetadataStore, base_duration): index_total = None index_num = 0 t0 = t1 = now() # SQLite callback handler to update progress bar during index creation def index_callback_handler(): try: t2 = now() index_percentage = calc_progress(t2 - t1, base_duration / 8.0) total_percentage = (index_num * 100.0 + index_percentage) / index_total self.notification_callback(f"recreating indexes\n" f"{total_percentage:.2f}% done") except Exception as e: self._logger.error(f"Error in SQLite callback handler: {type(e).__name__}:{str(e)}") self.shutting_down = True # Recreate table indexes with db_session(ddl=True): connection = mds._db.get_connection() try: db_objects = mds.get_objects_to_create() index_total = len(db_objects) for i, obj in enumerate(db_objects): index_num = i t1 = now() connection.set_progress_handler(index_callback_handler, 5000) obj.create(mds._db.schema.provider, connection) duration = now() - t1 self._logger.info(f"Upgrade: created {obj.name} in {duration:.2f} seconds") finally: connection.set_progress_handler(None, 0) duration = now() - t0 self._logger.info(f'Recreated all indexes in {duration:.2f} seconds') t1 = now() # SQLite callback handler to update progress bar during FTS index creation def fts_callback_handler(): try: t2 = now() self.notification_callback("adding full text search index...\n" f"{calc_progress(t2 - t1, base_duration):.2f}% done") except Exception as e: self._logger.error(f"Error in SQLite callback handler: {type(e).__name__}:{str(e)}") self.shutting_down = True # Create FTS index with db_session(ddl=True): mds.create_fts_triggers() connection = mds._db.get_connection() connection.set_progress_handler(fts_callback_handler, 5000) try: t = now() mds.fill_fts_index() duration = now() - t self._logger.info(f'Upgrade: fill FTS in {duration:.2f} seconds') finally: connection.set_progress_handler(None, 0) mds.shutdown()
def upgrade_pony_db_10to11(self): """ Upgrade GigaChannel DB from version 10 (7.6.x) to version 11 (7.7.x). Version 11 adds a `self_checked` field to TorrentState table if it already does not exist. """ # We have to create the Metadata Store object because Session-managed Store has not been started yet database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db' if not database_path.exists(): return mds = MetadataStore(database_path, self.channels_dir, self.trustchain_keypair, disable_sync=True, check_tables=False, db_version=10) self.do_upgrade_pony_db_10to11(mds) mds.shutdown()
def upgrade_pony_db_11to12(self): """ Upgrade GigaChannel DB from version 11 (7.8.x) to version 12 (7.9.x). Version 12 adds a `json_text`, `binary_data` and `data_type` fields to TorrentState table if it already does not exist. """ # We have to create the Metadata Store object because Session-managed Store has not been started yet database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db' if not database_path.exists(): return mds = MetadataStore(database_path, self.channels_dir, self.trustchain_keypair, disable_sync=True, check_tables=False, db_version=11) self.do_upgrade_pony_db_11to12(mds) mds.shutdown()
def upgrade_pony_db_13to14(self): mds_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db' tagdb_path = self.state_dir / STATEDIR_DB_DIR / 'tags.db' mds = MetadataStore(mds_path, self.channels_dir, self.trustchain_keypair, disable_sync=True, check_tables=False, db_version=13) if mds_path.exists() else None tag_db = TagDatabase( str(tagdb_path), create_tables=False, check_tables=False) if tagdb_path.exists() else None self.do_upgrade_pony_db_13to14(mds, tag_db) if mds: mds.shutdown() if tag_db: tag_db.shutdown()
def upgrade_pony_db_8to10(self): """ Upgrade GigaChannel DB from version 8 (7.5.x) to version 10 (7.6.x). This will recreate the database anew, which can take quite some time. """ database_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db' if not database_path.exists() or get_db_version(database_path) >= 10: # Either no old db exists, or the old db version is up to date - nothing to do return # Otherwise, start upgrading self.update_status("STARTING") tmp_database_path = database_path.parent / 'metadata_upgraded.db' # Clean the previous temp database tmp_database_path.unlink(missing_ok=True) # Create the new database mds = MetadataStore(tmp_database_path, None, self.trustchain_keypair, disable_sync=True, db_version=10) with db_session(ddl=True): mds.drop_indexes() mds.drop_fts_triggers() mds.shutdown() self._pony2pony = PonyToPonyMigration( database_path, tmp_database_path, self.update_status, logger=self._logger, shutdown_set_callback=self.interrupt_upgrade_event) duration_base = self._pony2pony.do_migration() self._pony2pony.recreate_indexes(mds, duration_base) # Remove the old DB database_path.unlink(missing_ok=True) if not self._pony2pony.shutting_down: # Move the upgraded db in its place tmp_database_path.rename(database_path) else: # The upgrade process was either skipped or interrupted. Delete the temp upgrade DB. tmp_database_path.unlink(missing_ok=True) self.update_status("FINISHED")
title='internal collection') mds.TorrentMetadata.from_dict( dict(origin_id=coll.id_, **gen_random_entry())) mds.TorrentMetadata.from_dict( dict(origin_id=coll.id_, **gen_random_entry())) my_channel.commit_channel_torrent() t2.soft_delete() my_channel.commit_channel_torrent() # Rename files to stable names mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob") torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent") os.rename(mdblob_name, CHANNEL_METADATA) os.rename(torrent_name, CHANNEL_TORRENT) # Update channel mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) my_channel.commit_channel_torrent() # Rename updated files to stable names os.rename(mdblob_name, CHANNEL_METADATA_UPDATED) os.rename(torrent_name, CHANNEL_TORRENT_UPDATED) if __name__ == "__main__": mds = MetadataStore(MEMORY_DB, SAMPLE_DIR, my_key) gen_sample_channel(mds)