def _InitDB(self): create_db = False db_path = os.path.join(self._db_dir, self._db_filenames['main']) if not os.path.exists(db_path): create_db = True external_db_paths = [ os.path.join(self._db_dir, self._db_filenames[db_name]) for db_name in self._db_filenames if db_name != 'main' ] existing_external_db_paths = [ external_db_path for external_db_path in external_db_paths if os.path.exists(external_db_path) ] if len(existing_external_db_paths) > 0: message = 'Although the external files, "{}" do exist, the main database file, "{}", does not! This makes for an invalid database, and the program will now quit. Please contact hydrus_dev if you do not know how this happened or need help recovering from hard drive failure.' message = message.format(', '.join(existing_external_db_paths), db_path) raise HydrusExceptions.DBAccessException(message) self._InitDBCursor() result = self._c.execute( 'SELECT 1 FROM sqlite_master WHERE type = ? AND name = ?;', ('table', 'version')).fetchone() if result is None: create_db = True if create_db: self._is_first_start = True self._CreateDB() self._Commit() self._BeginImmediate()
def _AttachExternalDatabases(self): for (name, filename) in self._db_filenames.items(): if name == 'main': continue db_path = os.path.join(self._db_dir, filename) if os.path.exists( db_path) and not HydrusPaths.FileisWriteable(db_path): raise HydrusExceptions.DBAccessException( '"{}" seems to be read-only!'.format(db_path)) self._Execute('ATTACH ? AS ' + name + ';', (db_path, )) db_path = os.path.join(self._db_dir, self._durable_temp_db_filename) self._Execute('ATTACH ? AS durable_temp;', (db_path, ))
def _InitDBCursor( self ): self._CloseDBCursor() db_path = os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) db_just_created = not os.path.exists( db_path ) self._db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES ) self._connection_timestamp = HydrusData.GetNow() self._c = self._db.cursor() if HG.no_db_temp_files: self._c.execute( 'PRAGMA temp_store = 2;' ) # use memory for temp store exclusively self._c.execute( 'ATTACH ":memory:" AS mem;' ) self._AttachExternalDatabases() # if this is set to 1, transactions are not immediately synced to the journal so multiple can be undone following a power-loss # if set to 2, all transactions are synced, so once a new one starts you know the last one is on disk # corruption cannot occur either way, but since we have multiple ATTACH dbs with diff journals, let's not mess around when power-cut during heavy file import or w/e synchronous = 2 if HG.db_synchronous_override is not None: synchronous = HG.db_synchronous_override # durable_temp is not excluded here db_names = [ name for ( index, name, path ) in self._c.execute( 'PRAGMA database_list;' ) if name not in ( 'mem', 'temp' ) ] for db_name in db_names: self._c.execute( 'PRAGMA {}.cache_size = -10000;'.format( db_name ) ) if HG.db_memory_journaling: self._c.execute( 'PRAGMA {}.journal_mode = MEMORY;'.format( db_name ) ) elif HG.no_wal: self._c.execute( 'PRAGMA {}.journal_mode = TRUNCATE;'.format( db_name ) ) else: self._c.execute( 'PRAGMA {}.journal_mode = WAL;'.format( db_name ) ) self._c.execute( 'PRAGMA {}.synchronous = {};'.format( db_name, synchronous ) ) try: self._c.execute( 'SELECT * FROM {}.sqlite_master;'.format( db_name ) ).fetchone() except sqlite3.OperationalError as e: if HG.no_wal: message = 'The database failed to read any data. Please check your hard drive and perhaps \'help my db is broke.txt\' in the db directory. Full error information:' else: message = 'The database failed to read some data. You may need to run the program in no-wal mode using the --no_wal command parameter. Full error information:' message += os.linesep * 2 message += str( e ) HydrusData.DebugPrint( message ) raise HydrusExceptions.DBAccessException( message ) try: self._BeginImmediate() except Exception as e: raise HydrusExceptions.DBAccessException( str( e ) )
def __init__( self, controller, db_dir, db_name ): if HydrusPaths.GetFreeSpace( db_dir ) < 500 * 1048576: raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' ) self._controller = controller self._db_dir = db_dir self._db_name = db_name self._transaction_started = 0 self._in_transaction = False self._transaction_contains_writes = False self._connection_timestamp = 0 main_db_filename = db_name if not main_db_filename.endswith( '.db' ): main_db_filename += '.db' self._db_filenames = {} self._db_filenames[ 'main' ] = main_db_filename self._durable_temp_db_filename = db_name + '.temp.db' self._InitExternalDatabases() if distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' ): self._fast_big_transaction_wal = False else: self._fast_big_transaction_wal = True self._is_first_start = False self._is_db_updated = False self._local_shutdown = False self._pause_and_disconnect = False self._loop_finished = False self._ready_to_serve_requests = False self._could_not_initialise = False self._jobs = queue.Queue() self._pubsubs = [] self._currently_doing_job = False self._current_status = '' self._current_job_name = '' self._db = None self._c = None if os.path.exists( os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) ): # open and close to clean up in case last session didn't close well self._InitDB() self._CloseDBCursor() self._InitDB() self._RepairDB() ( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone() if version > HC.SOFTWARE_VERSION: self._ReportOverupdatedDB( version ) if version < ( HC.SOFTWARE_VERSION - 15 ): self._ReportUnderupdatedDB( version ) if version < HC.SOFTWARE_VERSION - 50: raise Exception( 'Your current database version of hydrus ' + str( version ) + ' is too old for this software version ' + str( HC.SOFTWARE_VERSION ) + ' to update. Please try updating with version ' + str( version + 45 ) + ' or earlier first.' ) while version < HC.SOFTWARE_VERSION: time.sleep( self.UPDATE_WAIT ) try: self._BeginImmediate() except Exception as e: raise HydrusExceptions.DBAccessException( str( e ) ) try: self._UpdateDB( version ) self._Commit() self._is_db_updated = True except: e = Exception( 'Updating the ' + self._db_name + ' db to version ' + str( version + 1 ) + ' caused this error:' + os.linesep + traceback.format_exc() ) try: self._Rollback() except Exception as rollback_e: HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' ) HydrusData.PrintException( rollback_e ) raise e ( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone() self._CloseDBCursor() self._controller.CallToThreadLongRunning( self.MainLoop ) while not self._ready_to_serve_requests: time.sleep( 0.1 ) if self._could_not_initialise: raise Exception( 'Could not initialise the db! Error written to the log!' )
def Repair( self, current_db_version, cursor_transaction_wrapper: HydrusDBBase.DBCursorTransactionWrapper ): # core, initial tables first table_generation_dict = self._GetInitialTableGenerationDict() missing_table_rows = [ (table_name, create_query_without_name) for (table_name, (create_query_without_name, version_added)) in table_generation_dict.items() if version_added <= current_db_version and not self._TableExists(table_name) ] if len(missing_table_rows) > 0: missing_table_names = sorted([ missing_table_row[0] for missing_table_row in missing_table_rows ]) critical_table_names = self._GetCriticalTableNames() missing_critical_table_names = set( missing_table_names).intersection(critical_table_names) if len(missing_critical_table_names) > 0: message = 'Unfortunately, this database is missing one or more critical tables! This database is non functional and cannot be repaired. Please check out "install_dir/db/help my db is broke.txt" for the next steps.' raise HydrusExceptions.DBAccessException(message) self._PresentMissingTablesWarningToUser(missing_table_names) for (table_name, create_query_without_name) in missing_table_rows: self._Execute(create_query_without_name.format(table_name)) cursor_transaction_wrapper.CommitAndBegin() self._RepairRepopulateTables(missing_table_names, cursor_transaction_wrapper) cursor_transaction_wrapper.CommitAndBegin() # now indices for those tables index_generation_dict = self._GetInitialIndexGenerationDict() missing_index_rows = [ (self._GenerateIndexName(table_name, columns), table_name, columns, unique) for (table_name, columns, unique, version_added ) in self._FlattenIndexGenerationDict(index_generation_dict) if version_added <= current_db_version and not self._IndexExists(table_name, columns) ] if len(missing_index_rows): self._PresentMissingIndicesWarningToUser( sorted([ index_name for (index_name, table_name, columns, unique) in missing_index_rows ])) for (index_name, table_name, columns, unique) in missing_index_rows: self._CreateIndex(table_name, columns, unique=unique) cursor_transaction_wrapper.CommitAndBegin() # now do service tables, same thing over again table_generation_dict = self._GetServicesTableGenerationDict() missing_table_rows = [ (table_name, create_query_without_name) for (table_name, (create_query_without_name, version_added)) in table_generation_dict.items() if version_added <= current_db_version and not self._TableExists(table_name) ] if len(missing_table_rows) > 0: missing_table_names = sorted([ missing_table_row[0] for missing_table_row in missing_table_rows ]) self._PresentMissingTablesWarningToUser(missing_table_names) for (table_name, create_query_without_name) in missing_table_rows: self._Execute(create_query_without_name.format(table_name)) cursor_transaction_wrapper.CommitAndBegin() self._RepairRepopulateTables(missing_table_names, cursor_transaction_wrapper) cursor_transaction_wrapper.CommitAndBegin() # now indices for those tables index_generation_dict = self._GetServicesIndexGenerationDict() missing_index_rows = [ (self._GenerateIndexName(table_name, columns), table_name, columns, unique) for (table_name, columns, unique, version_added ) in self._FlattenIndexGenerationDict(index_generation_dict) if version_added <= current_db_version and not self._IndexExists(table_name, columns) ] if len(missing_index_rows): self._PresentMissingIndicesWarningToUser( sorted([ index_name for (index_name, table_name, columns, unique) in missing_index_rows ])) for (index_name, table_name, columns, unique) in missing_index_rows: self._CreateIndex(table_name, columns, unique=unique) cursor_transaction_wrapper.CommitAndBegin()
def _InitDBCursor(self): self._CloseDBCursor() db_path = os.path.join(self._db_dir, self._db_filenames['main']) try: self._db = sqlite3.connect(db_path, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) self._c = self._db.cursor() self._cursor_transaction_wrapper = DBCursorTransactionWrapper( self._c, HG.db_transaction_commit_period) self._LoadModules() if HG.no_db_temp_files: self._c.execute('PRAGMA temp_store = 2;' ) # use memory for temp store exclusively self._AttachExternalDatabases() self._c.execute('ATTACH ":memory:" AS mem;') except Exception as e: raise HydrusExceptions.DBAccessException( 'Could not connect to database! This could be an issue related to WAL and network storage, or something else. If it is not obvious to you, please let hydrus dev know. Error follows:' + os.linesep * 2 + str(e)) TemporaryIntegerTableNameCache.instance().Clear() # durable_temp is not excluded here db_names = [ name for (index, name, path) in self._c.execute('PRAGMA database_list;') if name not in ('mem', 'temp') ] for db_name in db_names: # MB -> KB cache_size = HG.db_cache_size * 1024 self._c.execute('PRAGMA {}.cache_size = -{};'.format( db_name, cache_size)) self._c.execute('PRAGMA {}.journal_mode = {};'.format( db_name, HG.db_journal_mode)) if HG.db_journal_mode in ('PERSIST', 'WAL'): self._c.execute('PRAGMA {}.journal_size_limit = {};'.format( db_name, 1024**3)) # 1GB for now self._c.execute('PRAGMA {}.synchronous = {};'.format( db_name, HG.db_synchronous)) try: self._c.execute('SELECT * FROM {}.sqlite_master;'.format( db_name)).fetchone() except sqlite3.OperationalError as e: message = 'The database seemed valid, but hydrus failed to read basic data from it. You may need to run the program in a different journal mode using --db_journal_mode. Full error information:' message += os.linesep * 2 message += str(e) HydrusData.DebugPrint(message) raise HydrusExceptions.DBAccessException(message) try: self._cursor_transaction_wrapper.BeginImmediate() except Exception as e: raise HydrusExceptions.DBAccessException(str(e))
def _InitDBConnection(self): self._CloseDBConnection() db_path = os.path.join(self._db_dir, self._db_filenames['main']) try: if os.path.exists( db_path) and not HydrusPaths.FileisWriteable(db_path): raise HydrusExceptions.DBAccessException( '"{}" seems to be read-only!'.format(db_path)) self._db = sqlite3.connect(db_path, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) c = self._db.cursor() self._SetCursor(c) self._is_connected = True self._cursor_transaction_wrapper = HydrusDBBase.DBCursorTransactionWrapper( self._c, HG.db_transaction_commit_period) if HG.no_db_temp_files: self._Execute('PRAGMA temp_store = 2;' ) # use memory for temp store exclusively self._AttachExternalDatabases() self._LoadModules() self._Execute('ATTACH ":memory:" AS mem;') except HydrusExceptions.DBAccessException as e: raise except Exception as e: raise HydrusExceptions.DBAccessException( 'Could not connect to database! If the answer is not obvious to you, please let hydrus dev know. Error follows:' + os.linesep * 2 + str(e)) HydrusDBBase.TemporaryIntegerTableNameCache.instance().Clear() # durable_temp is not excluded here db_names = [ name for (index, name, path) in self._Execute('PRAGMA database_list;') if name not in ('mem', 'temp') ] for db_name in db_names: # MB -> KB cache_size = HG.db_cache_size * 1024 self._Execute('PRAGMA {}.cache_size = -{};'.format( db_name, cache_size)) self._Execute('PRAGMA {}.journal_mode = {};'.format( db_name, HG.db_journal_mode)) if HG.db_journal_mode in ('PERSIST', 'WAL'): # We tried 1GB here, but I have reports of larger ones that don't seem to truncate ever? # Not sure what that is about, but I guess the db sometimes doesn't want to (expensively?) recover pages from the journal and just appends more data # In any case, this pragma is not a 'don't allow it to grow larger than', it's a 'after commit, truncate back to this', so no need to make it so large # default is -1, which means no limit self._Execute('PRAGMA {}.journal_size_limit = {};'.format( db_name, HydrusDBBase.JOURNAL_SIZE_LIMIT)) self._Execute('PRAGMA {}.synchronous = {};'.format( db_name, HG.db_synchronous)) try: self._Execute('SELECT * FROM {}.sqlite_master;'.format( db_name)).fetchone() except sqlite3.OperationalError as e: message = 'The database seemed valid, but hydrus failed to read basic data from it. You may need to run the program in a different journal mode using --db_journal_mode. Full error information:' message += os.linesep * 2 message += str(e) HydrusData.DebugPrint(message) raise HydrusExceptions.DBAccessException(message) try: self._cursor_transaction_wrapper.BeginImmediate() except Exception as e: if 'locked' in str(e): raise HydrusExceptions.DBAccessException( 'Database appeared to be locked. Please ensure there is not another client already running on this database, and then try restarting the client.' ) raise HydrusExceptions.DBAccessException(str(e))
def __init__(self, controller, db_dir, db_name): if HydrusPaths.GetFreeSpace(db_dir) < 500 * 1048576: raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' ) HydrusDBBase.DBBase.__init__(self) self._controller = controller self._db_dir = db_dir self._db_name = db_name self._modules = [] HydrusDBBase.TemporaryIntegerTableNameCache() self._ssl_cert_filename = '{}.crt'.format(self._db_name) self._ssl_key_filename = '{}.key'.format(self._db_name) self._ssl_cert_path = os.path.join(self._db_dir, self._ssl_cert_filename) self._ssl_key_path = os.path.join(self._db_dir, self._ssl_key_filename) main_db_filename = db_name if not main_db_filename.endswith('.db'): main_db_filename += '.db' self._db_filenames = {} self._db_filenames['main'] = main_db_filename self._durable_temp_db_filename = db_name + '.temp.db' durable_temp_db_path = os.path.join(self._db_dir, self._durable_temp_db_filename) if os.path.exists(durable_temp_db_path): HydrusPaths.DeletePath(durable_temp_db_path) wal_lad = durable_temp_db_path + '-wal' if os.path.exists(wal_lad): HydrusPaths.DeletePath(wal_lad) shm_lad = durable_temp_db_path + '-shm' if os.path.exists(shm_lad): HydrusPaths.DeletePath(shm_lad) HydrusData.Print( 'Found and deleted the durable temporary database on boot. The last exit was probably not clean.' ) self._InitExternalDatabases() self._is_first_start = False self._is_db_updated = False self._local_shutdown = False self._pause_and_disconnect = False self._loop_finished = False self._ready_to_serve_requests = False self._could_not_initialise = False self._jobs = queue.Queue() self._currently_doing_job = False self._current_status = '' self._current_job_name = '' self._db = None self._is_connected = False self._cursor_transaction_wrapper = None if os.path.exists( os.path.join(self._db_dir, self._db_filenames['main'])): # open and close to clean up in case last session didn't close well self._InitDB() self._CloseDBConnection() self._InitDB() (version, ) = self._Execute('SELECT version FROM version;').fetchone() if version > HC.SOFTWARE_VERSION: self._ReportOverupdatedDB(version) if version < (HC.SOFTWARE_VERSION - 15): self._ReportUnderupdatedDB(version) if version < HC.SOFTWARE_VERSION - 50: raise Exception('Your current database version of hydrus ' + str(version) + ' is too old for this software version ' + str(HC.SOFTWARE_VERSION) + ' to update. Please try updating with version ' + str(version + 45) + ' or earlier first.') self._RepairDB(version) while version < HC.SOFTWARE_VERSION: time.sleep(self.UPDATE_WAIT) try: self._cursor_transaction_wrapper.BeginImmediate() except Exception as e: raise HydrusExceptions.DBAccessException(str(e)) try: self._UpdateDB(version) self._cursor_transaction_wrapper.Commit() self._is_db_updated = True except: e = Exception('Updating the ' + self._db_name + ' db to version ' + str(version + 1) + ' caused this error:' + os.linesep + traceback.format_exc()) try: self._cursor_transaction_wrapper.Rollback() except Exception as rollback_e: HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' ) HydrusData.PrintException(rollback_e) raise e (version, ) = self._Execute('SELECT version FROM version;').fetchone() self._CloseDBConnection() self._controller.CallToThreadLongRunning(self.MainLoop) while not self._ready_to_serve_requests: time.sleep(0.1) if self._could_not_initialise: raise Exception( 'Could not initialise the db! Error written to the log!')