def _InitDB(self):
create_db = False
db_path = os.path.join(self._db_dir, self._db_filenames['main'])
if not os.path.exists(db_path):
create_db = True
external_db_paths = [
os.path.join(self._db_dir, self._db_filenames[db_name])
for db_name in self._db_filenames if db_name != 'main'
]
existing_external_db_paths = [
external_db_path for external_db_path in external_db_paths
if os.path.exists(external_db_path)
]
if len(existing_external_db_paths) > 0:
message = 'Although the external files, "{}" do exist, the main database file, "{}", does not! This makes for an invalid database, and the program will now quit. Please contact hydrus_dev if you do not know how this happened or need help recovering from hard drive failure.'
message = message.format(', '.join(existing_external_db_paths),
db_path)
raise HydrusExceptions.DBAccessException(message)
self._InitDBCursor()
result = self._c.execute(
'SELECT 1 FROM sqlite_master WHERE type = ? AND name = ?;',
('table', 'version')).fetchone()
if result is None:
create_db = True
if create_db:
self._is_first_start = True
self._CreateDB()
self._Commit()
self._BeginImmediate()
def _AttachExternalDatabases(self):
for (name, filename) in self._db_filenames.items():
if name == 'main':
continue
db_path = os.path.join(self._db_dir, filename)
if os.path.exists(
db_path) and not HydrusPaths.FileisWriteable(db_path):
raise HydrusExceptions.DBAccessException(
'"{}" seems to be read-only!'.format(db_path))
self._Execute('ATTACH ? AS ' + name + ';', (db_path, ))
db_path = os.path.join(self._db_dir, self._durable_temp_db_filename)
self._Execute('ATTACH ? AS durable_temp;', (db_path, ))
def _InitDBCursor( self ):
self._CloseDBCursor()
db_path = os.path.join( self._db_dir, self._db_filenames[ 'main' ] )
db_just_created = not os.path.exists( db_path )
self._db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
self._connection_timestamp = HydrusData.GetNow()
self._c = self._db.cursor()
if HG.no_db_temp_files:
self._c.execute( 'PRAGMA temp_store = 2;' ) # use memory for temp store exclusively
self._c.execute( 'ATTACH ":memory:" AS mem;' )
self._AttachExternalDatabases()
# if this is set to 1, transactions are not immediately synced to the journal so multiple can be undone following a power-loss
# if set to 2, all transactions are synced, so once a new one starts you know the last one is on disk
# corruption cannot occur either way, but since we have multiple ATTACH dbs with diff journals, let's not mess around when power-cut during heavy file import or w/e
synchronous = 2
if HG.db_synchronous_override is not None:
synchronous = HG.db_synchronous_override
# durable_temp is not excluded here
db_names = [ name for ( index, name, path ) in self._c.execute( 'PRAGMA database_list;' ) if name not in ( 'mem', 'temp' ) ]
for db_name in db_names:
self._c.execute( 'PRAGMA {}.cache_size = -10000;'.format( db_name ) )
if HG.db_memory_journaling:
self._c.execute( 'PRAGMA {}.journal_mode = MEMORY;'.format( db_name ) )
elif HG.no_wal:
self._c.execute( 'PRAGMA {}.journal_mode = TRUNCATE;'.format( db_name ) )
else:
self._c.execute( 'PRAGMA {}.journal_mode = WAL;'.format( db_name ) )
self._c.execute( 'PRAGMA {}.synchronous = {};'.format( db_name, synchronous ) )
try:
self._c.execute( 'SELECT * FROM {}.sqlite_master;'.format( db_name ) ).fetchone()
except sqlite3.OperationalError as e:
if HG.no_wal:
message = 'The database failed to read any data. Please check your hard drive and perhaps \'help my db is broke.txt\' in the db directory. Full error information:'
else:
message = 'The database failed to read some data. You may need to run the program in no-wal mode using the --no_wal command parameter. Full error information:'
message += os.linesep * 2
message += str( e )
HydrusData.DebugPrint( message )
raise HydrusExceptions.DBAccessException( message )
try:
self._BeginImmediate()
except Exception as e:
raise HydrusExceptions.DBAccessException( str( e ) )
def __init__( self, controller, db_dir, db_name ):
if HydrusPaths.GetFreeSpace( db_dir ) < 500 * 1048576:
raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' )
self._controller = controller
self._db_dir = db_dir
self._db_name = db_name
self._transaction_started = 0
self._in_transaction = False
self._transaction_contains_writes = False
self._connection_timestamp = 0
main_db_filename = db_name
if not main_db_filename.endswith( '.db' ):
main_db_filename += '.db'
self._db_filenames = {}
self._db_filenames[ 'main' ] = main_db_filename
self._durable_temp_db_filename = db_name + '.temp.db'
self._InitExternalDatabases()
if distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' ):
self._fast_big_transaction_wal = False
else:
self._fast_big_transaction_wal = True
self._is_first_start = False
self._is_db_updated = False
self._local_shutdown = False
self._pause_and_disconnect = False
self._loop_finished = False
self._ready_to_serve_requests = False
self._could_not_initialise = False
self._jobs = queue.Queue()
self._pubsubs = []
self._currently_doing_job = False
self._current_status = ''
self._current_job_name = ''
self._db = None
self._c = None
if os.path.exists( os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) ):
# open and close to clean up in case last session didn't close well
self._InitDB()
self._CloseDBCursor()
self._InitDB()
self._RepairDB()
( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone()
if version > HC.SOFTWARE_VERSION:
self._ReportOverupdatedDB( version )
if version < ( HC.SOFTWARE_VERSION - 15 ):
self._ReportUnderupdatedDB( version )
if version < HC.SOFTWARE_VERSION - 50:
raise Exception( 'Your current database version of hydrus ' + str( version ) + ' is too old for this software version ' + str( HC.SOFTWARE_VERSION ) + ' to update. Please try updating with version ' + str( version + 45 ) + ' or earlier first.' )
while version < HC.SOFTWARE_VERSION:
time.sleep( self.UPDATE_WAIT )
try:
self._BeginImmediate()
except Exception as e:
raise HydrusExceptions.DBAccessException( str( e ) )
try:
self._UpdateDB( version )
self._Commit()
self._is_db_updated = True
except:
e = Exception( 'Updating the ' + self._db_name + ' db to version ' + str( version + 1 ) + ' caused this error:' + os.linesep + traceback.format_exc() )
try:
self._Rollback()
except Exception as rollback_e:
HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' )
HydrusData.PrintException( rollback_e )
raise e
( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone()
self._CloseDBCursor()
self._controller.CallToThreadLongRunning( self.MainLoop )
while not self._ready_to_serve_requests:
time.sleep( 0.1 )
if self._could_not_initialise:
raise Exception( 'Could not initialise the db! Error written to the log!' )
def Repair(
self, current_db_version,
cursor_transaction_wrapper: HydrusDBBase.DBCursorTransactionWrapper
):
# core, initial tables first
table_generation_dict = self._GetInitialTableGenerationDict()
missing_table_rows = [
(table_name, create_query_without_name)
for (table_name, (create_query_without_name,
version_added)) in table_generation_dict.items()
if version_added <= current_db_version
and not self._TableExists(table_name)
]
if len(missing_table_rows) > 0:
missing_table_names = sorted([
missing_table_row[0]
for missing_table_row in missing_table_rows
])
critical_table_names = self._GetCriticalTableNames()
missing_critical_table_names = set(
missing_table_names).intersection(critical_table_names)
if len(missing_critical_table_names) > 0:
message = 'Unfortunately, this database is missing one or more critical tables! This database is non functional and cannot be repaired. Please check out "install_dir/db/help my db is broke.txt" for the next steps.'
raise HydrusExceptions.DBAccessException(message)
self._PresentMissingTablesWarningToUser(missing_table_names)
for (table_name, create_query_without_name) in missing_table_rows:
self._Execute(create_query_without_name.format(table_name))
cursor_transaction_wrapper.CommitAndBegin()
self._RepairRepopulateTables(missing_table_names,
cursor_transaction_wrapper)
cursor_transaction_wrapper.CommitAndBegin()
# now indices for those tables
index_generation_dict = self._GetInitialIndexGenerationDict()
missing_index_rows = [
(self._GenerateIndexName(table_name,
columns), table_name, columns, unique)
for (table_name, columns, unique, version_added
) in self._FlattenIndexGenerationDict(index_generation_dict)
if version_added <= current_db_version
and not self._IndexExists(table_name, columns)
]
if len(missing_index_rows):
self._PresentMissingIndicesWarningToUser(
sorted([
index_name for (index_name, table_name, columns,
unique) in missing_index_rows
]))
for (index_name, table_name, columns,
unique) in missing_index_rows:
self._CreateIndex(table_name, columns, unique=unique)
cursor_transaction_wrapper.CommitAndBegin()
# now do service tables, same thing over again
table_generation_dict = self._GetServicesTableGenerationDict()
missing_table_rows = [
(table_name, create_query_without_name)
for (table_name, (create_query_without_name,
version_added)) in table_generation_dict.items()
if version_added <= current_db_version
and not self._TableExists(table_name)
]
if len(missing_table_rows) > 0:
missing_table_names = sorted([
missing_table_row[0]
for missing_table_row in missing_table_rows
])
self._PresentMissingTablesWarningToUser(missing_table_names)
for (table_name, create_query_without_name) in missing_table_rows:
self._Execute(create_query_without_name.format(table_name))
cursor_transaction_wrapper.CommitAndBegin()
self._RepairRepopulateTables(missing_table_names,
cursor_transaction_wrapper)
cursor_transaction_wrapper.CommitAndBegin()
# now indices for those tables
index_generation_dict = self._GetServicesIndexGenerationDict()
missing_index_rows = [
(self._GenerateIndexName(table_name,
columns), table_name, columns, unique)
for (table_name, columns, unique, version_added
) in self._FlattenIndexGenerationDict(index_generation_dict)
if version_added <= current_db_version
and not self._IndexExists(table_name, columns)
]
if len(missing_index_rows):
self._PresentMissingIndicesWarningToUser(
sorted([
index_name for (index_name, table_name, columns,
unique) in missing_index_rows
]))
for (index_name, table_name, columns,
unique) in missing_index_rows:
self._CreateIndex(table_name, columns, unique=unique)
cursor_transaction_wrapper.CommitAndBegin()
def _InitDBCursor(self):
self._CloseDBCursor()
db_path = os.path.join(self._db_dir, self._db_filenames['main'])
try:
self._db = sqlite3.connect(db_path,
isolation_level=None,
detect_types=sqlite3.PARSE_DECLTYPES)
self._c = self._db.cursor()
self._cursor_transaction_wrapper = DBCursorTransactionWrapper(
self._c, HG.db_transaction_commit_period)
self._LoadModules()
if HG.no_db_temp_files:
self._c.execute('PRAGMA temp_store = 2;'
) # use memory for temp store exclusively
self._AttachExternalDatabases()
self._c.execute('ATTACH ":memory:" AS mem;')
except Exception as e:
raise HydrusExceptions.DBAccessException(
'Could not connect to database! This could be an issue related to WAL and network storage, or something else. If it is not obvious to you, please let hydrus dev know. Error follows:'
+ os.linesep * 2 + str(e))
TemporaryIntegerTableNameCache.instance().Clear()
# durable_temp is not excluded here
db_names = [
name
for (index, name, path) in self._c.execute('PRAGMA database_list;')
if name not in ('mem', 'temp')
]
for db_name in db_names:
# MB -> KB
cache_size = HG.db_cache_size * 1024
self._c.execute('PRAGMA {}.cache_size = -{};'.format(
db_name, cache_size))
self._c.execute('PRAGMA {}.journal_mode = {};'.format(
db_name, HG.db_journal_mode))
if HG.db_journal_mode in ('PERSIST', 'WAL'):
self._c.execute('PRAGMA {}.journal_size_limit = {};'.format(
db_name, 1024**3)) # 1GB for now
self._c.execute('PRAGMA {}.synchronous = {};'.format(
db_name, HG.db_synchronous))
try:
self._c.execute('SELECT * FROM {}.sqlite_master;'.format(
db_name)).fetchone()
except sqlite3.OperationalError as e:
message = 'The database seemed valid, but hydrus failed to read basic data from it. You may need to run the program in a different journal mode using --db_journal_mode. Full error information:'
message += os.linesep * 2
message += str(e)
HydrusData.DebugPrint(message)
raise HydrusExceptions.DBAccessException(message)
try:
self._cursor_transaction_wrapper.BeginImmediate()
except Exception as e:
raise HydrusExceptions.DBAccessException(str(e))
def _InitDBConnection(self):
self._CloseDBConnection()
db_path = os.path.join(self._db_dir, self._db_filenames['main'])
try:
if os.path.exists(
db_path) and not HydrusPaths.FileisWriteable(db_path):
raise HydrusExceptions.DBAccessException(
'"{}" seems to be read-only!'.format(db_path))
self._db = sqlite3.connect(db_path,
isolation_level=None,
detect_types=sqlite3.PARSE_DECLTYPES)
c = self._db.cursor()
self._SetCursor(c)
self._is_connected = True
self._cursor_transaction_wrapper = HydrusDBBase.DBCursorTransactionWrapper(
self._c, HG.db_transaction_commit_period)
if HG.no_db_temp_files:
self._Execute('PRAGMA temp_store = 2;'
) # use memory for temp store exclusively
self._AttachExternalDatabases()
self._LoadModules()
self._Execute('ATTACH ":memory:" AS mem;')
except HydrusExceptions.DBAccessException as e:
raise
except Exception as e:
raise HydrusExceptions.DBAccessException(
'Could not connect to database! If the answer is not obvious to you, please let hydrus dev know. Error follows:'
+ os.linesep * 2 + str(e))
HydrusDBBase.TemporaryIntegerTableNameCache.instance().Clear()
# durable_temp is not excluded here
db_names = [
name
for (index, name, path) in self._Execute('PRAGMA database_list;')
if name not in ('mem', 'temp')
]
for db_name in db_names:
# MB -> KB
cache_size = HG.db_cache_size * 1024
self._Execute('PRAGMA {}.cache_size = -{};'.format(
db_name, cache_size))
self._Execute('PRAGMA {}.journal_mode = {};'.format(
db_name, HG.db_journal_mode))
if HG.db_journal_mode in ('PERSIST', 'WAL'):
# We tried 1GB here, but I have reports of larger ones that don't seem to truncate ever?
# Not sure what that is about, but I guess the db sometimes doesn't want to (expensively?) recover pages from the journal and just appends more data
# In any case, this pragma is not a 'don't allow it to grow larger than', it's a 'after commit, truncate back to this', so no need to make it so large
# default is -1, which means no limit
self._Execute('PRAGMA {}.journal_size_limit = {};'.format(
db_name, HydrusDBBase.JOURNAL_SIZE_LIMIT))
self._Execute('PRAGMA {}.synchronous = {};'.format(
db_name, HG.db_synchronous))
try:
self._Execute('SELECT * FROM {}.sqlite_master;'.format(
db_name)).fetchone()
except sqlite3.OperationalError as e:
message = 'The database seemed valid, but hydrus failed to read basic data from it. You may need to run the program in a different journal mode using --db_journal_mode. Full error information:'
message += os.linesep * 2
message += str(e)
HydrusData.DebugPrint(message)
raise HydrusExceptions.DBAccessException(message)
try:
self._cursor_transaction_wrapper.BeginImmediate()
except Exception as e:
if 'locked' in str(e):
raise HydrusExceptions.DBAccessException(
'Database appeared to be locked. Please ensure there is not another client already running on this database, and then try restarting the client.'
)
raise HydrusExceptions.DBAccessException(str(e))
def __init__(self, controller, db_dir, db_name):
if HydrusPaths.GetFreeSpace(db_dir) < 500 * 1048576:
raise Exception(
'Sorry, it looks like the db partition has less than 500MB, please free up some space.'
)
HydrusDBBase.DBBase.__init__(self)
self._controller = controller
self._db_dir = db_dir
self._db_name = db_name
self._modules = []
HydrusDBBase.TemporaryIntegerTableNameCache()
self._ssl_cert_filename = '{}.crt'.format(self._db_name)
self._ssl_key_filename = '{}.key'.format(self._db_name)
self._ssl_cert_path = os.path.join(self._db_dir,
self._ssl_cert_filename)
self._ssl_key_path = os.path.join(self._db_dir, self._ssl_key_filename)
main_db_filename = db_name
if not main_db_filename.endswith('.db'):
main_db_filename += '.db'
self._db_filenames = {}
self._db_filenames['main'] = main_db_filename
self._durable_temp_db_filename = db_name + '.temp.db'
durable_temp_db_path = os.path.join(self._db_dir,
self._durable_temp_db_filename)
if os.path.exists(durable_temp_db_path):
HydrusPaths.DeletePath(durable_temp_db_path)
wal_lad = durable_temp_db_path + '-wal'
if os.path.exists(wal_lad):
HydrusPaths.DeletePath(wal_lad)
shm_lad = durable_temp_db_path + '-shm'
if os.path.exists(shm_lad):
HydrusPaths.DeletePath(shm_lad)
HydrusData.Print(
'Found and deleted the durable temporary database on boot. The last exit was probably not clean.'
)
self._InitExternalDatabases()
self._is_first_start = False
self._is_db_updated = False
self._local_shutdown = False
self._pause_and_disconnect = False
self._loop_finished = False
self._ready_to_serve_requests = False
self._could_not_initialise = False
self._jobs = queue.Queue()
self._currently_doing_job = False
self._current_status = ''
self._current_job_name = ''
self._db = None
self._is_connected = False
self._cursor_transaction_wrapper = None
if os.path.exists(
os.path.join(self._db_dir, self._db_filenames['main'])):
# open and close to clean up in case last session didn't close well
self._InitDB()
self._CloseDBConnection()
self._InitDB()
(version, ) = self._Execute('SELECT version FROM version;').fetchone()
if version > HC.SOFTWARE_VERSION:
self._ReportOverupdatedDB(version)
if version < (HC.SOFTWARE_VERSION - 15):
self._ReportUnderupdatedDB(version)
if version < HC.SOFTWARE_VERSION - 50:
raise Exception('Your current database version of hydrus ' +
str(version) +
' is too old for this software version ' +
str(HC.SOFTWARE_VERSION) +
' to update. Please try updating with version ' +
str(version + 45) + ' or earlier first.')
self._RepairDB(version)
while version < HC.SOFTWARE_VERSION:
time.sleep(self.UPDATE_WAIT)
try:
self._cursor_transaction_wrapper.BeginImmediate()
except Exception as e:
raise HydrusExceptions.DBAccessException(str(e))
try:
self._UpdateDB(version)
self._cursor_transaction_wrapper.Commit()
self._is_db_updated = True
except:
e = Exception('Updating the ' + self._db_name +
' db to version ' + str(version + 1) +
' caused this error:' + os.linesep +
traceback.format_exc())
try:
self._cursor_transaction_wrapper.Rollback()
except Exception as rollback_e:
HydrusData.Print(
'When the update failed, attempting to rollback the database failed.'
)
HydrusData.PrintException(rollback_e)
raise e
(version,
) = self._Execute('SELECT version FROM version;').fetchone()
self._CloseDBConnection()
self._controller.CallToThreadLongRunning(self.MainLoop)
while not self._ready_to_serve_requests:
time.sleep(0.1)
if self._could_not_initialise:
raise Exception(
'Could not initialise the db! Error written to the log!')