def MakeFileWriteable(path):
if not os.path.exists(path):
return
try:
stat_result = os.stat(path)
current_bits = stat_result.st_mode
if HC.PLATFORM_WINDOWS:
# this is actually the same value as S_IWUSR, but let's not try to second guess ourselves
desired_bits = stat.S_IREAD | stat.S_IWRITE
else:
# guarantee 644 for regular files m8
desired_bits = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
if not (desired_bits & current_bits) == desired_bits:
os.chmod(path, current_bits | desired_bits)
except Exception as e:
HydrusData.Print(
'Wanted to add write permission to "{}", but had an error: {}'.
format(path, str(e)))
def CatchExceptionClient(etype, value, tb):
try:
trace_list = traceback.format_tb(tb)
trace = ''.join(trace_list)
pretty_value = str(value)
if os.linesep in pretty_value:
(first_line, anything_else) = pretty_value.split(os.linesep, 1)
trace = trace + os.linesep + anything_else
else:
first_line = pretty_value
job_key = ClientThreading.JobKey()
if etype == HydrusExceptions.ShutdownException:
return
else:
try:
job_key.SetVariable('popup_title', str(etype.__name__))
except:
job_key.SetVariable('popup_title', str(etype))
job_key.SetVariable('popup_text_1', first_line)
job_key.SetVariable('popup_traceback', trace)
text = job_key.ToString()
HydrusData.Print('Uncaught exception:')
HydrusData.DebugPrint(text)
HG.client_controller.pub('message', job_key)
except:
text = 'Encountered an error I could not parse:'
text += os.linesep
text += str((etype, value, tb))
try:
text += traceback.format_exc()
except:
pass
HydrusData.ShowText(text)
time.sleep(1)
def CleanUpTempPath(os_file_handle, temp_path):
try:
os.close(os_file_handle)
except OSError:
gc.collect()
try:
os.close(os_file_handle)
except OSError:
HydrusData.Print('Could not close the temporary file ' + temp_path)
return
try:
os.remove(temp_path)
except OSError:
with TEMP_PATH_LOCK:
IN_USE_TEMP_PATHS.add((HydrusData.GetNow(), temp_path))
def _Commit( self ):
if self._in_transaction:
self._c.execute( 'COMMIT;' )
self._in_transaction = False
if HG.db_journal_mode == 'WAL' and HydrusData.TimeHasPassed( self._last_wal_checkpoint_time + 1800 ):
self._c.execute( 'PRAGMA wal_checkpoint(PASSIVE);' )
self._last_wal_checkpoint_time = HydrusData.GetNow()
if HydrusData.TimeHasPassed( self._last_mem_refresh_time + 600 ):
self._c.execute( 'DETACH mem;' )
self._c.execute( 'ATTACH ":memory:" AS mem;' )
TemporaryIntegerTableNameCache.instance().Clear()
self._last_mem_refresh_time = HydrusData.GetNow()
else:
HydrusData.Print( 'Received a call to commit, but was not in a transaction!' )
def SetYAMLDump( self, dump_type, dump_name, data ):
if dump_type == YAML_DUMP_ID_LOCAL_BOORU:
dump_name = dump_name.hex()
self._c.execute( 'DELETE FROM yaml_dumps WHERE dump_type = ? AND dump_name = ?;', ( dump_type, dump_name ) )
try:
self._c.execute( 'INSERT INTO yaml_dumps ( dump_type, dump_name, dump ) VALUES ( ?, ?, ? );', ( dump_type, dump_name, data ) )
except:
HydrusData.Print( ( dump_type, dump_name, data ) )
raise
if dump_type == YAML_DUMP_ID_LOCAL_BOORU:
service_id = self.modules_services.GetServiceId( CC.LOCAL_BOORU_SERVICE_KEY )
self._c.execute( 'DELETE FROM service_info WHERE service_id = ? AND info_type = ?;', ( service_id, HC.SERVICE_INFO_NUM_SHARES ) )
HG.client_controller.pub( 'refresh_local_booru_shares' )
def InitView(self):
HydrusController.HydrusController.InitView(self)
port = self._admin_service.GetPort()
if HydrusNetworking.LocalPortInUse(port):
HydrusData.Print(
'Something is already bound to port ' + str(port) +
', so your administration service cannot be started. Please quit the server and retry once the port is clear.'
)
else:
self.SetRunningTwistedServices(self._services)
#
job = self.CallRepeating(5.0, 600.0, self.SyncRepositories)
self._daemon_jobs['sync_repositories'] = job
job = self.CallRepeating(0.0, 30.0, self.SaveDirtyObjects)
self._daemon_jobs['save_dirty_objects'] = job
job = self.CallRepeating(0.0, 86400.0, self.DeleteOrphans)
self._daemon_jobs['delete_orphans'] = job
def InitialiseDefaults():
global DEFAULT_HYDRUS_STYLESHEET
try:
with open( os.path.join( STYLESHEET_DIR, 'default_hydrus.qss' ), 'r', encoding = 'utf-8' ) as f:
DEFAULT_HYDRUS_STYLESHEET = f.read()
except Exception as e:
HydrusData.Print( 'Failed to load default hydrus qss:' )
HydrusData.PrintException( e )
DEFAULT_HYDRUS_STYLESHEET = ''
global ORIGINAL_STYLE_NAME
global CURRENT_STYLE_NAME
ORIGINAL_STYLE_NAME = QW.QApplication.instance().style().objectName()
CURRENT_STYLE_NAME = ORIGINAL_STYLE_NAME
global ORIGINAL_STYLESHEET
global CURRENT_STYLESHEET
ORIGINAL_STYLESHEET = QW.QApplication.instance().styleSheet()
CURRENT_STYLESHEET = ORIGINAL_STYLESHEET
def _threadDoPOSTJob( self, request: HydrusServerRequest.HydrusRequest ):
update_period = request.parsed_request_args[ 'update_period' ]
if update_period < HydrusNetwork.MIN_UPDATE_PERIOD:
raise HydrusExceptions.BadRequestException( 'The update period was too low. It needs to be at least {}.'.format( HydrusData.TimeDeltaToPrettyTimeDelta( HydrusNetwork.MIN_UPDATE_PERIOD ) ) )
if update_period > HydrusNetwork.MAX_UPDATE_PERIOD:
raise HydrusExceptions.BadRequestException( 'The update period was too high. It needs to be lower than {}.'.format( HydrusData.TimeDeltaToPrettyTimeDelta( HydrusNetwork.MAX_UPDATE_PERIOD ) ) )
old_update_period = self._service.GetUpdatePeriod()
if old_update_period != update_period:
self._service.SetUpdatePeriod( update_period )
HydrusData.Print(
'Account {} changed the update period to from "{}" to "{}".'.format(
request.hydrus_account.GetAccountKey().hex(),
HydrusData.TimeDeltaToPrettyTimeDelta( old_update_period ),
HydrusData.TimeDeltaToPrettyTimeDelta( update_period )
)
)
response_context = HydrusServerResources.ResponseContext( 200 )
return response_context
def qt_code( win: QW.QWidget, job_key: ClientThreading.JobKey ):
try:
if win is not None and not QP.isValid( win ):
raise HydrusExceptions.QtDeadWindowException('Parent Window was destroyed before Qt command was called!')
result = func( *args, **kwargs )
job_key.SetVariable( 'result', result )
except ( HydrusExceptions.QtDeadWindowException, HydrusExceptions.DBCredentialsException, HydrusExceptions.ShutdownException ) as e:
job_key.SetErrorException( e )
except Exception as e:
job_key.SetErrorException( e )
HydrusData.Print( 'CallBlockingToQt just caught this error:' )
HydrusData.DebugPrint( traceback.format_exc() )
finally:
job_key.Finish()
def GetSSLPaths(self):
# create ssl keys
cert_here = os.path.exists(self._ssl_cert_path)
key_here = os.path.exists(self._ssl_key_path)
if cert_here ^ key_here:
raise Exception(
'While creating the server database, only one of the paths "{}" and "{}" existed. You can create a db with these files already in place, but please either delete the existing file (to have hydrus generate its own pair) or find the other in the pair (to use your own).'
.format(self._ssl_cert_path, self._ssl_key_path))
elif not (cert_here or key_here):
HydrusData.Print('Generating new cert/key files.')
if not HydrusEncryption.OPENSSL_OK:
raise Exception(
'The database was asked for ssl cert and keys to start either the server or the client api in https. The files do not exist yet, so the database wanted to create new ones, but unfortunately PyOpenSSL is not available, so this cannot be done. If you are running from source, please install this module using pip. Or drop in your own client.crt/client.key or server.crt/server.key files in the db directory.'
)
HydrusEncryption.GenerateOpenSSLCertAndKeyFile(
self._ssl_cert_path, self._ssl_key_path)
return (self._ssl_cert_path, self._ssl_key_path)
def run(self):
try:
while True:
while self._queue.empty():
CheckIfThreadShuttingDown()
self._event.wait(10.0)
self._event.clear()
CheckIfThreadShuttingDown()
try:
try:
(callable, args, kwargs) = self._queue.get(1.0)
except queue.Empty:
# https://github.com/hydrusnetwork/hydrus/issues/750
# this shouldn't happen, but...
# even if we assume we'll never get this, we don't want to make a business of hanging forever on things
continue
self._DoPreCall()
self._callable = (callable, args, kwargs)
callable(*args, **kwargs)
self._callable = None
del callable
except HydrusExceptions.ShutdownException:
return
except Exception as e:
HydrusData.Print(traceback.format_exc())
HydrusData.ShowException(e)
finally:
self._currently_working = False
time.sleep(0.00001)
except HydrusExceptions.ShutdownException:
return
def SetStatus(self, status, note='', exception=None):
if exception is not None:
first_line = str(exception).split(os.linesep)[0]
note = first_line + '\u2026 (Copy note to see full error)'
note += os.linesep
note += traceback.format_exc()
HydrusData.Print('Error when processing ' + self.url + ' !')
HydrusData.Print(traceback.format_exc())
self.status = status
self.note = note
self._UpdateModified()
def ShowExceptionTupleClient(etype, value, tb, do_wait=True):
if etype is None:
etype = HydrusExceptions.UnknownException
if value is None:
value = 'Unknown error'
if tb is None:
trace = 'No error trace--here is the stack:' + os.linesep + ''.join(
traceback.format_stack())
else:
trace = ''.join(traceback.format_exception(etype, value, tb))
pretty_value = str(value)
if os.linesep in pretty_value:
(first_line, anything_else) = pretty_value.split(os.linesep, 1)
trace = trace + os.linesep + anything_else
else:
first_line = pretty_value
job_key = ClientThreading.JobKey()
if etype == HydrusExceptions.ShutdownException:
return
else:
title = str(getattr(etype, '__name__', etype))
job_key.SetStatusTitle(title)
job_key.SetVariable('popup_text_1', first_line)
job_key.SetTraceback(trace)
text = job_key.ToString()
HydrusData.Print('Exception:')
HydrusData.DebugPrint(text)
HG.client_controller.pub('message', job_key)
if do_wait:
time.sleep(1)
def _Rollback( self ):
if self._in_transaction:
self._c.execute( 'ROLLBACK TO hydrus_savepoint;' )
else:
HydrusData.Print( 'Received a call to rollback, but was not in a transaction!' )
def Save(self):
if self._in_transaction:
try:
self._Execute('RELEASE hydrus_savepoint;')
except sqlite3.OperationalError:
HydrusData.Print(
'Tried to release a database savepoint, but failed!')
self._Execute('SAVEPOINT hydrus_savepoint;')
else:
HydrusData.Print(
'Received a call to save, but was not in a transaction!')
def run( self ) -> None:
while True:
try:
while self._NoWorkToStart():
if IsThreadShuttingDown():
return
#
if self._cancel_filter_needed.is_set():
self._FilterCancelled()
self._cancel_filter_needed.clear()
if self._sort_needed.is_set():
self._SortWaiting()
self._sort_needed.clear()
continue # if some work is now due, let's do it!
#
wait_time = self._GetLoopWaitTime()
self._new_job_arrived.wait( wait_time )
self._new_job_arrived.clear()
self._StartWork()
except HydrusExceptions.ShutdownException:
return
except Exception as e:
HydrusData.Print( traceback.format_exc() )
HydrusData.ShowException( e )
time.sleep( 0.00001 )
def ShowTextClient(text):
job_key = ClientThreading.JobKey()
job_key.SetVariable('popup_text_1', str(text))
text = job_key.ToString()
HydrusData.Print(text)
HG.client_controller.pub('message', job_key)
def SetSubtext(self, text):
if HG.boot_debug and self._updater is not None and len(text) > 0:
HydrusData.Print(text)
with self._lock:
self._status_subtext = text
self._NotifyUI()
def _Commit( self ):
if self._in_transaction:
self._c.execute( 'COMMIT;' )
self._in_transaction = False
else:
HydrusData.Print( 'Received a call to commit, but was not in a transaction!' )
def Commit(self):
if self._in_transaction:
self.DoPubSubs()
self.CleanPubSubs()
self._Execute('COMMIT;')
self._in_transaction = False
self._transaction_contains_writes = False
if HG.db_journal_mode == 'WAL' and HydrusData.TimeHasPassed(
self._last_wal_passive_checkpoint_time +
WAL_PASSIVE_CHECKPOINT_PERIOD):
if HydrusData.TimeHasPassed(
self._last_wal_truncate_checkpoint_time +
WAL_TRUNCATE_CHECKPOINT_PERIOD):
self._Execute('PRAGMA wal_checkpoint(TRUNCATE);')
self._last_wal_truncate_checkpoint_time = HydrusData.GetNow(
)
else:
self._Execute('PRAGMA wal_checkpoint(PASSIVE);')
self._last_wal_passive_checkpoint_time = HydrusData.GetNow()
if HydrusData.TimeHasPassed(self._last_mem_refresh_time +
MEM_REFRESH_PERIOD):
self._Execute('DETACH mem;')
self._Execute('ATTACH ":memory:" AS mem;')
TemporaryIntegerTableNameCache.instance().Clear()
self._last_mem_refresh_time = HydrusData.GetNow()
if HG.db_journal_mode == 'PERSIST' and HydrusData.TimeHasPassed(
self._last_journal_zero_time + JOURNAL_ZERO_PERIOD):
self._ZeroJournal()
self._last_journal_zero_time = HydrusData.GetNow()
else:
HydrusData.Print(
'Received a call to commit, but was not in a transaction!')
def SetText(self, text, print_to_log=True):
if self._updater is not None and print_to_log and len(text) > 0:
HydrusData.Print(text)
with self._lock:
self._status_text = text
self._status_subtext = ''
self._NotifyUI()
def run( self ):
try:
while True:
while self._queue.empty():
CheckIfThreadShuttingDown()
self._event.wait( 10.0 )
self._event.clear()
CheckIfThreadShuttingDown()
self._DoPreCall()
try:
( callable, args, kwargs ) = self._queue.get()
self._callable = ( callable, args, kwargs )
callable( *args, **kwargs )
self._callable = None
del callable
except HydrusExceptions.ShutdownException:
return
except Exception as e:
HydrusData.Print( traceback.format_exc() )
HydrusData.ShowException( e )
finally:
self._currently_working = False
time.sleep( 0.00001 )
except HydrusExceptions.ShutdownException:
return
def Rollback(self):
if self._in_transaction:
self._c.execute('ROLLBACK TO hydrus_savepoint;')
# still in transaction
# transaction may no longer contain writes, but it isn't important to figure out that it doesn't
else:
HydrusData.Print(
'Received a call to rollback, but was not in a transaction!')
def GetQtPixmap(self, clip_rect=None, target_resolution=None):
# colourspace conversions seem to be exclusively QImage territory
if self._icc_profile_bytes is not None:
qt_image = self.GetQtImage(clip_rect=clip_rect,
target_resolution=target_resolution)
return QG.QPixmap.fromImage(qt_image)
(my_width, my_height) = self._resolution
if clip_rect is None:
clip_rect = QC.QRect(QC.QPoint(0, 0),
QC.QSize(my_width, my_height))
if target_resolution is None:
target_resolution = clip_rect.size()
my_full_rect = QC.QRect(0, 0, my_width, my_height)
if my_full_rect.contains(clip_rect):
try:
numpy_image = self._GetNumPyImage(clip_rect, target_resolution)
(height, width, depth) = numpy_image.shape
data = numpy_image.data
return HG.client_controller.bitmap_manager.GetQtPixmapFromBuffer(
width, height, depth * 8, data)
except Exception as e:
HydrusData.PrintException(e, do_wait=False)
HydrusData.Print(
'Failed to produce a tile! Info is: {}, {}, {}, {}'.format(
self._hash.hex(), (my_width, my_height), clip_rect,
target_resolution))
pixmap = QG.QPixmap(target_resolution)
pixmap.fill(QC.Qt.black)
return pixmap
def GetQtImage(self, clip_rect=None, target_resolution=None):
if clip_rect is None:
(width, height) = self._resolution
clip_rect = QC.QRect(QC.QPoint(0, 0), QC.QSize(width, height))
if target_resolution is None:
target_resolution = clip_rect.size()
numpy_image = self._GetNumPyImage(clip_rect, target_resolution)
(height, width, depth) = numpy_image.shape
data = numpy_image.data
qt_image = HG.client_controller.bitmap_manager.GetQtImageFromBuffer(
width, height, depth * 8, data)
# ok this stuff was originally for image ICC, as loaded using PIL's image.info dict
# ultimately I figured out how to do the conversion with PIL itself, which was more universal
# however if we end up pulling display ICC or taking user-set ICC, we may want this qt code somewhere
if self._icc_profile_bytes is not None:
try:
if self._qt_colourspace is None:
self._qt_colourspace = QG.QColorSpace.fromIccProfile(
self._icc_profile_bytes)
# originally this was converting image ICC to sRGB, but I think in the 'display' sense, we'd be setting sRGB and then converting to the user-set ICC
# 'hey, Qt, this QImage is in sRGB (I already normalised it), now convert it to xxx, thanks!'
qt_image.setColorSpace(self._qt_colourspace)
qt_image.convertToColorSpace(QG.QColorSpace.SRgb)
except:
HydrusData.Print(
'Failed to load the ICC Profile for {} into a Qt Colourspace!'
.format(self._path))
self._icc_profile_bytes = None
return qt_image
def Rollback( self ):
if self._in_transaction:
self._c.execute( 'ROLLBACK TO hydrus_savepoint;' )
# any temp int tables created in this lad will be rolled back, so 'initialised' can't be trusted. just reset, no big deal
TemporaryIntegerTableNameCache.instance().Clear()
# still in transaction
# transaction may no longer contain writes, but it isn't important to figure out that it doesn't
else:
HydrusData.Print( 'Received a call to rollback, but was not in a transaction!' )
def _HandleThumbnailException(self, e, summary):
if self._thumbnail_error_occurred:
HydrusData.Print(summary)
else:
self._thumbnail_error_occurred = True
message = 'A thumbnail error has occurred. The problem thumbnail will appear with the default \'hydrus\' symbol. You may need to take hard drive recovery actions, and if the error is not obviously fixable, you can contact hydrus dev for additional help. Specific information for this first error follows. Subsequent thumbnail errors in this session will be silently printed to the log.'
message += os.linesep * 2
message += str(e)
message += os.linesep * 2
message += summary
HydrusData.ShowText(message)
def InitModel(self):
try:
self._InitTempDir()
except:
HydrusData.Print('Failed to initialise temp folder.')
self._fast_job_scheduler = HydrusThreading.JobScheduler(self)
self._slow_job_scheduler = HydrusThreading.JobScheduler(self)
self._fast_job_scheduler.start()
self._slow_job_scheduler.start()
self.db = self._InitDB()
def DoDeferredPhysicalDeletes(self):
num_files_deleted = 0
num_thumbnails_deleted = 0
pauser = HydrusData.BigJobPauser()
(file_hash, thumbnail_hash) = self.Read('deferred_physical_delete')
while (file_hash is not None
or thumbnail_hash is not None) and not HG.view_shutdown:
if file_hash is not None:
path = ServerFiles.GetExpectedFilePath(file_hash)
if os.path.exists(path):
HydrusPaths.RecyclePath(path)
num_files_deleted += 1
if thumbnail_hash is not None:
path = ServerFiles.GetExpectedThumbnailPath(thumbnail_hash)
if os.path.exists(path):
HydrusPaths.RecyclePath(path)
num_thumbnails_deleted += 1
self.WriteSynchronous('clear_deferred_physical_delete',
file_hash=file_hash,
thumbnail_hash=thumbnail_hash)
(file_hash, thumbnail_hash) = self.Read('deferred_physical_delete')
pauser.Pause()
if num_files_deleted > 0 or num_thumbnails_deleted > 0:
HydrusData.Print(
'Physically deleted {} files and {} thumbnails from file storage.'
.format(HydrusData.ToHumanInt(num_files_deleted),
HydrusData.ToHumanInt(num_files_deleted)))
def UpdateConf(self):
mpv_config_path = HG.client_controller.GetMPVConfPath()
if not os.path.exists(mpv_config_path):
default_mpv_config_path = HG.client_controller.GetDefaultMPVConfPath(
)
if not os.path.exists(default_mpv_config_path):
HydrusData.ShowText(
'There is no default mpv configuration file to load! Perhaps there is a problem with your install?'
)
return
else:
HydrusPaths.MirrorFile(default_mpv_config_path,
mpv_config_path)
#To load an existing config file (by default it doesn't load the user/global config like standalone mpv does):
load_f = getattr(mpv, '_mpv_load_config_file', None)
if load_f is not None and callable(load_f):
try:
load_f(self._player.handle, mpv_config_path.encode('utf-8')) # pylint: disable=E1102
except Exception as e:
HydrusData.ShowText(
'MPV could not load its configuration file! This was probably due to an invalid parameter value inside the conf. The error follows:'
)
HydrusData.ShowException(e)
else:
HydrusData.Print(
'Was unable to load mpv.conf--has the MPV API changed?')