def DAEMONSynchroniseRepositories( controller ):
if not controller.options[ 'pause_repo_sync' ]:
services = controller.services_manager.GetServices( HC.REPOSITORIES )
for service in services:
if HydrusThreading.IsThreadShuttingDown():
return
if controller.options[ 'pause_repo_sync' ]:
return
service.Sync( only_process_when_idle = True )
if HydrusThreading.IsThreadShuttingDown():
return
time.sleep( 3 )
def InitView(self):
if not self._no_daemons:
self._daemons.append(
HydrusThreading.DAEMONWorker(self,
'SleepCheck',
HydrusDaemons.DAEMONSleepCheck,
period=120))
self._daemons.append(
HydrusThreading.DAEMONWorker(
self,
'MaintainMemoryFast',
HydrusDaemons.DAEMONMaintainMemoryFast,
period=60))
self._daemons.append(
HydrusThreading.DAEMONWorker(
self,
'MaintainMemorySlow',
HydrusDaemons.DAEMONMaintainMemorySlow,
period=300))
self._daemons.append(
HydrusThreading.DAEMONBackgroundWorker(
self,
'MaintainDB',
HydrusDaemons.DAEMONMaintainDB,
period=300,
init_wait=60))
def wait_for_free_slot( controller, subs_jobs, max_simultaneous_subscriptions ):
time.sleep( 0.1 )
while True:
p1 = controller.options[ 'pause_subs_sync' ]
p2 = HydrusThreading.IsThreadShuttingDown()
if p1 or p2:
if HG.subscription_report_mode:
HydrusData.ShowText( 'Subscriptions cancelling. Global sub pause is ' + str( p1 ) + ' and sub daemon thread shutdown status is ' + str( p2 ) + '.' )
if p2:
for ( thread, job ) in subs_jobs:
HydrusThreading.ShutdownThread( thread )
raise HydrusExceptions.CancelledException( 'subs cancelling or thread shutting down' )
filter_finished_jobs( subs_jobs )
if len( subs_jobs ) < max_simultaneous_subscriptions:
return
time.sleep( 1.0 )
def DAEMONMaintainTrash( controller ):
if HC.options[ 'trash_max_size' ] is not None:
max_size = HC.options[ 'trash_max_size' ] * 1048576
service_info = controller.Read( 'service_info', CC.TRASH_SERVICE_KEY )
while service_info[ HC.SERVICE_INFO_TOTAL_SIZE ] > max_size:
if HydrusThreading.IsThreadShuttingDown():
return
hashes = controller.Read( 'trash_hashes', limit = 10 )
if len( hashes ) == 0:
return
content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, hashes )
service_keys_to_content_updates = { CC.TRASH_SERVICE_KEY : [ content_update ] }
controller.WaitUntilModelFree()
controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
service_info = controller.Read( 'service_info', CC.TRASH_SERVICE_KEY )
time.sleep( 2 )
if HC.options[ 'trash_max_age' ] is not None:
max_age = HC.options[ 'trash_max_age' ] * 3600
hashes = controller.Read( 'trash_hashes', limit = 10, minimum_age = max_age )
while len( hashes ) > 0:
if HydrusThreading.IsThreadShuttingDown():
return
content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, hashes )
service_keys_to_content_updates = { CC.TRASH_SERVICE_KEY : [ content_update ] }
controller.WaitUntilModelFree()
controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
hashes = controller.Read( 'trash_hashes', limit = 10, minimum_age = max_age )
time.sleep( 2 )
def InitView( self ):
if not self._no_daemons:
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SleepCheck', HydrusDaemons.DAEMONSleepCheck, period = 120 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainMemory', HydrusDaemons.DAEMONMaintainMemory, period = 300 ) )
self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300 ) )
def _CheckCancelTests(self):
if not self._cancelled.is_set():
should_cancel = False
if self._cancel_on_shutdown and HydrusThreading.IsThreadShuttingDown(
):
should_cancel = True
if self._only_when_idle and not HG.client_controller.CurrentlyIdle(
):
should_cancel = True
if self._stop_time is not None:
if HydrusData.TimeHasPassed(self._stop_time):
should_cancel = True
if should_cancel:
self.Cancel()
if not self._deleted.is_set():
if self._deletion_time is not None:
if HydrusData.TimeHasPassed(self._deletion_time):
self.Finish()
self._deleted.set()
def __init__(self, media, target_resolution=None):
RasterContainer.__init__(self, media, target_resolution)
self._hydrus_bitmap = None
HydrusThreading.CallToThread(self.THREADRender)
def DAEMONCheckImportFolders(controller):
if not controller.options['pause_import_folders_sync']:
HG.import_folders_running = True
import_folder_names = controller.Read(
'serialisable_names',
HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER)
try:
for name in import_folder_names:
import_folder = controller.Read(
'serialisable_named',
HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER, name)
if controller.options[
'pause_import_folders_sync'] or HydrusThreading.IsThreadShuttingDown(
):
break
import_folder.DoWork()
finally:
HG.import_folders_running = False
def _GetCallToThread(self):
with self._call_to_thread_lock:
for call_to_thread in self._call_to_threads:
if not call_to_thread.CurrentlyWorking():
return call_to_thread
# all the threads in the pool are currently busy
calling_from_the_thread_pool = threading.current_thread(
) in self._call_to_threads
if calling_from_the_thread_pool or len(
self._call_to_threads) < 200:
call_to_thread = HydrusThreading.THREADCallToThread(
self, 'CallToThread')
self._call_to_threads.append(call_to_thread)
call_to_thread.start()
else:
call_to_thread = random.choice(self._call_to_threads)
return call_to_thread
def InitModel(self):
try:
self._InitTempDir()
except:
HydrusData.Print('Failed to initialise temp folder.')
self._fast_job_scheduler = HydrusThreading.JobScheduler(self)
self._slow_job_scheduler = HydrusThreading.JobScheduler(self)
self._fast_job_scheduler.start()
self._slow_job_scheduler.start()
self.db = self._InitDB()
def StartDaemons(self):
HydrusThreading.DAEMONQueue('FlushRequestsMade',
ServerDB.DAEMONFlushRequestsMade,
'request_made',
period=60)
HydrusThreading.DAEMONWorker('CheckMonthlyData',
ServerDB.DAEMONCheckMonthlyData,
period=3600)
HydrusThreading.DAEMONWorker('ClearBans',
ServerDB.DAEMONClearBans,
period=3600)
HydrusThreading.DAEMONWorker('DeleteOrphans',
ServerDB.DAEMONDeleteOrphans,
period=86400)
HydrusThreading.DAEMONWorker('GenerateUpdates',
ServerDB.DAEMONGenerateUpdates,
period=1200)
HydrusThreading.DAEMONWorker('CheckDataUsage',
ServerDB.DAEMONCheckDataUsage,
period=86400)
HydrusThreading.DAEMONWorker('UPnP',
ServerDB.DAEMONUPnP,
('notify_new_options', ),
period=43200)
def InitModel(self):
self.temp_dir = HydrusPaths.GetTempDir()
self._job_scheduler = HydrusThreading.JobScheduler(self)
self._job_scheduler.start()
self.db = self._InitDB()
def CallLater( self, delay, func, *args, **kwargs ):
call = HydrusData.Call( func, *args, **kwargs )
job = HydrusThreading.SchedulableJob( self, self._job_scheduler, call, initial_delay = delay )
self._job_scheduler.AddJob( job )
return job
def _RENDERERSetRenderToPosition( self, index ):
with self._render_lock:
if self._render_to_index != index:
self._render_to_index = index
HydrusThreading.CallToThread( self.THREADRender )
def InitView( self ):
if not self._no_daemons:
self._daemons.append( HydrusThreading.DAEMONBackgroundWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300, init_wait = 60 ) )
self.CallRepeating( 10.0, 120.0, self.SleepCheck )
self.CallRepeating( 10.0, 60.0, self.MaintainMemoryFast )
self.CallRepeating( 10.0, 300.0, self.MaintainMemorySlow )
def InitView( self ):
HydrusController.HydrusController.InitView( self )
if not self._no_daemons:
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'DeleteOrphans', ServerDaemons.DAEMONDeleteOrphans, period = 86400 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'GenerateUpdates', ServerDaemons.DAEMONGenerateUpdates, period = 600, init_wait = 10 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SaveDirtyObjects', ServerDaemons.DAEMONSaveDirtyObjects, period = 30 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'UPnP', ServerDaemons.DAEMONUPnP, ( 'notify_new_options', ), period = 43200 ) )
#
self._services = self.Read( 'services' )
[ self._admin_service ] = [ service for service in self._services if service.GetServiceType() == HC.SERVER_ADMIN ]
port = self._admin_service.GetPort()
already_bound = False
try:
connection = HydrusNetworking.GetLocalConnection( port )
connection.close()
already_bound = True
except:
pass
if already_bound:
HydrusData.Print( 'Something is already bound to port ' + str( port ) + ', so your administration service cannot be started. Please quit the server and retry once the port is clear.' )
else:
for service in self._services:
self.StartService( service )
def DAEMONSynchroniseAccounts(controller):
services = controller.services_manager.GetServices(HC.RESTRICTED_SERVICES)
for service in services:
if HydrusThreading.IsThreadShuttingDown():
return
service.SyncAccount()
def CallRepeating( self, initial_delay, period, func, *args, **kwargs ):
job_scheduler = self._GetAppropriateJobScheduler( period )
call = HydrusData.Call( func, *args, **kwargs )
job = HydrusThreading.RepeatingJob( self, job_scheduler, initial_delay, period, call )
job_scheduler.AddJob( job )
return job
def CallLater( self, initial_delay, func, *args, **kwargs ):
job_scheduler = self._GetAppropriateJobScheduler( initial_delay )
call = HydrusData.Call( func, *args, **kwargs )
job = HydrusThreading.SchedulableJob( self, job_scheduler, initial_delay, call )
job_scheduler.AddJob( job )
return job
def InitView(self):
HydrusController.HydrusController.InitView(self)
if not self._no_daemons:
self._daemons.append(
HydrusThreading.DAEMONQueue(
self,
'FlushRequestsMade',
ServerDaemons.DAEMONFlushRequestsMade,
'request_made',
period=60))
self._daemons.append(
HydrusThreading.DAEMONWorker(
self,
'CheckMonthlyData',
ServerDaemons.DAEMONCheckMonthlyData,
period=3600))
self._daemons.append(
HydrusThreading.DAEMONWorker(self,
'ClearBans',
ServerDaemons.DAEMONClearBans,
period=3600))
self._daemons.append(
HydrusThreading.DAEMONWorker(self,
'DeleteOrphans',
ServerDaemons.DAEMONDeleteOrphans,
period=86400))
self._daemons.append(
HydrusThreading.DAEMONWorker(
self,
'GenerateUpdates',
ServerDaemons.DAEMONGenerateUpdates,
period=600))
self._daemons.append(
HydrusThreading.DAEMONWorker(
self,
'CheckDataUsage',
ServerDaemons.DAEMONCheckDataUsage,
period=86400))
self._daemons.append(
HydrusThreading.DAEMONWorker(self,
'UPnP',
ServerDaemons.DAEMONUPnP,
('notify_new_options', ),
period=43200))
self.CheckIfAdminPortInUse()
service_keys = self.Read('service_keys')
for service_key in service_keys:
self.ActionService(service_key, 'start')
def CallRepeating(self, period, delay, func, *args, **kwargs):
call = HydrusData.Call(func, *args, **kwargs)
job = HydrusThreading.RepeatingJob(self,
self._job_scheduler,
call,
period,
initial_delay=delay)
self._job_scheduler.AddJob(job)
return job
def FilterFreePaths(paths):
free_paths = []
for path in paths:
if HydrusThreading.IsThreadShuttingDown():
raise HydrusExceptions.ShutdownException()
if PathIsFree(path):
free_paths.append(path)
return free_paths
def _GetCallToThreadLongRunning(self):
for call_to_thread in self._long_running_call_to_threads:
if not call_to_thread.CurrentlyWorking():
return call_to_thread
call_to_thread = HydrusThreading.THREADCallToThread(self)
self._long_running_call_to_threads.append(call_to_thread)
call_to_thread.start()
return call_to_thread
def _GetCallToThread(self):
for call_to_thread in self._call_to_threads:
if not call_to_thread.CurrentlyWorking():
return call_to_thread
if len(self._call_to_threads) > 100:
raise Exception('Too many call to threads!')
call_to_thread = HydrusThreading.THREADCallToThread(self)
self._call_to_threads.append(call_to_thread)
call_to_thread.start()
return call_to_thread
def _GetCallToThread(self):
for call_to_thread in self._call_to_threads:
if not call_to_thread.CurrentlyWorking():
return call_to_thread
if len(self._call_to_threads) < 10:
call_to_thread = HydrusThreading.THREADCallToThread(self)
self._call_to_threads.append(call_to_thread)
call_to_thread.start()
else:
call_to_thread = random.choice(self._call_to_threads)
return call_to_thread
def __init__(self):
HydrusGlobals.controller = self
self._no_daemons = False
self._no_wal = False
self._InitArgsBools()
self._no_wal_path = os.path.join(HC.DB_DIR, 'no-wal')
if os.path.exists(self._no_wal_path):
self._no_wal = True
self._model_shutdown = False
self._view_shutdown = False
self._db = None
self._pubsub = HydrusPubSub.HydrusPubSub(
self, self.pubsub_binding_errors_to_ignore)
self._currently_doing_pubsub = False
self._daemons = []
self._caches = {}
self._managers = {}
self._call_to_threads = [
HydrusThreading.THREADCallToThread(self) for i in range(10)
]
self._timestamps = collections.defaultdict(lambda: 0)
self._timestamps['boot'] = HydrusData.GetNow()
self._just_woke_from_sleep = False
self._system_busy = False
def FilterFreePaths( paths ):
free_paths = []
for path in paths:
if HydrusThreading.IsThreadShuttingDown():
raise HydrusExceptions.ShutdownException()
try:
os.rename( path, path ) # rename a path to itself
free_paths.append( path )
except OSError as e: # 'already in use by another process'
HydrusData.Print( 'Already in use: ' + path )
return free_paths
def DAEMONSynchroniseSubscriptions(controller):
subscription_names = list(
controller.Read('serialisable_names',
HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION))
if controller.new_options.GetBoolean('process_subs_in_random_order'):
random.shuffle(subscription_names)
else:
subscription_names.sort()
HG.subscriptions_running = True
try:
for name in subscription_names:
p1 = controller.options['pause_subs_sync']
p2 = HydrusThreading.IsThreadShuttingDown()
if p1 or p2:
return
subscription = controller.Read(
'serialisable_named',
HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION, name)
subscription.Sync()
finally:
HG.subscriptions_running = False
def InitView( self ):
if self.options[ 'password' ] is not None:
self.pub( 'splash_set_status_text', 'waiting for password' )
def wx_code_password():
while True:
with wx.PasswordEntryDialog( self._splash, 'Enter your password', 'Enter password' ) as dlg:
if dlg.ShowModal() == wx.ID_OK:
# this can produce unicode with cyrillic or w/e keyboards, which hashlib can't handle
password = HydrusData.ToByteString( dlg.GetValue() )
if hashlib.sha256( password ).digest() == self.options[ 'password' ]: break
else:
raise HydrusExceptions.PermissionException( 'Bad password check' )
self.CallBlockingToWx( wx_code_password )
self.pub( 'splash_set_title_text', u'booting gui\u2026' )
def wx_code_gui():
self.gui = ClientGUI.FrameGUI( self )
self.ResetIdleTimer()
self.CallBlockingToWx( wx_code_gui )
# ShowText will now popup as a message, as popup message manager has overwritten the hooks
HydrusController.HydrusController.InitView( self )
self._booru_port_connection = None
self.RestartBooru()
if not self._no_daemons:
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckMouseIdle', ClientDaemons.DAEMONCheckMouseIdle, period = 10 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SynchroniseAccounts', ClientDaemons.DAEMONSynchroniseAccounts, ( 'notify_unknown_accounts', ) ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SaveDirtyObjects', ClientDaemons.DAEMONSaveDirtyObjects, ( 'important_dirt_to_clean', ), period = 30 ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'DownloadFiles', ClientDaemons.DAEMONDownloadFiles, ( 'notify_new_downloads', 'notify_new_permissions' ) ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'SynchroniseSubscriptions', ClientDaemons.DAEMONSynchroniseSubscriptions, ( 'notify_restart_subs_sync_daemon', 'notify_new_subscriptions' ), period = 14400, init_wait = 60, pre_call_wait = 3 ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'CheckImportFolders', ClientDaemons.DAEMONCheckImportFolders, ( 'notify_restart_import_folders_daemon', 'notify_new_import_folders' ), period = 180 ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'CheckExportFolders', ClientDaemons.DAEMONCheckExportFolders, ( 'notify_restart_export_folders_daemon', 'notify_new_export_folders' ), period = 180 ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'MaintainTrash', ClientDaemons.DAEMONMaintainTrash, init_wait = 120 ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'SynchroniseRepositories', ClientDaemons.DAEMONSynchroniseRepositories, ( 'notify_restart_repo_sync_daemon', 'notify_new_permissions' ), period = 4 * 3600, pre_call_wait = 1 ) )
self._daemons.append( HydrusThreading.DAEMONBackgroundWorker( self, 'UPnP', ClientDaemons.DAEMONUPnP, ( 'notify_new_upnp_mappings', ), init_wait = 120, pre_call_wait = 6 ) )
if self.db.IsFirstStart():
message = 'Hi, this looks like the first time you have started the hydrus client.'
message += os.linesep * 2
message += 'Don\'t forget to check out the help if you haven\'t already.'
message += os.linesep * 2
message += 'To dismiss popup messages like this, right-click them.'
HydrusData.ShowText( message )
if self.db.IsDBUpdated():
HydrusData.ShowText( 'The client has updated to version ' + str( HC.SOFTWARE_VERSION ) + '!' )
for message in self.db.GetInitialMessages():
HydrusData.ShowText( message )
def DAEMONDownloadFiles(controller):
hashes = controller.Read('downloads')
num_downloads = len(hashes)
if num_downloads > 0:
client_files_manager = controller.client_files_manager
successful_hashes = set()
job_key = ClientThreading.JobKey()
job_key.SetVariable('popup_text_1', 'initialising downloader')
controller.pub('message', job_key)
for hash in hashes:
job_key.SetVariable(
'popup_text_1', 'downloading ' +
HydrusData.ConvertIntToPrettyString(num_downloads -
len(successful_hashes)) +
' files from repositories')
(media_result, ) = controller.Read('media_results', (hash, ))
service_keys = list(
media_result.GetLocationsManager().GetCurrent())
random.shuffle(service_keys)
for service_key in service_keys:
if service_key == CC.LOCAL_FILE_SERVICE_KEY: break
elif service_key == CC.TRASH_SERVICE_KEY: continue
try:
service = controller.services_manager.GetService(
service_key)
except:
continue
if service.GetServiceType() == HC.FILE_REPOSITORY:
file_repository = service
if file_repository.IsFunctional():
try:
(os_file_handle,
temp_path) = HydrusPaths.GetTempPath()
try:
file_repository.Request(HC.GET,
'file', {'hash': hash},
temp_path=temp_path)
controller.WaitUntilModelFree()
automatic_archive = False
exclude_deleted = False # this is the important part here
min_size = None
min_resolution = None
file_import_options = ClientImporting.FileImportOptions(
automatic_archive=automatic_archive,
exclude_deleted=exclude_deleted,
min_size=min_size,
min_resolution=min_resolution)
file_import_job = ClientImporting.FileImportJob(
temp_path, file_import_options)
client_files_manager.ImportFile(
file_import_job)
successful_hashes.add(hash)
break
finally:
HydrusPaths.CleanUpTempPath(
os_file_handle, temp_path)
except HydrusExceptions.ServerBusyException:
job_key.SetVariable(
'popup_text_1',
file_repository.GetName() +
' was busy. waiting 30s before trying again')
time.sleep(30)
job_key.Delete()
controller.pub('notify_new_downloads')
return
except Exception as e:
HydrusData.ShowText('Error downloading file!')
HydrusData.ShowException(e)
elif service.GetServiceType() == HC.IPFS:
multihashes = HG.client_controller.Read(
'service_filenames', service_key, {hash})
if len(multihashes) > 0:
multihash = multihashes[0]
# this actually calls to a thread that can launch gui 'select from tree' stuff, so let's just break at this point
service.ImportFile(multihash)
break
if HydrusThreading.IsThreadShuttingDown():
return
if len(successful_hashes) > 0:
job_key.SetVariable(
'popup_text_1',
HydrusData.ConvertIntToPrettyString(len(successful_hashes)) +
' files downloaded')
job_key.Delete()