def DAEMONSynchroniseRepositories( controller ): if not controller.options[ 'pause_repo_sync' ]: services = controller.services_manager.GetServices( HC.REPOSITORIES ) for service in services: if HydrusThreading.IsThreadShuttingDown(): return if controller.options[ 'pause_repo_sync' ]: return service.Sync( only_process_when_idle = True ) if HydrusThreading.IsThreadShuttingDown(): return time.sleep( 3 )
def InitView(self): if not self._no_daemons: self._daemons.append( HydrusThreading.DAEMONWorker(self, 'SleepCheck', HydrusDaemons.DAEMONSleepCheck, period=120)) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainMemoryFast', HydrusDaemons.DAEMONMaintainMemoryFast, period=60)) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainMemorySlow', HydrusDaemons.DAEMONMaintainMemorySlow, period=300)) self._daemons.append( HydrusThreading.DAEMONBackgroundWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period=300, init_wait=60))
def wait_for_free_slot( controller, subs_jobs, max_simultaneous_subscriptions ): time.sleep( 0.1 ) while True: p1 = controller.options[ 'pause_subs_sync' ] p2 = HydrusThreading.IsThreadShuttingDown() if p1 or p2: if HG.subscription_report_mode: HydrusData.ShowText( 'Subscriptions cancelling. Global sub pause is ' + str( p1 ) + ' and sub daemon thread shutdown status is ' + str( p2 ) + '.' ) if p2: for ( thread, job ) in subs_jobs: HydrusThreading.ShutdownThread( thread ) raise HydrusExceptions.CancelledException( 'subs cancelling or thread shutting down' ) filter_finished_jobs( subs_jobs ) if len( subs_jobs ) < max_simultaneous_subscriptions: return time.sleep( 1.0 )
def DAEMONMaintainTrash( controller ): if HC.options[ 'trash_max_size' ] is not None: max_size = HC.options[ 'trash_max_size' ] * 1048576 service_info = controller.Read( 'service_info', CC.TRASH_SERVICE_KEY ) while service_info[ HC.SERVICE_INFO_TOTAL_SIZE ] > max_size: if HydrusThreading.IsThreadShuttingDown(): return hashes = controller.Read( 'trash_hashes', limit = 10 ) if len( hashes ) == 0: return content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, hashes ) service_keys_to_content_updates = { CC.TRASH_SERVICE_KEY : [ content_update ] } controller.WaitUntilModelFree() controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates ) service_info = controller.Read( 'service_info', CC.TRASH_SERVICE_KEY ) time.sleep( 2 ) if HC.options[ 'trash_max_age' ] is not None: max_age = HC.options[ 'trash_max_age' ] * 3600 hashes = controller.Read( 'trash_hashes', limit = 10, minimum_age = max_age ) while len( hashes ) > 0: if HydrusThreading.IsThreadShuttingDown(): return content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, hashes ) service_keys_to_content_updates = { CC.TRASH_SERVICE_KEY : [ content_update ] } controller.WaitUntilModelFree() controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates ) hashes = controller.Read( 'trash_hashes', limit = 10, minimum_age = max_age ) time.sleep( 2 )
def InitView( self ): if not self._no_daemons: self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SleepCheck', HydrusDaemons.DAEMONSleepCheck, period = 120 ) ) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainMemory', HydrusDaemons.DAEMONMaintainMemory, period = 300 ) ) self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300 ) )
def _CheckCancelTests(self): if not self._cancelled.is_set(): should_cancel = False if self._cancel_on_shutdown and HydrusThreading.IsThreadShuttingDown( ): should_cancel = True if self._only_when_idle and not HG.client_controller.CurrentlyIdle( ): should_cancel = True if self._stop_time is not None: if HydrusData.TimeHasPassed(self._stop_time): should_cancel = True if should_cancel: self.Cancel() if not self._deleted.is_set(): if self._deletion_time is not None: if HydrusData.TimeHasPassed(self._deletion_time): self.Finish() self._deleted.set()
def __init__(self, media, target_resolution=None): RasterContainer.__init__(self, media, target_resolution) self._hydrus_bitmap = None HydrusThreading.CallToThread(self.THREADRender)
def DAEMONCheckImportFolders(controller): if not controller.options['pause_import_folders_sync']: HG.import_folders_running = True import_folder_names = controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER) try: for name in import_folder_names: import_folder = controller.Read( 'serialisable_named', HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER, name) if controller.options[ 'pause_import_folders_sync'] or HydrusThreading.IsThreadShuttingDown( ): break import_folder.DoWork() finally: HG.import_folders_running = False
def _GetCallToThread(self): with self._call_to_thread_lock: for call_to_thread in self._call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread # all the threads in the pool are currently busy calling_from_the_thread_pool = threading.current_thread( ) in self._call_to_threads if calling_from_the_thread_pool or len( self._call_to_threads) < 200: call_to_thread = HydrusThreading.THREADCallToThread( self, 'CallToThread') self._call_to_threads.append(call_to_thread) call_to_thread.start() else: call_to_thread = random.choice(self._call_to_threads) return call_to_thread
def InitModel(self): try: self._InitTempDir() except: HydrusData.Print('Failed to initialise temp folder.') self._fast_job_scheduler = HydrusThreading.JobScheduler(self) self._slow_job_scheduler = HydrusThreading.JobScheduler(self) self._fast_job_scheduler.start() self._slow_job_scheduler.start() self.db = self._InitDB()
def StartDaemons(self): HydrusThreading.DAEMONQueue('FlushRequestsMade', ServerDB.DAEMONFlushRequestsMade, 'request_made', period=60) HydrusThreading.DAEMONWorker('CheckMonthlyData', ServerDB.DAEMONCheckMonthlyData, period=3600) HydrusThreading.DAEMONWorker('ClearBans', ServerDB.DAEMONClearBans, period=3600) HydrusThreading.DAEMONWorker('DeleteOrphans', ServerDB.DAEMONDeleteOrphans, period=86400) HydrusThreading.DAEMONWorker('GenerateUpdates', ServerDB.DAEMONGenerateUpdates, period=1200) HydrusThreading.DAEMONWorker('CheckDataUsage', ServerDB.DAEMONCheckDataUsage, period=86400) HydrusThreading.DAEMONWorker('UPnP', ServerDB.DAEMONUPnP, ('notify_new_options', ), period=43200)
def InitModel(self): self.temp_dir = HydrusPaths.GetTempDir() self._job_scheduler = HydrusThreading.JobScheduler(self) self._job_scheduler.start() self.db = self._InitDB()
def CallLater( self, delay, func, *args, **kwargs ): call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.SchedulableJob( self, self._job_scheduler, call, initial_delay = delay ) self._job_scheduler.AddJob( job ) return job
def _RENDERERSetRenderToPosition( self, index ): with self._render_lock: if self._render_to_index != index: self._render_to_index = index HydrusThreading.CallToThread( self.THREADRender )
def InitView( self ): if not self._no_daemons: self._daemons.append( HydrusThreading.DAEMONBackgroundWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300, init_wait = 60 ) ) self.CallRepeating( 10.0, 120.0, self.SleepCheck ) self.CallRepeating( 10.0, 60.0, self.MaintainMemoryFast ) self.CallRepeating( 10.0, 300.0, self.MaintainMemorySlow )
def InitView( self ): HydrusController.HydrusController.InitView( self ) if not self._no_daemons: self._daemons.append( HydrusThreading.DAEMONWorker( self, 'DeleteOrphans', ServerDaemons.DAEMONDeleteOrphans, period = 86400 ) ) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'GenerateUpdates', ServerDaemons.DAEMONGenerateUpdates, period = 600, init_wait = 10 ) ) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SaveDirtyObjects', ServerDaemons.DAEMONSaveDirtyObjects, period = 30 ) ) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'UPnP', ServerDaemons.DAEMONUPnP, ( 'notify_new_options', ), period = 43200 ) ) # self._services = self.Read( 'services' ) [ self._admin_service ] = [ service for service in self._services if service.GetServiceType() == HC.SERVER_ADMIN ] port = self._admin_service.GetPort() already_bound = False try: connection = HydrusNetworking.GetLocalConnection( port ) connection.close() already_bound = True except: pass if already_bound: HydrusData.Print( 'Something is already bound to port ' + str( port ) + ', so your administration service cannot be started. Please quit the server and retry once the port is clear.' ) else: for service in self._services: self.StartService( service )
def DAEMONSynchroniseAccounts(controller): services = controller.services_manager.GetServices(HC.RESTRICTED_SERVICES) for service in services: if HydrusThreading.IsThreadShuttingDown(): return service.SyncAccount()
def CallRepeating( self, initial_delay, period, func, *args, **kwargs ): job_scheduler = self._GetAppropriateJobScheduler( period ) call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.RepeatingJob( self, job_scheduler, initial_delay, period, call ) job_scheduler.AddJob( job ) return job
def CallLater( self, initial_delay, func, *args, **kwargs ): job_scheduler = self._GetAppropriateJobScheduler( initial_delay ) call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.SchedulableJob( self, job_scheduler, initial_delay, call ) job_scheduler.AddJob( job ) return job
def InitView(self): HydrusController.HydrusController.InitView(self) if not self._no_daemons: self._daemons.append( HydrusThreading.DAEMONQueue( self, 'FlushRequestsMade', ServerDaemons.DAEMONFlushRequestsMade, 'request_made', period=60)) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckMonthlyData', ServerDaemons.DAEMONCheckMonthlyData, period=3600)) self._daemons.append( HydrusThreading.DAEMONWorker(self, 'ClearBans', ServerDaemons.DAEMONClearBans, period=3600)) self._daemons.append( HydrusThreading.DAEMONWorker(self, 'DeleteOrphans', ServerDaemons.DAEMONDeleteOrphans, period=86400)) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'GenerateUpdates', ServerDaemons.DAEMONGenerateUpdates, period=600)) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckDataUsage', ServerDaemons.DAEMONCheckDataUsage, period=86400)) self._daemons.append( HydrusThreading.DAEMONWorker(self, 'UPnP', ServerDaemons.DAEMONUPnP, ('notify_new_options', ), period=43200)) self.CheckIfAdminPortInUse() service_keys = self.Read('service_keys') for service_key in service_keys: self.ActionService(service_key, 'start')
def CallRepeating(self, period, delay, func, *args, **kwargs): call = HydrusData.Call(func, *args, **kwargs) job = HydrusThreading.RepeatingJob(self, self._job_scheduler, call, period, initial_delay=delay) self._job_scheduler.AddJob(job) return job
def FilterFreePaths(paths): free_paths = [] for path in paths: if HydrusThreading.IsThreadShuttingDown(): raise HydrusExceptions.ShutdownException() if PathIsFree(path): free_paths.append(path) return free_paths
def _GetCallToThreadLongRunning(self): for call_to_thread in self._long_running_call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread call_to_thread = HydrusThreading.THREADCallToThread(self) self._long_running_call_to_threads.append(call_to_thread) call_to_thread.start() return call_to_thread
def _GetCallToThread(self): for call_to_thread in self._call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread if len(self._call_to_threads) > 100: raise Exception('Too many call to threads!') call_to_thread = HydrusThreading.THREADCallToThread(self) self._call_to_threads.append(call_to_thread) call_to_thread.start() return call_to_thread
def _GetCallToThread(self): for call_to_thread in self._call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread if len(self._call_to_threads) < 10: call_to_thread = HydrusThreading.THREADCallToThread(self) self._call_to_threads.append(call_to_thread) call_to_thread.start() else: call_to_thread = random.choice(self._call_to_threads) return call_to_thread
def __init__(self): HydrusGlobals.controller = self self._no_daemons = False self._no_wal = False self._InitArgsBools() self._no_wal_path = os.path.join(HC.DB_DIR, 'no-wal') if os.path.exists(self._no_wal_path): self._no_wal = True self._model_shutdown = False self._view_shutdown = False self._db = None self._pubsub = HydrusPubSub.HydrusPubSub( self, self.pubsub_binding_errors_to_ignore) self._currently_doing_pubsub = False self._daemons = [] self._caches = {} self._managers = {} self._call_to_threads = [ HydrusThreading.THREADCallToThread(self) for i in range(10) ] self._timestamps = collections.defaultdict(lambda: 0) self._timestamps['boot'] = HydrusData.GetNow() self._just_woke_from_sleep = False self._system_busy = False
def FilterFreePaths( paths ): free_paths = [] for path in paths: if HydrusThreading.IsThreadShuttingDown(): raise HydrusExceptions.ShutdownException() try: os.rename( path, path ) # rename a path to itself free_paths.append( path ) except OSError as e: # 'already in use by another process' HydrusData.Print( 'Already in use: ' + path ) return free_paths
def DAEMONSynchroniseSubscriptions(controller): subscription_names = list( controller.Read('serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION)) if controller.new_options.GetBoolean('process_subs_in_random_order'): random.shuffle(subscription_names) else: subscription_names.sort() HG.subscriptions_running = True try: for name in subscription_names: p1 = controller.options['pause_subs_sync'] p2 = HydrusThreading.IsThreadShuttingDown() if p1 or p2: return subscription = controller.Read( 'serialisable_named', HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION, name) subscription.Sync() finally: HG.subscriptions_running = False
def InitView( self ): if self.options[ 'password' ] is not None: self.pub( 'splash_set_status_text', 'waiting for password' ) def wx_code_password(): while True: with wx.PasswordEntryDialog( self._splash, 'Enter your password', 'Enter password' ) as dlg: if dlg.ShowModal() == wx.ID_OK: # this can produce unicode with cyrillic or w/e keyboards, which hashlib can't handle password = HydrusData.ToByteString( dlg.GetValue() ) if hashlib.sha256( password ).digest() == self.options[ 'password' ]: break else: raise HydrusExceptions.PermissionException( 'Bad password check' ) self.CallBlockingToWx( wx_code_password ) self.pub( 'splash_set_title_text', u'booting gui\u2026' ) def wx_code_gui(): self.gui = ClientGUI.FrameGUI( self ) self.ResetIdleTimer() self.CallBlockingToWx( wx_code_gui ) # ShowText will now popup as a message, as popup message manager has overwritten the hooks HydrusController.HydrusController.InitView( self ) self._booru_port_connection = None self.RestartBooru() if not self._no_daemons: self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckMouseIdle', ClientDaemons.DAEMONCheckMouseIdle, period = 10 ) ) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SynchroniseAccounts', ClientDaemons.DAEMONSynchroniseAccounts, ( 'notify_unknown_accounts', ) ) ) self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SaveDirtyObjects', ClientDaemons.DAEMONSaveDirtyObjects, ( 'important_dirt_to_clean', ), period = 30 ) ) self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'DownloadFiles', ClientDaemons.DAEMONDownloadFiles, ( 'notify_new_downloads', 'notify_new_permissions' ) ) ) self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'SynchroniseSubscriptions', ClientDaemons.DAEMONSynchroniseSubscriptions, ( 'notify_restart_subs_sync_daemon', 'notify_new_subscriptions' ), period = 14400, init_wait = 60, pre_call_wait = 3 ) ) self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'CheckImportFolders', ClientDaemons.DAEMONCheckImportFolders, ( 'notify_restart_import_folders_daemon', 'notify_new_import_folders' ), period = 180 ) ) self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'CheckExportFolders', ClientDaemons.DAEMONCheckExportFolders, ( 'notify_restart_export_folders_daemon', 'notify_new_export_folders' ), period = 180 ) ) self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'MaintainTrash', ClientDaemons.DAEMONMaintainTrash, init_wait = 120 ) ) self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'SynchroniseRepositories', ClientDaemons.DAEMONSynchroniseRepositories, ( 'notify_restart_repo_sync_daemon', 'notify_new_permissions' ), period = 4 * 3600, pre_call_wait = 1 ) ) self._daemons.append( HydrusThreading.DAEMONBackgroundWorker( self, 'UPnP', ClientDaemons.DAEMONUPnP, ( 'notify_new_upnp_mappings', ), init_wait = 120, pre_call_wait = 6 ) ) if self.db.IsFirstStart(): message = 'Hi, this looks like the first time you have started the hydrus client.' message += os.linesep * 2 message += 'Don\'t forget to check out the help if you haven\'t already.' message += os.linesep * 2 message += 'To dismiss popup messages like this, right-click them.' HydrusData.ShowText( message ) if self.db.IsDBUpdated(): HydrusData.ShowText( 'The client has updated to version ' + str( HC.SOFTWARE_VERSION ) + '!' ) for message in self.db.GetInitialMessages(): HydrusData.ShowText( message )
def DAEMONDownloadFiles(controller): hashes = controller.Read('downloads') num_downloads = len(hashes) if num_downloads > 0: client_files_manager = controller.client_files_manager successful_hashes = set() job_key = ClientThreading.JobKey() job_key.SetVariable('popup_text_1', 'initialising downloader') controller.pub('message', job_key) for hash in hashes: job_key.SetVariable( 'popup_text_1', 'downloading ' + HydrusData.ConvertIntToPrettyString(num_downloads - len(successful_hashes)) + ' files from repositories') (media_result, ) = controller.Read('media_results', (hash, )) service_keys = list( media_result.GetLocationsManager().GetCurrent()) random.shuffle(service_keys) for service_key in service_keys: if service_key == CC.LOCAL_FILE_SERVICE_KEY: break elif service_key == CC.TRASH_SERVICE_KEY: continue try: service = controller.services_manager.GetService( service_key) except: continue if service.GetServiceType() == HC.FILE_REPOSITORY: file_repository = service if file_repository.IsFunctional(): try: (os_file_handle, temp_path) = HydrusPaths.GetTempPath() try: file_repository.Request(HC.GET, 'file', {'hash': hash}, temp_path=temp_path) controller.WaitUntilModelFree() automatic_archive = False exclude_deleted = False # this is the important part here min_size = None min_resolution = None file_import_options = ClientImporting.FileImportOptions( automatic_archive=automatic_archive, exclude_deleted=exclude_deleted, min_size=min_size, min_resolution=min_resolution) file_import_job = ClientImporting.FileImportJob( temp_path, file_import_options) client_files_manager.ImportFile( file_import_job) successful_hashes.add(hash) break finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path) except HydrusExceptions.ServerBusyException: job_key.SetVariable( 'popup_text_1', file_repository.GetName() + ' was busy. waiting 30s before trying again') time.sleep(30) job_key.Delete() controller.pub('notify_new_downloads') return except Exception as e: HydrusData.ShowText('Error downloading file!') HydrusData.ShowException(e) elif service.GetServiceType() == HC.IPFS: multihashes = HG.client_controller.Read( 'service_filenames', service_key, {hash}) if len(multihashes) > 0: multihash = multihashes[0] # this actually calls to a thread that can launch gui 'select from tree' stuff, so let's just break at this point service.ImportFile(multihash) break if HydrusThreading.IsThreadShuttingDown(): return if len(successful_hashes) > 0: job_key.SetVariable( 'popup_text_1', HydrusData.ConvertIntToPrettyString(len(successful_hashes)) + ' files downloaded') job_key.Delete()