def DAEMONSynchroniseRepositories(controller): if not controller.options['pause_repo_sync']: services = controller.services_manager.GetServices(HC.REPOSITORIES, randomised=True) for service in services: if HydrusThreading.IsThreadShuttingDown(): return if controller.options['pause_repo_sync']: return service.SyncRemote() service.SyncProcessUpdates(maintenance_mode=HC.MAINTENANCE_IDLE) if HydrusThreading.IsThreadShuttingDown(): return time.sleep(1)
def DAEMONMaintainTrash( controller ): if HC.options[ 'trash_max_size' ] is not None: max_size = HC.options[ 'trash_max_size' ] * 1048576 service_info = controller.Read( 'service_info', CC.TRASH_SERVICE_KEY ) while service_info[ HC.SERVICE_INFO_TOTAL_SIZE ] > max_size: if HydrusThreading.IsThreadShuttingDown(): return hashes = controller.Read( 'trash_hashes', limit = 10 ) if len( hashes ) == 0: return content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, hashes ) service_keys_to_content_updates = { CC.TRASH_SERVICE_KEY : [ content_update ] } controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates ) service_info = controller.Read( 'service_info', CC.TRASH_SERVICE_KEY ) time.sleep( 2 ) if HC.options[ 'trash_max_age' ] is not None: max_age = HC.options[ 'trash_max_age' ] * 3600 hashes = controller.Read( 'trash_hashes', limit = 10, minimum_age = max_age ) while len( hashes ) > 0: if HydrusThreading.IsThreadShuttingDown(): return content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, hashes ) service_keys_to_content_updates = { CC.TRASH_SERVICE_KEY : [ content_update ] } controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates ) hashes = controller.Read( 'trash_hashes', limit = 10, minimum_age = max_age ) time.sleep( 2 )
def RemoveUPnPMapping(external_port, protocol): cmd = [UPNPC_PATH, '-d', str(external_port), protocol] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() try: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) except FileNotFoundError: RaiseMissingUPnPcError('remove UPnP port forward') HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) if stderr is not None and len(stderr) > 0: raise Exception('Problem while trying to remove UPnP mapping:' + os.linesep * 2 + stderr)
def GetUPnPMappings(): cmd = [UPNPC_PATH, '-l'] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() try: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) except FileNotFoundError: RaiseMissingUPnPcError('get current UPnP port forward mappings') HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) if stderr is not None and len(stderr) > 0: raise Exception( 'Problem while trying to fetch UPnP mappings (if it says No IGD UPnP Device, you are either on a VPN or your router does not seem to support UPnP):' + os.linesep * 2 + stderr) else: return GetUPnPMappingsParseResponse(stdout)
def RenderPageToFile( path, temp_path, page_index ): cmd = [ SWFRENDER_PATH, path, '-o', temp_path, '-p', str( page_index ) ] timeout = HydrusData.GetNow() + 60 sbp_kwargs = HydrusData.GetSubprocessKWArgs() HydrusData.CheckProgramIsNotShuttingDown() p = subprocess.Popen( cmd, **sbp_kwargs ) while p.poll() is None: if HydrusData.TimeHasPassed( timeout ): p.terminate() raise Exception( 'Could not render the swf page within 60 seconds!' ) time.sleep( 0.5 ) HydrusThreading.SubprocessCommunicate( p )
def do_it(hash_ids): for group_of_hash_ids in HydrusData.SplitListIntoChunks( hash_ids, 256): if HydrusThreading.IsThreadShuttingDown(): return hash_ids_to_tags_managers = HG.client_controller.Read( 'force_refresh_tags_managers', group_of_hash_ids) with self._lock: for (hash_id, tags_manager) in hash_ids_to_tags_managers.items(): media_result = self._hash_ids_to_media_results.get( hash_id, None) if media_result is not None: media_result.SetTagsManager(tags_manager) HG.client_controller.pub('refresh_all_tag_presentation_gui')
def DAEMONCheckExportFolders(): controller = HG.client_controller if not controller.new_options.GetBoolean('pause_export_folders_sync'): HG.export_folders_running = True try: export_folder_names = controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_EXPORT_FOLDER) for name in export_folder_names: export_folder = controller.Read( 'serialisable_named', HydrusSerialisable.SERIALISABLE_TYPE_EXPORT_FOLDER, name) if controller.new_options.GetBoolean( 'pause_export_folders_sync' ) or HydrusThreading.IsThreadShuttingDown(): break export_folder.DoWork() finally: HG.export_folders_running = False
def do_it(): if HC.PLATFORM_WINDOWS: cmd = ['explorer', '/select,', path] elif HC.PLATFORM_MACOS: cmd = ['open', '-R', path] elif HC.PLATFORM_LINUX: raise NotImplementedError('Linux cannot open file locations!') elif HC.PLATFORM_HAIKU: raise NotImplementedError('Haiku cannot open file locations!') sbp_kwargs = HydrusData.GetSubprocessKWArgs(hide_terminal=False) HydrusData.CheckProgramIsNotShuttingDown() process = subprocess.Popen(cmd, **sbp_kwargs) HydrusThreading.SubprocessCommunicate(process)
def do_it(): if HC.PLATFORM_WINDOWS: os.startfile(path) else: if HC.PLATFORM_MACOS: cmd = ['open', path] elif HC.PLATFORM_LINUX: cmd = ['xdg-open', path] elif HC.PLATFORM_HAIKU: cmd = ['open', path] # setsid call un-childs this new process sbp_kwargs = HydrusData.GetSubprocessKWArgs() preexec_fn = getattr(os, 'setsid', None) HydrusData.CheckProgramIsNotShuttingDown() process = subprocess.Popen(cmd, preexec_fn=preexec_fn, **sbp_kwargs) HydrusThreading.SubprocessCommunicate(process)
def DAEMONCheckImportFolders(): controller = HG.client_controller if not controller.options['pause_import_folders_sync']: HG.import_folders_running = True try: import_folder_names = controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER) for name in import_folder_names: import_folder = controller.Read( 'serialisable_named', HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER, name) if controller.options[ 'pause_import_folders_sync'] or HydrusThreading.IsThreadShuttingDown( ): break import_folder.DoWork() finally: HG.import_folders_running = False
def __enter__(self): # let all the readers know that we are bumping up to the front of the queue with self.parent.lock: self.parent.num_waiting_writers += 1 while not HydrusThreading.IsThreadShuttingDown(): with self.parent.lock: # if nothing reading or writing atm, sieze the opportunity if self.parent.num_readers == 0 and not self.parent.there_is_an_active_writer: self.parent.there_is_an_active_writer = True return # otherwise wait a bit self.parent.write_available_event.wait(1) self.parent.write_available_event.clear()
def _CheckCancelTests(self): if not self._cancelled.is_set(): should_cancel = False if self._cancel_on_shutdown and HydrusThreading.IsThreadShuttingDown( ): should_cancel = True if HG.client_controller.ShouldStopThisWork(self._maintenance_mode, self._stop_time): should_cancel = True if should_cancel: self.Cancel() if not self._deleted.is_set(): if self._deletion_time is not None: if HydrusData.TimeHasPassed(self._deletion_time): self.Finish() self._deleted.set()
def _GetCallToThread( self ): with self._call_to_thread_lock: for call_to_thread in self._call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread # all the threads in the pool are currently busy calling_from_the_thread_pool = threading.current_thread() in self._call_to_threads if calling_from_the_thread_pool or len( self._call_to_threads ) < 200: call_to_thread = HydrusThreading.THREADCallToThread( self, 'CallToThread' ) self._call_to_threads.append( call_to_thread ) call_to_thread.start() else: call_to_thread = random.choice( self._call_to_threads ) return call_to_thread
def AddUPnPMapping(internal_client, internal_port, external_port, protocol, description, duration=3600): cmd = [ upnpc_path, '-e', description, '-a', internal_client, str(internal_port), str(external_port), protocol, str(duration) ] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) AddUPnPMappingCheckResponse(internal_client, internal_port, external_port, protocol, stdout, stderr)
def GetUPnPMappings(): cmd = [upnpc_path, '-l'] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) if stderr is not None and len(stderr) > 0: raise Exception('Problem while trying to fetch UPnP mappings:' + os.linesep * 2 + stderr) else: return GetUPnPMappingsParseResponse(stdout)
def GetExternalIP(): if HydrusData.TimeHasPassed(EXTERNAL_IP['time'] + (3600 * 24)): cmd = [UPNPC_PATH, '-l'] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() try: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) except FileNotFoundError: RaiseMissingUPnPcError('fetch external IP') HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) if stderr is not None and len(stderr) > 0: raise Exception( 'Problem while trying to fetch External IP (if it says No IGD UPnP Device, you are either on a VPN or your router does not seem to support UPnP):' + os.linesep * 2 + str(stderr)) else: try: lines = HydrusText.DeserialiseNewlinedTexts(stdout) i = lines.index( 'i protocol exPort->inAddr:inPort description remoteHost leaseTime' ) # ExternalIPAddress = ip (gumpf, external_ip_address) = lines[i - 1].split(' = ') except ValueError: raise Exception('Could not parse external IP!') if external_ip_address == '0.0.0.0': raise Exception( 'Your UPnP device returned your external IP as 0.0.0.0! Try rebooting it, or overwrite it in options!' ) EXTERNAL_IP['ip'] = external_ip_address EXTERNAL_IP['time'] = HydrusData.GetNow() return EXTERNAL_IP['ip']
def InitModel(self): try: self._InitTempDir() except: HydrusData.Print('Failed to initialise temp folder.') self._fast_job_scheduler = HydrusThreading.JobScheduler(self) self._slow_job_scheduler = HydrusThreading.JobScheduler(self) self._fast_job_scheduler.start() self._slow_job_scheduler.start() self.db = self._InitDB()
def CallLater( self, initial_delay, func, *args, **kwargs ): call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.SingleJob( self, self._job_scheduler, initial_delay, call ) self._job_scheduler.AddJob( job ) return job
def CallRepeating( self, initial_delay, period, func, *args, **kwargs ): call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.RepeatingJob( self, self._job_scheduler, initial_delay, period, call ) self._job_scheduler.AddJob( job ) return job
def AddUPnPMapping(internal_client, internal_port, external_port, protocol, description, duration=3600): cmd = [ upnpc_path, '-e', description, '-a', internal_client, str(internal_port), str(external_port), protocol, str(duration) ] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) if 'x.x.x.x:' + str( external_port ) + ' TCP is redirected to internal ' + internal_client + ':' + str( internal_port) in stdout: raise HydrusExceptions.FirewallException( 'The UPnP mapping of ' + internal_client + ':' + str(internal_port) + '->external:' + str(external_port) + ' already exists as a port forward. If this UPnP mapping is automatic, please disable it.' ) if stdout is not None and 'failed with code' in stdout: if 'UnknownError' in stdout: raise HydrusExceptions.FirewallException( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + stdout) else: raise Exception('Problem while trying to add UPnP mapping:' + os.linesep * 2 + stdout) if stderr is not None and len(stderr) > 0: raise Exception('Problem while trying to add UPnP mapping:' + os.linesep * 2 + stderr)
def CallRepeating( self, initial_delay, period, func, *args, **kwargs ) -> HydrusThreading.RepeatingJob: job_scheduler = self._GetAppropriateJobScheduler( period ) call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.RepeatingJob( self, job_scheduler, initial_delay, period, call ) job_scheduler.AddJob( job ) return job
def CallLater( self, initial_delay, func, *args, **kwargs ) -> HydrusThreading.SingleJob: job_scheduler = self._GetAppropriateJobScheduler( initial_delay ) call = HydrusData.Call( func, *args, **kwargs ) job = HydrusThreading.SingleJob( self, job_scheduler, initial_delay, call ) job_scheduler.AddJob( job ) return job
def FilterFreePaths(paths): free_paths = [] for path in paths: HydrusThreading.CheckIfThreadShuttingDown() if PathIsFree(path): free_paths.append(path) return free_paths
def RenderImageToPNGPath(path, temp_png_path): # -y to overwrite the temp path cmd = [FFMPEG_PATH, '-y', "-i", path, temp_png_path] sbp_kwargs = HydrusData.GetSubprocessKWArgs() HydrusData.CheckProgramIsNotShuttingDown() try: process = subprocess.Popen(cmd, bufsize=10**5, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) except FileNotFoundError as e: global FFMPEG_MISSING_ERROR_PUBBED if not FFMPEG_MISSING_ERROR_PUBBED: message = 'FFMPEG, which hydrus uses to parse and render video, was not found! This may be due to it not being available on your system, or hydrus being unable to find it.' message += os.linesep * 2 if HC.PLATFORM_WINDOWS: message += 'You are on Windows, so there should be a copy of ffmpeg.exe in your install_dir/bin folder. If not, please check if your anti-virus has removed it and restore it through a new install.' else: message += 'If you are certain that FFMPEG is installed on your OS and accessible in your PATH, please let hydrus_dev know, as this problem is likely due to an environment problem. You may be able to solve this problem immediately by putting a static build of the ffmpeg executable in your install_dir/bin folder.' message += os.linesep * 2 message += 'You can check your current FFMPEG status through help->about.' HydrusData.ShowText(message) FFMPEG_MISSING_ERROR_PUBBED = True raise FileNotFoundError( 'Cannot interact with video because FFMPEG not found--are you sure it is installed? Full error: ' + str(e)) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(process)
def _GetCallToThreadLongRunning(self): with self._call_to_thread_lock: for call_to_thread in self._long_running_call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread call_to_thread = HydrusThreading.THREADCallToThread( self, 'CallToThreadLongRunning') self._long_running_call_to_threads.append(call_to_thread) call_to_thread.start() return call_to_thread
def __enter__(self): while not HydrusThreading.IsThreadShuttingDown(): with self.parent.lock: # if there are no writers, we can start reading if self.parent.num_waiting_writers == 0: self.parent.num_readers += 1 return # otherwise wait a bit self.parent.read_available_event.wait(1) self.parent.read_available_event.clear()
def _GetCallToThread(self): for call_to_thread in self._call_to_threads: if not call_to_thread.CurrentlyWorking(): return call_to_thread if len(self._call_to_threads) > 100: raise Exception('Too many call to threads!') call_to_thread = HydrusThreading.THREADCallToThread( self, 'CallToThread') self._call_to_threads.append(call_to_thread) call_to_thread.start() return call_to_thread
def AddUPnPMapping(internal_client, internal_port, external_port, protocol, description, duration=3600): if UPNPC_IS_MISSING: RaiseMissingUPnPcError('add UPnP port forward') cmd = [ UPNPC_PATH, '-e', description, '-a', internal_client, str(internal_port), str(external_port), protocol, str(duration) ] sbp_kwargs = HydrusData.GetSubprocessKWArgs(text=True) HydrusData.CheckProgramIsNotShuttingDown() try: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **sbp_kwargs) except FileNotFoundError: RaiseMissingUPnPcError('add UPnP port forward') HydrusData.WaitForProcessToFinish(p, 30) (stdout, stderr) = HydrusThreading.SubprocessCommunicate(p) AddUPnPMappingCheckResponse(internal_client, internal_port, external_port, protocol, stdout, stderr)
def _ImportFiles(self, job_key): did_work = False time_to_save = HydrusData.GetNow() + 600 num_files_imported = 0 presentation_hashes = [] presentation_hashes_fast = set() i = 0 num_total = len(self._file_seed_cache) num_total_unknown = self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN) num_total_done = num_total - num_total_unknown while True: file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN) p1 = HC.options['pause_import_folders_sync'] or self._paused p2 = HydrusThreading.IsThreadShuttingDown() p3 = job_key.IsCancelled() if file_seed is None or p1 or p2 or p3: break did_work = True if HydrusData.TimeHasPassed(time_to_save): HG.client_controller.WriteSynchronous('serialisable', self) time_to_save = HydrusData.GetNow() + 600 gauge_num_done = num_total_done + num_files_imported + 1 job_key.SetVariable( 'popup_text_1', 'importing file ' + HydrusData.ConvertValueRangeToPrettyString( gauge_num_done, num_total)) job_key.SetVariable('popup_gauge_1', (gauge_num_done, num_total)) path = file_seed.file_seed_data file_seed.ImportPath(self._file_seed_cache, self._file_import_options, limited_mimes=self._mimes) if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES: if file_seed.HasHash(): hash = file_seed.GetHash() if self._tag_import_options.HasAdditionalTags(): media_result = HG.client_controller.Read( 'media_result', hash) downloaded_tags = [] service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, media_result, downloaded_tags) # additional tags if len(service_keys_to_content_updates) > 0: HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates) service_keys_to_tags = ClientTags.ServiceKeysToTags() for (tag_service_key, filename_tagging_options) in list( self._tag_service_keys_to_filename_tagging_options. items()): if not HG.client_controller.services_manager.ServiceExists( tag_service_key): continue try: tags = filename_tagging_options.GetTags( tag_service_key, path) if len(tags) > 0: service_keys_to_tags[tag_service_key] = tags except Exception as e: HydrusData.ShowText( 'Trying to parse filename tags in the import folder "' + self._name + '" threw an error!') HydrusData.ShowException(e) if len(service_keys_to_tags) > 0: service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( {hash}, service_keys_to_tags) HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates) num_files_imported += 1 if hash not in presentation_hashes_fast: if file_seed.ShouldPresent(self._file_import_options): presentation_hashes.append(hash) presentation_hashes_fast.add(hash) elif file_seed.status == CC.STATUS_ERROR: HydrusData.Print( 'A file failed to import from import folder ' + self._name + ':' + path) i += 1 if i % 10 == 0: self._ActionPaths() if num_files_imported > 0: HydrusData.Print('Import folder ' + self._name + ' imported ' + HydrusData.ToHumanInt(num_files_imported) + ' files.') if len(presentation_hashes) > 0: ClientImporting.PublishPresentationHashes( self._name, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page) self._ActionPaths() return did_work
def GetFFMPEGVersion(): cmd = [ FFMPEG_PATH, '-version' ] HydrusData.CheckProgramIsNotShuttingDown() try: sbp_kwargs = HydrusData.GetSubprocessKWArgs( text = True ) process = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs ) except FileNotFoundError: return 'no ffmpeg found at path "{}"'.format( FFMPEG_PATH ) except Exception as e: HydrusData.ShowException( e ) return 'unable to execute ffmpeg at path "{}"'.format( FFMPEG_PATH ) ( stdout, stderr ) = HydrusThreading.SubprocessCommunicate( process ) del process lines = stdout.splitlines() if len( lines ) > 0: # typically 'ffmpeg version [VERSION] Copyright ... top_line = lines[0] if top_line.startswith( 'ffmpeg version ' ): top_line = top_line.replace( 'ffmpeg version ', '' ) if ' ' in top_line: version_string = top_line.split( ' ' )[0] return version_string message = 'FFMPEG was recently contacted to fetch version information. While FFMPEG could be found, the response could not be understood. Significant debug information has been printed to the log, which hydrus_dev would be interested in.' HydrusData.ShowText( message ) message += os.linesep * 2 message += str( sbp_kwargs ) message += os.linesep * 2 message += str( os.environ ) message += os.linesep * 2 message += 'STDOUT Response: {}'.format( stdout ) message += os.linesep * 2 message += 'STDERR Response: {}'.format( stderr ) HydrusData.Print( message ) global FFMPEG_NO_CONTENT_ERROR_PUBBED FFMPEG_NO_CONTENT_ERROR_PUBBED = True return 'unknown'