def CallBlockingToWx( self, func, *args, **kwargs ): def wx_code( job_key ): try: result = func( *args, **kwargs ) job_key.SetVariable( 'result', result ) except HydrusExceptions.PermissionException as e: job_key.SetVariable( 'error', e ) except Exception as e: job_key.SetVariable( 'error', e ) HydrusData.Print( 'CallBlockingToWx just caught this error:' ) HydrusData.DebugPrint( traceback.format_exc() ) finally: job_key.Finish() job_key = ClientThreading.JobKey() job_key.Begin() wx.CallAfter( wx_code, job_key ) while not job_key.IsDone(): if self._model_shutdown: raise HydrusExceptions.ShutdownException( 'Application is shutting down!' ) time.sleep( 0.05 ) if job_key.HasVariable( 'result' ): # result can be None, for wx_code that has no return variable result = job_key.GetIfHasVariable( 'result' ) return result error = job_key.GetIfHasVariable( 'error' ) if error is not None: raise error raise HydrusExceptions.ShutdownException()
def GetResult(self): time.sleep( 0.00001 ) # this one neat trick can save hassle on superquick jobs as event.wait can be laggy while True: if self._result_ready.wait(2) == True: break elif HG.model_shutdown: raise HydrusExceptions.ShutdownException( 'Application quit before db could serve result!') if isinstance(self._result, Exception): e = self._result raise e else: return self._result
def _WriteResponseToPath(self, response, temp_path, report_hooks): content_length = response.getheader('Content-Length') if content_length is not None: content_length = int(content_length) size_of_response = 0 with open(temp_path, 'wb') as f: for block in HydrusPaths.ReadFileLikeAsBlocks(response): if HydrusGlobals.model_shutdown: raise HydrusExceptions.ShutdownException( 'Application is shutting down!') size_of_response += len(block) if content_length is not None and size_of_response > content_length: raise Exception('Response was longer than suggested!') f.write(block) for hook in report_hooks: if content_length is not None: hook(content_length, size_of_response) return size_of_response
def GetAllPaths(raw_paths): file_paths = [] paths_to_process = raw_paths while len(paths_to_process) > 0: next_paths_to_process = [] for path in paths_to_process: if HG.view_shutdown: raise HydrusExceptions.ShutdownException() if os.path.isdir(path): subpaths = [ os.path.join(path, filename) for filename in os.listdir(path) ] next_paths_to_process.extend(subpaths) else: file_paths.append(path) paths_to_process = next_paths_to_process gc.collect() return file_paths
def Write( self, action, priority, synchronous, *args, **kwargs ): job_type = 'write' job = HydrusData.JobDatabase( job_type, synchronous, action, *args, **kwargs ) if self._controller.ModelIsShutdown(): raise HydrusExceptions.ShutdownException( 'Application has shut down!' ) self._jobs.put( ( priority, job ) ) if synchronous: return job.GetResult()
def CallBlockingToWx(self, callable, *args, **kwargs): def wx_code(job_key): try: result = callable(*args, **kwargs) job_key.SetVariable('result', result) except HydrusExceptions.PermissionException as e: job_key.SetVariable('error', e) except Exception as e: job_key.SetVariable('error', e) HydrusData.Print('CallBlockingToWx just caught this error:') HydrusData.DebugPrint(traceback.format_exc()) finally: job_key.Finish() job_key = ClientThreading.JobKey() job_key.Begin() wx.CallAfter(wx_code, job_key) while not job_key.IsDone(): if self._model_shutdown: return time.sleep(0.05) if job_key.HasVariable('result'): return job_key.GetVariable('result') elif job_key.HasVariable('error'): raise job_key.GetVariable('error') else: raise HydrusExceptions.ShutdownException()
def WaitUntilDone( self ): while True: self._is_done_event.wait( 5 ) with self._lock: if not self._is_started and self._death_time is not None and HydrusData.TimeHasPassed( self._death_time ): raise Exception( 'Network job death time reached--not sure what the error was. Maybe a paused service?' ) if self.IsDone(): break with self._lock: if self.engine.controller.ModelIsShutdown(): raise HydrusExceptions.ShutdownException() elif self._error_exception is not None: if isinstance( self._error_exception, Exception ): raise self._error_exception else: raise Exception( 'Problem in network error handling.' ) elif self._IsCancelled(): if self._method == 'POST': message = 'Upload cancelled: ' + self._status_text else: message = 'Download cancelled: ' + self._status_text raise HydrusExceptions.CancelledException( message )
def WaitUntilDBEmpty( self ): while True: if self._model_shutdown: raise HydrusExceptions.ShutdownException( 'Application shutting down!' ) elif self.db.JobsQueueEmpty() and not self.db.CurrentlyDoingJob(): return else: time.sleep( 0.00001 )
def WaitUntilPubSubsEmpty( self ): while True: if self._view_shutdown: raise HydrusExceptions.ShutdownException( 'Application shutting down!' ) elif self._pubsub.NoJobsQueued() and not self._currently_doing_pubsub: return else: time.sleep( 0.00001 )
def WaitUntilThumbnailsFree( self ): while True: if self._view_shutdown: raise HydrusExceptions.ShutdownException( 'Application shutting down!' ) elif not self._caches[ 'thumbnail' ].DoingWork(): return else: time.sleep( 0.00001 )
def FilterFreePaths(paths): free_paths = [] for path in paths: if HydrusThreading.IsThreadShuttingDown(): raise HydrusExceptions.ShutdownException() if PathIsFree(path): free_paths.append(path) return free_paths
def WaitUntilPubSubsEmpty( self ): while True: if self._model_shutdown: raise HydrusExceptions.ShutdownException( 'Application shutting down!' ) elif not self._pubsub.WorkToDo() and not self._pubsub.DoingWork(): return else: time.sleep( 0.00001 )
def Read( self, action, priority, *args, **kwargs ): if action in self.READ_WRITE_ACTIONS: job_type = 'read_write' else: job_type = 'read' synchronous = True job = HydrusData.JobDatabase( job_type, synchronous, action, *args, **kwargs ) if self._controller.ModelIsShutdown(): raise HydrusExceptions.ShutdownException( 'Application has shut down!' ) self._jobs.put( ( priority + 1, job ) ) # +1 so all writes of equal priority can clear out first return job.GetResult()
def WaitUntilDone( self ): while True: self._is_done_event.wait( 5 ) if self.IsDone(): break with self._lock: if self.engine.controller.ModelIsShutdown(): raise HydrusExceptions.ShutdownException() elif self._error_exception is not None: if isinstance( self._error_exception, Exception ): raise self._error_exception else: raise Exception( 'Problem in network error handling.' ) elif self._IsCancelled(): if self._method == 'POST': message = 'Upload cancelled!' else: message = 'Download cancelled!' raise HydrusExceptions.CancelledException( message )
def GetResult(self): while True: if self._result_ready.wait(2) == True: break elif HG.model_shutdown: raise HydrusExceptions.ShutdownException( 'Application quit before db could serve result!') if isinstance(self._result, Exception): e = self._result raise e else: return self._result
def FilterFreePaths( paths ): free_paths = [] for path in paths: if HydrusThreading.IsThreadShuttingDown(): raise HydrusExceptions.ShutdownException() try: os.rename( path, path ) # rename a path to itself free_paths.append( path ) except OSError as e: # 'already in use by another process' HydrusData.Print( 'Already in use: ' + path ) return free_paths
def GetAllPaths(raw_paths, do_human_sort=True): file_paths = [] paths_to_process = raw_paths while len(paths_to_process) > 0: next_paths_to_process = [] for path in paths_to_process: if HG.view_shutdown: raise HydrusExceptions.ShutdownException() if os.path.isdir(path): subpaths = [ os.path.join(path, filename) for filename in os.listdir(path) ] next_paths_to_process.extend(subpaths) else: file_paths.append(path) paths_to_process = next_paths_to_process if do_human_sort: HydrusData.HumanTextSort(file_paths) return file_paths
def _ReadResponse(self, response, stream_dest, max_allowed=None): with self._lock: if self._content_type is not None and self._content_type in HC.mime_enum_lookup: mime = HC.mime_enum_lookup[self._content_type] else: mime = None if 'content-length' in response.headers: self._num_bytes_to_read = int( response.headers['content-length']) if max_allowed is not None and self._num_bytes_to_read > max_allowed: raise HydrusExceptions.NetworkException( 'The url was apparently ' + HydrusData.ConvertIntToBytes(self._num_bytes_to_read) + ' but the max network size for this type of job is ' + HydrusData.ConvertIntToBytes(max_allowed) + '!') if self._file_import_options is not None: certain = True self._file_import_options.CheckNetworkDownload( mime, self._num_bytes_to_read, certain) else: self._num_bytes_to_read = None for chunk in response.iter_content(chunk_size=65536): if self._IsCancelled(): return stream_dest.write(chunk) chunk_length = len(chunk) with self._lock: self._num_bytes_read += chunk_length if max_allowed is not None and self._num_bytes_read > max_allowed: raise HydrusExceptions.NetworkException( 'The url exceeded the max network size for this type of job, which is ' + HydrusData.ConvertIntToBytes(max_allowed) + '!') if self._file_import_options is not None: certain = False self._file_import_options.CheckNetworkDownload( mime, self._num_bytes_to_read, certain) self._ReportDataUsed(chunk_length) self._WaitOnOngoingBandwidth() if HG.view_shutdown: raise HydrusExceptions.ShutdownException() if self._num_bytes_to_read is not None and self._num_bytes_read < self._num_bytes_to_read * 0.8: raise HydrusExceptions.ShouldReattemptNetworkException( 'Was expecting ' + HydrusData.ConvertIntToBytes(self._num_bytes_to_read) + ' but only got ' + HydrusData.ConvertIntToBytes(self._num_bytes_read) + '.')
def _ParseResponse( self, response, report_hooks ): server_header = response.getheader( 'Server' ) if server_header is not None and 'hydrus' in server_header: hydrus_service = True else: hydrus_service = False content_length = response.getheader( 'Content-Length' ) if content_length is not None: content_length = int( content_length ) for hook in report_hooks: hook( content_length, 0 ) data = '' for block in HydrusPaths.ReadFileLikeAsBlocks( response ): if HydrusGlobals.model_shutdown: raise HydrusExceptions.ShutdownException( 'Application is shutting down!' ) data += block if content_length is not None: for hook in report_hooks: hook( content_length, len( data ) ) if len( data ) > content_length: raise Exception( 'Response was longer than suggested!' ) size_of_response = len( data ) content_type = response.getheader( 'Content-Type' ) if content_type is None: parsed_response = data else: if '; ' in content_type: ( mime_string, additional_info ) = content_type.split( '; ', 1 ) else: ( mime_string, additional_info ) = ( content_type, '' ) if 'charset=' in additional_info: # this does utf-8, ISO-8859-4, whatever ( gumpf, charset ) = additional_info.split( '=' ) try: parsed_response = data.decode( charset ) except: parsed_response = data elif content_type == 'application/x-yaml': try: parsed_response = yaml.safe_load( data ) except yaml.error.YAMLError as e: raise HydrusExceptions.NetworkVersionException( 'Failed to parse a response object!' + os.linesep + HydrusData.ToUnicode( e ) ) elif content_type == 'application/json': if hydrus_service: parsed_response = HydrusSerialisable.CreateFromNetworkString( data ) else: parsed_response = data elif content_type == 'text/html': try: parsed_response = data.decode( 'utf-8' ) except: parsed_response = data else: parsed_response = data return ( parsed_response, size_of_response )