def CallBlockingToWx( self, func, *args, **kwargs ):
def wx_code( job_key ):
try:
result = func( *args, **kwargs )
job_key.SetVariable( 'result', result )
except HydrusExceptions.PermissionException as e:
job_key.SetVariable( 'error', e )
except Exception as e:
job_key.SetVariable( 'error', e )
HydrusData.Print( 'CallBlockingToWx just caught this error:' )
HydrusData.DebugPrint( traceback.format_exc() )
finally:
job_key.Finish()
job_key = ClientThreading.JobKey()
job_key.Begin()
wx.CallAfter( wx_code, job_key )
while not job_key.IsDone():
if self._model_shutdown:
raise HydrusExceptions.ShutdownException( 'Application is shutting down!' )
time.sleep( 0.05 )
if job_key.HasVariable( 'result' ):
# result can be None, for wx_code that has no return variable
result = job_key.GetIfHasVariable( 'result' )
return result
error = job_key.GetIfHasVariable( 'error' )
if error is not None:
raise error
raise HydrusExceptions.ShutdownException()
def GetResult(self):
time.sleep(
0.00001
) # this one neat trick can save hassle on superquick jobs as event.wait can be laggy
while True:
if self._result_ready.wait(2) == True:
break
elif HG.model_shutdown:
raise HydrusExceptions.ShutdownException(
'Application quit before db could serve result!')
if isinstance(self._result, Exception):
e = self._result
raise e
else:
return self._result
def _WriteResponseToPath(self, response, temp_path, report_hooks):
content_length = response.getheader('Content-Length')
if content_length is not None: content_length = int(content_length)
size_of_response = 0
with open(temp_path, 'wb') as f:
for block in HydrusPaths.ReadFileLikeAsBlocks(response):
if HydrusGlobals.model_shutdown:
raise HydrusExceptions.ShutdownException(
'Application is shutting down!')
size_of_response += len(block)
if content_length is not None and size_of_response > content_length:
raise Exception('Response was longer than suggested!')
f.write(block)
for hook in report_hooks:
if content_length is not None:
hook(content_length, size_of_response)
return size_of_response
def GetAllPaths(raw_paths):
file_paths = []
paths_to_process = raw_paths
while len(paths_to_process) > 0:
next_paths_to_process = []
for path in paths_to_process:
if HG.view_shutdown:
raise HydrusExceptions.ShutdownException()
if os.path.isdir(path):
subpaths = [
os.path.join(path, filename)
for filename in os.listdir(path)
]
next_paths_to_process.extend(subpaths)
else:
file_paths.append(path)
paths_to_process = next_paths_to_process
gc.collect()
return file_paths
def Write( self, action, priority, synchronous, *args, **kwargs ):
job_type = 'write'
job = HydrusData.JobDatabase( job_type, synchronous, action, *args, **kwargs )
if self._controller.ModelIsShutdown():
raise HydrusExceptions.ShutdownException( 'Application has shut down!' )
self._jobs.put( ( priority, job ) )
if synchronous: return job.GetResult()
def CallBlockingToWx(self, callable, *args, **kwargs):
def wx_code(job_key):
try:
result = callable(*args, **kwargs)
job_key.SetVariable('result', result)
except HydrusExceptions.PermissionException as e:
job_key.SetVariable('error', e)
except Exception as e:
job_key.SetVariable('error', e)
HydrusData.Print('CallBlockingToWx just caught this error:')
HydrusData.DebugPrint(traceback.format_exc())
finally:
job_key.Finish()
job_key = ClientThreading.JobKey()
job_key.Begin()
wx.CallAfter(wx_code, job_key)
while not job_key.IsDone():
if self._model_shutdown:
return
time.sleep(0.05)
if job_key.HasVariable('result'):
return job_key.GetVariable('result')
elif job_key.HasVariable('error'):
raise job_key.GetVariable('error')
else:
raise HydrusExceptions.ShutdownException()
def WaitUntilDone( self ):
while True:
self._is_done_event.wait( 5 )
with self._lock:
if not self._is_started and self._death_time is not None and HydrusData.TimeHasPassed( self._death_time ):
raise Exception( 'Network job death time reached--not sure what the error was. Maybe a paused service?' )
if self.IsDone():
break
with self._lock:
if self.engine.controller.ModelIsShutdown():
raise HydrusExceptions.ShutdownException()
elif self._error_exception is not None:
if isinstance( self._error_exception, Exception ):
raise self._error_exception
else:
raise Exception( 'Problem in network error handling.' )
elif self._IsCancelled():
if self._method == 'POST':
message = 'Upload cancelled: ' + self._status_text
else:
message = 'Download cancelled: ' + self._status_text
raise HydrusExceptions.CancelledException( message )
def WaitUntilDBEmpty( self ):
while True:
if self._model_shutdown:
raise HydrusExceptions.ShutdownException( 'Application shutting down!' )
elif self.db.JobsQueueEmpty() and not self.db.CurrentlyDoingJob():
return
else:
time.sleep( 0.00001 )
def WaitUntilPubSubsEmpty( self ):
while True:
if self._view_shutdown:
raise HydrusExceptions.ShutdownException( 'Application shutting down!' )
elif self._pubsub.NoJobsQueued() and not self._currently_doing_pubsub:
return
else:
time.sleep( 0.00001 )
def WaitUntilThumbnailsFree( self ):
while True:
if self._view_shutdown:
raise HydrusExceptions.ShutdownException( 'Application shutting down!' )
elif not self._caches[ 'thumbnail' ].DoingWork():
return
else:
time.sleep( 0.00001 )
def FilterFreePaths(paths):
free_paths = []
for path in paths:
if HydrusThreading.IsThreadShuttingDown():
raise HydrusExceptions.ShutdownException()
if PathIsFree(path):
free_paths.append(path)
return free_paths
def WaitUntilPubSubsEmpty( self ):
while True:
if self._model_shutdown:
raise HydrusExceptions.ShutdownException( 'Application shutting down!' )
elif not self._pubsub.WorkToDo() and not self._pubsub.DoingWork():
return
else:
time.sleep( 0.00001 )
def Read( self, action, priority, *args, **kwargs ):
if action in self.READ_WRITE_ACTIONS: job_type = 'read_write'
else: job_type = 'read'
synchronous = True
job = HydrusData.JobDatabase( job_type, synchronous, action, *args, **kwargs )
if self._controller.ModelIsShutdown():
raise HydrusExceptions.ShutdownException( 'Application has shut down!' )
self._jobs.put( ( priority + 1, job ) ) # +1 so all writes of equal priority can clear out first
return job.GetResult()
def WaitUntilDone( self ):
while True:
self._is_done_event.wait( 5 )
if self.IsDone():
break
with self._lock:
if self.engine.controller.ModelIsShutdown():
raise HydrusExceptions.ShutdownException()
elif self._error_exception is not None:
if isinstance( self._error_exception, Exception ):
raise self._error_exception
else:
raise Exception( 'Problem in network error handling.' )
elif self._IsCancelled():
if self._method == 'POST':
message = 'Upload cancelled!'
else:
message = 'Download cancelled!'
raise HydrusExceptions.CancelledException( message )
def GetResult(self):
while True:
if self._result_ready.wait(2) == True:
break
elif HG.model_shutdown:
raise HydrusExceptions.ShutdownException(
'Application quit before db could serve result!')
if isinstance(self._result, Exception):
e = self._result
raise e
else:
return self._result
def FilterFreePaths( paths ):
free_paths = []
for path in paths:
if HydrusThreading.IsThreadShuttingDown():
raise HydrusExceptions.ShutdownException()
try:
os.rename( path, path ) # rename a path to itself
free_paths.append( path )
except OSError as e: # 'already in use by another process'
HydrusData.Print( 'Already in use: ' + path )
return free_paths
def GetAllPaths(raw_paths, do_human_sort=True):
file_paths = []
paths_to_process = raw_paths
while len(paths_to_process) > 0:
next_paths_to_process = []
for path in paths_to_process:
if HG.view_shutdown:
raise HydrusExceptions.ShutdownException()
if os.path.isdir(path):
subpaths = [
os.path.join(path, filename)
for filename in os.listdir(path)
]
next_paths_to_process.extend(subpaths)
else:
file_paths.append(path)
paths_to_process = next_paths_to_process
if do_human_sort:
HydrusData.HumanTextSort(file_paths)
return file_paths
def _ReadResponse(self, response, stream_dest, max_allowed=None):
with self._lock:
if self._content_type is not None and self._content_type in HC.mime_enum_lookup:
mime = HC.mime_enum_lookup[self._content_type]
else:
mime = None
if 'content-length' in response.headers:
self._num_bytes_to_read = int(
response.headers['content-length'])
if max_allowed is not None and self._num_bytes_to_read > max_allowed:
raise HydrusExceptions.NetworkException(
'The url was apparently ' +
HydrusData.ConvertIntToBytes(self._num_bytes_to_read) +
' but the max network size for this type of job is ' +
HydrusData.ConvertIntToBytes(max_allowed) + '!')
if self._file_import_options is not None:
certain = True
self._file_import_options.CheckNetworkDownload(
mime, self._num_bytes_to_read, certain)
else:
self._num_bytes_to_read = None
for chunk in response.iter_content(chunk_size=65536):
if self._IsCancelled():
return
stream_dest.write(chunk)
chunk_length = len(chunk)
with self._lock:
self._num_bytes_read += chunk_length
if max_allowed is not None and self._num_bytes_read > max_allowed:
raise HydrusExceptions.NetworkException(
'The url exceeded the max network size for this type of job, which is '
+ HydrusData.ConvertIntToBytes(max_allowed) + '!')
if self._file_import_options is not None:
certain = False
self._file_import_options.CheckNetworkDownload(
mime, self._num_bytes_to_read, certain)
self._ReportDataUsed(chunk_length)
self._WaitOnOngoingBandwidth()
if HG.view_shutdown:
raise HydrusExceptions.ShutdownException()
if self._num_bytes_to_read is not None and self._num_bytes_read < self._num_bytes_to_read * 0.8:
raise HydrusExceptions.ShouldReattemptNetworkException(
'Was expecting ' +
HydrusData.ConvertIntToBytes(self._num_bytes_to_read) +
' but only got ' +
HydrusData.ConvertIntToBytes(self._num_bytes_read) + '.')
def _ParseResponse( self, response, report_hooks ):
server_header = response.getheader( 'Server' )
if server_header is not None and 'hydrus' in server_header:
hydrus_service = True
else:
hydrus_service = False
content_length = response.getheader( 'Content-Length' )
if content_length is not None:
content_length = int( content_length )
for hook in report_hooks:
hook( content_length, 0 )
data = ''
for block in HydrusPaths.ReadFileLikeAsBlocks( response ):
if HydrusGlobals.model_shutdown:
raise HydrusExceptions.ShutdownException( 'Application is shutting down!' )
data += block
if content_length is not None:
for hook in report_hooks:
hook( content_length, len( data ) )
if len( data ) > content_length:
raise Exception( 'Response was longer than suggested!' )
size_of_response = len( data )
content_type = response.getheader( 'Content-Type' )
if content_type is None: parsed_response = data
else:
if '; ' in content_type: ( mime_string, additional_info ) = content_type.split( '; ', 1 )
else: ( mime_string, additional_info ) = ( content_type, '' )
if 'charset=' in additional_info:
# this does utf-8, ISO-8859-4, whatever
( gumpf, charset ) = additional_info.split( '=' )
try: parsed_response = data.decode( charset )
except: parsed_response = data
elif content_type == 'application/x-yaml':
try: parsed_response = yaml.safe_load( data )
except yaml.error.YAMLError as e:
raise HydrusExceptions.NetworkVersionException( 'Failed to parse a response object!' + os.linesep + HydrusData.ToUnicode( e ) )
elif content_type == 'application/json':
if hydrus_service:
parsed_response = HydrusSerialisable.CreateFromNetworkString( data )
else:
parsed_response = data
elif content_type == 'text/html':
try: parsed_response = data.decode( 'utf-8' )
except: parsed_response = data
else: parsed_response = data
return ( parsed_response, size_of_response )
