def wait_for_free_slot( controller, subs_jobs, max_simultaneous_subscriptions ):
time.sleep( 0.1 )
while True:
p1 = controller.options[ 'pause_subs_sync' ]
p2 = HydrusThreading.IsThreadShuttingDown()
if p1 or p2:
if HG.subscription_report_mode:
HydrusData.ShowText( 'Subscriptions cancelling. Global sub pause is ' + str( p1 ) + ' and sub daemon thread shutdown status is ' + str( p2 ) + '.' )
if p2:
for ( thread, job ) in subs_jobs:
HydrusThreading.ShutdownThread( thread )
raise HydrusExceptions.CancelledException( 'subs cancelling or thread shutting down' )
filter_finished_jobs( subs_jobs )
if len( subs_jobs ) < max_simultaneous_subscriptions:
return
time.sleep( 1.0 )
def Parse(self, job_key, data, referral_url, desired_content):
search_urls = self.ParseURLs(job_key, data, referral_url)
content = []
for search_url in search_urls:
try:
job_key.SetVariable('script_status', 'fetching ' + search_url)
headers = {'Referer': referral_url}
response = ClientNetworking.RequestsGet(search_url,
headers=headers)
except HydrusExceptions.NotFoundException:
job_key.SetVariable('script_status', '404 - nothing found')
time.sleep(2)
continue
except HydrusExceptions.NetworkException as e:
job_key.SetVariable('script_status',
'Network error! Details written to log.')
HydrusData.Print('Problem fetching ' + search_url + ':')
HydrusData.PrintException(e)
time.sleep(2)
continue
linked_data = response.text
children_content = GetChildrenContent(job_key, self._children,
linked_data, search_url,
desired_content)
content.extend(children_content)
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
return content
def WaitUntilDone( self ):
while True:
self._is_done_event.wait( 5 )
with self._lock:
if not self._is_started and self._death_time is not None and HydrusData.TimeHasPassed( self._death_time ):
raise Exception( 'Network job death time reached--not sure what the error was. Maybe a paused service?' )
if self.IsDone():
break
with self._lock:
if self.engine.controller.ModelIsShutdown():
raise HydrusExceptions.ShutdownException()
elif self._error_exception is not None:
if isinstance( self._error_exception, Exception ):
raise self._error_exception
else:
raise Exception( 'Problem in network error handling.' )
elif self._IsCancelled():
if self._method == 'POST':
message = 'Upload cancelled: ' + self._status_text
else:
message = 'Download cancelled: ' + self._status_text
raise HydrusExceptions.CancelledException( message )
def SelectFromList( win, title, choice_tuples, value_to_select = None, sort_tuples = True ):
with ClientGUITopLevelWindows.DialogEdit( win, title ) as dlg:
panel = ClientGUIScrolledPanelsEdit.EditSelectFromListPanel( dlg, choice_tuples, value_to_select = value_to_select, sort_tuples = sort_tuples )
dlg.SetPanel( panel )
if dlg.ShowModal() == wx.ID_OK:
result = panel.GetValue()
return result
else:
raise HydrusExceptions.CancelledException()
def StreamResponseToFile(job_key, response, f):
if 'content-length' in response.headers:
gauge_range = int(response.headers['content-length'])
else:
gauge_range = None
gauge_value = 0
try:
for chunk in response.iter_content(chunk_size=65536):
(i_paused, should_quit) = job_key.WaitIfNeeded()
if should_quit:
raise HydrusExceptions.CancelledException()
f.write(chunk)
gauge_value += len(chunk)
if gauge_range is None:
text = 'downloading - ' + HydrusData.ConvertIntToBytes(
gauge_value)
else:
text = 'downloading - ' + HydrusData.ConvertValueRangeToBytes(
gauge_value, gauge_range)
job_key.SetVariable('popup_download',
(text, gauge_value, gauge_range))
finally:
job_key.DeleteVariable('popup_download')
def WaitUntilDone( self ):
while True:
self._is_done_event.wait( 5 )
if self.IsDone():
break
with self._lock:
if self.engine.controller.ModelIsShutdown():
raise HydrusExceptions.ShutdownException()
elif self._error_exception is not None:
if isinstance( self._error_exception, Exception ):
raise self._error_exception
else:
raise Exception( 'Problem in network error handling.' )
elif self._IsCancelled():
if self._method == 'POST':
message = 'Upload cancelled!'
else:
message = 'Download cancelled!'
raise HydrusExceptions.CancelledException( message )
def FetchData( self, job_key, file_identifier ):
# add gauge report hook and in-stream cancel support to the get/post calls
request_args = dict( self._static_args )
if self._file_identifier_type != FILE_IDENTIFIER_TYPE_FILE:
request_args[ self._file_identifier_arg_name ] = self._file_identifier_string_converter.Convert( file_identifier )
if self._query_type == HC.GET:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
raise Exception( 'Cannot have a file as an argument on a GET query!' )
full_request_url = ClientNetworking.CombineGETURLWithParameters( self._url, request_args )
job_key.SetVariable( 'script_status', 'fetching ' + full_request_url )
job_key.AddURL( full_request_url )
network_job = ClientNetworking.NetworkJob( 'GET', full_request_url )
elif self._query_type == HC.POST:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
job_key.SetVariable( 'script_status', 'uploading file' )
path = file_identifier
files = { self._file_identifier_arg_name : open( path, 'rb' ) }
else:
job_key.SetVariable( 'script_status', 'uploading identifier' )
files = None
network_job = ClientNetworking.NetworkJob( 'POST', self._url, body = request_args )
network_job.SetFiles( files )
# send nj to nj control on this panel here
network_job.OverrideBandwidth()
HG.client_controller.network_engine.AddJob( network_job )
try:
network_job.WaitUntilDone()
except HydrusExceptions.NotFoundException:
job_key.SetVariable( 'script_status', '404 - nothing found' )
raise
except HydrusExceptions.NetworkException as e:
job_key.SetVariable( 'script_status', 'Network error!' )
HydrusData.ShowException( e )
raise
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
data = network_job.GetContent()
return data
def Parse( self, job_key, data, referral_url, desired_content ):
search_urls = self.ParseURLs( job_key, data, referral_url )
content = []
for search_url in search_urls:
job_key.SetVariable( 'script_status', 'fetching ' + search_url )
network_job = ClientNetworking.NetworkJob( 'GET', search_url, referral_url = referral_url )
network_job.OverrideBandwidth()
HG.client_controller.network_engine.AddJob( network_job )
try:
network_job.WaitUntilDone()
except HydrusExceptions.CancelledException:
break
except HydrusExceptions.NetworkException as e:
if isinstance( e, HydrusExceptions.NotFoundException ):
job_key.SetVariable( 'script_status', '404 - nothing found' )
time.sleep( 2 )
continue
elif isinstance( e, HydrusExceptions.NetworkException ):
job_key.SetVariable( 'script_status', 'Network error! Details written to log.' )
HydrusData.Print( 'Problem fetching ' + search_url + ':' )
HydrusData.PrintException( e )
time.sleep( 2 )
continue
else:
raise
linked_data = network_job.GetContent()
children_content = GetChildrenContent( job_key, self._children, linked_data, search_url, desired_content )
content.extend( children_content )
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
return content
def FetchData(self, job_key, file_identifier):
try:
# add gauge report hook and in-stream cancel support to the get/post calls
request_args = dict(self._static_args)
if self._file_identifier_type != FILE_IDENTIFIER_TYPE_FILE:
request_args[
self._file_identifier_arg_name] = HydrusData.EncodeBytes(
self._file_identifier_encoding, file_identifier)
if self._query_type == HC.GET:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
raise Exception(
'Cannot have a file as an argument on a GET query!')
rendered_url = self._url + '?' + '&'.join(
(HydrusData.ToByteString(key) + '=' +
HydrusData.ToByteString(value)
for (key, value) in request_args.items()))
job_key.SetVariable('script_status',
'fetching ' + rendered_url)
job_key.AddURL(rendered_url)
response = ClientNetworking.RequestsGet(self._url,
params=request_args)
elif self._query_type == HC.POST:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
job_key.SetVariable('script_status', 'uploading file')
path = file_identifier
files = {self._file_identifier_arg_name: open(path, 'rb')}
else:
job_key.SetVariable('script_status',
'uploading identifier')
files = None
response = ClientNetworking.RequestsPost(self._url,
data=request_args,
files=files)
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
data = response.text
return data
except HydrusExceptions.NotFoundException:
job_key.SetVariable('script_status', '404 - nothing found')
raise
except HydrusExceptions.NetworkException as e:
job_key.SetVariable('script_status', 'Network error!')
HydrusData.ShowException(e)
raise
