def test_engine_simple_job(self):
mock_controller = TestConstants.MockController()
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
self.assertFalse(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
mock_controller.CallToThread(engine.MainLoop)
#
with HTTMock(catch_all):
with HTTMock(catch_wew_ok):
job = ClientNetworking.NetworkJob('GET', MOCK_URL)
engine.AddJob(job)
time.sleep(0.1)
self.assertTrue(job.IsDone())
self.assertFalse(job.HasError())
engine._new_work_to_do.set()
time.sleep(0.1)
self.assertEqual(len(engine._jobs_bandwidth_throttled), 0)
self.assertEqual(len(engine._jobs_login_throttled), 0)
self.assertEqual(len(engine._jobs_ready_to_start), 0)
self.assertEqual(len(engine._jobs_downloading), 0)
#
engine.Shutdown()
def _GetJob(self, for_login=False):
job = ClientNetworking.NetworkJob('GET', MOCK_URL)
job.SetForLogin(for_login)
mock_controller = TestConstants.MockController()
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
job.engine = engine
return job
def FetchData( self, job_key, file_identifier ):
# add gauge report hook and in-stream cancel support to the get/post calls
request_args = dict( self._static_args )
if self._file_identifier_type != FILE_IDENTIFIER_TYPE_FILE:
request_args[ self._file_identifier_arg_name ] = self._file_identifier_string_converter.Convert( file_identifier )
if self._query_type == HC.GET:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
raise Exception( 'Cannot have a file as an argument on a GET query!' )
full_request_url = ClientNetworking.CombineGETURLWithParameters( self._url, request_args )
job_key.SetVariable( 'script_status', 'fetching ' + full_request_url )
job_key.AddURL( full_request_url )
network_job = ClientNetworking.NetworkJob( 'GET', full_request_url )
elif self._query_type == HC.POST:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
job_key.SetVariable( 'script_status', 'uploading file' )
path = file_identifier
files = { self._file_identifier_arg_name : open( path, 'rb' ) }
else:
job_key.SetVariable( 'script_status', 'uploading identifier' )
files = None
network_job = ClientNetworking.NetworkJob( 'POST', self._url, body = request_args )
network_job.SetFiles( files )
# send nj to nj control on this panel here
network_job.OverrideBandwidth()
HG.client_controller.network_engine.AddJob( network_job )
try:
network_job.WaitUntilDone()
except HydrusExceptions.NotFoundException:
job_key.SetVariable( 'script_status', '404 - nothing found' )
raise
except HydrusExceptions.NetworkException as e:
job_key.SetVariable( 'script_status', 'Network error!' )
HydrusData.ShowException( e )
raise
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
data = network_job.GetContent()
return data
def Parse( self, job_key, data, referral_url, desired_content ):
search_urls = self.ParseURLs( job_key, data, referral_url )
content = []
for search_url in search_urls:
job_key.SetVariable( 'script_status', 'fetching ' + search_url )
network_job = ClientNetworking.NetworkJob( 'GET', search_url, referral_url = referral_url )
network_job.OverrideBandwidth()
HG.client_controller.network_engine.AddJob( network_job )
try:
network_job.WaitUntilDone()
except HydrusExceptions.CancelledException:
break
except HydrusExceptions.NetworkException as e:
if isinstance( e, HydrusExceptions.NotFoundException ):
job_key.SetVariable( 'script_status', '404 - nothing found' )
time.sleep( 2 )
continue
elif isinstance( e, HydrusExceptions.NetworkException ):
job_key.SetVariable( 'script_status', 'Network error! Details written to log.' )
HydrusData.Print( 'Problem fetching ' + search_url + ':' )
HydrusData.PrintException( e )
time.sleep( 2 )
continue
else:
raise
linked_data = network_job.GetContent()
children_content = GetChildrenContent( job_key, self._children, linked_data, search_url, desired_content )
content.extend( children_content )
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
return content