def test_engine_shutdown_manual(self):
mock_controller = TestConstants.MockController()
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
self.assertFalse(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
mock_controller.CallToThread(engine.MainLoop)
time.sleep(0.1)
self.assertTrue(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
engine.Shutdown()
time.sleep(0.1)
self.assertFalse(engine.IsRunning())
self.assertTrue(engine.IsShutdown())
def _GetJob(self, for_login=False):
job = ClientNetworking.NetworkJobHydrus(MOCK_HYDRUS_SERVICE_KEY, 'GET',
MOCK_HYDRUS_URL)
job.SetForLogin(for_login)
mock_controller = TestConstants.MockController()
mock_service = ClientServices.GenerateService(MOCK_HYDRUS_SERVICE_KEY,
HC.TAG_REPOSITORY,
'test tag repo')
mock_services_manager = TestConstants.MockServicesManager(
(mock_service, ))
mock_controller.services_manager = mock_services_manager
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
job.engine = engine
return job
def EnsureLoggedIn( self, name ):
with self._lock:
if name in self._error_names:
raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
if name == 'hentai foundry':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ]
elif name == 'pixiv':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
required_cookies = [ 'PHPSESSID' ]
if self._IsLoggedIn( network_context, required_cookies ):
return
try:
if name == 'hentai foundry':
self.LoginHF( network_context )
elif name == 'pixiv':
result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' )
if result is None:
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
( pixiv_id, password ) = result
self.LoginPixiv( network_context, pixiv_id, password )
if not self._IsLoggedIn( network_context, required_cookies ):
raise Exception( name + ' login did not work correctly!' )
HydrusData.Print( 'Successfully logged into ' + name + '.' )
except:
self._error_names.add( name )
raise
def InitModel(self):
self.pub('splash_set_title_text', u'booting db\u2026')
self._http = ClientNetworking.HTTPConnectionManager()
HydrusController.HydrusController.InitModel(self)
self._options = self.Read('options')
self._new_options = self.Read(
'serialisable',
HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS)
HC.options = self._options
if self._new_options.GetBoolean('use_system_ffmpeg'):
if HydrusVideoHandling.FFMPEG_PATH.startswith(HC.BIN_DIR):
HydrusVideoHandling.FFMPEG_PATH = os.path.basename(
HydrusVideoHandling.FFMPEG_PATH)
self._services_manager = ClientCaches.ServicesManager(self)
self._client_files_manager = ClientCaches.ClientFilesManager(self)
self._client_session_manager = ClientCaches.HydrusSessionManager(self)
self._managers['local_booru'] = ClientCaches.LocalBooruCache(self)
self._managers['tag_censorship'] = ClientCaches.TagCensorshipManager(
self)
self._managers['tag_siblings'] = ClientCaches.TagSiblingsManager(self)
self._managers['tag_parents'] = ClientCaches.TagParentsManager(self)
self._managers['undo'] = ClientCaches.UndoManager(self)
self._managers['web_sessions'] = ClientCaches.WebSessionManagerClient(
self)
if HC.options['proxy'] is not None:
(proxytype, host, port, username, password) = HC.options['proxy']
ClientNetworking.SetProxy(proxytype, host, port, username,
password)
def wx_code():
self._caches['images'] = ClientCaches.RenderedImageCache(self)
self._caches['thumbnail'] = ClientCaches.ThumbnailCache(self)
CC.GlobalBMPs.STATICInitialise()
self.CallBlockingToWx(wx_code)
self.sub(self, 'Clipboard', 'clipboard')
self.sub(self, 'RestartBooru', 'restart_booru')
def InitModel(self):
self.pub('splash_set_title_text', 'booting db...')
self._http = ClientNetworking.HTTPConnectionManager()
HydrusController.HydrusController.InitModel(self)
self._options = self.Read('options')
self._new_options = self.Read(
'serialisable',
HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS)
HC.options = self._options
self._services_manager = ClientCaches.ServicesManager(self)
self._client_files_manager = ClientCaches.ClientFilesManager(self)
self._client_session_manager = ClientCaches.HydrusSessionManager(self)
self._managers['local_booru'] = ClientCaches.LocalBooruCache(self)
self._managers['tag_censorship'] = ClientCaches.TagCensorshipManager(
self)
self._managers['tag_siblings'] = ClientCaches.TagSiblingsManager(self)
self._managers['tag_parents'] = ClientCaches.TagParentsManager(self)
self._managers['undo'] = ClientCaches.UndoManager(self)
self._managers['web_sessions'] = ClientCaches.WebSessionManagerClient(
self)
if HC.options['proxy'] is not None:
(proxytype, host, port, username, password) = HC.options['proxy']
ClientNetworking.SetProxy(proxytype, host, port, username,
password)
def wx_code():
self._caches['fullscreen'] = ClientCaches.RenderedImageCache(
self, 'fullscreen')
self._caches['preview'] = ClientCaches.RenderedImageCache(
self, 'preview')
self._caches['thumbnail'] = ClientCaches.ThumbnailCache(self)
CC.GlobalBMPs.STATICInitialise()
self.CallBlockingToWx(wx_code)
self.sub(self, 'Clipboard', 'clipboard')
self.sub(self, 'RestartServer', 'restart_server')
self.sub(self, 'RestartBooru', 'restart_booru')
def SetDefaultDomainManagerData( domain_manager ):
network_contexts_to_custom_header_dicts = {}
#
custom_header_dict = {}
custom_header_dict[ 'User-Agent' ] = ( 'hydrus client', ClientNetworkingDomain.VALID_APPROVED, 'This is the default User-Agent identifier for the client for all network connections.' )
network_contexts_to_custom_header_dicts[ ClientNetworking.GLOBAL_NETWORK_CONTEXT ] = custom_header_dict
#
custom_header_dict = {}
custom_header_dict[ 'User-Agent' ] = ( 'SCChannelApp/2.0.1 (Android; black)', ClientNetworkingDomain.VALID_UNKNOWN, 'Sankaku seem to currently have a User-Agent whitelist on file requests. Setting this User-Agent allows the sankaku downloader to work.' )
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'sankakucomplex.com' )
network_contexts_to_custom_header_dicts[ network_context ] = custom_header_dict
#
domain_manager.SetNetworkContextsToCustomHeaderDicts( network_contexts_to_custom_header_dicts )
def test_engine_simple_job(self):
mock_controller = TestConstants.MockController()
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
self.assertFalse(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
mock_controller.CallToThread(engine.MainLoop)
#
with HTTMock(catch_all):
with HTTMock(catch_wew_ok):
job = ClientNetworking.NetworkJob('GET', MOCK_URL)
engine.AddJob(job)
time.sleep(0.1)
self.assertTrue(job.IsDone())
self.assertFalse(job.HasError())
engine._new_work_to_do.set()
time.sleep(0.1)
self.assertEqual(len(engine._jobs_bandwidth_throttled), 0)
self.assertEqual(len(engine._jobs_login_throttled), 0)
self.assertEqual(len(engine._jobs_ready_to_start), 0)
self.assertEqual(len(engine._jobs_downloading), 0)
#
engine.Shutdown()
def _GetJob(self, for_login=False):
job = ClientNetworking.NetworkJob('GET', MOCK_URL)
job.SetForLogin(for_login)
mock_controller = TestConstants.MockController()
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
job.engine = engine
return job
def Parse(self, job_key, data, referral_url, desired_content):
search_urls = self.ParseURLs(job_key, data, referral_url)
content = []
for search_url in search_urls:
try:
job_key.SetVariable('script_status', 'fetching ' + search_url)
headers = {'Referer': referral_url}
response = ClientNetworking.RequestsGet(search_url,
headers=headers)
except HydrusExceptions.NotFoundException:
job_key.SetVariable('script_status', '404 - nothing found')
time.sleep(2)
continue
except HydrusExceptions.NetworkException as e:
job_key.SetVariable('script_status',
'Network error! Details written to log.')
HydrusData.Print('Problem fetching ' + search_url + ':')
HydrusData.PrintException(e)
time.sleep(2)
continue
linked_data = response.text
children_content = GetChildrenContent(job_key, self._children,
linked_data, search_url,
desired_content)
content.extend(children_content)
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
return content
def test_multipart(self):
hash = '5a1ba880a043e6207dca5f5407089fb0a0b0a588c8a6cd6a2807d173f59223d9'.decode(
'hex')
fields = []
fields.append(('sub', CC.FIELD_TEXT, 'subject'))
fields.append(('com', CC.FIELD_COMMENT, u'\u269COP is a f*g'))
fields.append(('pass', CC.FIELD_PASSWORD, '123456'))
fields.append(('recaptcha', CC.FIELD_VERIFICATION_RECAPTCHA,
'reticulating splines'))
fields.append(('resto', CC.FIELD_THREAD_ID, '1000000'))
fields.append(('spoiler/on', CC.FIELD_CHECKBOX, True))
fields.append(('upfile', CC.FIELD_FILE, (hash, HC.IMAGE_GIF,
TestConstants.tiniest_gif)))
result = ClientGUIManagement.GenerateDumpMultipartFormDataCTAndBody(
fields)
expected_result = (
'multipart/form-data; boundary=----------AaB03x',
'------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="sub"\r\n\r\nsubject\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="com"\r\n\r\n\xe2\x9a\x9cOP is a f*g\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="pass"\r\n\r\n123456\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="recaptcha"\r\n\r\nreticulating splines\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="resto"\r\n\r\n1000000\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="spoiler"\r\n\r\non\r\n------------AaB03x\r\nContent-Type: image/gif\r\nContent-Disposition: form-data; name="upfile"; filename="5a1ba880a043e6207dca5f5407089fb0a0b0a588c8a6cd6a2807d173f59223d9.gif"\r\n\r\nGIF89a\x01\x00\x01\x00\x00\xff\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x00;\r\n------------AaB03x--\r\n'
)
self.assertEqual(result, expected_result)
#
fields = {}
fields['act'] = 'do_login'
fields['id'] = 'token'
fields['pin'] = 'pin'
fields['long_login'] = '******'
fields['random_unicode'] = u'\u269C'
result = ClientNetworking.GenerateMultipartFormDataCTAndBodyFromDict(
fields)
expected_result = (
'multipart/form-data; boundary=----------AaB03x',
'------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="long_login"\r\n\r\nyes\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="random_unicode"\r\n\r\n\xe2\x9a\x9c\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="id"\r\n\r\ntoken\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="pin"\r\n\r\npin\r\n------------AaB03x\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name="act"\r\n\r\ndo_login\r\n------------AaB03x--\r\n'
)
self.assertEqual(result, expected_result)
def _PrepEngine(self):
mock_controller = TestConstants.MockController()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
ClientDefaults.SetDefaultDomainManagerData(domain_manager)
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
mock_controller.CallToThread(engine.MainLoop)
return (mock_controller, engine)
def Start(self, engine, network_context):
service_key = network_context.context_data
service = engine.controller.services_manager.GetService(service_key)
base_url = service.GetBaseURL()
url = base_url + 'session_key'
access_key = service.GetCredentials().GetAccessKey()
network_job = ClientNetworking.NetworkJobHydrus(
service_key, 'GET', url)
network_job.SetForLogin(True)
network_job.AddAdditionalHeader('Hydrus-Key', access_key.encode('hex'))
engine.AddJob(network_job)
try:
network_job.WaitUntilDone()
session = engine.session_manager.GetSession(network_context)
if self._IsLoggedIn(session):
HydrusData.Print('Successfully logged into ' +
service.GetName() + '.')
else:
service.DelayFutureRequests(
'Could not log in for unknown reason.')
except Exception as e:
e_string = str(e)
service.DelayFutureRequests(e_string)
def test_bandwidth_exceeded(self):
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
DOMAIN_NETWORK_CONTEXT = ClientNetworking.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
#
job = self._GetJob()
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.ReportDataUsed([DOMAIN_NETWORK_CONTEXT],
50)
job.engine.bandwidth_manager.SetRules(DOMAIN_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), False)
#
job = self._GetJob(for_login=True)
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.ReportDataUsed([DOMAIN_NETWORK_CONTEXT],
50)
job.engine.bandwidth_manager.SetRules(DOMAIN_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), True)
def test_bandwidth_ok(self):
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
HYDRUS_NETWORK_CONTEXT = ClientNetworking.NetworkContext(
CC.NETWORK_CONTEXT_HYDRUS, MOCK_HYDRUS_SERVICE_KEY)
#
job = self._GetJob()
job.engine.bandwidth_manager.ReportDataUsed([HYDRUS_NETWORK_CONTEXT],
50)
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.SetRules(HYDRUS_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), True)
#
job = self._GetJob(for_login=True)
job.engine.bandwidth_manager.ReportDataUsed([HYDRUS_NETWORK_CONTEXT],
50)
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.SetRules(HYDRUS_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), True)
def FetchData( self, job_key, file_identifier ):
# add gauge report hook and in-stream cancel support to the get/post calls
request_args = dict( self._static_args )
if self._file_identifier_type != FILE_IDENTIFIER_TYPE_FILE:
request_args[ self._file_identifier_arg_name ] = self._file_identifier_string_converter.Convert( file_identifier )
if self._query_type == HC.GET:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
raise Exception( 'Cannot have a file as an argument on a GET query!' )
full_request_url = ClientNetworking.CombineGETURLWithParameters( self._url, request_args )
job_key.SetVariable( 'script_status', 'fetching ' + full_request_url )
job_key.AddURL( full_request_url )
network_job = ClientNetworking.NetworkJob( 'GET', full_request_url )
elif self._query_type == HC.POST:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
job_key.SetVariable( 'script_status', 'uploading file' )
path = file_identifier
files = { self._file_identifier_arg_name : open( path, 'rb' ) }
else:
job_key.SetVariable( 'script_status', 'uploading identifier' )
files = None
network_job = ClientNetworking.NetworkJob( 'POST', self._url, body = request_args )
network_job.SetFiles( files )
# send nj to nj control on this panel here
network_job.OverrideBandwidth()
HG.client_controller.network_engine.AddJob( network_job )
try:
network_job.WaitUntilDone()
except HydrusExceptions.NotFoundException:
job_key.SetVariable( 'script_status', '404 - nothing found' )
raise
except HydrusExceptions.NetworkException as e:
job_key.SetVariable( 'script_status', 'Network error!' )
HydrusData.ShowException( e )
raise
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
data = network_job.GetContent()
return data
def SetDefaultBandwidthManagerRules( bandwidth_manager ):
KB = 1024
MB = 1024 ** 2
GB = 1024 ** 3
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 1, 5 ) # stop accidental spam
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 60, 120 ) # smooth out heavy usage/bugspam. db and gui prob need a break
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 86400, 10 * GB ) # check your inbox lad
bandwidth_manager.SetRules( ClientNetworking.GLOBAL_NETWORK_CONTEXT, rules )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 1, 1 ) # don't ever hammer a domain
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 86400, 2 * GB ) # don't go nuts on a site in a single day
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN ), rules )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 86400, 64 * MB ) # don't sync a giant db in one day
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS ), rules )
#
rules = HydrusNetworking.BandwidthRules()
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOWNLOADER ), rules )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 300, 100 ) # after that first sample of small files, take it easy
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 300, 128 * MB ) # after that first sample of big files, take it easy
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOWNLOADER_QUERY ), rules )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 86400, 200 ) # catch up on a big sub in little chunks every day
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 86400, 256 * MB ) # catch up on a big sub in little chunks every day
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_SUBSCRIPTION ), rules )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 300, 100 ) # after that first sample of small files, take it easy
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 300, 128 * MB ) # after that first sample of big files, take it easy
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_THREAD_WATCHER_THREAD ), rules )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 60 * 7, 80 )
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 4, 1 )
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'sankakucomplex.com' ), rules )
def Parse( self, job_key, data, referral_url, desired_content ):
search_urls = self.ParseURLs( job_key, data, referral_url )
content = []
for search_url in search_urls:
job_key.SetVariable( 'script_status', 'fetching ' + search_url )
network_job = ClientNetworking.NetworkJob( 'GET', search_url, referral_url = referral_url )
network_job.OverrideBandwidth()
HG.client_controller.network_engine.AddJob( network_job )
try:
network_job.WaitUntilDone()
except HydrusExceptions.CancelledException:
break
except HydrusExceptions.NetworkException as e:
if isinstance( e, HydrusExceptions.NotFoundException ):
job_key.SetVariable( 'script_status', '404 - nothing found' )
time.sleep( 2 )
continue
elif isinstance( e, HydrusExceptions.NetworkException ):
job_key.SetVariable( 'script_status', 'Network error! Details written to log.' )
HydrusData.Print( 'Problem fetching ' + search_url + ':' )
HydrusData.PrintException( e )
time.sleep( 2 )
continue
else:
raise
linked_data = network_job.GetContent()
children_content = GetChildrenContent( job_key, self._children, linked_data, search_url, desired_content )
content.extend( children_content )
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
return content
def FetchData(self, job_key, file_identifier):
try:
# add gauge report hook and in-stream cancel support to the get/post calls
request_args = dict(self._static_args)
if self._file_identifier_type != FILE_IDENTIFIER_TYPE_FILE:
request_args[
self._file_identifier_arg_name] = HydrusData.EncodeBytes(
self._file_identifier_encoding, file_identifier)
if self._query_type == HC.GET:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
raise Exception(
'Cannot have a file as an argument on a GET query!')
rendered_url = self._url + '?' + '&'.join(
(HydrusData.ToByteString(key) + '=' +
HydrusData.ToByteString(value)
for (key, value) in request_args.items()))
job_key.SetVariable('script_status',
'fetching ' + rendered_url)
job_key.AddURL(rendered_url)
response = ClientNetworking.RequestsGet(self._url,
params=request_args)
elif self._query_type == HC.POST:
if self._file_identifier_type == FILE_IDENTIFIER_TYPE_FILE:
job_key.SetVariable('script_status', 'uploading file')
path = file_identifier
files = {self._file_identifier_arg_name: open(path, 'rb')}
else:
job_key.SetVariable('script_status',
'uploading identifier')
files = None
response = ClientNetworking.RequestsPost(self._url,
data=request_args,
files=files)
if job_key.IsCancelled():
raise HydrusExceptions.CancelledException()
data = response.text
return data
except HydrusExceptions.NotFoundException:
job_key.SetVariable('script_status', '404 - nothing found')
raise
except HydrusExceptions.NetworkException as e:
job_key.SetVariable('script_status', 'Network error!')
HydrusData.ShowException(e)
raise
def InitModel( self ):
self.pub( 'splash_set_title_text', u'booting db\u2026' )
HydrusController.HydrusController.InitModel( self )
self.pub( 'splash_set_status_text', u'initialising managers' )
self.pub( 'splash_set_status_subtext', u'services' )
self.services_manager = ClientCaches.ServicesManager( self )
self.pub( 'splash_set_status_subtext', u'options' )
self.options = self.Read( 'options' )
self.new_options = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS )
HC.options = self.options
if self.new_options.GetBoolean( 'use_system_ffmpeg' ):
if HydrusVideoHandling.FFMPEG_PATH.startswith( HC.BIN_DIR ):
HydrusVideoHandling.FFMPEG_PATH = os.path.basename( HydrusVideoHandling.FFMPEG_PATH )
self.pub( 'splash_set_status_subtext', u'client files' )
self.InitClientFilesManager()
#
self.pub( 'splash_set_status_subtext', u'network' )
bandwidth_manager = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER )
if bandwidth_manager is None:
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
ClientDefaults.SetDefaultBandwidthManagerRules( bandwidth_manager )
bandwidth_manager._dirty = True
wx.MessageBox( 'Your bandwidth manager was missing on boot! I have recreated a new empty one with default rules. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )
session_manager = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER )
if session_manager is None:
session_manager = ClientNetworking.NetworkSessionManager()
session_manager._dirty = True
wx.MessageBox( 'Your session manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )
domain_manager = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
if domain_manager is None:
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
ClientDefaults.SetDefaultDomainManagerData( domain_manager )
domain_manager._dirty = True
wx.MessageBox( 'Your domain manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )
domain_manager.Initialise()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
self.network_engine = ClientNetworking.NetworkEngine( self, bandwidth_manager, session_manager, domain_manager, login_manager )
self.CallToThreadLongRunning( self.network_engine.MainLoop )
#
self._shortcuts_manager = ClientCaches.ShortcutsManager( self )
self.local_booru_manager = ClientCaches.LocalBooruCache( self )
self.pub( 'splash_set_status_subtext', u'tag censorship' )
self._managers[ 'tag_censorship' ] = ClientCaches.TagCensorshipManager( self )
self.pub( 'splash_set_status_subtext', u'tag siblings' )
self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
self.pub( 'splash_set_status_subtext', u'tag parents' )
self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
def wx_code():
self._caches[ 'images' ] = ClientCaches.RenderedImageCache( self )
self._caches[ 'thumbnail' ] = ClientCaches.ThumbnailCache( self )
CC.GlobalBMPs.STATICInitialise()
self.pub( 'splash_set_status_subtext', u'image caches' )
self.CallBlockingToWx( wx_code )
self.sub( self, 'ToClipboard', 'clipboard' )
self.sub( self, 'RestartBooru', 'restart_booru' )
def test_can_start(self):
EMPTY_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
PERMISSIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None,
10000)
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
RESTRICTIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None, 1)
DOMAIN_NETWORK_CONTEXT = ClientNetworking.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
SUBDOMAIN_NETWORK_CONTEXT = ClientNetworking.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_SUBDOMAIN)
GLOBAL_NETWORK_CONTEXTS = [ClientNetworking.GLOBAL_NETWORK_CONTEXT]
DOMAIN_NETWORK_CONTEXTS = [
ClientNetworking.GLOBAL_NETWORK_CONTEXT, DOMAIN_NETWORK_CONTEXT
]
SUBDOMAIN_NETWORK_CONTEXTS = [
ClientNetworking.GLOBAL_NETWORK_CONTEXT, DOMAIN_NETWORK_CONTEXT,
SUBDOMAIN_NETWORK_CONTEXT
]
#
fast_forward = HydrusData.GetNow() + 3600
with patch.object(HydrusData, 'GetNow', return_value=fast_forward):
bm = ClientNetworking.NetworkBandwidthManager()
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.ReportRequestUsed(DOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(DOMAIN_NETWORK_CONTEXTS, 50)
bm.ReportRequestUsed(SUBDOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(SUBDOMAIN_NETWORK_CONTEXTS, 25)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT, EMPTY_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_REQUEST_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworking.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))