def ConvertLegacyToNewBandwidth(
legacy_bandwidth_manager: NetworkBandwidthManagerLegacy):
tracker_containers = []
(network_contexts_to_bandwidth_trackers,
network_contexts_to_bandwidth_rules) = legacy_bandwidth_manager.GetData()
for (network_context,
bandwidth_tracker) in network_contexts_to_bandwidth_trackers.items():
tracker_container_name = HydrusData.GenerateKey().hex()
tracker_container = ClientNetworkingBandwidth.NetworkBandwidthManagerTrackerContainer(
tracker_container_name,
network_context=network_context,
bandwidth_tracker=bandwidth_tracker)
tracker_containers.append(tracker_container)
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
for (network_context,
bandwidth_rules) in network_contexts_to_bandwidth_rules.items():
bandwidth_manager.SetRules(network_context, bandwidth_rules)
bandwidth_manager.SetTrackerContainers(tracker_containers,
set_all_trackers_dirty=True)
bandwidth_manager.SetDirty()
return bandwidth_manager
def _GetJob(self, for_login=False):
job = ClientNetworkingJobs.NetworkJobHydrus(MOCK_HYDRUS_SERVICE_KEY,
'GET', MOCK_HYDRUS_URL)
job.SetForLogin(for_login)
mock_controller = TestController.MockController()
mock_service = ClientServices.GenerateService(MOCK_HYDRUS_SERVICE_KEY,
HC.TAG_REPOSITORY,
'test tag repo')
mock_services_manager = TestController.MockServicesManager(
(mock_service, ))
mock_controller.services_manager = mock_services_manager
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
job.engine = engine
return job
def test_engine_shutdown_manual(self):
mock_controller = TestController.MockController()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
self.assertFalse(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
mock_controller.CallToThread(engine.MainLoop)
time.sleep(0.1)
self.assertTrue(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
engine.Shutdown()
time.sleep(0.1)
self.assertFalse(engine.IsRunning())
self.assertTrue(engine.IsShutdown())
def _PrepEngine( self ):
mock_controller = TestController.MockController()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
ClientDefaults.SetDefaultDomainManagerData( domain_manager )
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
mock_controller.CallToThread( engine.MainLoop )
return ( mock_controller, engine )
def test_engine_simple_job(self):
mock_controller = TestController.MockController()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
self.assertFalse(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
mock_controller.CallToThread(engine.MainLoop)
#
with HTTMock(catch_all):
with HTTMock(catch_wew_ok):
job = ClientNetworkingJobs.NetworkJob('GET', MOCK_URL)
engine.AddJob(job)
time.sleep(0.25)
self.assertTrue(job.IsDone())
self.assertFalse(job.HasError())
engine._new_work_to_do.set()
time.sleep(0.25)
self.assertEqual(len(engine._jobs_awaiting_validity), 0)
self.assertEqual(len(engine._jobs_awaiting_bandwidth), 0)
self.assertEqual(len(engine._jobs_awaiting_login), 0)
self.assertEqual(len(engine._jobs_awaiting_slot), 0)
self.assertEqual(len(engine._jobs_running), 0)
#
engine.Shutdown()
def _GetJob(self, for_login=False):
job = ClientNetworkingJobs.NetworkJob('GET', MOCK_URL)
job.SetForLogin(for_login)
mock_controller = TestController.MockController()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
job.engine = engine
return job
def test_engine_shutdown_app(self):
mock_controller = TestController.MockController()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine(mock_controller,
bandwidth_manager,
session_manager,
domain_manager, login_manager)
self.assertFalse(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
mock_controller.CallToThread(engine.MainLoop)
time.sleep(0.1)
self.assertTrue(engine.IsRunning())
self.assertFalse(engine.IsShutdown())
HG.model_shutdown = True
try:
engine._new_work_to_do.set()
time.sleep(0.1)
self.assertFalse(engine.IsRunning())
self.assertTrue(engine.IsShutdown())
finally:
HG.model_shutdown = False
def test_can_start(self):
EMPTY_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
PERMISSIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None,
10000)
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
RESTRICTIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None, 1)
DOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
SUBDOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_SUBDOMAIN)
GLOBAL_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT
]
DOMAIN_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
DOMAIN_NETWORK_CONTEXT
]
SUBDOMAIN_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
DOMAIN_NETWORK_CONTEXT, SUBDOMAIN_NETWORK_CONTEXT
]
#
fast_forward = HydrusData.GetNow() + 3600
with patch.object(HydrusData, 'GetNow', return_value=fast_forward):
bm = ClientNetworkingBandwidth.NetworkBandwidthManager()
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.ReportRequestUsed(DOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(DOMAIN_NETWORK_CONTEXTS, 50)
bm.ReportRequestUsed(SUBDOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(SUBDOMAIN_NETWORK_CONTEXTS, 25)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
EMPTY_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_REQUEST_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
def __init__(self, win, only_run):
self.app = win
self.win = win
self.only_run = only_run
self.db_dir = tempfile.mkdtemp()
global DB_DIR
DB_DIR = self.db_dir
self._server_files_dir = os.path.join(self.db_dir, 'server_files')
self._updates_dir = os.path.join(self.db_dir, 'test_updates')
client_files_default = os.path.join(self.db_dir, 'client_files')
HydrusPaths.MakeSureDirectoryExists(self._server_files_dir)
HydrusPaths.MakeSureDirectoryExists(self._updates_dir)
HydrusPaths.MakeSureDirectoryExists(client_files_default)
HG.controller = self
HG.client_controller = self
HG.server_controller = self
HG.test_controller = self
self.db = self
self.gui = self
self.frame_splash_status = ClientGUISplash.FrameSplashStatus()
self._call_to_threads = []
self._pubsub = HydrusPubSub.HydrusPubSub(self, lambda o: True)
self.new_options = ClientOptions.ClientOptions()
HC.options = ClientDefaults.GetClientDefaultOptions()
self.options = HC.options
def show_text(text):
pass
HydrusData.ShowText = show_text
self._reads = {}
self._reads['local_booru_share_keys'] = []
self._reads['messaging_sessions'] = []
self._reads['options'] = ClientDefaults.GetClientDefaultOptions()
self._reads['file_system_predicates'] = []
self._reads['media_results'] = []
self._param_reads = {}
self.example_tag_repo_service_key = HydrusData.GenerateKey()
services = []
services.append(
ClientServices.GenerateService(CC.LOCAL_BOORU_SERVICE_KEY,
HC.LOCAL_BOORU, 'local booru'))
services.append(
ClientServices.GenerateService(CC.CLIENT_API_SERVICE_KEY,
HC.CLIENT_API_SERVICE,
'client api'))
services.append(
ClientServices.GenerateService(CC.COMBINED_LOCAL_FILE_SERVICE_KEY,
HC.COMBINED_LOCAL_FILE,
'all local files'))
services.append(
ClientServices.GenerateService(CC.LOCAL_FILE_SERVICE_KEY,
HC.LOCAL_FILE_DOMAIN, 'my files'))
services.append(
ClientServices.GenerateService(CC.TRASH_SERVICE_KEY,
HC.LOCAL_FILE_TRASH_DOMAIN,
'trash'))
services.append(
ClientServices.GenerateService(CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
HC.LOCAL_TAG, 'my tags'))
services.append(
ClientServices.GenerateService(self.example_tag_repo_service_key,
HC.TAG_REPOSITORY,
'example tag repo'))
services.append(
ClientServices.GenerateService(CC.COMBINED_TAG_SERVICE_KEY,
HC.COMBINED_TAG, 'all known tags'))
services.append(
ClientServices.GenerateService(
LOCAL_RATING_LIKE_SERVICE_KEY, HC.LOCAL_RATING_LIKE,
'example local rating like service'))
services.append(
ClientServices.GenerateService(
LOCAL_RATING_NUMERICAL_SERVICE_KEY, HC.LOCAL_RATING_NUMERICAL,
'example local rating numerical service'))
self._reads['services'] = services
client_files_locations = {}
for prefix in HydrusData.IterateHexPrefixes():
for c in ('f', 't'):
client_files_locations[c + prefix] = client_files_default
self._reads['client_files_locations'] = client_files_locations
self._reads['sessions'] = []
self._reads['tag_parents'] = {}
self._reads['tag_siblings_all_ideals'] = {}
self._reads['in_inbox'] = False
self._writes = collections.defaultdict(list)
self._managers = {}
self.column_list_manager = ClientGUIListManager.ColumnListManager()
self.services_manager = ClientServices.ServicesManager(self)
self.client_files_manager = ClientFiles.ClientFilesManager(self)
self.parsing_cache = ClientCaches.ParsingCache()
bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager()
session_manager = ClientNetworkingSessions.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
ClientDefaults.SetDefaultDomainManagerData(domain_manager)
login_manager = ClientNetworkingLogin.NetworkLoginManager()
self.network_engine = ClientNetworking.NetworkEngine(
self, bandwidth_manager, session_manager, domain_manager,
login_manager)
self.CallToThreadLongRunning(self.network_engine.MainLoop)
self.tag_display_manager = ClientTagsHandling.TagDisplayManager()
self._managers['undo'] = ClientManagers.UndoManager(self)
self.server_session_manager = HydrusSessions.HydrusSessionManagerServer(
)
self.bitmap_manager = ClientManagers.BitmapManager(self)
self.local_booru_manager = ClientCaches.LocalBooruCache(self)
self.client_api_manager = ClientAPI.APIManager()
self._cookies = {}
self._job_scheduler = HydrusThreading.JobScheduler(self)
self._job_scheduler.start()