def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
self.engine = None
self._dirty = False
self._lock = threading.Lock()
self._last_pages_gallery_query_timestamps = collections.defaultdict( lambda: 0 )
self._last_subscriptions_gallery_query_timestamps = collections.defaultdict( lambda: 0 )
self._last_watchers_query_timestamps = collections.defaultdict( lambda: 0 )
self._tracker_container_names_to_tracker_containers = {}
self._network_contexts_to_tracker_containers = {}
self._tracker_container_names = set()
self._dirty_tracker_container_names = set()
self._deletee_tracker_container_names = set()
self._my_bandwidth_tracker = HydrusNetworking.BandwidthTracker()
self._network_contexts_to_bandwidth_rules = collections.defaultdict( HydrusNetworking.BandwidthRules )
for context_type in [ CC.NETWORK_CONTEXT_GLOBAL, CC.NETWORK_CONTEXT_HYDRUS, CC.NETWORK_CONTEXT_DOMAIN, CC.NETWORK_CONTEXT_DOWNLOADER_PAGE, CC.NETWORK_CONTEXT_SUBSCRIPTION, CC.NETWORK_CONTEXT_WATCHER_PAGE ]:
self._network_contexts_to_bandwidth_rules[ ClientNetworkingContexts.NetworkContext( context_type ) ] = HydrusNetworking.BandwidthRules()
def _GetTracker( self, network_context: ClientNetworkingContexts.NetworkContext, making_it_dirty = False ):
if network_context not in self._network_contexts_to_tracker_containers:
bandwidth_tracker = HydrusNetworking.BandwidthTracker()
tracker_container_name = HydrusData.GenerateKey().hex()
tracker_container = NetworkBandwidthManagerTrackerContainer( tracker_container_name, network_context = network_context, bandwidth_tracker = bandwidth_tracker )
self._tracker_container_names_to_tracker_containers[ tracker_container_name ] = tracker_container
self._network_contexts_to_tracker_containers[ network_context ] = tracker_container
# note this discards ephemeral network contexts, which have temporary identifiers that are generally invisible to the user
if not network_context.IsEphemeral():
self._tracker_container_names.add( tracker_container_name )
self._dirty_tracker_container_names.add( tracker_container_name )
self._SetDirty()
tracker_container = self._network_contexts_to_tracker_containers[ network_context ]
if making_it_dirty and not network_context.IsEphemeral():
self._dirty_tracker_container_names.add( tracker_container.GetName() )
return tracker_container.bandwidth_tracker
def SetServices(self, services):
# doesn't need the dirty_object_lock because the caller takes it
# first test available ports
my_ports = {s.GetPort() for s in self._services}
for service in services:
port = service.GetPort()
if port not in my_ports and HydrusNetworking.LocalPortInUse(port):
raise HydrusExceptions.ServerException(
'Something was already bound to port ' + str(port))
#
self._services = services
self.CallToThread(self.services_upnp_manager.SetServices,
self._services)
[self._admin_service] = [
service for service in self._services
if service.GetServiceType() == HC.SERVER_ADMIN
]
self.SetRunningTwistedServices(self._services)
def InitView(self):
HydrusController.HydrusController.InitView(self)
port = self._admin_service.GetPort()
if HydrusNetworking.LocalPortInUse(port):
HydrusData.Print(
'Something is already bound to port ' + str(port) +
', so your administration service cannot be started. Please quit the server and retry once the port is clear.'
)
else:
self.SetRunningTwistedServices(self._services)
#
job = self.CallRepeating(5.0, 600.0, self.SyncRepositories)
self._daemon_jobs['sync_repositories'] = job
job = self.CallRepeating(0.0, 30.0, self.SaveDirtyObjects)
self._daemon_jobs['save_dirty_objects'] = job
job = self.CallRepeating(0.0, 86400.0, self.DeleteOrphans)
self._daemon_jobs['delete_orphans'] = job
def GetTracker( self, network_context ):
with self._lock:
if network_context in self._network_contexts_to_tracker_containers:
return self._GetTracker( network_context )
else:
return HydrusNetworking.BandwidthTracker()
def GetValue(self):
bandwidth_rules = HydrusNetworking.BandwidthRules()
for rule in self._listctrl.GetData():
(bandwidth_type, time_delta, max_allowed) = rule
bandwidth_rules.AddRule(bandwidth_type, time_delta, max_allowed)
return bandwidth_rules
def test_no_rules( self ):
rules = HydrusNetworking.BandwidthRules()
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
def __init__(self, name, network_context=None, bandwidth_tracker=None):
if network_context is None:
network_context = ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT
if bandwidth_tracker is None:
bandwidth_tracker = HydrusNetworking.BandwidthTracker()
HydrusSerialisable.SerialisableBaseNamed.__init__(self, name)
self.network_context = network_context
self.bandwidth_tracker = bandwidth_tracker
def _Add( self ):
title = 'new account type'
permissions = {}
bandwidth_rules = HydrusNetworking.BandwidthRules()
account_type = HydrusNetwork.AccountType.GenerateNewAccountType( title, permissions, bandwidth_rules )
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit account type' ) as dlg_edit:
panel = EditAccountTypePanel( dlg_edit, self._service_type, account_type )
dlg_edit.SetPanel( panel )
if dlg_edit.exec() == QW.QDialog.Accepted:
new_account_type = panel.GetValue()
self._account_types_listctrl.AddDatas( ( new_account_type, ) )
def InitView(self):
HydrusController.HydrusController.InitView(self)
port = self._admin_service.GetPort()
if HydrusNetworking.LocalPortInUse(port):
HydrusData.Print(
'Something is already bound to port ' + str(port) +
', so your administration service cannot be started. Please quit the server and retry once the port is clear.'
)
else:
self.SetRunningTwistedServices(self._services)
#
job = self.CallRepeating(5.0, HydrusNetwork.UPDATE_CHECKING_PERIOD,
self.SyncRepositories)
job.WakeOnPubSub('notify_new_repo_sync')
self._daemon_jobs['sync_repositories'] = job
job = self.CallRepeating(0.0, 30.0, self.SaveDirtyObjects)
self._daemon_jobs['save_dirty_objects'] = job
job = self.CallRepeating(30.0, 86400.0, self.DoDeferredPhysicalDeletes)
job.WakeOnPubSub('notify_new_physical_file_deletes')
self._daemon_jobs['deferred_physical_deletes'] = job
job = self.CallRepeating(120.0, 3600.0 * 4, self.NullifyHistory)
job.WakeOnPubSub('notify_new_nullification')
self._daemon_jobs['nullify_history'] = job
def test_bandwidth_exceeded(self):
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
HYDRUS_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_HYDRUS, MOCK_HYDRUS_SERVICE_KEY)
#
job = self._GetJob()
self.assertEqual(job.TryToStartBandwidth(), True)
job.engine.bandwidth_manager.ReportDataUsed([HYDRUS_NETWORK_CONTEXT],
50)
job.engine.bandwidth_manager.SetRules(HYDRUS_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertEqual(job.TryToStartBandwidth(), False)
#
job = self._GetJob(for_login=True)
self.assertEqual(job.TryToStartBandwidth(), True)
job.engine.bandwidth_manager.ReportDataUsed([HYDRUS_NETWORK_CONTEXT],
50)
job.engine.bandwidth_manager.SetRules(HYDRUS_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertEqual(job.TryToStartBandwidth(), True)
def test_bandwidth_ok(self):
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
DOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
#
job = self._GetJob()
job.engine.bandwidth_manager.ReportDataUsed([DOMAIN_NETWORK_CONTEXT],
50)
self.assertEqual(job.TryToStartBandwidth(), True)
job.engine.bandwidth_manager.SetRules(DOMAIN_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
self.assertEqual(job.TryToStartBandwidth(), True)
#
job = self._GetJob(for_login=True)
job.engine.bandwidth_manager.ReportDataUsed([DOMAIN_NETWORK_CONTEXT],
50)
self.assertEqual(job.TryToStartBandwidth(), True)
job.engine.bandwidth_manager.SetRules(DOMAIN_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
self.assertEqual(job.TryToStartBandwidth(), True)
def _test_restricted(self, service):
# access_key
registration_key = HydrusData.GenerateKey()
HG.test_controller.SetRead('access_key', self._access_key)
response = service.Request(HC.GET, 'access_key',
{'registration_key': registration_key})
self.assertEqual(response['access_key'], self._access_key)
# set up session
last_error = 0
account = self._account
HG.test_controller.SetRead('service', service)
HG.test_controller.SetRead('account_key_from_access_key',
HydrusData.GenerateKey())
HG.test_controller.SetRead('account', self._account)
# account
response = service.Request(HC.GET, 'account')
self.assertEqual(repr(response['account']), repr(self._account))
# account from access key
HG.test_controller.SetRead('account', self._account)
subject_account_identifier = HydrusNetwork.AccountIdentifier(
account_key=self._account.GetAccountKey())
response = service.Request(
HC.GET, 'other_account',
{'subject_identifier': subject_account_identifier})
self.assertEqual(repr(response['account']), repr(self._account))
# account from file
HG.test_controller.SetRead('account_from_content', self._account)
content = HydrusNetwork.Content(
content_type=HC.CONTENT_TYPE_FILES,
content_data=(HydrusData.GenerateKey(), ))
subject_account_identifier = HydrusNetwork.AccountIdentifier(
content=content)
response = service.Request(
HC.GET, 'other_account',
{'subject_identifier': subject_account_identifier})
self.assertEqual(repr(response['account']), repr(self._account))
# account from mapping
HG.test_controller.SetRead('account_from_content', self._account)
content = HydrusNetwork.Content(
content_type=HC.CONTENT_TYPE_MAPPING,
content_data=('hello', HydrusData.GenerateKey()))
subject_account_identifier = HydrusNetwork.AccountIdentifier(
content=content)
response = service.Request(
HC.GET, 'other_account',
{'subject_identifier': subject_account_identifier})
self.assertEqual(repr(response['account']), repr(self._account))
# account_info
account_info = {'message': 'hello'}
HG.test_controller.SetRead('account_info', account_info)
subject_account_identifier = HydrusNetwork.AccountIdentifier(
account_key=HydrusData.GenerateKey())
response = service.Request(
HC.GET, 'account_info',
{'subject_identifier': subject_account_identifier})
self.assertEqual(response['account_info'], account_info)
#
content = HydrusNetwork.Content(
content_type=HC.CONTENT_TYPE_FILES,
content_data=(HydrusData.GenerateKey(), ))
subject_account_identifier = HydrusNetwork.AccountIdentifier(
content=content)
with self.assertRaises(HydrusExceptions.BadRequestException):
# can only do it with an account id
response = service.Request(
HC.GET, 'account_info',
{'subject_identifier': subject_account_identifier})
content = HydrusNetwork.Content(
content_type=HC.CONTENT_TYPE_MAPPING,
content_data=('hello', HydrusData.GenerateKey()))
subject_account_identifier = HydrusNetwork.AccountIdentifier(
content=content)
with self.assertRaises(HydrusExceptions.BadRequestException):
# can only do it with an account id
response = service.Request(
HC.GET, 'account_info',
{'subject_identifier': subject_account_identifier})
# account_types
account_types = [
HydrusNetwork.AccountType.GenerateAdminAccountType(
service.GetServiceType())
]
HG.test_controller.SetRead('account_types', account_types)
HG.test_controller.ClearWrites('account_types')
response = service.Request(HC.GET, 'account_types')
self.assertEqual(response['account_types'][0].GetAccountTypeKey(),
account_types[0].GetAccountTypeKey())
empty_account_type = HydrusNetwork.AccountType.GenerateNewAccountType(
'empty account', {}, HydrusNetworking.BandwidthRules())
account_types.append(empty_account_type)
service.Request(
HC.POST, 'account_types', {
'account_types': account_types,
'deletee_account_type_keys_to_new_account_type_keys': {}
})
written = HG.test_controller.GetWrite('account_types')
[(args, kwargs)] = written
(written_service_key, written_account, written_account_types,
written_deletee_account_type_keys_to_new_account_type_keys) = args
self.assertEqual(
{wat.GetAccountTypeKey()
for wat in written_account_types},
{at.GetAccountTypeKey()
for at in account_types})
self.assertEqual(
written_deletee_account_type_keys_to_new_account_type_keys, {})
# registration_keys
registration_key = HydrusData.GenerateKey()
HG.test_controller.SetRead('registration_keys', [registration_key])
response = service.Request(
HC.GET, 'registration_keys', {
'num': 1,
'account_type_key': os.urandom(32),
'expires': HydrusData.GetNow() + 1200
})
self.assertEqual(response['registration_keys'], [registration_key])
def StartServices(*args, **kwargs):
HydrusData.Print('Starting services\u2026')
for service in services:
service_key = service.GetServiceKey()
service_type = service.GetServiceType()
name = service.GetName()
try:
port = service.GetPort()
if service_type == HC.SERVER_ADMIN:
http_factory = ServerServer.HydrusServiceAdmin(
service)
elif service_type == HC.FILE_REPOSITORY:
http_factory = ServerServer.HydrusServiceRepositoryFile(
service)
elif service_type == HC.TAG_REPOSITORY:
http_factory = ServerServer.HydrusServiceRepositoryTag(
service)
else:
return
(ssl_cert_path, ssl_key_path) = self.db.GetSSLPaths()
sslmethod = twisted.internet.ssl.SSL.TLSv1_2_METHOD
context_factory = twisted.internet.ssl.DefaultOpenSSLContextFactory(
ssl_key_path, ssl_cert_path, sslmethod)
ipv6_port = None
try:
ipv6_port = reactor.listenSSL(port,
http_factory,
context_factory,
interface='::')
except Exception as e:
HydrusData.Print('Could not bind to IPv6:')
HydrusData.Print(str(e))
ipv4_port = None
try:
ipv4_port = reactor.listenSSL(
port, http_factory, context_factory)
except:
if ipv6_port is None:
raise
self._service_keys_to_connected_ports[service_key] = (
ipv4_port, ipv6_port)
if HydrusNetworking.LocalPortInUse(port):
HydrusData.Print('Running "{}" on port {}.'.format(
name, port))
else:
raise Exception(
'Tried to bind port {} for "{}" but it failed.'
.format(port, name))
except Exception as e:
HydrusData.Print(traceback.format_exc())
HydrusData.Print('Services started')
def test_can_start(self):
EMPTY_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
PERMISSIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None,
10000)
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
RESTRICTIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None, 1)
DOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
SUBDOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_SUBDOMAIN)
GLOBAL_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT
]
DOMAIN_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
DOMAIN_NETWORK_CONTEXT
]
SUBDOMAIN_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
DOMAIN_NETWORK_CONTEXT, SUBDOMAIN_NETWORK_CONTEXT
]
#
fast_forward = HydrusData.GetNow() + 3600
with patch.object(HydrusData, 'GetNow', return_value=fast_forward):
bm = ClientNetworkingBandwidth.NetworkBandwidthManager()
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.ReportRequestUsed(DOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(DOMAIN_NETWORK_CONTEXTS, 50)
bm.ReportRequestUsed(SUBDOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(SUBDOMAIN_NETWORK_CONTEXTS, 25)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
EMPTY_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_REQUEST_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
def _test_account_creation(self):
result = self._read('account_types', self._tag_service_key,
self._tag_service_account)
(self._tag_service_admin_account_type, ) = result
self.assertEqual(self._tag_service_admin_account_type.GetTitle(),
'administrator')
#
self._regular_user_account_type = HydrusNetwork.AccountType.GenerateNewAccountType(
'regular user',
{HC.CONTENT_TYPE_MAPPINGS: HC.PERMISSION_ACTION_CREATE},
HydrusNetworking.BandwidthRules())
self._deletee_user_account_type = HydrusNetwork.AccountType.GenerateNewAccountType(
'deletee user', {}, HydrusNetworking.BandwidthRules())
new_account_types = [
self._tag_service_admin_account_type,
self._regular_user_account_type, self._deletee_user_account_type
]
#
self._write('account_types', self._tag_service_key,
self._tag_service_account, new_account_types, {})
edited_account_types = self._read('account_types',
self._tag_service_key,
self._tag_service_account)
self.assertEqual(
{at.GetAccountTypeKey()
for at in edited_account_types}, {
at.GetAccountTypeKey()
for at in (self._tag_service_admin_account_type,
self._regular_user_account_type,
self._deletee_user_account_type)
})
#
r_keys = self._read(
'registration_keys', self._tag_service_key,
self._tag_service_account, 5,
self._deletee_user_account_type.GetAccountTypeKey(), 86400 * 365)
access_keys = [
self._read('access_key', self._tag_service_key, r_key)
for r_key in r_keys
]
account_keys = [
self._read('account_key_from_access_key', self._tag_service_key,
access_key) for access_key in access_keys
]
accounts = [
self._read('account', self._tag_service_key, account_key)
for account_key in account_keys
]
for account in accounts:
self.assertEqual(
account.GetAccountType().GetAccountTypeKey(),
self._deletee_user_account_type.GetAccountTypeKey())
#
deletee_account_type_keys_to_replacement_account_type_keys = {
self._deletee_user_account_type.GetAccountTypeKey():
self._regular_user_account_type.GetAccountTypeKey()
}
new_account_types = [
self._tag_service_admin_account_type,
self._regular_user_account_type
]
self._write(
'account_types', self._tag_service_key, self._tag_service_account,
new_account_types,
deletee_account_type_keys_to_replacement_account_type_keys)
accounts = [
self._read('account', self._tag_service_key, account_key)
for account_key in account_keys
]
self._tag_service_regular_account = accounts[0]
for account in accounts:
self.assertEqual(
account.GetAccountType().GetAccountTypeKey(),
self._regular_user_account_type.GetAccountTypeKey())
#
r_keys = self._read(
'registration_keys', self._tag_service_key,
self._tag_service_account, 5,
self._regular_user_account_type.GetAccountTypeKey(), 86400 * 365)
self.assertEqual(len(r_keys), 5)
for r_key in r_keys:
self.assertEqual(len(r_key), 32)
r_key = r_keys[0]
access_key = self._read('access_key', self._tag_service_key, r_key)
access_key_2 = self._read('access_key', self._tag_service_key, r_key)
self.assertNotEqual(access_key, access_key_2)
with self.assertRaises(
HydrusExceptions.InsufficientCredentialsException):
# this access key has been replaced
self._read('account_key_from_access_key', self._tag_service_key,
access_key)
account_key = self._read('account_key_from_access_key',
self._tag_service_key, access_key_2)
with self.assertRaises(
HydrusExceptions.InsufficientCredentialsException):
# this registration key has been deleted
self._read('access_key', self._tag_service_key, r_key)
from mock import patch
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core.networking import HydrusNetworking
now = HydrusData.GetNow()
now_10 = now + 10
now_20 = now + 20
with patch.object( HydrusData, 'GetNow', return_value = now ):
HIGH_USAGE = HydrusNetworking.BandwidthTracker()
for i in range( 100 ):
HIGH_USAGE.ReportRequestUsed()
HIGH_USAGE.ReportDataUsed( random.randint( 512, 1024 ) )
LOW_USAGE = HydrusNetworking.BandwidthTracker()
LOW_USAGE.ReportRequestUsed()
LOW_USAGE.ReportDataUsed( 1024 )
ZERO_USAGE = HydrusNetworking.BandwidthTracker()
class TestBandwidthRules( unittest.TestCase ):
def test_per_sec( self ):
# at short time deltas, we can always start based on data alone
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 1, 10240 )
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertFalse( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_10 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_20 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 1, 1 )
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertFalse( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_10 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_20 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 1, 10240 )
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 1, 1 )
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertFalse( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertFalse( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_10 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_20 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertTrue( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
def test_bandwidth_tracker( self ):
bandwidth_tracker = HydrusNetworking.BandwidthTracker()
self.assertEqual( bandwidth_tracker.GetCurrentMonthSummary(), 'used 0B in 0 requests this month' )
now = HydrusData.GetNow()
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 1 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 2 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 2 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 6 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 6 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 3600 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 3600 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None ), 0 )
#
bandwidth_tracker.ReportDataUsed( 1024 )
bandwidth_tracker.ReportRequestUsed()
self.assertEqual( bandwidth_tracker.GetCurrentMonthSummary(), 'used 1.0KB in 1 requests this month' )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 1 ), 1 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 2 ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 2 ), 1 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 6 ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 6 ), 1 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 3600 ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 3600 ), 1 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None ), 1 )
#
five_secs_from_now = now + 5
with patch.object( HydrusData, 'GetNow', return_value = five_secs_from_now ):
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 1 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 2 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 2 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 6 ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 6 ), 1 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 3600 ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 3600 ), 1 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None ), 1024 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None ), 1 )
#
bandwidth_tracker.ReportDataUsed( 32 )
bandwidth_tracker.ReportRequestUsed()
bandwidth_tracker.ReportDataUsed( 32 )
bandwidth_tracker.ReportRequestUsed()
self.assertEqual( bandwidth_tracker.GetCurrentMonthSummary(), 'used 1.1KB in 3 requests this month' )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 0 ), 0 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ), 64 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 1 ), 2 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 2 ), 64 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 2 ), 2 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 6 ), 1088 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 6 ), 3 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 3600 ), 1088 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, 3600 ), 3 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None ), 1088 )
self.assertEqual( bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None ), 3 )
def test_per_month( self ):
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, None, 10240 )
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_10 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_20 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, None, 10 )
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_10 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_20 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
#
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, None, 10240 )
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, None, 10 )
with patch.object( HydrusData, 'GetNow', return_value = now ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_10 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
with patch.object( HydrusData, 'GetNow', return_value = now_20 ):
self.assertTrue( rules.CanStartRequest( ZERO_USAGE ) )
self.assertTrue( rules.CanStartRequest( LOW_USAGE ) )
self.assertFalse( rules.CanStartRequest( HIGH_USAGE ) )
self.assertTrue( rules.CanContinueDownload( ZERO_USAGE ) )
self.assertTrue( rules.CanContinueDownload( LOW_USAGE ) )
self.assertTrue( rules.CanContinueDownload( HIGH_USAGE ) )
