def _GenerateSpecificNetworkContexts( self ):
# we always store cookies in the larger session (even if the cookie itself refers to a subdomain in the session object)
# but we can login to a specific subdomain
session_network_context = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, self._second_level_domain )
login_network_context = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, self._domain )
return ( session_network_context, login_network_context )
def _GenerateSpecificNetworkContexts( self ):
# we always store cookies in the larger session (even if the cookie itself refers to a subdomain in the session object)
# but we can login to a specific subdomain
domain = ClientNetworkingDomain.ConvertURLIntoDomain( self._url )
domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( domain )
session_network_context = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, domains[-1] )
login_network_context = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, domain )
return ( session_network_context, login_network_context )
def _GenerateNetworkContexts( self ):
network_contexts = NetworkJob._GenerateNetworkContexts( self )
network_contexts.append( ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_SUBSCRIPTION, self._subscription_key ) )
return network_contexts
def __init__(self):
HydrusSerialisable.SerialisableBase.__init__(self)
self.engine = None
self._dirty = False
self._lock = threading.Lock()
self._network_contexts_to_bandwidth_trackers = collections.defaultdict(
HydrusNetworking.BandwidthTracker)
self._network_contexts_to_bandwidth_rules = collections.defaultdict(
HydrusNetworking.BandwidthRules)
for context_type in [
CC.NETWORK_CONTEXT_GLOBAL, CC.NETWORK_CONTEXT_HYDRUS,
CC.NETWORK_CONTEXT_DOMAIN, CC.NETWORK_CONTEXT_DOWNLOADER_PAGE,
CC.NETWORK_CONTEXT_SUBSCRIPTION,
CC.NETWORK_CONTEXT_WATCHER_PAGE
]:
self._network_contexts_to_bandwidth_rules[
ClientNetworkingContexts.NetworkContext(
context_type)] = HydrusNetworking.BandwidthRules()
def _GenerateNetworkContexts( self ):
network_contexts = NetworkJob._GenerateNetworkContexts( self )
network_contexts.append( ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_WATCHER_PAGE, self._watcher_key ) )
return network_contexts
def _GenerateNetworkContexts( self ):
network_contexts = NetworkJob._GenerateNetworkContexts( self )
network_contexts.append( ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOWNLOADER_PAGE, self._downloader_page_key ) )
return network_contexts
def _GetRules(self, network_context):
if network_context not in self._network_contexts_to_bandwidth_rules:
network_context = ClientNetworkingContexts.NetworkContext(
network_context.context_type) # i.e. the default
return self._network_contexts_to_bandwidth_rules[network_context]
def _GenerateSpecificNetworkContexts( self ):
# we store cookies on and login to the same hydrus-specific context
session_network_context = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, self._service_key )
login_network_context = session_network_context
return ( session_network_context, login_network_context )
def _GenerateNetworkContexts( self ):
network_contexts = []
network_contexts.append( ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT )
network_contexts.append( ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, self._service_key ) )
return network_contexts
def _GenerateNetworkContexts( self ):
network_contexts = []
network_contexts.append( ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT )
domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( self._domain )
network_contexts.extend( ( ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, domain ) for domain in domains ) )
return network_contexts
def _GetSessionNetworkContext(self, network_context):
# just in case one of these slips through somehow
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
second_level_domain = ClientNetworkingDomain.ConvertDomainIntoSecondLevelDomain(
network_context.context_data)
network_context = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, second_level_domain)
return network_context
def AutoAddDomainMetadatas(self, domain_metadatas):
for domain_metadata in domain_metadatas:
if not domain_metadata.HasBandwidthRules():
return
with self._lock:
domain = domain_metadata.GetDomain()
network_context = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, domain)
bandwidth_rules = domain_metadata.GetBandwidthRules()
self._network_contexts_to_bandwidth_rules[
network_context] = bandwidth_rules
def GetSession(self, network_context):
with self._lock:
# just in case one of these slips through somehow
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
second_level_domain = ClientNetworkingDomain.ConvertDomainIntoSecondLevelDomain(
network_context.context_data)
network_context = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, second_level_domain)
if network_context not in self._network_contexts_to_sessions:
self._network_contexts_to_sessions[
network_context] = self._GenerateSession(network_context)
session = self._network_contexts_to_sessions[network_context]
#
self._CleanSessionCookies(network_context, session)
#
# tumblr can't into ssl for some reason, and the data subdomain they use has weird cert properties, looking like amazon S3
# perhaps it is inward-facing somehow? whatever the case, let's just say f**k it for tumblr
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN and network_context.context_data == 'tumblr.com':
session.verify = False
#
self._SetDirty()
return session
def test_bandwidth_ok(self):
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
HYDRUS_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_HYDRUS, MOCK_HYDRUS_SERVICE_KEY)
#
job = self._GetJob()
job.engine.bandwidth_manager.ReportDataUsed([HYDRUS_NETWORK_CONTEXT],
50)
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.SetRules(HYDRUS_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), True)
#
job = self._GetJob(for_login=True)
job.engine.bandwidth_manager.ReportDataUsed([HYDRUS_NETWORK_CONTEXT],
50)
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.SetRules(HYDRUS_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), True)
def test_bandwidth_exceeded(self):
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
DOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
#
job = self._GetJob()
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.ReportDataUsed([DOMAIN_NETWORK_CONTEXT],
50)
job.engine.bandwidth_manager.SetRules(DOMAIN_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), False)
#
job = self._GetJob(for_login=True)
self.assertEqual(job.BandwidthOK(), True)
job.engine.bandwidth_manager.ReportDataUsed([DOMAIN_NETWORK_CONTEXT],
50)
job.engine.bandwidth_manager.SetRules(DOMAIN_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertEqual(job.BandwidthOK(), True)
def EnsureLoggedIn(self, name):
with self._lock:
if name in self._error_names:
raise Exception(
name +
' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.'
)
if name == 'hentai foundry':
network_context = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com')
required_cookies = ['PHPSESSID', 'YII_CSRF_TOKEN']
elif name == 'pixiv':
network_context = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net')
required_cookies = ['PHPSESSID']
if self._IsLoggedIn(network_context, required_cookies):
return
try:
if name == 'hentai foundry':
self.LoginHF(network_context)
elif name == 'pixiv':
result = self.engine.controller.Read(
'serialisable_simple', 'pixiv_account')
if result is None:
raise HydrusExceptions.DataMissing(
'You need to set up your pixiv credentials in services->manage pixiv account.'
)
(pixiv_id, password) = result
self.LoginPixiv(network_context, pixiv_id, password)
if not self._IsLoggedIn(network_context, required_cookies):
raise Exception(name + ' login did not work correctly!')
HydrusData.Print('Successfully logged into ' + name + '.')
except:
self._error_names.add(name)
raise
def test_can_start(self):
EMPTY_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 1048576)
PERMISSIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
PERMISSIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None,
10000)
RESTRICTIVE_DATA_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_DATA_RULES.AddRule(HC.BANDWIDTH_TYPE_DATA, None, 10)
RESTRICTIVE_REQUEST_RULES = HydrusNetworking.BandwidthRules()
RESTRICTIVE_REQUEST_RULES.AddRule(HC.BANDWIDTH_TYPE_REQUESTS, None, 1)
DOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_DOMAIN)
SUBDOMAIN_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext(
CC.NETWORK_CONTEXT_DOMAIN, MOCK_SUBDOMAIN)
GLOBAL_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT
]
DOMAIN_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
DOMAIN_NETWORK_CONTEXT
]
SUBDOMAIN_NETWORK_CONTEXTS = [
ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
DOMAIN_NETWORK_CONTEXT, SUBDOMAIN_NETWORK_CONTEXT
]
#
fast_forward = HydrusData.GetNow() + 3600
with patch.object(HydrusData, 'GetNow', return_value=fast_forward):
bm = ClientNetworkingBandwidth.NetworkBandwidthManager()
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.ReportRequestUsed(DOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(DOMAIN_NETWORK_CONTEXTS, 50)
bm.ReportRequestUsed(SUBDOMAIN_NETWORK_CONTEXTS)
bm.ReportDataUsed(SUBDOMAIN_NETWORK_CONTEXTS, 25)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
EMPTY_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(SUBDOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_DATA_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(DOMAIN_NETWORK_CONTEXT, PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_REQUEST_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
PERMISSIVE_REQUEST_RULES)
self.assertTrue(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertTrue(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
#
bm.SetRules(ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT,
RESTRICTIVE_DATA_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, RESTRICTIVE_REQUEST_RULES)
bm.SetRules(DOMAIN_NETWORK_CONTEXT, EMPTY_RULES)
self.assertFalse(bm.CanStartRequest(GLOBAL_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(DOMAIN_NETWORK_CONTEXTS))
self.assertFalse(bm.CanStartRequest(SUBDOMAIN_NETWORK_CONTEXTS))
# make this serialisable
class LoginCredentials( object ):
def __init__( self ):
self._credentials = {} # user-facing name (unique) : string
self._validity = VALIDITY_UNTESTED
def GetCredential( self, name ):
return self._credentials[ name ]
PIXIV_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
HENTAI_FOUNDRY_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
SERIALISABLE_NAME = 'Login Manager'
SERIALISABLE_VERSION = 1
SESSION_TIMEOUT = 60 * 45
def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
# needs _dirty and setdirty and be on that serialisation check and so on
def GetSessionForDomain(self, domain):
network_context = ClientNetworkingContexts.NetworkContext(
context_type=CC.NETWORK_CONTEXT_DOMAIN, context_data=domain)
return self.GetSession(network_context)