def _callbackParseGETArgs( self, request ):
parsed_request_args = HydrusNetwork.ParseHydrusNetworkGETArgs( request.args )
request.parsed_request_args = parsed_request_args
return request
def _threadDoPOSTJob(self, request):
services = request.parsed_request_args['services']
unique_ports = {service.GetPort() for service in services}
if len(unique_ports) < len(services):
raise HydrusExceptions.BadRequestException(
'It looks like some of those services share ports! Please give them unique ports!'
)
with HG.dirty_object_lock:
HG.server_controller.SetServices(services)
service_keys_to_access_keys = HG.server_controller.WriteSynchronous(
'services', request.hydrus_account, services)
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes(
{'service_keys_to_access_keys': service_keys_to_access_keys})
response_context = HydrusServerResources.ResponseContext(200,
body=body)
return response_context
def _threadDoGETJob(self, request):
num = request.parsed_request_args['num']
account_type_key = request.parsed_request_args['account_type_key']
if 'expires' in request.parsed_request_args:
expires = request.parsed_request_args['expires']
else:
expires = None
registration_keys = HG.server_controller.Read('registration_keys',
self._service_key,
request.hydrus_account,
num, account_type_key,
expires)
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes(
{'registration_keys': registration_keys})
response_context = HydrusServerResources.ResponseContext(200,
body=body)
return response_context
def _callbackParsePOSTArgs( self, request ):
request.content.seek( 0 )
if not request.requestHeaders.hasHeader( 'Content-Type' ):
parsed_request_args = HydrusNetworking.ParsedRequestArguments()
else:
content_types = request.requestHeaders.getRawHeaders( 'Content-Type' )
content_type = content_types[0]
try:
mime = HC.mime_enum_lookup[ content_type ]
except:
raise HydrusExceptions.BadRequestException( 'Did not recognise Content-Type header!' )
total_bytes_read = 0
if mime == HC.APPLICATION_JSON:
json_string = request.content.read()
total_bytes_read += len( json_string )
parsed_request_args = HydrusNetwork.ParseNetworkBytesToParsedHydrusArgs( json_string )
else:
( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
request.temp_file_info = ( os_file_handle, temp_path )
with open( temp_path, 'wb' ) as f:
for block in HydrusPaths.ReadFileLikeAsBlocks( request.content ):
f.write( block )
total_bytes_read += len( block )
decompression_bombs_ok = self._DecompressionBombsOK( request )
parsed_request_args = HydrusServerResources.ParseFileArguments( temp_path, decompression_bombs_ok )
self._reportDataUsed( request, total_bytes_read )
request.parsed_request_args = parsed_request_args
return request
def _threadDoGETJob(self, request):
subject_identifier = request.parsed_request_args['subject_identifier']
if subject_identifier.HasAccountKey():
subject_account_key = subject_identifier.GetData()
else:
raise HydrusExceptions.MissingCredentialsException(
'I was expecting an account key, but did not get one!')
subject_account = HG.server_controller.Read('account',
self._service_key,
subject_account_key)
account_info = HG.server_controller.Read('account_info',
self._service_key,
request.hydrus_account,
subject_account)
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes(
{'account_info': account_info})
response_context = HydrusServerResources.ResponseContext(200,
body=body)
return response_context
def _threadDoGETJob( self, request ):
petition_count_info = HG.server_controller.Read( 'num_petitions', self._service_key, request.hydrus_account )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'num_petitions' : petition_count_info } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
account = request.hydrus_account
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'account' : account } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
services = HG.server_controller.Read( 'services_from_account', request.hydrus_account )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'services' : services } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def setUpClass( cls ):
cls._access_key = HydrusData.GenerateKey()
cls._serverside_file_service = HydrusNetwork.GenerateService( HydrusData.GenerateKey(), HC.FILE_REPOSITORY, 'file repo', HC.DEFAULT_SERVICE_PORT + 1 )
cls._serverside_tag_service = HydrusNetwork.GenerateService( HydrusData.GenerateKey(), HC.TAG_REPOSITORY, 'tag repo', HC.DEFAULT_SERVICE_PORT )
cls._serverside_admin_service = HydrusNetwork.GenerateService( HydrusData.GenerateKey(), HC.SERVER_ADMIN, 'server admin', HC.DEFAULT_SERVER_ADMIN_PORT )
cls._clientside_file_service = ClientServices.GenerateService( HydrusData.GenerateKey(), HC.FILE_REPOSITORY, 'file repo' )
cls._clientside_tag_service = ClientServices.GenerateService( HydrusData.GenerateKey(), HC.TAG_REPOSITORY, 'tag repo' )
cls._clientside_admin_service = ClientServices.GenerateService( HydrusData.GenerateKey(), HC.SERVER_ADMIN, 'server admin' )
cls._clientside_file_service.SetCredentials( HydrusNetwork.Credentials( '127.0.0.1', HC.DEFAULT_SERVICE_PORT + 1, cls._access_key ) )
cls._clientside_tag_service.SetCredentials( HydrusNetwork.Credentials( '127.0.0.1', HC.DEFAULT_SERVICE_PORT, cls._access_key ) )
cls._clientside_admin_service.SetCredentials( HydrusNetwork.Credentials( '127.0.0.1', HC.DEFAULT_SERVER_ADMIN_PORT, cls._access_key ) )
cls._local_booru = ClientServices.GenerateService( CC.LOCAL_BOORU_SERVICE_KEY, HC.LOCAL_BOORU, 'local booru' )
services_manager = HG.test_controller.services_manager
services_manager._keys_to_services[ cls._clientside_file_service.GetServiceKey() ] = cls._clientside_file_service
services_manager._keys_to_services[ cls._clientside_tag_service.GetServiceKey() ] = cls._clientside_tag_service
services_manager._keys_to_services[ cls._clientside_admin_service.GetServiceKey() ] = cls._clientside_admin_service
permissions = [ HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS, HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES ]
account_key = HydrusData.GenerateKey()
account_type = HydrusNetwork.AccountType.GenerateAdminAccountType( HC.SERVER_ADMIN )
created = HydrusData.GetNow() - 100000
expires = None
cls._account = HydrusNetwork.Account( account_key, account_type, created, expires )
cls._file_hash = HydrusData.GenerateKey()
def TWISTEDSetup():
cls._ssl_cert_path = os.path.join( TestController.DB_DIR, 'server.crt' )
cls._ssl_key_path = os.path.join( TestController.DB_DIR, 'server.key' )
# if db test ran, this is still hanging around and read-only, so don't bother to fail overwriting
if not os.path.exists( cls._ssl_cert_path ):
HydrusEncryption.GenerateOpenSSLCertAndKeyFile( cls._ssl_cert_path, cls._ssl_key_path )
context_factory = twisted.internet.ssl.DefaultOpenSSLContextFactory( cls._ssl_key_path, cls._ssl_cert_path )
reactor.listenSSL( HC.DEFAULT_SERVER_ADMIN_PORT, ServerServer.HydrusServiceAdmin( cls._serverside_admin_service ), context_factory )
reactor.listenSSL( HC.DEFAULT_SERVICE_PORT + 1, ServerServer.HydrusServiceRepositoryFile( cls._serverside_file_service ), context_factory )
reactor.listenSSL( HC.DEFAULT_SERVICE_PORT, ServerServer.HydrusServiceRepositoryTag( cls._serverside_tag_service ), context_factory )
reactor.listenTCP( 45866, ClientLocalServer.HydrusServiceBooru( cls._local_booru, allow_non_local_connections = False ) )
reactor.callFromThread( TWISTEDSetup )
time.sleep( 3 )
def _threadDoGETJob( self, request ):
access_key = self._parseHydrusNetworkAccessKey( request )
verified = HG.server_controller.Read( 'verify_access_key', self._service_key, access_key )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'verified' : verified } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
registration_key = request.parsed_request_args[ 'registration_key' ]
access_key = HG.server_controller.Read( 'access_key', self._service_key, registration_key )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'access_key' : access_key } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
hash = request.parsed_request_args[ 'hash' ]
( ip, timestamp ) = HG.server_controller.Read( 'ip', self._service_key, request.hydrus_account, hash )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'ip' : ip, 'timestamp' : timestamp } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
updates = HG.server_controller.Read( 'immediate_update', self._service_key, request.hydrus_account )
updates = HydrusSerialisable.SerialisableList( updates )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'updates' : updates } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
content_type = request.parsed_request_args[ 'content_type' ]
status = request.parsed_request_args[ 'status' ]
petition = HG.server_controller.Read( 'petition', self._service_key, request.hydrus_account, content_type, status )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'petition' : petition } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _threadDoGETJob( self, request ):
# no permissions check as any functional account can get metadata slices
since = request.parsed_request_args[ 'since' ]
metadata_slice = self._service.GetMetadataSlice( since )
body = HydrusNetwork.DumpHydrusArgsToNetworkBytes( { 'metadata_slice' : metadata_slice } )
response_context = HydrusServerResources.ResponseContext( 200, body = body )
return response_context
def _test_repo(self, service):
service_key = service.GetServiceKey()
# num_petitions
num_petitions = [[
HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_STATUS_PETITIONED, 23
], [HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_STATUS_PENDING, 0]]
HG.test_controller.SetRead('num_petitions', num_petitions)
response = service.Request(HC.GET, 'num_petitions')
self.assertEqual(response['num_petitions'], num_petitions)
# petition
action = HC.CONTENT_UPDATE_PETITION
petitioner_account = HydrusNetwork.Account.GenerateUnknownAccount()
reason = 'it sucks'
contents = [
HydrusNetwork.Content(
HC.CONTENT_TYPE_FILES,
[HydrusData.GenerateKey() for i in range(10)])
]
petition = HydrusNetwork.Petition(action, petitioner_account, reason,
contents)
HG.test_controller.SetRead('petition', petition)
response = service.Request(
HC.GET, 'petition', {
'content_type': HC.CONTENT_TYPE_FILES,
'status': HC.CONTENT_UPDATE_PETITION
})
self.assertEqual(response['petition'].GetSerialisableTuple(),
petition.GetSerialisableTuple())
# definitions
definitions_update = HydrusNetwork.DefinitionsUpdate()
for i in range(100, 200):
definitions_update.AddRow(
(HC.DEFINITIONS_TYPE_TAGS, i, 'series:test ' + str(i)))
definitions_update.AddRow((HC.DEFINITIONS_TYPE_HASHES, i + 500,
HydrusData.GenerateKey()))
definitions_update_network_bytes = definitions_update.DumpToNetworkBytes(
)
definitions_update_hash = hashlib.sha256(
definitions_update_network_bytes).digest()
path = ServerFiles.GetExpectedFilePath(definitions_update_hash)
HydrusPaths.MakeSureDirectoryExists(path)
with open(path, 'wb') as f:
f.write(definitions_update_network_bytes)
response = service.Request(HC.GET, 'update',
{'update_hash': definitions_update_hash})
try:
os.remove(path)
except:
pass
self.assertEqual(response, definitions_update_network_bytes)
# content
rows = [(random.randint(100, 1000),
[random.randint(100, 1000) for i in range(50)])
for j in range(20)]
content_update = HydrusNetwork.ContentUpdate()
for row in rows:
content_update.AddRow(
(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, row))
content_update_network_bytes = content_update.DumpToNetworkBytes()
content_update_hash = hashlib.sha256(
content_update_network_bytes).digest()
path = ServerFiles.GetExpectedFilePath(content_update_hash)
with open(path, 'wb') as f:
f.write(content_update_network_bytes)
response = service.Request(HC.GET, 'update',
{'update_hash': content_update_hash})
try:
os.remove(path)
except:
pass
self.assertEqual(response, content_update_network_bytes)
# metadata
metadata = HydrusNetwork.Metadata()
metadata.AppendUpdate([definitions_update_hash, content_update_hash],
HydrusData.GetNow() - 101000,
HydrusData.GetNow() - 1000,
HydrusData.GetNow() + 100000)
service._metadata = metadata
response = service.Request(HC.GET, 'metadata_slice', {'since': 0})
self.assertEqual(response['metadata_slice'].GetSerialisableTuple(),
metadata.GetSerialisableTuple())
# post content
raise NotImplementedError()
'''
def test_server(self):
discard = HG.test_controller.GetWrite(
'session') # just to discard gumph from testserver
session_key_1 = HydrusData.GenerateKey()
service_key = HydrusData.GenerateKey()
permissions = [
HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS,
HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES
]
account_type = HydrusNetwork.AccountType.GenerateAdminAccountType(
HC.SERVER_ADMIN)
created = HydrusData.GetNow() - 100000
expires = HydrusData.GetNow() + 300
account_key_1 = HydrusData.GenerateKey()
account_key_2 = HydrusData.GenerateKey()
access_key_1 = HydrusData.GenerateKey()
hashed_access_key_1 = hashlib.sha256(access_key_1).digest()
access_key_2 = HydrusData.GenerateKey()
hashed_access_key_2 = hashlib.sha256(access_key_2).digest()
account = HydrusNetwork.Account(account_key_1, account_type, created,
expires)
account_2 = HydrusNetwork.Account(account_key_2, account_type, created,
expires)
# test timeout
expires = HydrusData.GetNow() - 10
HG.test_controller.SetRead('sessions', [
(session_key_1, service_key, account, hashed_access_key_1, expires)
])
session_manager = HydrusSessions.HydrusSessionManagerServer()
with self.assertRaises(HydrusExceptions.SessionException):
session_manager.GetAccount(service_key, session_key_1)
# test missing
with self.assertRaises(HydrusExceptions.SessionException):
session_manager.GetAccount(service_key, HydrusData.GenerateKey())
# test fetching a session already in db, after bootup
expires = HydrusData.GetNow() + 300
HG.test_controller.SetRead('sessions', [
(session_key_1, service_key, account, hashed_access_key_1, expires)
])
session_manager = HydrusSessions.HydrusSessionManagerServer()
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, account)
read_account = session_manager.GetAccountFromAccessKey(
service_key, access_key_1)
self.assertIs(read_account, account)
# test too busy to add a new session for a new account it doesn't know about
HG.server_busy.acquire()
with self.assertRaises(HydrusExceptions.ServerBusyException):
session_manager.AddSession(service_key, HydrusData.GenerateKey())
session_manager.GetAccountFromAccessKey(service_key,
HydrusData.GenerateKey())
# but ok to get for a session that already exists while busy
session_manager.GetAccount(service_key, session_key_1)
session_manager.GetAccountFromAccessKey(service_key, access_key_1)
HG.server_busy.release()
# test adding a session
HG.test_controller.ClearWrites('session')
expires = HydrusData.GetNow() + 300
HG.test_controller.SetRead('account_key_from_access_key',
account_key_2)
HG.test_controller.SetRead('account', account_2)
(session_key_2,
expires_2) = session_manager.AddSession(service_key, access_key_2)
[(args, kwargs)] = HG.test_controller.GetWrite('session')
(written_session_key, written_service_key, written_account_key,
written_expires) = args
self.assertEqual(
(session_key_2, service_key, account_key_2, expires_2),
(written_session_key, written_service_key, written_account_key,
written_expires))
read_account = session_manager.GetAccount(service_key, session_key_2)
self.assertIs(read_account, account_2)
read_account = session_manager.GetAccountFromAccessKey(
service_key, access_key_2)
self.assertIs(read_account, account_2)
# test adding a new session for an account already in the manager
HG.test_controller.SetRead('account_key_from_access_key',
account_key_1)
HG.test_controller.SetRead('account', account)
(session_key_3,
expires_3) = session_manager.AddSession(service_key, access_key_1)
[(args, kwargs)] = HG.test_controller.GetWrite('session')
(written_session_key, written_service_key, written_account_key,
written_expires) = args
self.assertEqual(
(session_key_3, service_key, account_key_1, expires_3),
(written_session_key, written_service_key, written_account_key,
written_expires))
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, account)
read_account = session_manager.GetAccount(service_key, session_key_3)
self.assertIs(read_account, account)
read_account = session_manager.GetAccountFromAccessKey(
service_key, access_key_1)
self.assertIs(read_account, account)
# test individual account refresh
expires = HydrusData.GetNow() + 300
new_obj_account_1 = HydrusNetwork.Account(account_key_1, account_type,
created, expires)
HG.test_controller.SetRead('account', new_obj_account_1)
session_manager.RefreshAccounts(service_key, [account_key_1])
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, new_obj_account_1)
read_account = session_manager.GetAccount(service_key, session_key_3)
self.assertIs(read_account, new_obj_account_1)
read_account = session_manager.GetAccountFromAccessKey(
service_key, access_key_1)
self.assertIs(read_account, new_obj_account_1)
# test all account refresh
expires = HydrusData.GetNow() + 300
new_obj_account_2 = HydrusNetwork.Account(account_key_2, account_type,
created, expires)
HG.test_controller.SetRead(
'sessions', [(session_key_1, service_key, new_obj_account_2,
hashed_access_key_2, expires),
(session_key_2, service_key, new_obj_account_1,
hashed_access_key_1, expires),
(session_key_3, service_key, new_obj_account_2,
hashed_access_key_2, expires)])
session_manager.RefreshAllAccounts()
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, new_obj_account_2)
read_account = session_manager.GetAccount(service_key, session_key_2)
self.assertIs(read_account, new_obj_account_1)
read_account = session_manager.GetAccount(service_key, session_key_3)
self.assertIs(read_account, new_obj_account_2)
read_account = session_manager.GetAccountFromAccessKey(
service_key, access_key_1)
self.assertIs(read_account, new_obj_account_1)
read_account = session_manager.GetAccountFromAccessKey(
service_key, access_key_2)
self.assertIs(read_account, new_obj_account_2)