def test_throughput_while_adjusting_servicecount(self): with self.database.transaction(): ServiceManager.createOrUpdateService(TestService, "TestService", target_count=0) emptyThroughputs = [self.measureThroughput(1.0)] fullThroughputs = [] for i in range(2): with self.database.transaction(): ServiceManager.startService("TestService", 20) self.waitForCount(20) fullThroughputs.append(self.measureThroughput(1.0)) with self.database.transaction(): ServiceManager.startService("TestService", 0) self.waitForCount(0) emptyThroughputs.append(self.measureThroughput(1.0)) print("Throughput with no workers: ", emptyThroughputs) print("Throughput with 20 workers: ", fullThroughputs) # we want to ensure that we don't have some problem where our transaction throughput # goes down because we have left-over connections or something similar in the server, # which would be a real problem! self.assertTrue(emptyThroughputs[-1] * 2 > emptyThroughputs[0], (emptyThroughputs))
def configurableSetUp(self, auth_type="LDAP", auth_hostname=None, authorized_groups=(), ldap_base_dn=None, ldap_ntlm_domain=None, company_name=None): self.token = genToken() log_level = self._logger.getEffectiveLevel() loglevel_name = logging.getLevelName(log_level) self.server, self.cleanupFn = autoconfigure_and_start_service_manager( port=DATABASE_SERVER_PORT, auth_token=self.token, loglevel_name=loglevel_name) try: self.database = connect("localhost", DATABASE_SERVER_PORT, self.token, retry=True) self.database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) with self.database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) optional_args = [] if len(authorized_groups) > 0: optional_args.extend( ['--authorized-groups', *authorized_groups]) if auth_hostname: optional_args.extend(['--auth-hostname', auth_hostname]) if ldap_base_dn: optional_args.extend(['--ldap-base-dn', ldap_base_dn]) if ldap_ntlm_domain: optional_args.extend(['--ldap-ntlm-domain', ldap_ntlm_domain]) if company_name: optional_args.extend(['--company-name', company_name]) ActiveWebService.configureFromCommandline(self.database, service, [ '--port', str(WEB_SERVER_PORT), '--host', 'localhost', '--log-level', loglevel_name, '--auth', auth_type ] + optional_args) with self.database.transaction(): ServiceManager.startService("ActiveWebService", 1) self.waitUntilUp() except Exception: self.cleanupFn(error=True) raise
def test_racheting_service_count_up_and_down(self): with self.database.transaction(): ServiceManager.createOrUpdateService(TestService, "TestService", target_count=1) numpy.random.seed(42) for count in numpy.random.choice(6, size=20): logging.getLogger(__name__).info( "Setting count for TestService to %s and waiting for it to be alive.", count) with self.database.transaction(): ServiceManager.startService("TestService", int(count)) self.waitForCount(count) with self.database.transaction(): ServiceManager.startService("TestService", 0) self.waitForCount(0) # make sure we don't have a bunch of zombie processes hanging underneath the service manager time.sleep(1.0) self.assertEqual(len(psutil.Process().children()[0].children()), 0)
def test_once(timeout=6.0): with self.database.transaction(): self.assertIsNone(Initialized.lookupAny()) self.assertIsNone(Stopped.lookupAny()) ServiceManager.createOrUpdateService(WaitForService, svcName) ServiceManager.startService(svcName) ServiceManager.waitRunning(self.database, svcName, timeout=timeout) with self.database.view(): self.assertIsNotNone(Initialized.lookupAny()) with self.database.transaction(): ServiceManager.stopService(svcName) ServiceManager.waitStopped(self.database, svcName, timeout=timeout) with self.database.view(): self.assertIsNotNone(Stopped.lookupAny()) with self.database.transaction(): for obj in Initialized.lookupAll(): obj.delete() for obj in Stopped.lookupAll(): obj.delete()
def DISABLEDtest_throughput_with_many_workers(self): with self.database.transaction(): ServiceManager.createOrUpdateService(TestService, "TestService", target_count=0) throughputs = [] for ct in [16,18,20,22,24,26,28,30,32,34,0]: with self.database.transaction(): ServiceManager.startService("TestService", ct) self.waitForCount(ct) throughputs.append(self.measureThroughput(5.0)) print("Total throughput was", throughputs, " transactions per second")
def main(argv=None): if argv is not None: argv = sys.argv token = genToken() port = 8020 with tempfile.TemporaryDirectory() as tmpDirName: try: server = start_service_manager(tmpDirName, port, token) database = connect("localhost", port, token, retry=True) database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) with database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) ActiveWebService.configureFromCommandline(database, service, [ '--port', '8000', '--host', '0.0.0.0', '--auth', 'NONE', '--log-level', 'INFO' ]) with database.transaction(): ServiceManager.startService("ActiveWebService", 1) with database.transaction(): service = ServiceManager.createOrUpdateService( UninitializableService, "UninitializableService", target_count=1) with database.transaction(): service = ServiceManager.createOrUpdateService(HappyService, "HappyService", target_count=1) with database.transaction(): service = ServiceManager.createOrUpdateService( GraphDisplayService, "GraphDisplayService", target_count=1) while True: time.sleep(.1) finally: server.terminate() server.wait() return 0
def test_shutdown_hanging_services(self): with self.database.transaction(): ServiceManager.createOrUpdateService(HangingService, "HangingService", target_count=10) self.waitForCount(10) t0 = time.time() with self.database.transaction(): ServiceManager.startService("HangingService", 0) self.waitForCount(0) self.assertTrue(time.time() - t0 < 2.0) # make sure we don't have a bunch of zombie processes hanging underneath the service manager time.sleep(1.0) self.assertEqual(len(psutil.Process().children()[0].children()), 0)
def configurableSetUp( self, hostname='localhost', login_plugin_factory=None, # default: LoginIpPlugin, login_config=None, auth_plugins=(None), module=None, db_init_fun=None): self.base_url = "http://{host}:{port}".format(host=hostname, port=WEB_SERVER_PORT) login_plugin_factory = login_plugin_factory or LoginIpPlugin self.token = genToken() log_level = self._logger.getEffectiveLevel() loglevel_name = logging.getLevelName(log_level) self.server, self.cleanupFn = autoconfigure_and_start_service_manager( port=DATABASE_SERVER_PORT, auth_token=self.token, loglevel_name=loglevel_name, own_hostname=hostname, db_hostname=hostname) try: self.database = connect(hostname, DATABASE_SERVER_PORT, self.token, retry=True) self.database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) if db_init_fun is not None: db_init_fun(self.database) codebase = None if module is not None and not module.__name__.startswith( "object_database."): self.database.serializeFromModule(module) root_path = TypedPythonCodebase.rootlevelPathFromModule(module) tpcodebase = TypedPythonCodebase.FromRootlevelPath(root_path) with self.database.transaction(): codebase = service_schema.Codebase.createFromCodebase( tpcodebase) with self.database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) ActiveWebService.configureFromCommandline(self.database, service, [ '--port', str(WEB_SERVER_PORT), '--host', hostname, '--log-level', loglevel_name, ]) if login_config is None: login_config = self.login_config ActiveWebService.setLoginPlugin(self.database, service, login_plugin_factory, auth_plugins, codebase=codebase, config=login_config) with self.database.transaction(): ServiceManager.startService("ActiveWebService", 1) self.waitUntilUp() except Exception: self.cleanupFn(error=True) raise
def main(argv=None): if argv is not None: argv = sys.argv token = genToken() port = 8020 loglevel_name = 'INFO' with tempfile.TemporaryDirectory() as tmpDirName: try: server = start_service_manager(tmpDirName, port, token, loglevel_name=loglevel_name) database = connect("localhost", port, token, retry=True) database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) with database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) ActiveWebService.configureFromCommandline( database, service, ['--port', '8000', '--host', '0.0.0.0', '--log-level', loglevel_name] ) ActiveWebService.setLoginPlugin( database, service, LoginIpPlugin, [None], config={'company_name': 'A Testing Company'} ) with database.transaction(): ServiceManager.startService("ActiveWebService", 1) with database.transaction(): service = ServiceManager.createOrUpdateService( UninitializableService, "UninitializableService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( HappyService, "HappyService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( GraphDisplayService, "GraphDisplayService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( TextEditorService, "TextEditorService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( DropdownTestService, "DropdownTestService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( BigGridTestService, "BigGridTestService", target_count=1 ) with database.transaction(): ServiceManager.createOrUpdateServiceWithCodebase( service_schema.Codebase.createFromFiles({ 'test_service/__init__.py': '', 'test_service/service.py': textwrap.dedent(""" from object_database.service_manager.ServiceBase import ServiceBase class TestService(ServiceBase): gbRamUsed = 0 coresUsed = 0 def initialize(self): with self.db.transaction(): self.runtimeConfig.serviceInstance.statusMessage = "Loaded" def doWork(self, shouldStop): shouldStop.wait() def display(self, queryParams=None): return "test service display message" """) }), "test_service.service.TestService", "TestService", 10 ) print("SERVER IS BOOTED") while True: time.sleep(.1) finally: server.terminate() server.wait() return 0
def setCountAndBlock(self, count): with self.database.transaction(): ServiceManager.startService("TestService", count) self.waitForCount(count)