def configurableSetUp(self, auth_type="LDAP", auth_hostname=None, authorized_groups=(), ldap_base_dn=None, ldap_ntlm_domain=None, company_name=None): self.token = genToken() log_level = self._logger.getEffectiveLevel() loglevel_name = logging.getLevelName(log_level) self.server, self.cleanupFn = autoconfigure_and_start_service_manager( port=DATABASE_SERVER_PORT, auth_token=self.token, loglevel_name=loglevel_name) try: self.database = connect("localhost", DATABASE_SERVER_PORT, self.token, retry=True) self.database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) with self.database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) optional_args = [] if len(authorized_groups) > 0: optional_args.extend( ['--authorized-groups', *authorized_groups]) if auth_hostname: optional_args.extend(['--auth-hostname', auth_hostname]) if ldap_base_dn: optional_args.extend(['--ldap-base-dn', ldap_base_dn]) if ldap_ntlm_domain: optional_args.extend(['--ldap-ntlm-domain', ldap_ntlm_domain]) if company_name: optional_args.extend(['--company-name', company_name]) ActiveWebService.configureFromCommandline(self.database, service, [ '--port', str(WEB_SERVER_PORT), '--host', 'localhost', '--log-level', loglevel_name, '--auth', auth_type ] + optional_args) with self.database.transaction(): ServiceManager.startService("ActiveWebService", 1) self.waitUntilUp() except Exception: self.cleanupFn(error=True) raise
def helper_memory_leak(self, cell, initFn, workFn, thresholdMB): port = 8021 server, cleanupFn = autoconfigure_and_start_service_manager( port=port, auth_token=self.token, ) try: db = connect("localhost", port, self.token, retry=True) db.subscribeToSchema(test_schema) cells = Cells(db) cells.withRoot(cell) initFn(db, cells) rss0 = currentMemUsageMb() log_cells_stats(cells, self._logger.info, indentation=4) workFn(db, cells) log_cells_stats(cells, self._logger.info, indentation=4) rss = currentMemUsageMb() self._logger.info("Initial Memory Usage: {} MB".format(rss0)) self._logger.info("Final Memory Usage: {} MB".format(rss)) self.assertTrue( rss - rss0 < thresholdMB, "Memory Usage Increased by {} MB.".format(rss - rss0)) finally: cleanupFn()
def main(argv): parser = argparse.ArgumentParser("Run a database throughput test") parser.add_argument("host") parser.add_argument("port") parser.add_argument("--service-token", type=str, required=True, help="the auth token to be used with this service") parser.add_argument("seconds", type=float) parser.add_argument("--threads", dest='threads', type=int, default=1) parsedArgs = parser.parse_args(argv[1:]) db = connect(parsedArgs.host, parsedArgs.port, parsedArgs.service_token) schema = Schema("database_throughput_test") @schema.define class Counter: k = int db.subscribeToSchema(schema) t0 = time.time() transactionCount = [] def doWork(): with db.transaction(): c = Counter() while time.time() - t0 < parsedArgs.seconds: with db.transaction(): c.k = c.k + 1 with db.view(): transactionCount.append(c.k) threads = [ threading.Thread(target=doWork) for _ in range(parsedArgs.threads) ] for t in threads: t.start() for t in threads: t.join() print( sum(transactionCount) / parsedArgs.seconds, " transactions per second") return 0
def main(argv=None): if argv is not None: argv = sys.argv token = genToken() port = 8020 with tempfile.TemporaryDirectory() as tmpDirName: try: server = start_service_manager(tmpDirName, port, token) database = connect("localhost", port, token, retry=True) database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) with database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) ActiveWebService.configureFromCommandline(database, service, [ '--port', '8000', '--host', '0.0.0.0', '--auth', 'NONE', '--log-level', 'INFO' ]) with database.transaction(): ServiceManager.startService("ActiveWebService", 1) with database.transaction(): service = ServiceManager.createOrUpdateService( UninitializableService, "UninitializableService", target_count=1) with database.transaction(): service = ServiceManager.createOrUpdateService(HappyService, "HappyService", target_count=1) with database.transaction(): service = ServiceManager.createOrUpdateService( GraphDisplayService, "GraphDisplayService", target_count=1) while True: time.sleep(.1) finally: server.terminate() server.wait() return 0
def setUp(self): self.test_start_time = time.time() self.token = genToken() self.tempDirObj = tempfile.TemporaryDirectory() self.tempDirectoryName = self.tempDirObj.name object_database.service_manager.Codebase.setCodebaseInstantiationDirectory( self.tempDirectoryName, forceReset=True) os.makedirs(os.path.join(self.tempDirectoryName, 'source')) os.makedirs(os.path.join(self.tempDirectoryName, 'storage')) os.makedirs(os.path.join(self.tempDirectoryName, 'logs')) self.logDir = os.path.join(self.tempDirectoryName, 'logs') logLevelName = logging.getLevelName( logging.getLogger(__name__).getEffectiveLevel()) if VERBOSE: kwargs = {} else: kwargs = { 'stdout': subprocess.DEVNULL, 'stderr': subprocess.DEVNULL } self.server = subprocess.Popen([ sys.executable, os.path.join(ownDir, '..', 'frontends', 'service_manager.py'), 'localhost', 'localhost', "8023", "--run_db", '--logdir', os.path.join(self.tempDirectoryName, 'logs'), '--source', os.path.join(self.tempDirectoryName, 'source'), '--storage', os.path.join(self.tempDirectoryName, 'storage'), '--service-token', self.token, '--shutdownTimeout', '1.0', '--ssl-path', os.path.join(ownDir, '..', '..', 'testcert.cert'), '--log-level', logLevelName ], **kwargs) try: self.database = connect("localhost", 8023, self.token, retry=True) self.database.subscribeToSchema(core_schema, service_schema, *self.schemasToSubscribeTo()) except Exception: self.server.terminate() self.server.wait() self.tempDirObj.cleanup() raise
def setUp(self): self.token = genToken() self.tempDirObj = tempfile.TemporaryDirectory() self.tempDirectoryName = self.tempDirObj.name object_database.service_manager.Codebase.setCodebaseInstantiationDirectory(self.tempDirectoryName, forceReset=True) os.makedirs(os.path.join(self.tempDirectoryName,'source')) os.makedirs(os.path.join(self.tempDirectoryName,'storage')) if not VERBOSE: kwargs = {'stdout': subprocess.DEVNULL, 'stderr': subprocess.DEVNULL} else: kwargs = {} try: self.server = subprocess.Popen( [sys.executable, os.path.join(ownDir, '..', 'frontends', 'service_manager.py'), 'localhost', 'localhost', "8023", "--run_db", '--source', os.path.join(self.tempDirectoryName, 'source'), '--storage', os.path.join(self.tempDirectoryName, 'storage'), '--service-token', self.token, '--shutdownTimeout', '1.0', '--ssl-path', os.path.join(ownDir, '..', '..', 'testcert.cert') ], **kwargs ) # this should throw a subprocess.TimeoutExpired exception if the service did not crash self.server.wait(1.3) except subprocess.TimeoutExpired: pass else: raise Exception("Failed to start service_manager (retcode:{})" .format(self.server.returncode) ) try: self.database = connect("localhost", 8023, self.token, retry=True) self.database.subscribeToSchema(core_schema, service_schema, *self.schemasToSubscribeTo()) except Exception: self.server.terminate() self.server.wait() self.tempDirObj.cleanup() raise
def main(argv=None): if argv is None: argv = sys.argv parser = argparse.ArgumentParser("Redeploy code to the simulation") subparsers = parser.add_subparsers() info_parser = subparsers.add_parser('info', help='list current services') info_parser.set_defaults(command='info') redeploy_parser = subparsers.add_parser( 'redeploy', help='redeploy a set of specific services') redeploy_parser.set_defaults(command='redeploy') redeploy_parser.add_argument('--watch', action='store_true') parsedArgs = parser.parse_args(argv[1:]) name = "Simulation" # add a fake deployment for 'Simulation' deploymentConfig = dict(name="Simulation", host="localhost", port=8887, token="fake_auth_token") try: database = connect(deploymentConfig['host'], deploymentConfig['port'], deploymentConfig['token']) database.subscribeToSchema(core_schema, service_schema) if parsedArgs.command == "info": info(database, deploymentConfig) elif parsedArgs.command == "redeploy": redeploy(database, deploymentConfig, parsedArgs.watch) else: raise UserWarning(f"Unknown command {parsedArgs.command}") except UserWarning as e: print("ERROR: ", e) return 1 return 0
def dbConnectionFactory(): return connect(parsedArgs.host, parsedArgs.port, parsedArgs.authToken)
def _main(argv): configureLogging() parser = argparse.ArgumentParser("Install and configure services.") parser.add_argument("--hostname", default=os.getenv("ODB_HOST", "localhost"), required=False) parser.add_argument("--port", type=int, default=int(os.getenv("ODB_PORT", 8000)), required=False) parser.add_argument("--auth", type=str, default=os.getenv("ODB_AUTH_TOKEN", ""), required=False, help="Auth token to use to connect.") subparsers = parser.add_subparsers() connections_parser = subparsers.add_parser('connections', help='list live connections') connections_parser.set_defaults(command='connections') install_parser = subparsers.add_parser('install', help='install a service') install_parser.set_defaults(command='install') install_parser.add_argument("--path", action='append', dest='paths') install_parser.add_argument("--class") install_parser.add_argument("--name", required=False) install_parser.add_argument("--placement", required=False, default='Any', choices=['Any','Master','Worker']) install_parser.add_argument("--singleton", required=False, action='store_true') reset_parser = subparsers.add_parser('reset', help='reset a service''s boot count') reset_parser.set_defaults(command='reset') reset_parser.add_argument("name") configure_parser = subparsers.add_parser('configure', help='configure a service') configure_parser.set_defaults(command='configure') configure_parser.add_argument("name") configure_parser.add_argument("args", nargs=argparse.REMAINDER) list_parser = subparsers.add_parser('list', help='list installed services') list_parser.set_defaults(command='list') instances_parser = subparsers.add_parser('instances', help='list running service instances') instances_parser.set_defaults(command='instances') hosts_parser = subparsers.add_parser('hosts', help='list running hosts') hosts_parser.set_defaults(command='hosts') start_parser = subparsers.add_parser('start', help='Start (or change target replicas for) a service') start_parser.set_defaults(command='start') start_parser.add_argument("name") start_parser.add_argument("--count", type=int, default=1, required=False) stop_parser = subparsers.add_parser('stop', help='Stop a service') stop_parser.set_defaults(command='stop') stop_parser.add_argument("name") parsedArgs = parser.parse_args(argv[1:]) db = connect(parsedArgs.hostname, parsedArgs.port, parsedArgs.auth) db.subscribeToSchema(core_schema, service_schema, lazySubscription=True) if parsedArgs.command == 'connections': table = [['Connection ID']] with db.view(): for c in sorted(core_schema.Connection.lookupAll(), key=lambda c: c._identity): table.append([c._identity]) print(formatTable(table)) if parsedArgs.command == 'configure': try: with db.transaction(): s = service_schema.Service.lookupAny(name=parsedArgs.name) svcClass = s.instantiateServiceType() svcClass.configureFromCommandline(db, s, parsedArgs.args) except Exception as e: traceback.print_exc() return 1 if parsedArgs.command == 'reset': with db.transaction(): service = service_schema.Service.lookupAny(name=parsedArgs.name) service.timesBootedUnsuccessfully = 0 if parsedArgs.command == 'install': if parsedArgs.paths: paths = parsedArgs.paths else: paths = [findGitParent(os.getcwd())] gbRamUsed = 1 coresUsed = 1 with db.transaction(): fullClassname = getattr(parsedArgs, 'class') modulename, classname = fullClassname.rsplit(".",1) if modulename.startswith("object_database"): def _getobject(modname, attribute): mod = __import__(modname, fromlist=[attribute]) return mod.__dict__[attribute] actualClass = _getobject(modulename, classname) if not isinstance(actualClass, type): print("Named class %s is not a type." % fullClassname) return 1 if not issubclass(actualClass, ServiceBase): print("Named class %s is not a ServiceBase subclass." % fullClassname) return 1 coresUsed = actualClass.coresUsed gbRamUsed = actualClass.gbRamUsed ServiceManager.createOrUpdateService(actualClass, classname, placement=parsedArgs.placement, isSingleton=parsedArgs.singleton, coresUsed=coresUsed, gbRamUsed=gbRamUsed ) else: codebase = service_schema.Codebase.create(paths) #make sure the codebase is importable, etc module = codebase.instantiate(modulename) if module is None: print("Can't find", module, "in the codebase") actualClass = module.__dict__.get(classname, None) if actualClass is None: print("Can't find", module, "in module", modulename) if actualClass is None: print("Can't find", classname, "in the codebase") return 1 if not isinstance(actualClass, type): print("Named class %s is not a type." % fullClassname) return 1 if not issubclass(actualClass, ServiceBase): print("Named class %s is not a ServiceBase subclass." % fullClassname) return 1 coresUsed = actualClass.coresUsed gbRamUsed = actualClass.gbRamUsed if not parsedArgs.name: name = fullClassname.split(".")[-1] else: name = parsedArgs.name ServiceManager.createOrUpdateServiceWithCodebase(codebase, fullClassname, name, targetCount=None, placement=parsedArgs.placement, coresUsed=coresUsed, gbRamUsed=gbRamUsed) if parsedArgs.command == 'list': table = [['Service', 'Codebase', 'Module', 'Class', 'Placement', 'TargetCount', 'Cores', 'RAM']] with db.view(): for s in sorted(service_schema.Service.lookupAll(), key=lambda s:s.name): table.append([ s.name, str(s.codebase), s.service_module_name, s.service_class_name, s.placement, str(s.target_count), s.coresUsed, s.gbRamUsed ]) print(formatTable(table)) if parsedArgs.command == 'instances': table = [['Service', 'Host', 'Connection', 'State']] with db.view(): for s in sorted(service_schema.ServiceInstance.lookupAll(), key=lambda s:(s.service.name, s.host.hostname, s.state)): table.append([ s.service.name, s.host.hostname, s.connection if s.connection.exists() else "<DEAD>", s.state ]) print(formatTable(table)) if parsedArgs.command == 'hosts': table = [['Connection', 'IsMaster', 'Hostname', 'RAM USAGE', 'CORE USAGE', 'SERVICE COUNT']] with db.view(): for s in sorted(service_schema.ServiceHost.lookupAll(), key=lambda s:s.hostname): table.append([ s.connection._identity, str(s.isMaster), s.hostname, "%.1f / %.1f" % (s.gbRamUsed, s.maxGbRam), "%s / %s" % (s.coresUsed, s.maxCores), str(len(service_schema.ServiceInstance.lookupAll(host=s))) ]) print(formatTable(table)) if parsedArgs.command == 'start': with db.transaction(): s = service_schema.Service.lookupAny(name=parsedArgs.name) if not s: print("Can't find a service named", s) return 1 s.target_count = max(parsedArgs.count, 0) if parsedArgs.command == 'stop': with db.transaction(): s = service_schema.Service.lookupAny(name=parsedArgs.name) if not s: print("Can't find a service named", s) return 1 s.target_count = 0 return 0
def dbConnectionFactory(): return connect(host, port, self.serviceToken)
def dbConnectionFactory(): return connect(host, port, self.authToken)
def configurableSetUp( self, hostname='localhost', login_plugin_factory=None, # default: LoginIpPlugin, login_config=None, auth_plugins=(None), module=None, db_init_fun=None): self.base_url = "http://{host}:{port}".format(host=hostname, port=WEB_SERVER_PORT) login_plugin_factory = login_plugin_factory or LoginIpPlugin self.token = genToken() log_level = self._logger.getEffectiveLevel() loglevel_name = logging.getLevelName(log_level) self.server, self.cleanupFn = autoconfigure_and_start_service_manager( port=DATABASE_SERVER_PORT, auth_token=self.token, loglevel_name=loglevel_name, own_hostname=hostname, db_hostname=hostname) try: self.database = connect(hostname, DATABASE_SERVER_PORT, self.token, retry=True) self.database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) if db_init_fun is not None: db_init_fun(self.database) codebase = None if module is not None and not module.__name__.startswith( "object_database."): self.database.serializeFromModule(module) root_path = TypedPythonCodebase.rootlevelPathFromModule(module) tpcodebase = TypedPythonCodebase.FromRootlevelPath(root_path) with self.database.transaction(): codebase = service_schema.Codebase.createFromCodebase( tpcodebase) with self.database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) ActiveWebService.configureFromCommandline(self.database, service, [ '--port', str(WEB_SERVER_PORT), '--host', hostname, '--log-level', loglevel_name, ]) if login_config is None: login_config = self.login_config ActiveWebService.setLoginPlugin(self.database, service, login_plugin_factory, auth_plugins, codebase=codebase, config=login_config) with self.database.transaction(): ServiceManager.startService("ActiveWebService", 1) self.waitUntilUp() except Exception: self.cleanupFn(error=True) raise
def main(argv=None): if argv is not None: argv = sys.argv token = genToken() port = 8020 loglevel_name = 'INFO' with tempfile.TemporaryDirectory() as tmpDirName: try: server = start_service_manager(tmpDirName, port, token, loglevel_name=loglevel_name) database = connect("localhost", port, token, retry=True) database.subscribeToSchema(core_schema, service_schema, active_webservice_schema) with database.transaction(): service = ServiceManager.createOrUpdateService( ActiveWebService, "ActiveWebService", target_count=0) ActiveWebService.configureFromCommandline( database, service, ['--port', '8000', '--host', '0.0.0.0', '--log-level', loglevel_name] ) ActiveWebService.setLoginPlugin( database, service, LoginIpPlugin, [None], config={'company_name': 'A Testing Company'} ) with database.transaction(): ServiceManager.startService("ActiveWebService", 1) with database.transaction(): service = ServiceManager.createOrUpdateService( UninitializableService, "UninitializableService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( HappyService, "HappyService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( GraphDisplayService, "GraphDisplayService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( TextEditorService, "TextEditorService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( DropdownTestService, "DropdownTestService", target_count=1 ) with database.transaction(): service = ServiceManager.createOrUpdateService( BigGridTestService, "BigGridTestService", target_count=1 ) with database.transaction(): ServiceManager.createOrUpdateServiceWithCodebase( service_schema.Codebase.createFromFiles({ 'test_service/__init__.py': '', 'test_service/service.py': textwrap.dedent(""" from object_database.service_manager.ServiceBase import ServiceBase class TestService(ServiceBase): gbRamUsed = 0 coresUsed = 0 def initialize(self): with self.db.transaction(): self.runtimeConfig.serviceInstance.statusMessage = "Loaded" def doWork(self, shouldStop): shouldStop.wait() def display(self, queryParams=None): return "test service display message" """) }), "test_service.service.TestService", "TestService", 10 ) print("SERVER IS BOOTED") while True: time.sleep(.1) finally: server.terminate() server.wait() return 0
def newDbConnection(self): return connect("localhost", 8023, self.token, retry=True)
def main(argv): parser = argparse.ArgumentParser("Control the AWS service") parser.add_argument("--hostname", default=os.getenv("ODB_HOST", "localhost"), required=False) parser.add_argument("--port", type=int, default=int(os.getenv("ODB_PORT", 8000)), required=False) parser.add_argument("--auth", type=str, default=os.getenv("ODB_AUTH_TOKEN"), required=False, help="Auth token to use to connect.") subparsers = parser.add_subparsers() config_parser = subparsers.add_parser('config', help='configure the service') config_parser.set_defaults(command='config') config_parser.add_argument('--region', required=False) config_parser.add_argument('--vpc_id', required=False) config_parser.add_argument('--subnet', required=False) config_parser.add_argument('--security_group', required=False) config_parser.add_argument('--keypair', required=False) config_parser.add_argument('--worker_name', required=False) config_parser.add_argument('--worker_iam_role_name', required=False) config_parser.add_argument('--docker_image', required=False) config_parser.add_argument('--defaultStorageSize', required=False, type=int) config_parser.add_argument('--max_to_boot', required=False, type=int) install_parser = subparsers.add_parser('install', help='install the service') install_parser.set_defaults(command='install') list_parser = subparsers.add_parser('list', help='list machines') list_parser.set_defaults(command='list') boot_parser = subparsers.add_parser('boot', help='set the number of desired boxes') boot_parser.set_defaults(command='boot') boot_parser.add_argument("instance_type") boot_parser.add_argument("count", type=int) killall_parser = subparsers.add_parser('killall', help='kill everything') killall_parser.set_defaults(command='killall') reset_parser = subparsers.add_parser('reset', help='kill everything') reset_parser.set_defaults(command='reset') configureLogging() parsedArgs = parser.parse_args(argv[1:]) db = connect(parsedArgs.hostname, parsedArgs.port, parsedArgs.auth) db.subscribeToSchema(service_schema, lazySubscription=True) db.subscribeToSchema(aws_worker_boot_schema) if parsedArgs.command == 'reset': with db.transaction(): for s in aws_worker_boot_schema.State.lookupAll(): s.delete() if parsedArgs.command == 'config': with db.transaction(): AwsWorkerBootService.configure( db_hostname=parsedArgs.hostname, db_port=parsedArgs.port, region=parsedArgs.region, vpc_id=parsedArgs.vpc_id, subnet=parsedArgs.subnet, security_group=parsedArgs.security_group, keypair=parsedArgs.keypair, worker_name=parsedArgs.worker_name, worker_iam_role_name=parsedArgs.worker_iam_role_name, docker_image=parsedArgs.docker_image, defaultStorageSize=parsedArgs.defaultStorageSize, max_to_boot=parsedArgs.max_to_boot) if parsedArgs.command == 'install': with db.transaction(): ServiceManager.createOrUpdateService(AwsWorkerBootService, "AwsWorkerBootService", placement="Master") if parsedArgs.command == 'list': print() print() with db.view(): api = AwsApi() booted = AwsWorkerBootService.currentBooted() targets = AwsWorkerBootService.currentTargets() table = [["Instance Type", "Booted", "Desired"]] for instance_type in sorted(set(list(booted) + list(targets))): table.append([ instance_type, booted.get(instance_type, 0), targets.get(instance_type, 0) ]) print(formatTable(table)) print() print() with db.view(): api = AwsApi() table = [["InstanceID", "InstanceType", "IP", "Uptime"]] for instanceId, instance in api.allRunningInstances( spot=None).items(): table.append([ instance['InstanceId'], instance['InstanceType'], instance['PrivateIpAddress'], secondsToHumanReadable(time.time() - instance['LaunchTime'].timestamp()) ]) print(formatTable(table)) print() print() if parsedArgs.command == 'boot': with db.transaction(): AwsWorkerBootService.setBootState(parsedArgs.instance_type, parsedArgs.count) if parsedArgs.command == 'killall': with db.transaction(): AwsWorkerBootService.shutdownAll() with db.view(): api = AwsApi() for i in api.allRunningInstances(): try: api.terminateInstanceById(i) except Exception: pass return 0