def configurableSetUp(self,
auth_type="LDAP",
auth_hostname=None,
authorized_groups=(),
ldap_base_dn=None,
ldap_ntlm_domain=None,
company_name=None):
self.token = genToken()
log_level = self._logger.getEffectiveLevel()
loglevel_name = logging.getLevelName(log_level)
self.server, self.cleanupFn = autoconfigure_and_start_service_manager(
port=DATABASE_SERVER_PORT,
auth_token=self.token,
loglevel_name=loglevel_name)
try:
self.database = connect("localhost",
DATABASE_SERVER_PORT,
self.token,
retry=True)
self.database.subscribeToSchema(core_schema, service_schema,
active_webservice_schema)
with self.database.transaction():
service = ServiceManager.createOrUpdateService(
ActiveWebService, "ActiveWebService", target_count=0)
optional_args = []
if len(authorized_groups) > 0:
optional_args.extend(
['--authorized-groups', *authorized_groups])
if auth_hostname:
optional_args.extend(['--auth-hostname', auth_hostname])
if ldap_base_dn:
optional_args.extend(['--ldap-base-dn', ldap_base_dn])
if ldap_ntlm_domain:
optional_args.extend(['--ldap-ntlm-domain', ldap_ntlm_domain])
if company_name:
optional_args.extend(['--company-name', company_name])
ActiveWebService.configureFromCommandline(self.database, service, [
'--port',
str(WEB_SERVER_PORT), '--host', 'localhost', '--log-level',
loglevel_name, '--auth', auth_type
] + optional_args)
with self.database.transaction():
ServiceManager.startService("ActiveWebService", 1)
self.waitUntilUp()
except Exception:
self.cleanupFn(error=True)
raise
def test_can_run_throughput_test(self):
try:
token = genToken()
server = subprocess.Popen([
sys.executable,
os.path.join(own_dir, "frontends", "database_server.py"),
"localhost", "8888",
"--service-token", token,
"--inmem"]
)
time.sleep(.5)
client = subprocess.run([
sys.executable,
os.path.join(own_dir, "frontends", "database_throughput_test.py"),
"localhost", "8888",
"--service-token", token,
"1"
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
self.assertEqual(client.returncode, 0)
finally:
server.terminate()
server.wait()
def autoconfigure_and_start_service_manager(port=None, auth_token=None, loglevel_name=None, **kwargs):
port = port or 8020
auth_token = auth_token or genToken()
if loglevel_name is None:
loglevel = logging.getLogger(__name__).getEffectiveLevel()
loglevel_name = logging.getLevelName(loglevel)
tempDirObj = tempfile.TemporaryDirectory()
tempDirectoryName = tempDirObj.name
server = start_service_manager(
tempDirectoryName,
port,
auth_token,
loglevel_name,
**kwargs
)
def cleanupFn(error=False):
server.terminate()
server.wait()
if error:
logging.getLogger(__name__).warning(
"Exited with an error. Leaving temporary directory around for inspection: {}"
.format(tempDirectoryName)
)
else:
tempDirObj.cleanup()
return server, cleanupFn
def setUp(self):
self.auth_token = genToken()
self.mem_store = InMemoryPersistence()
self.server = InMemServer(self.mem_store, self.auth_token)
self.server._gc_interval = .1
self.server.start()
def setUp(self):
self.token = genToken()
self.server = InMemServer(auth_token=self.token)
self.server.start()
self.db = self.server.connect(self.token)
self.db.subscribeToSchema(test_schema)
self.cells = Cells(self.db)
def configureApp(self):
instanceName = self.serviceObject.name
self.app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY') or genToken()
self.app.add_url_rule('/', endpoint=None, view_func=lambda: redirect("/services"))
self.app.add_url_rule('/content/<path:path>', endpoint=None, view_func=self.sendContent)
self.app.add_url_rule('/services', endpoint=None, view_func=self.sendPage)
self.app.add_url_rule('/services/<path:path>', endpoint=None, view_func=self.sendPage)
self.app.add_url_rule('/login', endpoint=None, view_func=self.login, methods=['GET', 'POST'])
self.app.add_url_rule('/logout', endpoint=None, view_func=self.logout)
self.sockets.add_url_rule('/socket/<path:path>', None, self.mainSocket)
def setUp(self):
self.mem_store = InMemoryPersistence()
self.auth_token = genToken()
sc = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
sc.load_cert_chain('testcert.cert', 'testcert.key')
self.server = TcpServer(host="localhost",
port=8888,
mem_store=self.mem_store,
ssl_context=sc,
auth_token=self.auth_token)
self.server._gc_interval = .1
self.server.start()
def main(argv=None):
if argv is not None:
argv = sys.argv
token = genToken()
port = 8020
with tempfile.TemporaryDirectory() as tmpDirName:
try:
server = start_service_manager(tmpDirName, port, token)
database = connect("localhost", port, token, retry=True)
database.subscribeToSchema(core_schema, service_schema,
active_webservice_schema)
with database.transaction():
service = ServiceManager.createOrUpdateService(
ActiveWebService, "ActiveWebService", target_count=0)
ActiveWebService.configureFromCommandline(database, service, [
'--port', '8000', '--host', '0.0.0.0', '--auth', 'NONE',
'--log-level', 'INFO'
])
with database.transaction():
ServiceManager.startService("ActiveWebService", 1)
with database.transaction():
service = ServiceManager.createOrUpdateService(
UninitializableService,
"UninitializableService",
target_count=1)
with database.transaction():
service = ServiceManager.createOrUpdateService(HappyService,
"HappyService",
target_count=1)
with database.transaction():
service = ServiceManager.createOrUpdateService(
GraphDisplayService, "GraphDisplayService", target_count=1)
while True:
time.sleep(.1)
finally:
server.terminate()
server.wait()
return 0
def setUp(self):
self.test_start_time = time.time()
self.token = genToken()
self.tempDirObj = tempfile.TemporaryDirectory()
self.tempDirectoryName = self.tempDirObj.name
object_database.service_manager.Codebase.setCodebaseInstantiationDirectory(
self.tempDirectoryName, forceReset=True)
os.makedirs(os.path.join(self.tempDirectoryName, 'source'))
os.makedirs(os.path.join(self.tempDirectoryName, 'storage'))
os.makedirs(os.path.join(self.tempDirectoryName, 'logs'))
self.logDir = os.path.join(self.tempDirectoryName, 'logs')
logLevelName = logging.getLevelName(
logging.getLogger(__name__).getEffectiveLevel())
if VERBOSE:
kwargs = {}
else:
kwargs = {
'stdout': subprocess.DEVNULL,
'stderr': subprocess.DEVNULL
}
self.server = subprocess.Popen([
sys.executable,
os.path.join(ownDir, '..', 'frontends', 'service_manager.py'),
'localhost', 'localhost', "8023", "--run_db", '--logdir',
os.path.join(self.tempDirectoryName, 'logs'), '--source',
os.path.join(self.tempDirectoryName, 'source'), '--storage',
os.path.join(self.tempDirectoryName, 'storage'), '--service-token',
self.token, '--shutdownTimeout', '1.0', '--ssl-path',
os.path.join(ownDir, '..', '..',
'testcert.cert'), '--log-level', logLevelName
], **kwargs)
try:
self.database = connect("localhost", 8023, self.token, retry=True)
self.database.subscribeToSchema(core_schema, service_schema,
*self.schemasToSubscribeTo())
except Exception:
self.server.terminate()
self.server.wait()
self.tempDirObj.cleanup()
raise
def configureApp(self):
self.app.config['SECRET_KEY'] = os.environ.get(
'SECRET_KEY') or genToken()
self.app.add_url_rule('/',
endpoint='index',
view_func=lambda: redirect("/services"))
self.app.add_url_rule('/content/<path:path>',
endpoint=None,
view_func=self.sendContent)
self.app.add_url_rule('/services',
endpoint=None,
view_func=self.sendPage)
self.app.add_url_rule('/services/<path:path>',
endpoint=None,
view_func=self.sendPage)
self.app.add_url_rule('/status', view_func=self.statusPage)
self.sockets.add_url_rule('/socket/<path:path>', None, self.mainSocket)
def setUp(self):
self.token = genToken()
self.tempDirObj = tempfile.TemporaryDirectory()
self.tempDirectoryName = self.tempDirObj.name
object_database.service_manager.Codebase.setCodebaseInstantiationDirectory(self.tempDirectoryName, forceReset=True)
os.makedirs(os.path.join(self.tempDirectoryName,'source'))
os.makedirs(os.path.join(self.tempDirectoryName,'storage'))
if not VERBOSE:
kwargs = {'stdout': subprocess.DEVNULL, 'stderr': subprocess.DEVNULL}
else:
kwargs = {}
try:
self.server = subprocess.Popen(
[sys.executable, os.path.join(ownDir, '..', 'frontends', 'service_manager.py'),
'localhost', 'localhost', "8023",
"--run_db",
'--source', os.path.join(self.tempDirectoryName, 'source'),
'--storage', os.path.join(self.tempDirectoryName, 'storage'),
'--service-token', self.token,
'--shutdownTimeout', '1.0',
'--ssl-path', os.path.join(ownDir, '..', '..', 'testcert.cert')
],
**kwargs
)
# this should throw a subprocess.TimeoutExpired exception if the service did not crash
self.server.wait(1.3)
except subprocess.TimeoutExpired:
pass
else:
raise Exception("Failed to start service_manager (retcode:{})"
.format(self.server.returncode)
)
try:
self.database = connect("localhost", 8023, self.token, retry=True)
self.database.subscribeToSchema(core_schema, service_schema, *self.schemasToSubscribeTo())
except Exception:
self.server.terminate()
self.server.wait()
self.tempDirObj.cleanup()
raise
def setUp(self):
self.tempDir = tempfile.TemporaryDirectory()
self.tempDirName = self.tempDir.__enter__()
self.auth_token = genToken()
if hasattr(self, 'redisProcess') and self.redisProcess:
self.redisProcess.terminate()
self.redisProcess.wait()
self.redisProcess = subprocess.Popen([
"/usr/bin/redis-server", '--port', '1115', '--logfile',
os.path.join(self.tempDirName, "log.txt"), "--dbfilename",
"db.rdb", "--dir",
os.path.join(self.tempDirName)
])
time.sleep(.5)
assert self.redisProcess.poll() is None
redis.StrictRedis(db=0, decode_responses=True, port=1115).echo("hi")
self.mem_store = RedisPersistence(port=1115)
self.server = InMemServer(self.mem_store, self.auth_token)
self.server._gc_interval = .1
self.server.start()
def configurableSetUp(
self,
hostname='localhost',
login_plugin_factory=None, # default: LoginIpPlugin,
login_config=None,
auth_plugins=(None),
module=None,
db_init_fun=None):
self.base_url = "http://{host}:{port}".format(host=hostname,
port=WEB_SERVER_PORT)
login_plugin_factory = login_plugin_factory or LoginIpPlugin
self.token = genToken()
log_level = self._logger.getEffectiveLevel()
loglevel_name = logging.getLevelName(log_level)
self.server, self.cleanupFn = autoconfigure_and_start_service_manager(
port=DATABASE_SERVER_PORT,
auth_token=self.token,
loglevel_name=loglevel_name,
own_hostname=hostname,
db_hostname=hostname)
try:
self.database = connect(hostname,
DATABASE_SERVER_PORT,
self.token,
retry=True)
self.database.subscribeToSchema(core_schema, service_schema,
active_webservice_schema)
if db_init_fun is not None:
db_init_fun(self.database)
codebase = None
if module is not None and not module.__name__.startswith(
"object_database."):
self.database.serializeFromModule(module)
root_path = TypedPythonCodebase.rootlevelPathFromModule(module)
tpcodebase = TypedPythonCodebase.FromRootlevelPath(root_path)
with self.database.transaction():
codebase = service_schema.Codebase.createFromCodebase(
tpcodebase)
with self.database.transaction():
service = ServiceManager.createOrUpdateService(
ActiveWebService, "ActiveWebService", target_count=0)
ActiveWebService.configureFromCommandline(self.database, service, [
'--port',
str(WEB_SERVER_PORT),
'--host',
hostname,
'--log-level',
loglevel_name,
])
if login_config is None:
login_config = self.login_config
ActiveWebService.setLoginPlugin(self.database,
service,
login_plugin_factory,
auth_plugins,
codebase=codebase,
config=login_config)
with self.database.transaction():
ServiceManager.startService("ActiveWebService", 1)
self.waitUntilUp()
except Exception:
self.cleanupFn(error=True)
raise
def main(argv=None):
if argv is not None:
argv = sys.argv
token = genToken()
port = 8020
loglevel_name = 'INFO'
with tempfile.TemporaryDirectory() as tmpDirName:
try:
server = start_service_manager(tmpDirName, port, token,
loglevel_name=loglevel_name)
database = connect("localhost", port, token, retry=True)
database.subscribeToSchema(core_schema, service_schema,
active_webservice_schema)
with database.transaction():
service = ServiceManager.createOrUpdateService(
ActiveWebService, "ActiveWebService", target_count=0)
ActiveWebService.configureFromCommandline(
database, service,
['--port', '8000',
'--host', '0.0.0.0',
'--log-level', loglevel_name]
)
ActiveWebService.setLoginPlugin(
database,
service,
LoginIpPlugin,
[None],
config={'company_name': 'A Testing Company'}
)
with database.transaction():
ServiceManager.startService("ActiveWebService", 1)
with database.transaction():
service = ServiceManager.createOrUpdateService(
UninitializableService, "UninitializableService",
target_count=1
)
with database.transaction():
service = ServiceManager.createOrUpdateService(
HappyService, "HappyService", target_count=1
)
with database.transaction():
service = ServiceManager.createOrUpdateService(
GraphDisplayService, "GraphDisplayService", target_count=1
)
with database.transaction():
service = ServiceManager.createOrUpdateService(
TextEditorService, "TextEditorService", target_count=1
)
with database.transaction():
service = ServiceManager.createOrUpdateService(
DropdownTestService, "DropdownTestService", target_count=1
)
with database.transaction():
service = ServiceManager.createOrUpdateService(
BigGridTestService, "BigGridTestService", target_count=1
)
with database.transaction():
ServiceManager.createOrUpdateServiceWithCodebase(
service_schema.Codebase.createFromFiles({
'test_service/__init__.py': '',
'test_service/service.py': textwrap.dedent("""
from object_database.service_manager.ServiceBase import ServiceBase
class TestService(ServiceBase):
gbRamUsed = 0
coresUsed = 0
def initialize(self):
with self.db.transaction():
self.runtimeConfig.serviceInstance.statusMessage = "Loaded"
def doWork(self, shouldStop):
shouldStop.wait()
def display(self, queryParams=None):
return "test service display message"
""")
}),
"test_service.service.TestService",
"TestService",
10
)
print("SERVER IS BOOTED")
while True:
time.sleep(.1)
finally:
server.terminate()
server.wait()
return 0
def setUp(self):
self.token = genToken()
self.mem_store = InMemoryPersistence()
self.server = InMemServer(self.mem_store, self.token)
self.server.start()