def main(args):
global logger
# parse args, return zero and exit if help string was printed
parser = PythonTestArgumentParser()
args, filter_actions = parser.parse_args(args[1:])
try:
if not args.skip_build:
result = buildModule(args)
if result:
return result
from object_database.util import configureLogging
configureLogging("test", logging.INFO)
logger = logging.getLogger(__name__)
# set the python path so that we load the right version of the library.
os.environ['PYTHONPATH'] = os.environ.get(
'PYTHONPATH', '') + ":" + os.path.abspath("./build/install")
return executeTests(args, filter_actions)
except Exception:
import traceback
logger.error("executeTests() threw an exception: \n%s",
traceback.format_exc())
return 1
def main(args):
global logger
# parse args, return zero and exit if help string was printed
parser = PythonTestArgumentParser()
args, filter_actions = parser.parse_args(args[1:])
try:
if not args.skip_build:
result = buildModule(args)
if result:
return result
if args.error_output_dir is not None:
os.makedirs(args.error_output_dir, exist_ok=True)
# export an env var; this is what the tests config look for in the
# OutputCapturePlugin class above
os.environ["TEST_ERROR_OUTPUT_DIRECTORY"] = args.error_output_dir
from object_database.util import configureLogging
configureLogging("test", logging.INFO)
logger = logging.getLogger(__name__)
# set the python path so that we load the right version of the library.
os.environ['PYTHONPATH'] = os.environ.get(
'PYTHONPATH', '') + ":" + os.path.abspath("./build/install")
return executeTests(args, filter_actions)
except Exception:
import traceback
logging.getLogger(__name__).error(
"executeTests() threw an exception: \n%s", traceback.format_exc())
return 1
def main(argv):
parser = argparse.ArgumentParser("Run a specific service.")
parser.add_argument("servicename")
parser.add_argument("host")
parser.add_argument("port", type=int)
parser.add_argument("instanceid", type=int)
parser.add_argument("sourceDir")
parser.add_argument("storageRoot")
parser.add_argument("authToken")
parser.add_argument("--log-level", required=False, default="INFO")
parser.add_argument("--ip", required=False)
parsedArgs = parser.parse_args(argv[1:])
level = parsedArgs.log_level.upper()
level = validateLogLevel(level, fallback='INFO')
configureLogging(preamble=str(parsedArgs.instanceid), level=level)
logger = logging.getLogger(__name__)
logger.info("service_entrypoint.py connecting to %s:%s for %s",
parsedArgs.host, parsedArgs.port, parsedArgs.instanceid)
setCodebaseInstantiationDirectory(parsedArgs.sourceDir)
try:
def dbConnectionFactory():
return connect(parsedArgs.host, parsedArgs.port,
parsedArgs.authToken)
mainDbConnection = dbConnectionFactory()
if parsedArgs.ip is not None:
logger.info("Our ip explicitly specified as %s", parsedArgs.ip)
ourIP = parsedArgs.ip
else:
ourIP = mainDbConnection.getConnectionMetadata()['sockname'][0]
logger.info(
"Our ip inferred from our connection back to the server as %s",
ourIP)
manager = ServiceWorker(mainDbConnection, dbConnectionFactory,
int(parsedArgs.instanceid),
parsedArgs.storageRoot, parsedArgs.authToken,
ourIP)
manager.runAndWaitForShutdown()
return 0
except Exception:
logger.error("service_entrypoint failed with an exception:\n%s",
traceback.format_exc())
return 1
def main(argv):
parser = argparse.ArgumentParser("Run a specific service.")
parser.add_argument("servicename")
parser.add_argument("host")
parser.add_argument("port", type=int)
parser.add_argument("instanceid")
parser.add_argument("sourceDir")
parser.add_argument("storageRoot")
parser.add_argument("serviceToken")
parser.add_argument("--log-level", required=False, default="INFO")
parsedArgs = parser.parse_args(argv[1:])
level = parsedArgs.log_level.upper()
checkLogLevelValidity(level)
configureLogging(preamble=parsedArgs.instanceid[:8], level=level)
logger = logging.getLogger(__name__)
logger.info("service_entrypoint.py connecting to %s:%s for %s",
parsedArgs.host,
parsedArgs.port,
parsedArgs.instanceid
)
def dbConnectionFactory():
return connect(parsedArgs.host, parsedArgs.port, parsedArgs.serviceToken)
setCodebaseInstantiationDirectory(parsedArgs.sourceDir)
try:
manager = ServiceWorker(
dbConnectionFactory, parsedArgs.instanceid,
parsedArgs.storageRoot, parsedArgs.serviceToken
)
manager.runAndWaitForShutdown()
return 0
except Exception:
logger.error("service_entrypoint failed with an exception:\n%s", traceback.format_exc())
return 1
def setUpClass(cls):
configureLogging(preamble="cells_html_test", level=logging.INFO)
cls._logger = logging.getLogger(__name__)
def _main(argv):
configureLogging()
parser = argparse.ArgumentParser("Install and configure services.")
parser.add_argument("--hostname", default=os.getenv("ODB_HOST", "localhost"), required=False)
parser.add_argument("--port", type=int, default=int(os.getenv("ODB_PORT", 8000)), required=False)
parser.add_argument("--auth", type=str, default=os.getenv("ODB_AUTH_TOKEN", ""), required=False, help="Auth token to use to connect.")
subparsers = parser.add_subparsers()
connections_parser = subparsers.add_parser('connections', help='list live connections')
connections_parser.set_defaults(command='connections')
install_parser = subparsers.add_parser('install', help='install a service')
install_parser.set_defaults(command='install')
install_parser.add_argument("--path", action='append', dest='paths')
install_parser.add_argument("--class")
install_parser.add_argument("--name", required=False)
install_parser.add_argument("--placement", required=False, default='Any', choices=['Any','Master','Worker'])
install_parser.add_argument("--singleton", required=False, action='store_true')
reset_parser = subparsers.add_parser('reset', help='reset a service''s boot count')
reset_parser.set_defaults(command='reset')
reset_parser.add_argument("name")
configure_parser = subparsers.add_parser('configure', help='configure a service')
configure_parser.set_defaults(command='configure')
configure_parser.add_argument("name")
configure_parser.add_argument("args", nargs=argparse.REMAINDER)
list_parser = subparsers.add_parser('list', help='list installed services')
list_parser.set_defaults(command='list')
instances_parser = subparsers.add_parser('instances', help='list running service instances')
instances_parser.set_defaults(command='instances')
hosts_parser = subparsers.add_parser('hosts', help='list running hosts')
hosts_parser.set_defaults(command='hosts')
start_parser = subparsers.add_parser('start', help='Start (or change target replicas for) a service')
start_parser.set_defaults(command='start')
start_parser.add_argument("name")
start_parser.add_argument("--count", type=int, default=1, required=False)
stop_parser = subparsers.add_parser('stop', help='Stop a service')
stop_parser.set_defaults(command='stop')
stop_parser.add_argument("name")
parsedArgs = parser.parse_args(argv[1:])
db = connect(parsedArgs.hostname, parsedArgs.port, parsedArgs.auth)
db.subscribeToSchema(core_schema, service_schema, lazySubscription=True)
if parsedArgs.command == 'connections':
table = [['Connection ID']]
with db.view():
for c in sorted(core_schema.Connection.lookupAll(), key=lambda c: c._identity):
table.append([c._identity])
print(formatTable(table))
if parsedArgs.command == 'configure':
try:
with db.transaction():
s = service_schema.Service.lookupAny(name=parsedArgs.name)
svcClass = s.instantiateServiceType()
svcClass.configureFromCommandline(db, s, parsedArgs.args)
except Exception as e:
traceback.print_exc()
return 1
if parsedArgs.command == 'reset':
with db.transaction():
service = service_schema.Service.lookupAny(name=parsedArgs.name)
service.timesBootedUnsuccessfully = 0
if parsedArgs.command == 'install':
if parsedArgs.paths:
paths = parsedArgs.paths
else:
paths = [findGitParent(os.getcwd())]
gbRamUsed = 1
coresUsed = 1
with db.transaction():
fullClassname = getattr(parsedArgs, 'class')
modulename, classname = fullClassname.rsplit(".",1)
if modulename.startswith("object_database"):
def _getobject(modname, attribute):
mod = __import__(modname, fromlist=[attribute])
return mod.__dict__[attribute]
actualClass = _getobject(modulename, classname)
if not isinstance(actualClass, type):
print("Named class %s is not a type." % fullClassname)
return 1
if not issubclass(actualClass, ServiceBase):
print("Named class %s is not a ServiceBase subclass." % fullClassname)
return 1
coresUsed = actualClass.coresUsed
gbRamUsed = actualClass.gbRamUsed
ServiceManager.createOrUpdateService(actualClass, classname,
placement=parsedArgs.placement, isSingleton=parsedArgs.singleton,
coresUsed=coresUsed, gbRamUsed=gbRamUsed
)
else:
codebase = service_schema.Codebase.create(paths)
#make sure the codebase is importable, etc
module = codebase.instantiate(modulename)
if module is None:
print("Can't find", module, "in the codebase")
actualClass = module.__dict__.get(classname, None)
if actualClass is None:
print("Can't find", module, "in module", modulename)
if actualClass is None:
print("Can't find", classname, "in the codebase")
return 1
if not isinstance(actualClass, type):
print("Named class %s is not a type." % fullClassname)
return 1
if not issubclass(actualClass, ServiceBase):
print("Named class %s is not a ServiceBase subclass." % fullClassname)
return 1
coresUsed = actualClass.coresUsed
gbRamUsed = actualClass.gbRamUsed
if not parsedArgs.name:
name = fullClassname.split(".")[-1]
else:
name = parsedArgs.name
ServiceManager.createOrUpdateServiceWithCodebase(codebase, fullClassname, name, targetCount=None,
placement=parsedArgs.placement, coresUsed=coresUsed, gbRamUsed=gbRamUsed)
if parsedArgs.command == 'list':
table = [['Service', 'Codebase', 'Module', 'Class', 'Placement', 'TargetCount', 'Cores', 'RAM']]
with db.view():
for s in sorted(service_schema.Service.lookupAll(), key=lambda s:s.name):
table.append([
s.name,
str(s.codebase),
s.service_module_name,
s.service_class_name,
s.placement,
str(s.target_count),
s.coresUsed,
s.gbRamUsed
])
print(formatTable(table))
if parsedArgs.command == 'instances':
table = [['Service', 'Host', 'Connection', 'State']]
with db.view():
for s in sorted(service_schema.ServiceInstance.lookupAll(), key=lambda s:(s.service.name, s.host.hostname, s.state)):
table.append([
s.service.name,
s.host.hostname,
s.connection if s.connection.exists() else "<DEAD>",
s.state
])
print(formatTable(table))
if parsedArgs.command == 'hosts':
table = [['Connection', 'IsMaster', 'Hostname', 'RAM USAGE', 'CORE USAGE', 'SERVICE COUNT']]
with db.view():
for s in sorted(service_schema.ServiceHost.lookupAll(), key=lambda s:s.hostname):
table.append([
s.connection._identity,
str(s.isMaster),
s.hostname,
"%.1f / %.1f" % (s.gbRamUsed, s.maxGbRam),
"%s / %s" % (s.coresUsed, s.maxCores),
str(len(service_schema.ServiceInstance.lookupAll(host=s)))
])
print(formatTable(table))
if parsedArgs.command == 'start':
with db.transaction():
s = service_schema.Service.lookupAny(name=parsedArgs.name)
if not s:
print("Can't find a service named", s)
return 1
s.target_count = max(parsedArgs.count, 0)
if parsedArgs.command == 'stop':
with db.transaction():
s = service_schema.Service.lookupAny(name=parsedArgs.name)
if not s:
print("Can't find a service named", s)
return 1
s.target_count = 0
return 0
def setUpClass(cls):
cls.cleanupFn = lambda error=None: None
cls.base_url = "http://localhost:{port}".format(port=WEB_SERVER_PORT)
configureLogging("aws_test")
cls._logger = logging.getLogger(__name__)
def setUpClass(cls):
cls.cleanupFn = lambda error=None: None
configureLogging("aws_test")
cls._logger = logging.getLogger(__name__)
cls.login_config = dict(company_name="Testing Company")
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
"Run the main service manager and the object_database_service.")
parser.add_argument("own_hostname")
parser.add_argument("db_hostname")
parser.add_argument("port", type=int)
parser.add_argument("--source",
help="path for the source trees used by services",
required=True)
parser.add_argument("--storage",
help="path for local storage used by services",
required=True)
parser.add_argument("--service-token",
type=str,
required=True,
help="the auth token to be used with this service")
parser.add_argument("--run_db", default=False, action='store_true')
parser.add_argument("--ssl-path",
default=None,
required=False,
help="path to (self-signed) SSL certificate")
parser.add_argument("--redis_port", type=int, default=None, required=False)
parser.add_argument("--max_gb_ram",
type=float,
default=None,
required=False)
parser.add_argument("--max_cores", type=int, default=None, required=False)
parser.add_argument("--shutdownTimeout",
type=float,
default=None,
required=False)
parser.add_argument('--logdir', default=None, required=False)
parser.add_argument("--log-level", required=False, default="INFO")
parsedArgs = parser.parse_args(argv[1:])
level_name = parsedArgs.log_level.upper()
checkLogLevelValidity(level_name)
# getLevelName returns a name when given an int and an int when given a name,
# and there doesn't seem to be an other way to get the int from the string.
configureLogging("service_manager", level=logging.getLevelName(level_name))
logger = logging.getLogger(__name__)
parsedArgs = parser.parse_args(argv[1:])
if parsedArgs.redis_port is not None and not parsedArgs.run_db:
sys.stderr.write(
'error: please add --run_db if you want to run a database\n')
parser.print_help()
return 2
logger.info("ServiceManager on %s connecting to %s:%s",
parsedArgs.own_hostname, parsedArgs.db_hostname,
parsedArgs.port)
shouldStop = threading.Event()
def shutdownCleanly(signalNumber, frame):
logger.info("Received signal %s. Stopping.", signalNumber)
shouldStop.set()
signal.signal(signal.SIGINT, shutdownCleanly)
signal.signal(signal.SIGTERM, shutdownCleanly)
resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 4096))
object_database_port = parsedArgs.port
databaseServer = None
serviceManager = None
try:
if parsedArgs.run_db:
ssl_ctx = sslContextFromCertPathOrNone(parsedArgs.ssl_path)
databaseServer = TcpServer(
parsedArgs.own_hostname,
object_database_port,
RedisPersistence(port=parsedArgs.redis_port) if
parsedArgs.redis_port is not None else InMemoryPersistence(),
ssl_context=ssl_ctx,
auth_token=parsedArgs.service_token)
databaseServer.start()
logger.info("Started a database server on %s:%s",
parsedArgs.own_hostname, object_database_port)
serviceManager = None
logger.info("Started object_database")
try:
while not shouldStop.is_set():
if serviceManager is None:
try:
serviceManager = SubprocessServiceManager(
parsedArgs.own_hostname,
parsedArgs.db_hostname,
parsedArgs.port,
parsedArgs.source,
parsedArgs.storage,
parsedArgs.service_token,
isMaster=parsedArgs.run_db,
maxGbRam=parsedArgs.max_gb_ram
or int(psutil.virtual_memory().total / 1024.0 /
1024.0 / 1024.0 + .1),
maxCores=parsedArgs.max_cores
or multiprocessing.cpu_count(),
logfileDirectory=parsedArgs.logdir,
shutdownTimeout=parsedArgs.shutdownTimeout)
logger.info("Connected the service-manager")
except (ConnectionRefusedError, DisconnectedException):
serviceManager = None
if serviceManager is None:
logger.error(
"Failed to connect to service manager. Sleeping and retrying"
)
time.sleep(10)
else:
serviceManager.start()
else:
shouldStop.wait(
timeout=max(.1, serviceManager.shutdownTimeout / 10))
try:
serviceManager.cleanup()
except (ConnectionRefusedError, DisconnectedException,
concurrent.futures._base.TimeoutError):
#try to reconnect
logger.error(
"Disconnected from object_database host. Attempting to reconnect."
)
serviceManager.stop(gracefully=False)
serviceManager = None
except Exception:
logger.error("Service manager cleanup failed:\n%s",
traceback.format_exc())
except KeyboardInterrupt:
return 0
return 0
finally:
if serviceManager is not None:
try:
serviceManager.stop(gracefully=False)
except Exception:
logger.error("Failed to stop the service manager:\n%s",
traceback.format_exc())
if databaseServer is not None:
try:
databaseServer.stop()
except Exception:
logger.error("Failed to stop the database server:\n%s",
traceback.format_exc())
def setUpClass(cls):
configureLogging('database_ring_invariant_test')
def main(argv):
parser = argparse.ArgumentParser("Control the AWS service")
parser.add_argument("--hostname",
default=os.getenv("ODB_HOST", "localhost"),
required=False)
parser.add_argument("--port",
type=int,
default=int(os.getenv("ODB_PORT", 8000)),
required=False)
parser.add_argument("--auth",
type=str,
default=os.getenv("ODB_AUTH_TOKEN"),
required=False,
help="Auth token to use to connect.")
subparsers = parser.add_subparsers()
config_parser = subparsers.add_parser('config',
help='configure the service')
config_parser.set_defaults(command='config')
config_parser.add_argument('--region', required=False)
config_parser.add_argument('--vpc_id', required=False)
config_parser.add_argument('--subnet', required=False)
config_parser.add_argument('--security_group', required=False)
config_parser.add_argument('--keypair', required=False)
config_parser.add_argument('--worker_name', required=False)
config_parser.add_argument('--worker_iam_role_name', required=False)
config_parser.add_argument('--docker_image', required=False)
config_parser.add_argument('--defaultStorageSize',
required=False,
type=int)
config_parser.add_argument('--max_to_boot', required=False, type=int)
install_parser = subparsers.add_parser('install',
help='install the service')
install_parser.set_defaults(command='install')
list_parser = subparsers.add_parser('list', help='list machines')
list_parser.set_defaults(command='list')
boot_parser = subparsers.add_parser('boot',
help='set the number of desired boxes')
boot_parser.set_defaults(command='boot')
boot_parser.add_argument("instance_type")
boot_parser.add_argument("count", type=int)
killall_parser = subparsers.add_parser('killall', help='kill everything')
killall_parser.set_defaults(command='killall')
reset_parser = subparsers.add_parser('reset', help='kill everything')
reset_parser.set_defaults(command='reset')
configureLogging()
parsedArgs = parser.parse_args(argv[1:])
db = connect(parsedArgs.hostname, parsedArgs.port, parsedArgs.auth)
db.subscribeToSchema(service_schema, lazySubscription=True)
db.subscribeToSchema(aws_worker_boot_schema)
if parsedArgs.command == 'reset':
with db.transaction():
for s in aws_worker_boot_schema.State.lookupAll():
s.delete()
if parsedArgs.command == 'config':
with db.transaction():
AwsWorkerBootService.configure(
db_hostname=parsedArgs.hostname,
db_port=parsedArgs.port,
region=parsedArgs.region,
vpc_id=parsedArgs.vpc_id,
subnet=parsedArgs.subnet,
security_group=parsedArgs.security_group,
keypair=parsedArgs.keypair,
worker_name=parsedArgs.worker_name,
worker_iam_role_name=parsedArgs.worker_iam_role_name,
docker_image=parsedArgs.docker_image,
defaultStorageSize=parsedArgs.defaultStorageSize,
max_to_boot=parsedArgs.max_to_boot)
if parsedArgs.command == 'install':
with db.transaction():
ServiceManager.createOrUpdateService(AwsWorkerBootService,
"AwsWorkerBootService",
placement="Master")
if parsedArgs.command == 'list':
print()
print()
with db.view():
api = AwsApi()
booted = AwsWorkerBootService.currentBooted()
targets = AwsWorkerBootService.currentTargets()
table = [["Instance Type", "Booted", "Desired"]]
for instance_type in sorted(set(list(booted) + list(targets))):
table.append([
instance_type,
booted.get(instance_type, 0),
targets.get(instance_type, 0)
])
print(formatTable(table))
print()
print()
with db.view():
api = AwsApi()
table = [["InstanceID", "InstanceType", "IP", "Uptime"]]
for instanceId, instance in api.allRunningInstances(
spot=None).items():
table.append([
instance['InstanceId'], instance['InstanceType'],
instance['PrivateIpAddress'],
secondsToHumanReadable(time.time() -
instance['LaunchTime'].timestamp())
])
print(formatTable(table))
print()
print()
if parsedArgs.command == 'boot':
with db.transaction():
AwsWorkerBootService.setBootState(parsedArgs.instance_type,
parsedArgs.count)
if parsedArgs.command == 'killall':
with db.transaction():
AwsWorkerBootService.shutdownAll()
with db.view():
api = AwsApi()
for i in api.allRunningInstances():
try:
api.terminateInstanceById(i)
except Exception:
pass
return 0
def setUpClass(cls):
configureLogging("database_test")
cls.PERFORMANCE_FACTOR = 1.0 if os.environ.get('TRAVIS_CI',
None) is None else 2.0