def clear(args):
if not args.yes_i_am_sure:
log.warn("This will remove all stored Calamari monitoring status and history. Use '--yes-i-am-sure' to proceed")
return
log.info("Loading configuration..")
config = CalamariConfig()
log.info("Dropping tables")
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.drop_all(engine)
Base.metadata.reflect(engine)
if ALEMBIC_TABLE in Base.metadata.tables:
Base.metadata.tables[ALEMBIC_TABLE].drop(engine)
log.info("Complete. Now run `%s initialize`" % os.path.basename(sys.argv[0]))
CliSerializer
from calamari_rest.views.database_view_set import DatabaseViewSet
from calamari_rest.views.exceptions import ServiceUnavailable
from calamari_rest.views.paginated_mixin import PaginatedMixin
from calamari_rest.views.remote_view_set import RemoteViewSet
from calamari_rest.views.rpc_view import RPCViewSet, DataObject
from calamari_rest.views.crush_node import lookup_ancestry
from calamari_common.config import CalamariConfig
from calamari_common.types import CRUSH_MAP, CRUSH_RULE, CRUSH_NODE, CRUSH_TYPE, POOL, OSD, USER_REQUEST_COMPLETE, USER_REQUEST_SUBMITTED, \
OSD_IMPLEMENTED_COMMANDS, MON, OSD_MAP, SYNC_OBJECT_TYPES, ServiceId
from calamari_common.db.event import Event, severity_from_str, SEVERITIES
from django.views.decorators.csrf import csrf_exempt
from calamari_rest.views.server_metadata import get_local_grains, get_remote_grains
config = CalamariConfig()
log = logging.getLogger('django.request')
if log.level <= logging.DEBUG:
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
for handler in log.handlers:
logging.getLogger('sqlalchemy.engine').addHandler(handler)
@api_view(['GET'])
@login_required
def grains(request):
"""
The salt grains for the host running Calamari server. These are variables
from Saltstack that tell us useful properties of the host.
import logging
logging.captureWarnings(True)
from calamari_common.config import CalamariConfig
config = CalamariConfig()
FORMAT = "%(asctime)s - %(levelname)s - %(name)s %(message)s"
log = logging.getLogger('calamari')
handler = logging.FileHandler(config.get('cthulhu', 'log_path'))
handler.setFormatter(logging.Formatter(FORMAT))
log.addHandler(handler)
log.setLevel(logging.getLevelName(config.get('cthulhu', 'log_level')))
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars))
# Configure postgres database
if os.path.exists(POSTGRES_SLS):
p = subprocess.Popen(["salt-call", "--local", "state.template",
POSTGRES_SLS],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
log.debug("Postgres salt stdout: %s" % out)
log.debug("Postgres salt stderr: %s" % err)
if p.returncode != 0:
raise RuntimeError("salt-call for postgres failed with rc={0}".format(p.returncode))
else:
# This is the path you take if you're running in a development environment
log.debug("Skipping postgres configuration, SLS not found")
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
# Django's database
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
log.info("Initializing web interface...")
user_model = get_user_model()
if args.admin_username and args.admin_password and args.admin_email:
if not user_model.objects.filter(username=args.admin_username).exists():
log.info("Creating user '%s'" % args.admin_username)
user_model.objects.create_superuser(
username=args.admin_username,
password=args.admin_password,
email=args.admin_email
)
else:
if not user_model.objects.all().count():
# When prompting for details, it's good to let the user know what the account
# is (especially that's a web UI one, not a linux system one)
log.info("You will now be prompted for login details for the administrative "
"user account. This is the account you will use to log into the web interface "
"once setup is complete.")
# Prompt for user details
execute_from_command_line(["", "createsuperuser"])
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid)
# Start services, configure to run on boot
if os.path.exists(SERVICES_SLS):
log.info("Starting/enabling services...")
p = subprocess.Popen(["salt-call", "--local", "state.template",
SERVICES_SLS],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
log.debug("Services salt stdout: %s" % out)
log.debug("Services salt stderr: %s" % err)
if p.returncode != 0:
raise RuntimeError("salt-call for services failed with rc={0}".format(p.returncode))
else:
# This is the path you take if you're running in a development environment
log.debug("Skipping services configuration")
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
in a single process.
"""
from django.core.servers.basehttp import get_internal_wsgi_application
import gevent.event
import gevent
import signal
from gevent.server import StreamServer
import os
from gevent.pywsgi import WSGIServer
import zerorpc
from calamari_common.config import CalamariConfig
from cthulhu.log import log
import sys
from gevent.hub import Hub
config = CalamariConfig()
TIMEOUT = 5 # seconds till we tick the cthulhu eventer
SALT_RESET_PERIOD = 300 # seconds till we teardown/setup our salt_caller. Do this because it's leaking memory
def patch_gevent_hub_error_handler():
Hub._origin_handle_error = Hub.handle_error
def custom_handle_error(self, context, type, value, tb):
if not issubclass(type, Hub.SYSTEM_ERROR + Hub.NOT_ERROR):
log.error("Uncaught exception", exc_info=(type, value, tb))
self._origin_handle_error(context, type, value, tb)
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars))
run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt')
run_local_salt(sls=POSTGRES_SLS, message='postgres')
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
# Django's database
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
create_default_roles()
create_admin_users(args)
log.info("Initializing web interface...")
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid)
# Start services, configure to run on boot
run_local_salt(sls=SERVICES_SLS, message='services')
# During an upgrade: update minions that were connected previously
update_connected_minions()
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'),
'w').write(get_random_string(50, chars))
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
# Django's database
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
create_default_roles()
create_admin_users(args)
log.info("Initializing web interface...")
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid,
apache_user.pw_gid)
# Create self-signed SSL certs only if they do not exist
ssl_key = config.get('calamari_web', 'ssl_key')
ssl_cert = config.get('calamari_web', 'ssl_cert')
if not os.path.exists(ssl_key):
run_cmd([
'openssl', 'req', '-new', '-nodes', '-x509', '-subj',
"/C=US/ST=Oregon/L=Portland/O=IT/CN=calamari-lite", '-days',
'3650', '-keyout', ssl_key, '-out', ssl_cert, '-extensions',
'v3_ca'
], "Generating self-signed SSL certificate...")
# ensure the bundled crt is readable
os.chmod(ssl_cert, 0644)
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
setup_supervisor()
log.info("Complete.")
import logging
from calamari_common.config import CalamariConfig
config = CalamariConfig()
FORMAT = "%(asctime)s - %(levelname)s - %(name)s %(message)s"
log = logging.getLogger('cthulhu')
handler = logging.FileHandler(config.get('cthulhu', 'log_path'))
handler.setFormatter(logging.Formatter(FORMAT))
log.addHandler(handler)
log.setLevel(logging.getLevelName(config.get('cthulhu', 'log_level')))
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars))
run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt')
run_local_salt(sls=POSTGRES_SLS, message='postgres')
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
# Django's database
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
create_default_roles()
create_admin_users(args)
log.info("Initializing web interface...")
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid)
# Start services, configure to run on boot
run_local_salt(sls=SERVICES_SLS, message='services')
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
# Django settings for calamari project.
import os
from os.path import dirname, abspath, join
import sys
from django.core.exceptions import ImproperlyConfigured
from calamari_common.config import CalamariConfig
config = CalamariConfig()
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': config.get("calamari_web", "db_engine"),
'NAME': config.get("calamari_web", "db_name"),
'USER': config.get("calamari_web", "db_user"),
'PASSWORD': config.get("calamari_web", "db_password"),
'HOST': config.get("calamari_web", "db_host"),
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
import logging
import re
import gevent
from calamari_common.remote.base import Remote, Unavailable, AUTH_REJECTED, AUTH_NEW, AUTH_ACCEPTED
from calamari_common.salt_wrapper import master_config, _create_loader, client_config, MasterPillarUtil, LocalClient, condition_kwarg, \
SaltEventSource, Key
from calamari_common.config import CalamariConfig
from calamari_common.types import SERVER, NotFound
log = logging.getLogger('calamari.remote.salt')
config = CalamariConfig()
salt_config = client_config(config.get('cthulhu', 'salt_config_path'))
# Issue up to this many disk I/Os to load grains at once
CONCURRENT_GRAIN_LOADS = 16
class SaltRemote(Remote):
def run_job_sync(self, fqdn, cmd, args, timeout=None):
client = LocalClient(config.get('cthulhu', 'salt_config_path'))
results = client.cmd(fqdn, cmd, args, timeout=timeout)
if results:
if isinstance(fqdn, list):
return results
else:
return results[fqdn]
else:
raise Unavailable()
def run_job(self, fqdn, cmd, args):
# Django settings for calamari project.
import os
from os.path import dirname, abspath, join
import sys
from django.core.exceptions import ImproperlyConfigured
from calamari_common.config import CalamariConfig
config = CalamariConfig()
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {}
try:
import sqlalchemy # noqa
except ImportError:
pass
else:
DATABASES['default'] = {
'ENGINE': config.get("calamari_web", "db_engine"),
'NAME': config.get("calamari_web", "db_name"),
}
# Django settings for calamari project.
import os
from os.path import dirname, abspath, join
import sys
from django.core.exceptions import ImproperlyConfigured
from calamari_common.config import CalamariConfig
config = CalamariConfig()
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': config.get("calamari_web", "db_engine"),
'NAME': config.get("calamari_web", "db_name"),
'USER': config.get("calamari_web", "db_user"),
'PASSWORD': config.get("calamari_web", "db_password"),
'HOST': config.get("calamari_web", "db_host"),
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
import yaml
import os
from calamari_common.config import CalamariConfig
config = CalamariConfig()
TGT_CONF_TEMPLATE_DIR = os.path.dirname(config.path)
TGT_CONF_TEMPLATE = 'tgt_conf.yaml'
# TGT_ETC_CONF_DIR = '/etc/tgt/conf.d'
TGT_ETC_CONF_DIR = TGT_CONF_TEMPLATE_DIR
TGT_ETC_EX_FILE = 'targets.futong.conf'
TGTADMIN_CMD = '/usr/sbin/tgt-admin'
CEPH_CONFIG = '/etc/ceph/ceph.conf'
SALT_CONFIG_PATH = config.get('cthulhu', 'salt_config_path')
class ConfigError(Exception):
pass
class Config(object):
def __init__(self, conf_file, conf=None, schema=None):
self.conf_file = conf_file
self.conf = conf
self.schema = schema
def parse(self):
if self.conf_file is not None and self.conf_file != "":
try:
with open(self.conf_file, "r") as f:
self.conf = yaml.load(f)
if self.schema:
in a single process.
"""
from django.core.servers.basehttp import get_internal_wsgi_application
import gevent.event
import gevent
import signal
from gevent.server import StreamServer
import os
from gevent.pywsgi import WSGIServer
import zerorpc
from calamari_common.config import CalamariConfig
from cthulhu.log import log
import sys
from gevent.hub import Hub
config = CalamariConfig()
TIMEOUT = 5 # seconds till we tick the cthulhu eventer
SALT_RESET_PERIOD = 300 # seconds till we teardown/setup our salt_caller. Do this because it's leaking memory
def patch_gevent_hub_error_handler():
Hub._origin_handle_error = Hub.handle_error
def custom_handle_error(self, context, type, value, tb):
if not issubclass(type, Hub.SYSTEM_ERROR + Hub.NOT_ERROR):
log.error("Uncaught exception", exc_info=(type, value, tb))
self._origin_handle_error(context, type, value, tb)
import logging
import re
import gevent
from calamari_common.remote.base import Remote, Unavailable, AUTH_REJECTED, AUTH_NEW, AUTH_ACCEPTED
from calamari_common.salt_wrapper import master_config, _create_loader, client_config, MasterPillarUtil, LocalClient, condition_kwarg, \
SaltEventSource, Key
from calamari_common.config import CalamariConfig
from calamari_common.types import SERVER, NotFound
log = logging.getLogger('calamari.remote.salt')
config = CalamariConfig()
salt_config = client_config(config.get('cthulhu', 'salt_config_path'))
# Issue up to this many disk I/Os to load grains at once
CONCURRENT_GRAIN_LOADS = 16
class SaltRemote(Remote):
def run_job_sync(self, fqdn, cmd, args, timeout=None):
client = LocalClient(config.get('cthulhu', 'salt_config_path'))
results = client.cmd(fqdn, cmd, args, timeout=timeout)
if results:
if isinstance(fqdn, list):
return results
else:
return results[fqdn]
else:
