def clear(args):
if not args.yes_i_am_sure:
log.warn("This will remove all stored Calamari monitoring status and history. Use '--yes-i-am-sure' to proceed")
return
log.info("Loading configuration..")
config = CalamariConfig()
log.info("Dropping tables")
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.drop_all(engine)
Base.metadata.reflect(engine)
if ALEMBIC_TABLE in Base.metadata.tables:
Base.metadata.tables[ALEMBIC_TABLE].drop(engine)
log.info("Complete. Now run `%s initialize`" % os.path.basename(sys.argv[0]))
import logging
logging.captureWarnings(True)
from calamari_common.config import CalamariConfig
config = CalamariConfig()
FORMAT = "%(asctime)s - %(levelname)s - %(name)s %(message)s"
log = logging.getLogger('calamari')
handler = logging.FileHandler(config.get('cthulhu', 'log_path'))
handler.setFormatter(logging.Formatter(FORMAT))
log.addHandler(handler)
log.setLevel(logging.getLevelName(config.get('cthulhu', 'log_level')))
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars))
# Configure postgres database
if os.path.exists(POSTGRES_SLS):
p = subprocess.Popen(["salt-call", "--local", "state.template",
POSTGRES_SLS],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
log.debug("Postgres salt stdout: %s" % out)
log.debug("Postgres salt stderr: %s" % err)
if p.returncode != 0:
raise RuntimeError("salt-call for postgres failed with rc={0}".format(p.returncode))
else:
# This is the path you take if you're running in a development environment
log.debug("Skipping postgres configuration, SLS not found")
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
# Django's database
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
log.info("Initializing web interface...")
user_model = get_user_model()
if args.admin_username and args.admin_password and args.admin_email:
if not user_model.objects.filter(username=args.admin_username).exists():
log.info("Creating user '%s'" % args.admin_username)
user_model.objects.create_superuser(
username=args.admin_username,
password=args.admin_password,
email=args.admin_email
)
else:
if not user_model.objects.all().count():
# When prompting for details, it's good to let the user know what the account
# is (especially that's a web UI one, not a linux system one)
log.info("You will now be prompted for login details for the administrative "
"user account. This is the account you will use to log into the web interface "
"once setup is complete.")
# Prompt for user details
execute_from_command_line(["", "createsuperuser"])
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid)
# Start services, configure to run on boot
if os.path.exists(SERVICES_SLS):
log.info("Starting/enabling services...")
p = subprocess.Popen(["salt-call", "--local", "state.template",
SERVICES_SLS],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
log.debug("Services salt stdout: %s" % out)
log.debug("Services salt stderr: %s" % err)
if p.returncode != 0:
raise RuntimeError("salt-call for services failed with rc={0}".format(p.returncode))
else:
# This is the path you take if you're running in a development environment
log.debug("Skipping services configuration")
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars))
run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt')
run_local_salt(sls=POSTGRES_SLS, message='postgres')
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
# Django's database
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
create_default_roles()
create_admin_users(args)
log.info("Initializing web interface...")
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid)
# Start services, configure to run on boot
run_local_salt(sls=SERVICES_SLS, message='services')
# During an upgrade: update minions that were connected previously
update_connected_minions()
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
Hub.handle_error = custom_handle_error
patch_gevent_hub_error_handler()
def handle_exception(exc_type, exc_value, exc_traceback):
log.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
sys.excepthook = handle_exception
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
ssl = {
'certfile': config.get('calamari_web', 'ssl_cert'),
'keyfile': config.get('calamari_web', 'ssl_key')
}
class ShallowCarbonCache(gevent.Greenlet):
def __init__(self):
super(ShallowCarbonCache, self).__init__()
self.complete = gevent.event.Event()
self.latest = {}
self.rpc = zerorpc.Server({
'get_latest': self.get_latest
})
self.rpc_thread = None
import logging
from calamari_common.config import CalamariConfig
config = CalamariConfig()
FORMAT = "%(asctime)s - %(levelname)s - %(name)s %(message)s"
log = logging.getLogger('cthulhu')
handler = logging.FileHandler(config.get('cthulhu', 'log_path'))
handler.setFormatter(logging.Formatter(FORMAT))
log.addHandler(handler)
log.setLevel(logging.getLevelName(config.get('cthulhu', 'log_level')))
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars))
run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt')
run_local_salt(sls=POSTGRES_SLS, message='postgres')
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
# Django's database
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
create_default_roles()
create_admin_users(args)
log.info("Initializing web interface...")
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid)
# Start services, configure to run on boot
run_local_salt(sls=SERVICES_SLS, message='services')
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
from calamari_common.config import CalamariConfig
config = CalamariConfig()
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': config.get("calamari_web", "db_engine"),
'NAME': config.get("calamari_web", "db_name"),
'USER': config.get("calamari_web", "db_user"),
'PASSWORD': config.get("calamari_web", "db_password"),
'HOST': config.get("calamari_web", "db_host"),
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['*']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
import logging
import re
import gevent
from calamari_common.remote.base import Remote, Unavailable, AUTH_REJECTED, AUTH_NEW, AUTH_ACCEPTED
from calamari_common.salt_wrapper import master_config, _create_loader, client_config, MasterPillarUtil, LocalClient, condition_kwarg, \
SaltEventSource, Key
from calamari_common.config import CalamariConfig
from calamari_common.types import SERVER, NotFound
log = logging.getLogger('calamari.remote.salt')
config = CalamariConfig()
salt_config = client_config(config.get('cthulhu', 'salt_config_path'))
# Issue up to this many disk I/Os to load grains at once
CONCURRENT_GRAIN_LOADS = 16
class SaltRemote(Remote):
def run_job_sync(self, fqdn, cmd, args, timeout=None):
client = LocalClient(config.get('cthulhu', 'salt_config_path'))
results = client.cmd(fqdn, cmd, args, timeout=timeout)
if results:
if isinstance(fqdn, list):
return results
else:
return results[fqdn]
else:
raise Unavailable()
def run_job(self, fqdn, cmd, args):
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {}
try:
import sqlalchemy # noqa
except ImportError:
pass
else:
DATABASES['default'] = {
'ENGINE': config.get("calamari_web", "db_engine"),
'NAME': config.get("calamari_web", "db_name"),
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['*']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
from calamari_common.config import CalamariConfig
config = CalamariConfig()
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': config.get("calamari_web", "db_engine"),
'NAME': config.get("calamari_web", "db_name"),
'USER': config.get("calamari_web", "db_user"),
'PASSWORD': config.get("calamari_web", "db_password"),
'HOST': config.get("calamari_web", "db_host"),
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['*']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
import yaml
import os
from calamari_common.config import CalamariConfig
config = CalamariConfig()
TGT_CONF_TEMPLATE_DIR = os.path.dirname(config.path)
TGT_CONF_TEMPLATE = 'tgt_conf.yaml'
# TGT_ETC_CONF_DIR = '/etc/tgt/conf.d'
TGT_ETC_CONF_DIR = TGT_CONF_TEMPLATE_DIR
TGT_ETC_EX_FILE = 'targets.futong.conf'
TGTADMIN_CMD = '/usr/sbin/tgt-admin'
CEPH_CONFIG = '/etc/ceph/ceph.conf'
SALT_CONFIG_PATH = config.get('cthulhu', 'salt_config_path')
class ConfigError(Exception):
pass
class Config(object):
def __init__(self, conf_file, conf=None, schema=None):
self.conf_file = conf_file
self.conf = conf
self.schema = schema
def parse(self):
if self.conf_file is not None and self.conf_file != "":
try:
with open(self.conf_file, "r") as f:
self.conf = yaml.load(f)
if self.schema:
patch_gevent_hub_error_handler()
def handle_exception(exc_type, exc_value, exc_traceback):
log.error("Uncaught exception",
exc_info=(exc_type, exc_value, exc_traceback))
sys.excepthook = handle_exception
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
ssl = {
'certfile': config.get('calamari_web', 'ssl_cert'),
'keyfile': config.get('calamari_web', 'ssl_key')
}
class ShallowCarbonCache(gevent.Greenlet):
def __init__(self):
super(ShallowCarbonCache, self).__init__()
self.complete = gevent.event.Event()
self.latest = {}
self.rpc = zerorpc.Server({'get_latest': self.get_latest})
self.rpc_thread = None
def get_latest(self, paths):
import logging
import re
import gevent
from calamari_common.remote.base import Remote, Unavailable, AUTH_REJECTED, AUTH_NEW, AUTH_ACCEPTED
from calamari_common.salt_wrapper import master_config, _create_loader, client_config, MasterPillarUtil, LocalClient, condition_kwarg, \
SaltEventSource, Key
from calamari_common.config import CalamariConfig
from calamari_common.types import SERVER, NotFound
log = logging.getLogger('calamari.remote.salt')
config = CalamariConfig()
salt_config = client_config(config.get('cthulhu', 'salt_config_path'))
# Issue up to this many disk I/Os to load grains at once
CONCURRENT_GRAIN_LOADS = 16
class SaltRemote(Remote):
def run_job_sync(self, fqdn, cmd, args, timeout=None):
client = LocalClient(config.get('cthulhu', 'salt_config_path'))
results = client.cmd(fqdn, cmd, args, timeout=timeout)
if results:
if isinstance(fqdn, list):
return results
else:
return results[fqdn]
else:
