def clear(args): if not args.yes_i_am_sure: log.warn("This will remove all stored Calamari monitoring status and history. Use '--yes-i-am-sure' to proceed") return log.info("Loading configuration..") config = CalamariConfig() log.info("Dropping tables") db_path = config.get('cthulhu', 'db_path') engine = create_engine(db_path) Base.metadata.drop_all(engine) Base.metadata.reflect(engine) if ALEMBIC_TABLE in Base.metadata.tables: Base.metadata.tables[ALEMBIC_TABLE].drop(engine) log.info("Complete. Now run `%s initialize`" % os.path.basename(sys.argv[0]))
CliSerializer from calamari_rest.views.database_view_set import DatabaseViewSet from calamari_rest.views.exceptions import ServiceUnavailable from calamari_rest.views.paginated_mixin import PaginatedMixin from calamari_rest.views.remote_view_set import RemoteViewSet from calamari_rest.views.rpc_view import RPCViewSet, DataObject from calamari_rest.views.crush_node import lookup_ancestry from calamari_common.config import CalamariConfig from calamari_common.types import CRUSH_MAP, CRUSH_RULE, CRUSH_NODE, CRUSH_TYPE, POOL, OSD, USER_REQUEST_COMPLETE, USER_REQUEST_SUBMITTED, \ OSD_IMPLEMENTED_COMMANDS, MON, OSD_MAP, SYNC_OBJECT_TYPES, ServiceId from calamari_common.db.event import Event, severity_from_str, SEVERITIES from django.views.decorators.csrf import csrf_exempt from calamari_rest.views.server_metadata import get_local_grains, get_remote_grains config = CalamariConfig() log = logging.getLogger('django.request') if log.level <= logging.DEBUG: logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) for handler in log.handlers: logging.getLogger('sqlalchemy.engine').addHandler(handler) @api_view(['GET']) @login_required def grains(request): """ The salt grains for the host running Calamari server. These are variables from Saltstack that tell us useful properties of the host.
import logging logging.captureWarnings(True) from calamari_common.config import CalamariConfig config = CalamariConfig() FORMAT = "%(asctime)s - %(levelname)s - %(name)s %(message)s" log = logging.getLogger('calamari') handler = logging.FileHandler(config.get('cthulhu', 'log_path')) handler.setFormatter(logging.Formatter(FORMAT)) log.addHandler(handler) log.setLevel(logging.getLevelName(config.get('cthulhu', 'log_level')))
def initialize(args): """ This command exists to: - Prevent the user having to type more than one thing - Prevent the user seeing internals like 'manage.py' which we would rather people were not messing with on production systems. """ log.info("Loading configuration..") config = CalamariConfig() # Generate django's SECRET_KEY setting # Do this first, otherwise subsequent django ops will raise ImproperlyConfigured. # Write into a file instead of directly, so that package upgrades etc won't spuriously # prompt for modified config unless it really is modified. if not os.path.exists(config.get('calamari_web', 'secret_key_path')): chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars)) # Configure postgres database if os.path.exists(POSTGRES_SLS): p = subprocess.Popen(["salt-call", "--local", "state.template", POSTGRES_SLS], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() log.debug("Postgres salt stdout: %s" % out) log.debug("Postgres salt stderr: %s" % err) if p.returncode != 0: raise RuntimeError("salt-call for postgres failed with rc={0}".format(p.returncode)) else: # This is the path you take if you're running in a development environment log.debug("Skipping postgres configuration, SLS not found") # Cthulhu's database db_path = config.get('cthulhu', 'db_path') engine = create_engine(db_path) Base.metadata.reflect(engine) alembic_config = AlembicConfig() if ALEMBIC_TABLE in Base.metadata.tables: log.info("Updating database...") # Database already populated, migrate forward command.upgrade(alembic_config, "head") else: log.info("Initializing database...") # Blank database, do initial population Base.metadata.create_all(engine) command.stamp(alembic_config, "head") # Django's database os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings") with quiet(): execute_from_command_line(["", "syncdb", "--noinput"]) log.info("Initializing web interface...") user_model = get_user_model() if args.admin_username and args.admin_password and args.admin_email: if not user_model.objects.filter(username=args.admin_username).exists(): log.info("Creating user '%s'" % args.admin_username) user_model.objects.create_superuser( username=args.admin_username, password=args.admin_password, email=args.admin_email ) else: if not user_model.objects.all().count(): # When prompting for details, it's good to let the user know what the account # is (especially that's a web UI one, not a linux system one) log.info("You will now be prompted for login details for the administrative " "user account. This is the account you will use to log into the web interface " "once setup is complete.") # Prompt for user details execute_from_command_line(["", "createsuperuser"]) # Django's static files with quiet(): execute_from_command_line(["", "collectstatic", "--noinput"]) # Because we've loaded Django, it will have written log files as # this user (probably root). Fix it so that apache can write them later. apache_user = pwd.getpwnam(config.get('calamari_web', 'username')) os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid) # Handle SQLite case, otherwise no chown is needed if config.get('calamari_web', 'db_engine').endswith("sqlite3"): os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid) # Start services, configure to run on boot if os.path.exists(SERVICES_SLS): log.info("Starting/enabling services...") p = subprocess.Popen(["salt-call", "--local", "state.template", SERVICES_SLS], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() log.debug("Services salt stdout: %s" % out) log.debug("Services salt stderr: %s" % err) if p.returncode != 0: raise RuntimeError("salt-call for services failed with rc={0}".format(p.returncode)) else: # This is the path you take if you're running in a development environment log.debug("Skipping services configuration") # Signal supervisor to restart cthulhu as we have created its database log.info("Restarting services...") subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # TODO: optionally generate or install HTTPS certs + hand to apache log.info("Complete.")
in a single process. """ from django.core.servers.basehttp import get_internal_wsgi_application import gevent.event import gevent import signal from gevent.server import StreamServer import os from gevent.pywsgi import WSGIServer import zerorpc from calamari_common.config import CalamariConfig from cthulhu.log import log import sys from gevent.hub import Hub config = CalamariConfig() TIMEOUT = 5 # seconds till we tick the cthulhu eventer SALT_RESET_PERIOD = 300 # seconds till we teardown/setup our salt_caller. Do this because it's leaking memory def patch_gevent_hub_error_handler(): Hub._origin_handle_error = Hub.handle_error def custom_handle_error(self, context, type, value, tb): if not issubclass(type, Hub.SYSTEM_ERROR + Hub.NOT_ERROR): log.error("Uncaught exception", exc_info=(type, value, tb)) self._origin_handle_error(context, type, value, tb)
def initialize(args): """ This command exists to: - Prevent the user having to type more than one thing - Prevent the user seeing internals like 'manage.py' which we would rather people were not messing with on production systems. """ log.info("Loading configuration..") config = CalamariConfig() # Generate django's SECRET_KEY setting # Do this first, otherwise subsequent django ops will raise ImproperlyConfigured. # Write into a file instead of directly, so that package upgrades etc won't spuriously # prompt for modified config unless it really is modified. if not os.path.exists(config.get('calamari_web', 'secret_key_path')): chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars)) run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt') run_local_salt(sls=POSTGRES_SLS, message='postgres') # Cthulhu's database db_path = config.get('cthulhu', 'db_path') engine = create_engine(db_path) Base.metadata.reflect(engine) alembic_config = AlembicConfig() if ALEMBIC_TABLE in Base.metadata.tables: log.info("Updating database...") # Database already populated, migrate forward command.upgrade(alembic_config, "head") else: log.info("Initializing database...") # Blank database, do initial population Base.metadata.create_all(engine) command.stamp(alembic_config, "head") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings") # Django's database with quiet(): execute_from_command_line(["", "syncdb", "--noinput"]) create_default_roles() create_admin_users(args) log.info("Initializing web interface...") # Django's static files with quiet(): execute_from_command_line(["", "collectstatic", "--noinput"]) # Because we've loaded Django, it will have written log files as # this user (probably root). Fix it so that apache can write them later. apache_user = pwd.getpwnam(config.get('calamari_web', 'username')) os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid) # Handle SQLite case, otherwise no chown is needed if config.get('calamari_web', 'db_engine').endswith("sqlite3"): os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid) # Start services, configure to run on boot run_local_salt(sls=SERVICES_SLS, message='services') # During an upgrade: update minions that were connected previously update_connected_minions() # Signal supervisor to restart cthulhu as we have created its database log.info("Restarting services...") subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # TODO: optionally generate or install HTTPS certs + hand to apache log.info("Complete.")
def initialize(args): """ This command exists to: - Prevent the user having to type more than one thing - Prevent the user seeing internals like 'manage.py' which we would rather people were not messing with on production systems. """ log.info("Loading configuration..") config = CalamariConfig() # Generate django's SECRET_KEY setting # Do this first, otherwise subsequent django ops will raise ImproperlyConfigured. # Write into a file instead of directly, so that package upgrades etc won't spuriously # prompt for modified config unless it really is modified. if not os.path.exists(config.get('calamari_web', 'secret_key_path')): chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars)) # Cthulhu's database db_path = config.get('cthulhu', 'db_path') engine = create_engine(db_path) Base.metadata.reflect(engine) alembic_config = AlembicConfig() if ALEMBIC_TABLE in Base.metadata.tables: log.info("Updating database...") # Database already populated, migrate forward command.upgrade(alembic_config, "head") else: log.info("Initializing database...") # Blank database, do initial population Base.metadata.create_all(engine) command.stamp(alembic_config, "head") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings") # Django's database with quiet(): execute_from_command_line(["", "syncdb", "--noinput"]) create_default_roles() create_admin_users(args) log.info("Initializing web interface...") # Django's static files with quiet(): execute_from_command_line(["", "collectstatic", "--noinput"]) # Because we've loaded Django, it will have written log files as # this user (probably root). Fix it so that apache can write them later. apache_user = pwd.getpwnam(config.get('calamari_web', 'username')) os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid) # Create self-signed SSL certs only if they do not exist ssl_key = config.get('calamari_web', 'ssl_key') ssl_cert = config.get('calamari_web', 'ssl_cert') if not os.path.exists(ssl_key): run_cmd([ 'openssl', 'req', '-new', '-nodes', '-x509', '-subj', "/C=US/ST=Oregon/L=Portland/O=IT/CN=calamari-lite", '-days', '3650', '-keyout', ssl_key, '-out', ssl_cert, '-extensions', 'v3_ca' ], "Generating self-signed SSL certificate...") # ensure the bundled crt is readable os.chmod(ssl_cert, 0644) # Signal supervisor to restart cthulhu as we have created its database log.info("Restarting services...") setup_supervisor() log.info("Complete.")
import logging from calamari_common.config import CalamariConfig config = CalamariConfig() FORMAT = "%(asctime)s - %(levelname)s - %(name)s %(message)s" log = logging.getLogger('cthulhu') handler = logging.FileHandler(config.get('cthulhu', 'log_path')) handler.setFormatter(logging.Formatter(FORMAT)) log.addHandler(handler) log.setLevel(logging.getLevelName(config.get('cthulhu', 'log_level')))
def initialize(args): """ This command exists to: - Prevent the user having to type more than one thing - Prevent the user seeing internals like 'manage.py' which we would rather people were not messing with on production systems. """ log.info("Loading configuration..") config = CalamariConfig() # Generate django's SECRET_KEY setting # Do this first, otherwise subsequent django ops will raise ImproperlyConfigured. # Write into a file instead of directly, so that package upgrades etc won't spuriously # prompt for modified config unless it really is modified. if not os.path.exists(config.get('calamari_web', 'secret_key_path')): chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' open(config.get('calamari_web', 'secret_key_path'), 'w').write(get_random_string(50, chars)) run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt') run_local_salt(sls=POSTGRES_SLS, message='postgres') # Cthulhu's database db_path = config.get('cthulhu', 'db_path') engine = create_engine(db_path) Base.metadata.reflect(engine) alembic_config = AlembicConfig() if ALEMBIC_TABLE in Base.metadata.tables: log.info("Updating database...") # Database already populated, migrate forward command.upgrade(alembic_config, "head") else: log.info("Initializing database...") # Blank database, do initial population Base.metadata.create_all(engine) command.stamp(alembic_config, "head") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings") # Django's database with quiet(): execute_from_command_line(["", "syncdb", "--noinput"]) create_default_roles() create_admin_users(args) log.info("Initializing web interface...") # Django's static files with quiet(): execute_from_command_line(["", "collectstatic", "--noinput"]) # Because we've loaded Django, it will have written log files as # this user (probably root). Fix it so that apache can write them later. apache_user = pwd.getpwnam(config.get('calamari_web', 'username')) os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid, apache_user.pw_gid) # Handle SQLite case, otherwise no chown is needed if config.get('calamari_web', 'db_engine').endswith("sqlite3"): os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid, apache_user.pw_gid) # Start services, configure to run on boot run_local_salt(sls=SERVICES_SLS, message='services') # Signal supervisor to restart cthulhu as we have created its database log.info("Restarting services...") subprocess.call(['supervisorctl', 'restart', 'cthulhu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # TODO: optionally generate or install HTTPS certs + hand to apache log.info("Complete.")
# Django settings for calamari project. import os from os.path import dirname, abspath, join import sys from django.core.exceptions import ImproperlyConfigured from calamari_common.config import CalamariConfig config = CalamariConfig() DEBUG = False TEMPLATE_DEBUG = DEBUG ADMINS = ( # ('Your Name', '*****@*****.**'), ) MANAGERS = ADMINS DATABASES = { 'default': { 'ENGINE': config.get("calamari_web", "db_engine"), 'NAME': config.get("calamari_web", "db_name"), 'USER': config.get("calamari_web", "db_user"), 'PASSWORD': config.get("calamari_web", "db_password"), 'HOST': config.get("calamari_web", "db_host"), } } # Hosts/domain names that are valid for this site; required if DEBUG is False # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
import logging import re import gevent from calamari_common.remote.base import Remote, Unavailable, AUTH_REJECTED, AUTH_NEW, AUTH_ACCEPTED from calamari_common.salt_wrapper import master_config, _create_loader, client_config, MasterPillarUtil, LocalClient, condition_kwarg, \ SaltEventSource, Key from calamari_common.config import CalamariConfig from calamari_common.types import SERVER, NotFound log = logging.getLogger('calamari.remote.salt') config = CalamariConfig() salt_config = client_config(config.get('cthulhu', 'salt_config_path')) # Issue up to this many disk I/Os to load grains at once CONCURRENT_GRAIN_LOADS = 16 class SaltRemote(Remote): def run_job_sync(self, fqdn, cmd, args, timeout=None): client = LocalClient(config.get('cthulhu', 'salt_config_path')) results = client.cmd(fqdn, cmd, args, timeout=timeout) if results: if isinstance(fqdn, list): return results else: return results[fqdn] else: raise Unavailable() def run_job(self, fqdn, cmd, args):
# Django settings for calamari project. import os from os.path import dirname, abspath, join import sys from django.core.exceptions import ImproperlyConfigured from calamari_common.config import CalamariConfig config = CalamariConfig() DEBUG = False TEMPLATE_DEBUG = DEBUG ADMINS = ( # ('Your Name', '*****@*****.**'), ) MANAGERS = ADMINS DATABASES = {} try: import sqlalchemy # noqa except ImportError: pass else: DATABASES['default'] = { 'ENGINE': config.get("calamari_web", "db_engine"), 'NAME': config.get("calamari_web", "db_name"), }
import yaml import os from calamari_common.config import CalamariConfig config = CalamariConfig() TGT_CONF_TEMPLATE_DIR = os.path.dirname(config.path) TGT_CONF_TEMPLATE = 'tgt_conf.yaml' # TGT_ETC_CONF_DIR = '/etc/tgt/conf.d' TGT_ETC_CONF_DIR = TGT_CONF_TEMPLATE_DIR TGT_ETC_EX_FILE = 'targets.futong.conf' TGTADMIN_CMD = '/usr/sbin/tgt-admin' CEPH_CONFIG = '/etc/ceph/ceph.conf' SALT_CONFIG_PATH = config.get('cthulhu', 'salt_config_path') class ConfigError(Exception): pass class Config(object): def __init__(self, conf_file, conf=None, schema=None): self.conf_file = conf_file self.conf = conf self.schema = schema def parse(self): if self.conf_file is not None and self.conf_file != "": try: with open(self.conf_file, "r") as f: self.conf = yaml.load(f) if self.schema:
import logging import re import gevent from calamari_common.remote.base import Remote, Unavailable, AUTH_REJECTED, AUTH_NEW, AUTH_ACCEPTED from calamari_common.salt_wrapper import master_config, _create_loader, client_config, MasterPillarUtil, LocalClient, condition_kwarg, \ SaltEventSource, Key from calamari_common.config import CalamariConfig from calamari_common.types import SERVER, NotFound log = logging.getLogger('calamari.remote.salt') config = CalamariConfig() salt_config = client_config(config.get('cthulhu', 'salt_config_path')) # Issue up to this many disk I/Os to load grains at once CONCURRENT_GRAIN_LOADS = 16 class SaltRemote(Remote): def run_job_sync(self, fqdn, cmd, args, timeout=None): client = LocalClient(config.get('cthulhu', 'salt_config_path')) results = client.cmd(fqdn, cmd, args, timeout=timeout) if results: if isinstance(fqdn, list): return results else: return results[fqdn] else: