Exemplo n.º 1
0
    def config(self):
        """Returns an opus.lib.conf.OpusConfig object for this project. This is
        automatically saved when the model's save() method is called.

        This will raise an error if the project doesn't exist (such as before
        it's deployed for the first time

        """
        if not self._conf:
            self._conf = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
        return self._conf
Exemplo n.º 2
0
    def __init__(self, projectdir):
        """Initialize the Project Deployer with the directory root of the
        project to be deployed.

        """
        self.projectdir = projectdir

        # Find the project name by taking the last component of the path
        path = os.path.abspath(self.projectdir)
        self.projectname = os.path.basename(path)

        self.uid = None

        # Go ahead and eat up the config file into memory
        self.config = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
Exemplo n.º 3
0
    def _setup_config(self):
        # Erases the normal settings.py and creates an opussettings.json
        # to replace it
        self.config = OpusConfig.new_from_template(
                os.path.join(self.projectdir, "opussettings.json"))
        
        with open(os.path.join(self.projectdir, "settings.py"), 'w') as s:
            s.write("""# Opus-built Project Settings file
# The settings for this project are not stored in this file, but rather in the
# file opussettings.json in JSON format. You may put your own values below to
# override Opus's configuration, but this is not recommended.
from opus.lib.conf import load_settings
load_settings(globals())

import djcelery
djcelery.setup_loader()
""")

        # Randomizing SECRET_KEY is taken care of for us by new_from_template,
        # but we still have to set ROOT_URLCONF
        self.config['ROOT_URLCONF'] = "urls"
Exemplo n.º 4
0
class DeployedProject(models.Model):
    """The actual model for a deployed project. The database doesn't contain
    too many fields, but this class contains lots of methods to query information and
    edit projects. Most of the state is stored in the filesystem and not the
    database. (For example, whether the project is activated is defined by the
    presence of an apache config file for the project)
    """
    name = IdentifierField(unique=True)
    owner = models.ForeignKey(django.contrib.auth.models.User)

    def __init__(self, *args, **kwargs):
        super(DeployedProject, self).__init__(*args, **kwargs)

        self._conf = None

    @property
    def projectdir(self):
        return os.path.join(settings.OPUS_BASE_DIR, self.name)

    @property
    def apache_conf(self):
        return os.path.join(settings.OPUS_APACHE_CONFD, "opus"+self.name+".conf")

    @models.permalink
    def get_absolute_url(self):
        return ("opus.project.deployment.views.edit_or_create",
                (),
                dict(projectname=self.name))

    @property
    def serve_http(self):
        port = settings.OPUS_HTTP_PORT
        if not port:
            return
        if port != 80:
            portline = ":"+str(port)
        else:
            portline = ""
        return "http://{0}{1}{2}/".format(
                self.name, settings.OPUS_APACHE_SERVERNAME_SUFFIX,
                portline)
    @property
    def serve_https(self):
        port = settings.OPUS_HTTPS_PORT
        if not port:
            return
        if port != 443:
            portline = ":"+str(port)
        else:
            portline = ""
        return "https://{0}{1}{2}/".format(
                self.name, settings.OPUS_APACHE_SERVERNAME_SUFFIX,
                portline)

    def get_urls(self):
        """Gets the urls that this project is being served from. This list is
        populated even if active is False.
        """
        urls = []
        http = self.serve_http
        if http:
            urls.append(http)
        https = self.serve_https
        if https:
            urls.append(https)
        return urls

    def get_apps(self):
        """Returns an iterator over application names that are currently
        installed"""
        for app in self.config['INSTALLED_APPS']:
            if os.path.exists(os.path.join(self.projectdir, app)):
                yield app

    @property
    def config(self):
        """Returns an opus.lib.conf.OpusConfig object for this project. This is
        automatically saved when the model's save() method is called.

        This will raise an error if the project doesn't exist (such as before
        it's deployed for the first time

        """
        if not self._conf:
            self._conf = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
        return self._conf

    def save(self, *args, **kwargs):
        if self._conf:
            self._conf.save()
            # Touch wsgi file, indicating to mod_wsgi to re-load modules and
            # therefore any changed configuration parameters
            wsgifile = os.path.join(self.projectdir, "wsgi", 'django.wsgi')
            if os.path.exists(wsgifile):
                # It may not exist if the project isn't active
                os.utime(wsgifile, None)
        super(DeployedProject, self).save(*args, **kwargs)

    def is_active(self):
        return os.path.exists(self.apache_conf)
    active = property(is_active)

    def _verify_project(self):
        """Verifies that the given project name corresponds to a real un-deployed
        project in the base dir.
        Returns False if something went wrong.

        """
        fullpath = self.projectdir
        if not os.path.isdir(fullpath):
            return False
        if os.path.exists(os.path.join(fullpath, "wsgi")):
            # Already deployed?
            return False
        if not os.path.exists(os.path.join(fullpath, "__init__.py")):
            return False
        if not os.path.exists(os.path.join(fullpath, "settings.py")):
            return False
        return True

    def deploy(self, info, active=True):
        """Call this to deploy a project. If successful, the model is saved and
        this method returns None. If something went wrong, a
        DeploymentException is raised with a description of the error, and the
        model is not saved. If something is wrong with the given information, a
        ValidationError is raised.

        Pass in a DeploymentInfo object with the appropriate attributes set.
        That information is used to deploy a project, but is not stored within
        the model itself.

        If active is not True, the deployment will be created inactive, and the
        apache configuration file will not be created.

        """
        # This should have been called externally before, but do it again just
        # to be sure nothing's changed.
        self.full_clean()

        # Do some validation checks to see if the given project name points to
        # a valid un-deployed django project
        if not self._verify_project():
            raise DeploymentException("Sanity check failed, will not create project with that name")

        d = opus.lib.deployer.ProjectDeployer(self.projectdir)

        d.create_environment()

        # Do this before settings the sensitive database information
        d.secure_project(settings.OPUS_SECUREOPS_COMMAND)

        d.configure_database(info.dbengine,
                info.dbname,
                info.dbuser,
                info.dbpassword,
                info.dbhost,
                info.dbport,
                )

        # This must go before sync_database, in case some settings that are
        # set by set_paths are used by a models.py at import time.
        d.set_paths()

        d.install_requirements(settings.OPUS_SECUREOPS_COMMAND)

        d.sync_database(info.superusername,
                info.superemail,
                info.superpassword,
                settings.OPUS_SECUREOPS_COMMAND
                )

        d.gen_cert(settings.OPUS_APACHE_SERVERNAME_SUFFIX)

        d.setup_celery(settings.OPUS_SECUREOPS_COMMAND,
                pythonpath=self._get_path_additions())

        if active:
            self.activate(d)

        self.save()

    def _get_path_additions(self):
        return "{0}".format(
                os.path.split(opus.__path__[0])[0],
                )

    def set_debug(self, d):
        """Sets debug mode on or off. Remember to save afterwards"""
        self.config['DEBUG'] = bool(d)
        self.config['TEMPLATE_DEBUG'] = bool(d)
        if d:
            self.config['LOG_LEVEL'] = "DEBUG"
        else:
            self.config['LOG_LEVEL'] = "INFO"

    def activate(self, d=None):
        """Activate this project. This writes out the apache config with the
        current parameters. Also writes out the wsgi file. Finally, starts the
        supervisord process which starts celeryd and celerybeat

        This is normally done during deployment, but this is useful to call
        after any change that affects the apache config so that the changes
        take effect. If you do this, don't forget to save() too.

        Pass in a deployer object, otherwise one will be created.

        """
        if not self.all_settings_set():
            raise DeploymentException("Tried to activate, but some applications still have settings to set")
        if not d:
            d = opus.lib.deployer.ProjectDeployer(self.projectdir)
        # The opus libraries should be in the path for the deployed app. TODO:
        # Find a better way to handle this.
        path_additions = self._get_path_additions()
        d.configure_apache(settings.OPUS_APACHE_CONFD,
                settings.OPUS_HTTP_PORT,
                settings.OPUS_HTTPS_PORT,
                settings.OPUS_APACHE_SERVERNAME_SUFFIX,
                secureops=settings.OPUS_SECUREOPS_COMMAND,
                pythonpath=path_additions,
                ssl_crt=settings.OPUS_SSL_CRT,
                ssl_key=settings.OPUS_SSL_KEY,
                ssl_chain=settings.OPUS_SSL_CHAIN,
                )

        # Schedule celery to start supervisord. Somehow if supervisord is
        # started directly by mod_wsgi, strange things happen to supervisord's
        # signal handlers
        opus.project.deployment.tasks.start_supervisord.delay(self.projectdir)


    def deactivate(self):
        """Removes the apache configuration file and restarts apache.

        """
        destroyer = opus.lib.deployer.ProjectUndeployer(self.projectdir)
        destroyer.remove_apache_conf(settings.OPUS_APACHE_CONFD,
                secureops=settings.OPUS_SECUREOPS_COMMAND)
        destroyer.stop_celery(
                secureops=settings.OPUS_SECUREOPS_COMMAND)

        # Make sure all processes are stopped
        opus.project.deployment.tasks.kill_processes.apply_async(
                args=[self.pk],
                countdown=5)

    def destroy(self):
        """Destroys the project. Deletes it off the drive, removes the system
        user, de-configures apache, and finally removes itself from the
        database.

        This method is idempotent, it can be called on a non-existant project
        or project in an inconsistant or intermediate state.

        This method will still error in these cases (not necessarily
        exaustive)

        * Apache can't be restarted
        * There's an error removing the user other than "user doesn't exist"
        * The project dir exists but cannot be removed

        """

        destroyer = opus.lib.deployer.ProjectUndeployer(self.projectdir)

        destroyer.remove_apache_conf(settings.OPUS_APACHE_CONFD,
                secureops=settings.OPUS_SECUREOPS_COMMAND)

        destroyer.stop_celery(
                secureops=settings.OPUS_SECUREOPS_COMMAND)

        destroyer.delete_celery(
                secureops=settings.OPUS_SECUREOPS_COMMAND)

        # This also kills off any remaining processes owned by that user
        destroyer.delete_user(
                secureops=settings.OPUS_SECUREOPS_COMMAND)

        # Remove database and user if automatically created
        try:
            if self.config['DATABASES']['default']['ENGINE'].endswith(\
                    "postgresql_psycopg2") and \
                    settings.OPUS_AUTO_POSTGRES_CONFIG:
                database.delete_postgres(self.name)
        except Exception, e:
            log.warning("Ignoring this error when trying to delete postgres user: %s", e)

        destroyer.remove_projectdir(
                secureops=settings.OPUS_SECUREOPS_COMMAND)

        if self.id is not None:
            self.delete()
Exemplo n.º 5
0
class ProjectDeployer(object):
    """The project deployer. Each method performs a specific deployment action.
    The typical workflow for using the deployer is to create a deployment
    object, and call these methods in roughly this order:
    
    * secure_project() should be called first to lock down the settings and set
      permissions before any sensitive information is pushed to the
      configuration files
    * configure_database() which will set the database configuration
      parameters, pushing sensitive information into config files.
    * sync-database() which will run django's syncdb function and create the
      admin superuser for the project. This pushes sensitive information to
      the database.
    * set_paths() pushes a few absolute directory paths to the configuration
    * configure_apache() creates a wsgi entry point file and apache
      configuration file, and restarts apache. This should be the last method
      called, apache will start serving project files right after this returns.

    If using Django, the deployment model DeployedProject's deploy() method
    does all of the above.

    """

    def __init__(self, projectdir):
        """Initialize the Project Deployer with the directory root of the
        project to be deployed.

        """
        self.projectdir = projectdir

        # Find the project name by taking the last component of the path
        path = os.path.abspath(self.projectdir)
        self.projectname = os.path.basename(path)

        self.uid = None

        # Go ahead and eat up the config file into memory
        self.config = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))

    def set_paths(self):
        """Sets the paths for the TEMPLATE_DIRS and the LOG_DIR
        settings
        
        """
        self.config['TEMPLATE_DIRS'] = (os.path.join(self.projectdir, "templates"),)
        self.config['LOG_DIR'] = os.path.join(self.projectdir, 'log')
        self.config['MEDIA_ROOT'] = os.path.join(self.projectdir, 'media/')
        self.config['OPUS_SECURE_UPLOADS'] = os.path.join(self.projectdir, "opus_secure_uploads/")
        self.config.save()

        # Set a new manage.py with appropriate paths
        manage = """#!/usr/bin/env python
from django.core.management import execute_from_command_line

import os, os.path
import sys
projectpath = {projectdir!r}
sys.path.append(projectpath)
sys.path.append({opuspath!r})
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
os.environ['OPUS_SETTINGS_FILE'] = os.path.join(projectpath, 'opussettings.json')
try:
    open(os.path.join(projectpath, "settings.py"), 'r')
except IOError, e:
    sys.stderr.write("Error: Can't open the file 'settings.py' in %r.\\n" % projectpath)
    sys.stderr.write(str(e)+'\\n')
    sys.exit(1)

if __name__ == "__main__":
    execute_from_command_line()
"""
        with open(os.path.join(self.projectdir, "manage.py"), 'w') as manageout:
            manageout.write(manage.format(
                projectdir=self.projectdir,
                opuspath=os.path.dirname(opus.__path__[0])))
        

    def configure_database(self, engine, *args):
        """Configure the Django database
        
        engine is one of the following:
        For 'postgresql_psycopg2', 'postgresql', 'mysql', or 'oracle' the next
        five parameters should be the name, user, password, host, and port

        For 'sqlite3' no other parameters are used, an sqlite3 database is
        created automatically.
        """

        dbuser = ''
        dbpassword = ''
        dbhost = ''
        dbport = ''
        if engine == "sqlite3":
            dbname = os.path.join(self.projectdir, "sqlite", "database.sqlite")
        elif engine in ('postgresql_psycopg2', 'postgresql', 'mysql', 'oracle'):
            if len(args) < 3:
                raise TypeError("You must specify the database username and password")
            dbname = args[0]
            if not dbname:
                raise ValueError("You must specify a value for database name")
            dbuser = args[1]
            dbpassword = args[2]
            if len(args) >= 4:
                dbhost = args[3]
            if len(args) >= 5:
                dbport = args[4]
        else:
            raise ValueError("Bad database engine")

        defaultdb = {}
        self.config['DATABASES'] = {'default': defaultdb}
        defaultdb['ENGINE'] = 'django.db.backends.{0}'.format(engine)
        defaultdb['NAME'] = dbname
        defaultdb['USER'] = dbuser
        defaultdb['PASSWORD'] = dbpassword
        defaultdb['HOST'] = dbhost
        defaultdb['PORT'] = dbport
        if defaultdb['ENGINE'].endswith("postgresql_psycopg2"):
            # Require SSL on the server
            defaultdb['OPTIONS'] = {"sslmode": "require"}

        self.config.save()

    def sync_database(self, username=None, email=None, password=None,
            secureops="secureops"):
        """Do the initial database sync. If a username, email, and password are
        provided, a superuser is created

        """
        self._sync_database(secureops)
        if username and email and password:
            self.create_superuser(username, email, password)

    def _getenv(self):
        """"Gets an environment with paths set up for a manage.py or
        djang-admin subprocess"""
        env = dict(os.environ)
        env['OPUS_SETTINGS_FILE'] = os.path.join(self.projectdir, "opussettings.json")
        env['PYTHONPATH'] = os.path.split(opus.__path__[0])[0] + ":" + \
                self.projectdir
        # Tells the logging module to disable logging, which would create
        # permission issues
        env['OPUS_LOGGING_DISABLE'] = "1"
        env['DJANGO_SETTINGS_MODULE'] = "settings"
        return env

    def _sync_database(self, secureops):
        # Runs sync on the database
        log.debug("Running syncdb")
        envpython = os.path.join(self.projectdir, "env", "bin", "python")
        # Search path for where django-admin is
        djangoadmin = which("django-admin.py")
        if not djangoadmin:
            raise DeploymentException("Could not find django-admin.py. Is it installed and in the system PATH?")
        proc = subprocess.Popen([secureops, "-y", "opus"+self.projectname,
            envpython,
            djangoadmin
            ],
                cwd=self.projectdir,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                env=self._getenv(),
                close_fds=True,
                )
        output = proc.communicate()[0]
        ret = proc.wait()
        if ret:
            raise DeploymentException("syncdb failed. Code {0}. {1}".format(ret, output))

    def create_superuser(self, username, email, password):
        """Creates a new superuser with given parameters in the target
        project"""
        # Create the user and set the password by invoking django code to
        # directly interface with the database. Do this in a sub process so as
        # not to have all the Django modules loaded and configured in this
        # interpreter, which may conflict with any Django settings already
        # imported.
        # This is done this way to avoid calling manage.py or running any
        # client code that the user has access to, since this happens with
        # Opus' permissions, not the deployed project permissions.
        log.debug("Creating superuser")
        dbconfig = self.config['DATABASES']['default']
        if dbconfig['ENGINE'].endswith("postgresql_psycopg2"):
            options = """'OPTIONS': {'sslmode': 'require'}"""
        else:
            options = ""
        program = """
import os
try:
    del os.environ['DJANGO_SETTINGS_MODULE']
except KeyError:
    pass
from django.conf import settings
settings.configure(DATABASES = {{'default':
    {{
        'ENGINE': '{engine}',
        'NAME': {name!r},
        'USER': {user!r},
        'PASSWORD': {password!r},
        'HOST': {host!r},
        'PORT': {port!r},
        {options}
    }}
}})
from django.contrib.auth.models import User
User.objects.create_superuser({suuser!r},{suemail!r},{supassword!r})
        """.format(
                engine=dbconfig['ENGINE'],
                name=dbconfig['NAME'],
                user=dbconfig['USER'],
                password=dbconfig['PASSWORD'],
                host=dbconfig['HOST'],
                port=dbconfig['PORT'],
                suuser=username,
                suemail=email,
                supassword=password,
                options=options
                )

        process = subprocess.Popen(["python"], stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                close_fds=True,
                )
        output = process.communicate(program)[0]
        ret = process.wait()
        if ret:
            raise DeploymentException("Setting super user password failed. {0}".format(output))

    def _pre_secure(self):
        # Creates a directory and empty file for the sqlite3 database,
        # and touches empty files for the ssl certificates and keyfiles.
        # Done so that permissions can be set on these items *before* sensitive
        # information is put into them
        os.mkdir(
                os.path.join(self.projectdir, "sqlite")
                )
        os.mkdir(
                os.path.join(self.projectdir, "run")
                )
        os.mkdir(
                os.path.join(self.projectdir, "opus_secure_uploads")
                )
        d = open(os.path.join(self.projectdir, "sqlite", "database.sqlite"), 'w')
        d.close()

        open(os.path.join(self.projectdir, "ssl.crt"), 'w').close()
        open(os.path.join(self.projectdir, "ssl.key"), 'w').close()

    def secure_project(self, secureops="secureops"):
        """Calling this does two things: It calls useradd to create a new Linux
        user, and it changes permissions on settings.py so only that user can
        access it. This is a necessary step before calling configure_apache()

        Pass in the path to the secureops binary, otherwise PATH is searched

        """
        # Touch certian files and directories so they can be secured before
        # they're filled with sensitive information
        self._pre_secure()

        # Attempt to create a linux user, and change user permissions
        # of the settings.py and the sqlite database 
        # Name the user after opus and the project name
        newname = "opus"+self.projectname
        command = [secureops, 
                "-c",
                newname,
                ]
        # Set sensitive files appropriately
        settingsfile = os.path.join(self.projectdir, "settings.py")
        command.append(settingsfile)
        # Also secure log directory
        command.append(os.path.join(self.projectdir, "log"))
        # And the opus settings
        command.append(os.path.join(self.projectdir, "opussettings.json"))
        # And sqlite dir and file
        command.append(os.path.join(self.projectdir, "sqlite"))
        command.append(os.path.join(self.projectdir, "sqlite", "database.sqlite"))
        command.append(os.path.join(self.projectdir, "ssl.crt"))
        command.append(os.path.join(self.projectdir, "ssl.key"))
        command.append(os.path.join(self.projectdir, "run"))
        command.append(os.path.join(self.projectdir, "opus_secure_uploads"))
        # Set writable several directories under the requirements env
        command.append(os.path.join(self.projectdir, "env"))
        for d in (['bin'], ['include'], ['lib','python*','site-packages']):
            p = glob(os.path.join(self.projectdir, "env", *d))[0]
            command.append(p)

        log.info("Calling secure operation with arguments {0!r}".format(command))
        log.debug("cwd: {0}".format(os.getcwd()))
        proc = subprocess.Popen(command,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                )
        output = proc.communicate()[0]
        ret = proc.wait()
        log.debug("Secure ops finished. Ret: {1}, Output: {0!r}".format(output, ret))
        if ret:
            raise DeploymentException("Could not create user and/or change file permissions. {0}. Ret: {1}".format(output, ret))

        # Also an important step: delete settings.pyc if it exists, which could
        # have sensitive information in it (although not likely, the usual
        # setup is to store settings in opussettings.json
        settingspyc = os.path.join(self.projectdir, "settings.pyc")
        if os.path.exists(settingspyc):
            try:
                os.unlink(settingspyc)
            except IOError, e:
                raise DeploymentException("Couldn't delete settings.pyc! {0}".format(e))

        # Generate a new secret key for the settings. One may have been set at
        # create time, but it should be considered public knowledge since the
        # permissions hadn't been set yet.
        self.config["SECRET_KEY"]  = ''.join([random.choice(
            'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)')
            for _ in range(50)])
        self.config.save()