def config(self): """Returns an opus.lib.conf.OpusConfig object for this project. This is automatically saved when the model's save() method is called. This will raise an error if the project doesn't exist (such as before it's deployed for the first time """ if not self._conf: self._conf = OpusConfig(os.path.join(self.projectdir, "opussettings.json")) return self._conf
def __init__(self, projectdir): """Initialize the Project Deployer with the directory root of the project to be deployed. """ self.projectdir = projectdir # Find the project name by taking the last component of the path path = os.path.abspath(self.projectdir) self.projectname = os.path.basename(path) self.uid = None # Go ahead and eat up the config file into memory self.config = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
def _setup_config(self): # Erases the normal settings.py and creates an opussettings.json # to replace it self.config = OpusConfig.new_from_template( os.path.join(self.projectdir, "opussettings.json")) with open(os.path.join(self.projectdir, "settings.py"), 'w') as s: s.write("""# Opus-built Project Settings file # The settings for this project are not stored in this file, but rather in the # file opussettings.json in JSON format. You may put your own values below to # override Opus's configuration, but this is not recommended. from opus.lib.conf import load_settings load_settings(globals()) import djcelery djcelery.setup_loader() """) # Randomizing SECRET_KEY is taken care of for us by new_from_template, # but we still have to set ROOT_URLCONF self.config['ROOT_URLCONF'] = "urls"
class DeployedProject(models.Model): """The actual model for a deployed project. The database doesn't contain too many fields, but this class contains lots of methods to query information and edit projects. Most of the state is stored in the filesystem and not the database. (For example, whether the project is activated is defined by the presence of an apache config file for the project) """ name = IdentifierField(unique=True) owner = models.ForeignKey(django.contrib.auth.models.User) def __init__(self, *args, **kwargs): super(DeployedProject, self).__init__(*args, **kwargs) self._conf = None @property def projectdir(self): return os.path.join(settings.OPUS_BASE_DIR, self.name) @property def apache_conf(self): return os.path.join(settings.OPUS_APACHE_CONFD, "opus"+self.name+".conf") @models.permalink def get_absolute_url(self): return ("opus.project.deployment.views.edit_or_create", (), dict(projectname=self.name)) @property def serve_http(self): port = settings.OPUS_HTTP_PORT if not port: return if port != 80: portline = ":"+str(port) else: portline = "" return "http://{0}{1}{2}/".format( self.name, settings.OPUS_APACHE_SERVERNAME_SUFFIX, portline) @property def serve_https(self): port = settings.OPUS_HTTPS_PORT if not port: return if port != 443: portline = ":"+str(port) else: portline = "" return "https://{0}{1}{2}/".format( self.name, settings.OPUS_APACHE_SERVERNAME_SUFFIX, portline) def get_urls(self): """Gets the urls that this project is being served from. This list is populated even if active is False. """ urls = [] http = self.serve_http if http: urls.append(http) https = self.serve_https if https: urls.append(https) return urls def get_apps(self): """Returns an iterator over application names that are currently installed""" for app in self.config['INSTALLED_APPS']: if os.path.exists(os.path.join(self.projectdir, app)): yield app @property def config(self): """Returns an opus.lib.conf.OpusConfig object for this project. This is automatically saved when the model's save() method is called. This will raise an error if the project doesn't exist (such as before it's deployed for the first time """ if not self._conf: self._conf = OpusConfig(os.path.join(self.projectdir, "opussettings.json")) return self._conf def save(self, *args, **kwargs): if self._conf: self._conf.save() # Touch wsgi file, indicating to mod_wsgi to re-load modules and # therefore any changed configuration parameters wsgifile = os.path.join(self.projectdir, "wsgi", 'django.wsgi') if os.path.exists(wsgifile): # It may not exist if the project isn't active os.utime(wsgifile, None) super(DeployedProject, self).save(*args, **kwargs) def is_active(self): return os.path.exists(self.apache_conf) active = property(is_active) def _verify_project(self): """Verifies that the given project name corresponds to a real un-deployed project in the base dir. Returns False if something went wrong. """ fullpath = self.projectdir if not os.path.isdir(fullpath): return False if os.path.exists(os.path.join(fullpath, "wsgi")): # Already deployed? return False if not os.path.exists(os.path.join(fullpath, "__init__.py")): return False if not os.path.exists(os.path.join(fullpath, "settings.py")): return False return True def deploy(self, info, active=True): """Call this to deploy a project. If successful, the model is saved and this method returns None. If something went wrong, a DeploymentException is raised with a description of the error, and the model is not saved. If something is wrong with the given information, a ValidationError is raised. Pass in a DeploymentInfo object with the appropriate attributes set. That information is used to deploy a project, but is not stored within the model itself. If active is not True, the deployment will be created inactive, and the apache configuration file will not be created. """ # This should have been called externally before, but do it again just # to be sure nothing's changed. self.full_clean() # Do some validation checks to see if the given project name points to # a valid un-deployed django project if not self._verify_project(): raise DeploymentException("Sanity check failed, will not create project with that name") d = opus.lib.deployer.ProjectDeployer(self.projectdir) d.create_environment() # Do this before settings the sensitive database information d.secure_project(settings.OPUS_SECUREOPS_COMMAND) d.configure_database(info.dbengine, info.dbname, info.dbuser, info.dbpassword, info.dbhost, info.dbport, ) # This must go before sync_database, in case some settings that are # set by set_paths are used by a models.py at import time. d.set_paths() d.install_requirements(settings.OPUS_SECUREOPS_COMMAND) d.sync_database(info.superusername, info.superemail, info.superpassword, settings.OPUS_SECUREOPS_COMMAND ) d.gen_cert(settings.OPUS_APACHE_SERVERNAME_SUFFIX) d.setup_celery(settings.OPUS_SECUREOPS_COMMAND, pythonpath=self._get_path_additions()) if active: self.activate(d) self.save() def _get_path_additions(self): return "{0}".format( os.path.split(opus.__path__[0])[0], ) def set_debug(self, d): """Sets debug mode on or off. Remember to save afterwards""" self.config['DEBUG'] = bool(d) self.config['TEMPLATE_DEBUG'] = bool(d) if d: self.config['LOG_LEVEL'] = "DEBUG" else: self.config['LOG_LEVEL'] = "INFO" def activate(self, d=None): """Activate this project. This writes out the apache config with the current parameters. Also writes out the wsgi file. Finally, starts the supervisord process which starts celeryd and celerybeat This is normally done during deployment, but this is useful to call after any change that affects the apache config so that the changes take effect. If you do this, don't forget to save() too. Pass in a deployer object, otherwise one will be created. """ if not self.all_settings_set(): raise DeploymentException("Tried to activate, but some applications still have settings to set") if not d: d = opus.lib.deployer.ProjectDeployer(self.projectdir) # The opus libraries should be in the path for the deployed app. TODO: # Find a better way to handle this. path_additions = self._get_path_additions() d.configure_apache(settings.OPUS_APACHE_CONFD, settings.OPUS_HTTP_PORT, settings.OPUS_HTTPS_PORT, settings.OPUS_APACHE_SERVERNAME_SUFFIX, secureops=settings.OPUS_SECUREOPS_COMMAND, pythonpath=path_additions, ssl_crt=settings.OPUS_SSL_CRT, ssl_key=settings.OPUS_SSL_KEY, ssl_chain=settings.OPUS_SSL_CHAIN, ) # Schedule celery to start supervisord. Somehow if supervisord is # started directly by mod_wsgi, strange things happen to supervisord's # signal handlers opus.project.deployment.tasks.start_supervisord.delay(self.projectdir) def deactivate(self): """Removes the apache configuration file and restarts apache. """ destroyer = opus.lib.deployer.ProjectUndeployer(self.projectdir) destroyer.remove_apache_conf(settings.OPUS_APACHE_CONFD, secureops=settings.OPUS_SECUREOPS_COMMAND) destroyer.stop_celery( secureops=settings.OPUS_SECUREOPS_COMMAND) # Make sure all processes are stopped opus.project.deployment.tasks.kill_processes.apply_async( args=[self.pk], countdown=5) def destroy(self): """Destroys the project. Deletes it off the drive, removes the system user, de-configures apache, and finally removes itself from the database. This method is idempotent, it can be called on a non-existant project or project in an inconsistant or intermediate state. This method will still error in these cases (not necessarily exaustive) * Apache can't be restarted * There's an error removing the user other than "user doesn't exist" * The project dir exists but cannot be removed """ destroyer = opus.lib.deployer.ProjectUndeployer(self.projectdir) destroyer.remove_apache_conf(settings.OPUS_APACHE_CONFD, secureops=settings.OPUS_SECUREOPS_COMMAND) destroyer.stop_celery( secureops=settings.OPUS_SECUREOPS_COMMAND) destroyer.delete_celery( secureops=settings.OPUS_SECUREOPS_COMMAND) # This also kills off any remaining processes owned by that user destroyer.delete_user( secureops=settings.OPUS_SECUREOPS_COMMAND) # Remove database and user if automatically created try: if self.config['DATABASES']['default']['ENGINE'].endswith(\ "postgresql_psycopg2") and \ settings.OPUS_AUTO_POSTGRES_CONFIG: database.delete_postgres(self.name) except Exception, e: log.warning("Ignoring this error when trying to delete postgres user: %s", e) destroyer.remove_projectdir( secureops=settings.OPUS_SECUREOPS_COMMAND) if self.id is not None: self.delete()
class ProjectDeployer(object): """The project deployer. Each method performs a specific deployment action. The typical workflow for using the deployer is to create a deployment object, and call these methods in roughly this order: * secure_project() should be called first to lock down the settings and set permissions before any sensitive information is pushed to the configuration files * configure_database() which will set the database configuration parameters, pushing sensitive information into config files. * sync-database() which will run django's syncdb function and create the admin superuser for the project. This pushes sensitive information to the database. * set_paths() pushes a few absolute directory paths to the configuration * configure_apache() creates a wsgi entry point file and apache configuration file, and restarts apache. This should be the last method called, apache will start serving project files right after this returns. If using Django, the deployment model DeployedProject's deploy() method does all of the above. """ def __init__(self, projectdir): """Initialize the Project Deployer with the directory root of the project to be deployed. """ self.projectdir = projectdir # Find the project name by taking the last component of the path path = os.path.abspath(self.projectdir) self.projectname = os.path.basename(path) self.uid = None # Go ahead and eat up the config file into memory self.config = OpusConfig(os.path.join(self.projectdir, "opussettings.json")) def set_paths(self): """Sets the paths for the TEMPLATE_DIRS and the LOG_DIR settings """ self.config['TEMPLATE_DIRS'] = (os.path.join(self.projectdir, "templates"),) self.config['LOG_DIR'] = os.path.join(self.projectdir, 'log') self.config['MEDIA_ROOT'] = os.path.join(self.projectdir, 'media/') self.config['OPUS_SECURE_UPLOADS'] = os.path.join(self.projectdir, "opus_secure_uploads/") self.config.save() # Set a new manage.py with appropriate paths manage = """#!/usr/bin/env python from django.core.management import execute_from_command_line import os, os.path import sys projectpath = {projectdir!r} sys.path.append(projectpath) sys.path.append({opuspath!r}) os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' os.environ['OPUS_SETTINGS_FILE'] = os.path.join(projectpath, 'opussettings.json') try: open(os.path.join(projectpath, "settings.py"), 'r') except IOError, e: sys.stderr.write("Error: Can't open the file 'settings.py' in %r.\\n" % projectpath) sys.stderr.write(str(e)+'\\n') sys.exit(1) if __name__ == "__main__": execute_from_command_line() """ with open(os.path.join(self.projectdir, "manage.py"), 'w') as manageout: manageout.write(manage.format( projectdir=self.projectdir, opuspath=os.path.dirname(opus.__path__[0]))) def configure_database(self, engine, *args): """Configure the Django database engine is one of the following: For 'postgresql_psycopg2', 'postgresql', 'mysql', or 'oracle' the next five parameters should be the name, user, password, host, and port For 'sqlite3' no other parameters are used, an sqlite3 database is created automatically. """ dbuser = '' dbpassword = '' dbhost = '' dbport = '' if engine == "sqlite3": dbname = os.path.join(self.projectdir, "sqlite", "database.sqlite") elif engine in ('postgresql_psycopg2', 'postgresql', 'mysql', 'oracle'): if len(args) < 3: raise TypeError("You must specify the database username and password") dbname = args[0] if not dbname: raise ValueError("You must specify a value for database name") dbuser = args[1] dbpassword = args[2] if len(args) >= 4: dbhost = args[3] if len(args) >= 5: dbport = args[4] else: raise ValueError("Bad database engine") defaultdb = {} self.config['DATABASES'] = {'default': defaultdb} defaultdb['ENGINE'] = 'django.db.backends.{0}'.format(engine) defaultdb['NAME'] = dbname defaultdb['USER'] = dbuser defaultdb['PASSWORD'] = dbpassword defaultdb['HOST'] = dbhost defaultdb['PORT'] = dbport if defaultdb['ENGINE'].endswith("postgresql_psycopg2"): # Require SSL on the server defaultdb['OPTIONS'] = {"sslmode": "require"} self.config.save() def sync_database(self, username=None, email=None, password=None, secureops="secureops"): """Do the initial database sync. If a username, email, and password are provided, a superuser is created """ self._sync_database(secureops) if username and email and password: self.create_superuser(username, email, password) def _getenv(self): """"Gets an environment with paths set up for a manage.py or djang-admin subprocess""" env = dict(os.environ) env['OPUS_SETTINGS_FILE'] = os.path.join(self.projectdir, "opussettings.json") env['PYTHONPATH'] = os.path.split(opus.__path__[0])[0] + ":" + \ self.projectdir # Tells the logging module to disable logging, which would create # permission issues env['OPUS_LOGGING_DISABLE'] = "1" env['DJANGO_SETTINGS_MODULE'] = "settings" return env def _sync_database(self, secureops): # Runs sync on the database log.debug("Running syncdb") envpython = os.path.join(self.projectdir, "env", "bin", "python") # Search path for where django-admin is djangoadmin = which("django-admin.py") if not djangoadmin: raise DeploymentException("Could not find django-admin.py. Is it installed and in the system PATH?") proc = subprocess.Popen([secureops, "-y", "opus"+self.projectname, envpython, djangoadmin ], cwd=self.projectdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=self._getenv(), close_fds=True, ) output = proc.communicate()[0] ret = proc.wait() if ret: raise DeploymentException("syncdb failed. Code {0}. {1}".format(ret, output)) def create_superuser(self, username, email, password): """Creates a new superuser with given parameters in the target project""" # Create the user and set the password by invoking django code to # directly interface with the database. Do this in a sub process so as # not to have all the Django modules loaded and configured in this # interpreter, which may conflict with any Django settings already # imported. # This is done this way to avoid calling manage.py or running any # client code that the user has access to, since this happens with # Opus' permissions, not the deployed project permissions. log.debug("Creating superuser") dbconfig = self.config['DATABASES']['default'] if dbconfig['ENGINE'].endswith("postgresql_psycopg2"): options = """'OPTIONS': {'sslmode': 'require'}""" else: options = "" program = """ import os try: del os.environ['DJANGO_SETTINGS_MODULE'] except KeyError: pass from django.conf import settings settings.configure(DATABASES = {{'default': {{ 'ENGINE': '{engine}', 'NAME': {name!r}, 'USER': {user!r}, 'PASSWORD': {password!r}, 'HOST': {host!r}, 'PORT': {port!r}, {options} }} }}) from django.contrib.auth.models import User User.objects.create_superuser({suuser!r},{suemail!r},{supassword!r}) """.format( engine=dbconfig['ENGINE'], name=dbconfig['NAME'], user=dbconfig['USER'], password=dbconfig['PASSWORD'], host=dbconfig['HOST'], port=dbconfig['PORT'], suuser=username, suemail=email, supassword=password, options=options ) process = subprocess.Popen(["python"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, ) output = process.communicate(program)[0] ret = process.wait() if ret: raise DeploymentException("Setting super user password failed. {0}".format(output)) def _pre_secure(self): # Creates a directory and empty file for the sqlite3 database, # and touches empty files for the ssl certificates and keyfiles. # Done so that permissions can be set on these items *before* sensitive # information is put into them os.mkdir( os.path.join(self.projectdir, "sqlite") ) os.mkdir( os.path.join(self.projectdir, "run") ) os.mkdir( os.path.join(self.projectdir, "opus_secure_uploads") ) d = open(os.path.join(self.projectdir, "sqlite", "database.sqlite"), 'w') d.close() open(os.path.join(self.projectdir, "ssl.crt"), 'w').close() open(os.path.join(self.projectdir, "ssl.key"), 'w').close() def secure_project(self, secureops="secureops"): """Calling this does two things: It calls useradd to create a new Linux user, and it changes permissions on settings.py so only that user can access it. This is a necessary step before calling configure_apache() Pass in the path to the secureops binary, otherwise PATH is searched """ # Touch certian files and directories so they can be secured before # they're filled with sensitive information self._pre_secure() # Attempt to create a linux user, and change user permissions # of the settings.py and the sqlite database # Name the user after opus and the project name newname = "opus"+self.projectname command = [secureops, "-c", newname, ] # Set sensitive files appropriately settingsfile = os.path.join(self.projectdir, "settings.py") command.append(settingsfile) # Also secure log directory command.append(os.path.join(self.projectdir, "log")) # And the opus settings command.append(os.path.join(self.projectdir, "opussettings.json")) # And sqlite dir and file command.append(os.path.join(self.projectdir, "sqlite")) command.append(os.path.join(self.projectdir, "sqlite", "database.sqlite")) command.append(os.path.join(self.projectdir, "ssl.crt")) command.append(os.path.join(self.projectdir, "ssl.key")) command.append(os.path.join(self.projectdir, "run")) command.append(os.path.join(self.projectdir, "opus_secure_uploads")) # Set writable several directories under the requirements env command.append(os.path.join(self.projectdir, "env")) for d in (['bin'], ['include'], ['lib','python*','site-packages']): p = glob(os.path.join(self.projectdir, "env", *d))[0] command.append(p) log.info("Calling secure operation with arguments {0!r}".format(command)) log.debug("cwd: {0}".format(os.getcwd())) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) output = proc.communicate()[0] ret = proc.wait() log.debug("Secure ops finished. Ret: {1}, Output: {0!r}".format(output, ret)) if ret: raise DeploymentException("Could not create user and/or change file permissions. {0}. Ret: {1}".format(output, ret)) # Also an important step: delete settings.pyc if it exists, which could # have sensitive information in it (although not likely, the usual # setup is to store settings in opussettings.json settingspyc = os.path.join(self.projectdir, "settings.pyc") if os.path.exists(settingspyc): try: os.unlink(settingspyc) except IOError, e: raise DeploymentException("Couldn't delete settings.pyc! {0}".format(e)) # Generate a new secret key for the settings. One may have been set at # create time, but it should be considered public knowledge since the # permissions hadn't been set yet. self.config["SECRET_KEY"] = ''.join([random.choice( 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for _ in range(50)]) self.config.save()