def config(self):
"""Returns an opus.lib.conf.OpusConfig object for this project. This is
automatically saved when the model's save() method is called.
This will raise an error if the project doesn't exist (such as before
it's deployed for the first time
"""
if not self._conf:
self._conf = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
return self._conf
def __init__(self, projectdir):
"""Initialize the Project Deployer with the directory root of the
project to be deployed.
"""
self.projectdir = projectdir
# Find the project name by taking the last component of the path
path = os.path.abspath(self.projectdir)
self.projectname = os.path.basename(path)
self.uid = None
# Go ahead and eat up the config file into memory
self.config = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
def _setup_config(self):
# Erases the normal settings.py and creates an opussettings.json
# to replace it
self.config = OpusConfig.new_from_template(
os.path.join(self.projectdir, "opussettings.json"))
with open(os.path.join(self.projectdir, "settings.py"), 'w') as s:
s.write("""# Opus-built Project Settings file
# The settings for this project are not stored in this file, but rather in the
# file opussettings.json in JSON format. You may put your own values below to
# override Opus's configuration, but this is not recommended.
from opus.lib.conf import load_settings
load_settings(globals())
import djcelery
djcelery.setup_loader()
""")
# Randomizing SECRET_KEY is taken care of for us by new_from_template,
# but we still have to set ROOT_URLCONF
self.config['ROOT_URLCONF'] = "urls"
class DeployedProject(models.Model):
"""The actual model for a deployed project. The database doesn't contain
too many fields, but this class contains lots of methods to query information and
edit projects. Most of the state is stored in the filesystem and not the
database. (For example, whether the project is activated is defined by the
presence of an apache config file for the project)
"""
name = IdentifierField(unique=True)
owner = models.ForeignKey(django.contrib.auth.models.User)
def __init__(self, *args, **kwargs):
super(DeployedProject, self).__init__(*args, **kwargs)
self._conf = None
@property
def projectdir(self):
return os.path.join(settings.OPUS_BASE_DIR, self.name)
@property
def apache_conf(self):
return os.path.join(settings.OPUS_APACHE_CONFD, "opus"+self.name+".conf")
@models.permalink
def get_absolute_url(self):
return ("opus.project.deployment.views.edit_or_create",
(),
dict(projectname=self.name))
@property
def serve_http(self):
port = settings.OPUS_HTTP_PORT
if not port:
return
if port != 80:
portline = ":"+str(port)
else:
portline = ""
return "http://{0}{1}{2}/".format(
self.name, settings.OPUS_APACHE_SERVERNAME_SUFFIX,
portline)
@property
def serve_https(self):
port = settings.OPUS_HTTPS_PORT
if not port:
return
if port != 443:
portline = ":"+str(port)
else:
portline = ""
return "https://{0}{1}{2}/".format(
self.name, settings.OPUS_APACHE_SERVERNAME_SUFFIX,
portline)
def get_urls(self):
"""Gets the urls that this project is being served from. This list is
populated even if active is False.
"""
urls = []
http = self.serve_http
if http:
urls.append(http)
https = self.serve_https
if https:
urls.append(https)
return urls
def get_apps(self):
"""Returns an iterator over application names that are currently
installed"""
for app in self.config['INSTALLED_APPS']:
if os.path.exists(os.path.join(self.projectdir, app)):
yield app
@property
def config(self):
"""Returns an opus.lib.conf.OpusConfig object for this project. This is
automatically saved when the model's save() method is called.
This will raise an error if the project doesn't exist (such as before
it's deployed for the first time
"""
if not self._conf:
self._conf = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
return self._conf
def save(self, *args, **kwargs):
if self._conf:
self._conf.save()
# Touch wsgi file, indicating to mod_wsgi to re-load modules and
# therefore any changed configuration parameters
wsgifile = os.path.join(self.projectdir, "wsgi", 'django.wsgi')
if os.path.exists(wsgifile):
# It may not exist if the project isn't active
os.utime(wsgifile, None)
super(DeployedProject, self).save(*args, **kwargs)
def is_active(self):
return os.path.exists(self.apache_conf)
active = property(is_active)
def _verify_project(self):
"""Verifies that the given project name corresponds to a real un-deployed
project in the base dir.
Returns False if something went wrong.
"""
fullpath = self.projectdir
if not os.path.isdir(fullpath):
return False
if os.path.exists(os.path.join(fullpath, "wsgi")):
# Already deployed?
return False
if not os.path.exists(os.path.join(fullpath, "__init__.py")):
return False
if not os.path.exists(os.path.join(fullpath, "settings.py")):
return False
return True
def deploy(self, info, active=True):
"""Call this to deploy a project. If successful, the model is saved and
this method returns None. If something went wrong, a
DeploymentException is raised with a description of the error, and the
model is not saved. If something is wrong with the given information, a
ValidationError is raised.
Pass in a DeploymentInfo object with the appropriate attributes set.
That information is used to deploy a project, but is not stored within
the model itself.
If active is not True, the deployment will be created inactive, and the
apache configuration file will not be created.
"""
# This should have been called externally before, but do it again just
# to be sure nothing's changed.
self.full_clean()
# Do some validation checks to see if the given project name points to
# a valid un-deployed django project
if not self._verify_project():
raise DeploymentException("Sanity check failed, will not create project with that name")
d = opus.lib.deployer.ProjectDeployer(self.projectdir)
d.create_environment()
# Do this before settings the sensitive database information
d.secure_project(settings.OPUS_SECUREOPS_COMMAND)
d.configure_database(info.dbengine,
info.dbname,
info.dbuser,
info.dbpassword,
info.dbhost,
info.dbport,
)
# This must go before sync_database, in case some settings that are
# set by set_paths are used by a models.py at import time.
d.set_paths()
d.install_requirements(settings.OPUS_SECUREOPS_COMMAND)
d.sync_database(info.superusername,
info.superemail,
info.superpassword,
settings.OPUS_SECUREOPS_COMMAND
)
d.gen_cert(settings.OPUS_APACHE_SERVERNAME_SUFFIX)
d.setup_celery(settings.OPUS_SECUREOPS_COMMAND,
pythonpath=self._get_path_additions())
if active:
self.activate(d)
self.save()
def _get_path_additions(self):
return "{0}".format(
os.path.split(opus.__path__[0])[0],
)
def set_debug(self, d):
"""Sets debug mode on or off. Remember to save afterwards"""
self.config['DEBUG'] = bool(d)
self.config['TEMPLATE_DEBUG'] = bool(d)
if d:
self.config['LOG_LEVEL'] = "DEBUG"
else:
self.config['LOG_LEVEL'] = "INFO"
def activate(self, d=None):
"""Activate this project. This writes out the apache config with the
current parameters. Also writes out the wsgi file. Finally, starts the
supervisord process which starts celeryd and celerybeat
This is normally done during deployment, but this is useful to call
after any change that affects the apache config so that the changes
take effect. If you do this, don't forget to save() too.
Pass in a deployer object, otherwise one will be created.
"""
if not self.all_settings_set():
raise DeploymentException("Tried to activate, but some applications still have settings to set")
if not d:
d = opus.lib.deployer.ProjectDeployer(self.projectdir)
# The opus libraries should be in the path for the deployed app. TODO:
# Find a better way to handle this.
path_additions = self._get_path_additions()
d.configure_apache(settings.OPUS_APACHE_CONFD,
settings.OPUS_HTTP_PORT,
settings.OPUS_HTTPS_PORT,
settings.OPUS_APACHE_SERVERNAME_SUFFIX,
secureops=settings.OPUS_SECUREOPS_COMMAND,
pythonpath=path_additions,
ssl_crt=settings.OPUS_SSL_CRT,
ssl_key=settings.OPUS_SSL_KEY,
ssl_chain=settings.OPUS_SSL_CHAIN,
)
# Schedule celery to start supervisord. Somehow if supervisord is
# started directly by mod_wsgi, strange things happen to supervisord's
# signal handlers
opus.project.deployment.tasks.start_supervisord.delay(self.projectdir)
def deactivate(self):
"""Removes the apache configuration file and restarts apache.
"""
destroyer = opus.lib.deployer.ProjectUndeployer(self.projectdir)
destroyer.remove_apache_conf(settings.OPUS_APACHE_CONFD,
secureops=settings.OPUS_SECUREOPS_COMMAND)
destroyer.stop_celery(
secureops=settings.OPUS_SECUREOPS_COMMAND)
# Make sure all processes are stopped
opus.project.deployment.tasks.kill_processes.apply_async(
args=[self.pk],
countdown=5)
def destroy(self):
"""Destroys the project. Deletes it off the drive, removes the system
user, de-configures apache, and finally removes itself from the
database.
This method is idempotent, it can be called on a non-existant project
or project in an inconsistant or intermediate state.
This method will still error in these cases (not necessarily
exaustive)
* Apache can't be restarted
* There's an error removing the user other than "user doesn't exist"
* The project dir exists but cannot be removed
"""
destroyer = opus.lib.deployer.ProjectUndeployer(self.projectdir)
destroyer.remove_apache_conf(settings.OPUS_APACHE_CONFD,
secureops=settings.OPUS_SECUREOPS_COMMAND)
destroyer.stop_celery(
secureops=settings.OPUS_SECUREOPS_COMMAND)
destroyer.delete_celery(
secureops=settings.OPUS_SECUREOPS_COMMAND)
# This also kills off any remaining processes owned by that user
destroyer.delete_user(
secureops=settings.OPUS_SECUREOPS_COMMAND)
# Remove database and user if automatically created
try:
if self.config['DATABASES']['default']['ENGINE'].endswith(\
"postgresql_psycopg2") and \
settings.OPUS_AUTO_POSTGRES_CONFIG:
database.delete_postgres(self.name)
except Exception, e:
log.warning("Ignoring this error when trying to delete postgres user: %s", e)
destroyer.remove_projectdir(
secureops=settings.OPUS_SECUREOPS_COMMAND)
if self.id is not None:
self.delete()
class ProjectDeployer(object):
"""The project deployer. Each method performs a specific deployment action.
The typical workflow for using the deployer is to create a deployment
object, and call these methods in roughly this order:
* secure_project() should be called first to lock down the settings and set
permissions before any sensitive information is pushed to the
configuration files
* configure_database() which will set the database configuration
parameters, pushing sensitive information into config files.
* sync-database() which will run django's syncdb function and create the
admin superuser for the project. This pushes sensitive information to
the database.
* set_paths() pushes a few absolute directory paths to the configuration
* configure_apache() creates a wsgi entry point file and apache
configuration file, and restarts apache. This should be the last method
called, apache will start serving project files right after this returns.
If using Django, the deployment model DeployedProject's deploy() method
does all of the above.
"""
def __init__(self, projectdir):
"""Initialize the Project Deployer with the directory root of the
project to be deployed.
"""
self.projectdir = projectdir
# Find the project name by taking the last component of the path
path = os.path.abspath(self.projectdir)
self.projectname = os.path.basename(path)
self.uid = None
# Go ahead and eat up the config file into memory
self.config = OpusConfig(os.path.join(self.projectdir, "opussettings.json"))
def set_paths(self):
"""Sets the paths for the TEMPLATE_DIRS and the LOG_DIR
settings
"""
self.config['TEMPLATE_DIRS'] = (os.path.join(self.projectdir, "templates"),)
self.config['LOG_DIR'] = os.path.join(self.projectdir, 'log')
self.config['MEDIA_ROOT'] = os.path.join(self.projectdir, 'media/')
self.config['OPUS_SECURE_UPLOADS'] = os.path.join(self.projectdir, "opus_secure_uploads/")
self.config.save()
# Set a new manage.py with appropriate paths
manage = """#!/usr/bin/env python
from django.core.management import execute_from_command_line
import os, os.path
import sys
projectpath = {projectdir!r}
sys.path.append(projectpath)
sys.path.append({opuspath!r})
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
os.environ['OPUS_SETTINGS_FILE'] = os.path.join(projectpath, 'opussettings.json')
try:
open(os.path.join(projectpath, "settings.py"), 'r')
except IOError, e:
sys.stderr.write("Error: Can't open the file 'settings.py' in %r.\\n" % projectpath)
sys.stderr.write(str(e)+'\\n')
sys.exit(1)
if __name__ == "__main__":
execute_from_command_line()
"""
with open(os.path.join(self.projectdir, "manage.py"), 'w') as manageout:
manageout.write(manage.format(
projectdir=self.projectdir,
opuspath=os.path.dirname(opus.__path__[0])))
def configure_database(self, engine, *args):
"""Configure the Django database
engine is one of the following:
For 'postgresql_psycopg2', 'postgresql', 'mysql', or 'oracle' the next
five parameters should be the name, user, password, host, and port
For 'sqlite3' no other parameters are used, an sqlite3 database is
created automatically.
"""
dbuser = ''
dbpassword = ''
dbhost = ''
dbport = ''
if engine == "sqlite3":
dbname = os.path.join(self.projectdir, "sqlite", "database.sqlite")
elif engine in ('postgresql_psycopg2', 'postgresql', 'mysql', 'oracle'):
if len(args) < 3:
raise TypeError("You must specify the database username and password")
dbname = args[0]
if not dbname:
raise ValueError("You must specify a value for database name")
dbuser = args[1]
dbpassword = args[2]
if len(args) >= 4:
dbhost = args[3]
if len(args) >= 5:
dbport = args[4]
else:
raise ValueError("Bad database engine")
defaultdb = {}
self.config['DATABASES'] = {'default': defaultdb}
defaultdb['ENGINE'] = 'django.db.backends.{0}'.format(engine)
defaultdb['NAME'] = dbname
defaultdb['USER'] = dbuser
defaultdb['PASSWORD'] = dbpassword
defaultdb['HOST'] = dbhost
defaultdb['PORT'] = dbport
if defaultdb['ENGINE'].endswith("postgresql_psycopg2"):
# Require SSL on the server
defaultdb['OPTIONS'] = {"sslmode": "require"}
self.config.save()
def sync_database(self, username=None, email=None, password=None,
secureops="secureops"):
"""Do the initial database sync. If a username, email, and password are
provided, a superuser is created
"""
self._sync_database(secureops)
if username and email and password:
self.create_superuser(username, email, password)
def _getenv(self):
""""Gets an environment with paths set up for a manage.py or
djang-admin subprocess"""
env = dict(os.environ)
env['OPUS_SETTINGS_FILE'] = os.path.join(self.projectdir, "opussettings.json")
env['PYTHONPATH'] = os.path.split(opus.__path__[0])[0] + ":" + \
self.projectdir
# Tells the logging module to disable logging, which would create
# permission issues
env['OPUS_LOGGING_DISABLE'] = "1"
env['DJANGO_SETTINGS_MODULE'] = "settings"
return env
def _sync_database(self, secureops):
# Runs sync on the database
log.debug("Running syncdb")
envpython = os.path.join(self.projectdir, "env", "bin", "python")
# Search path for where django-admin is
djangoadmin = which("django-admin.py")
if not djangoadmin:
raise DeploymentException("Could not find django-admin.py. Is it installed and in the system PATH?")
proc = subprocess.Popen([secureops, "-y", "opus"+self.projectname,
envpython,
djangoadmin
],
cwd=self.projectdir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=self._getenv(),
close_fds=True,
)
output = proc.communicate()[0]
ret = proc.wait()
if ret:
raise DeploymentException("syncdb failed. Code {0}. {1}".format(ret, output))
def create_superuser(self, username, email, password):
"""Creates a new superuser with given parameters in the target
project"""
# Create the user and set the password by invoking django code to
# directly interface with the database. Do this in a sub process so as
# not to have all the Django modules loaded and configured in this
# interpreter, which may conflict with any Django settings already
# imported.
# This is done this way to avoid calling manage.py or running any
# client code that the user has access to, since this happens with
# Opus' permissions, not the deployed project permissions.
log.debug("Creating superuser")
dbconfig = self.config['DATABASES']['default']
if dbconfig['ENGINE'].endswith("postgresql_psycopg2"):
options = """'OPTIONS': {'sslmode': 'require'}"""
else:
options = ""
program = """
import os
try:
del os.environ['DJANGO_SETTINGS_MODULE']
except KeyError:
pass
from django.conf import settings
settings.configure(DATABASES = {{'default':
{{
'ENGINE': '{engine}',
'NAME': {name!r},
'USER': {user!r},
'PASSWORD': {password!r},
'HOST': {host!r},
'PORT': {port!r},
{options}
}}
}})
from django.contrib.auth.models import User
User.objects.create_superuser({suuser!r},{suemail!r},{supassword!r})
""".format(
engine=dbconfig['ENGINE'],
name=dbconfig['NAME'],
user=dbconfig['USER'],
password=dbconfig['PASSWORD'],
host=dbconfig['HOST'],
port=dbconfig['PORT'],
suuser=username,
suemail=email,
supassword=password,
options=options
)
process = subprocess.Popen(["python"], stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True,
)
output = process.communicate(program)[0]
ret = process.wait()
if ret:
raise DeploymentException("Setting super user password failed. {0}".format(output))
def _pre_secure(self):
# Creates a directory and empty file for the sqlite3 database,
# and touches empty files for the ssl certificates and keyfiles.
# Done so that permissions can be set on these items *before* sensitive
# information is put into them
os.mkdir(
os.path.join(self.projectdir, "sqlite")
)
os.mkdir(
os.path.join(self.projectdir, "run")
)
os.mkdir(
os.path.join(self.projectdir, "opus_secure_uploads")
)
d = open(os.path.join(self.projectdir, "sqlite", "database.sqlite"), 'w')
d.close()
open(os.path.join(self.projectdir, "ssl.crt"), 'w').close()
open(os.path.join(self.projectdir, "ssl.key"), 'w').close()
def secure_project(self, secureops="secureops"):
"""Calling this does two things: It calls useradd to create a new Linux
user, and it changes permissions on settings.py so only that user can
access it. This is a necessary step before calling configure_apache()
Pass in the path to the secureops binary, otherwise PATH is searched
"""
# Touch certian files and directories so they can be secured before
# they're filled with sensitive information
self._pre_secure()
# Attempt to create a linux user, and change user permissions
# of the settings.py and the sqlite database
# Name the user after opus and the project name
newname = "opus"+self.projectname
command = [secureops,
"-c",
newname,
]
# Set sensitive files appropriately
settingsfile = os.path.join(self.projectdir, "settings.py")
command.append(settingsfile)
# Also secure log directory
command.append(os.path.join(self.projectdir, "log"))
# And the opus settings
command.append(os.path.join(self.projectdir, "opussettings.json"))
# And sqlite dir and file
command.append(os.path.join(self.projectdir, "sqlite"))
command.append(os.path.join(self.projectdir, "sqlite", "database.sqlite"))
command.append(os.path.join(self.projectdir, "ssl.crt"))
command.append(os.path.join(self.projectdir, "ssl.key"))
command.append(os.path.join(self.projectdir, "run"))
command.append(os.path.join(self.projectdir, "opus_secure_uploads"))
# Set writable several directories under the requirements env
command.append(os.path.join(self.projectdir, "env"))
for d in (['bin'], ['include'], ['lib','python*','site-packages']):
p = glob(os.path.join(self.projectdir, "env", *d))[0]
command.append(p)
log.info("Calling secure operation with arguments {0!r}".format(command))
log.debug("cwd: {0}".format(os.getcwd()))
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output = proc.communicate()[0]
ret = proc.wait()
log.debug("Secure ops finished. Ret: {1}, Output: {0!r}".format(output, ret))
if ret:
raise DeploymentException("Could not create user and/or change file permissions. {0}. Ret: {1}".format(output, ret))
# Also an important step: delete settings.pyc if it exists, which could
# have sensitive information in it (although not likely, the usual
# setup is to store settings in opussettings.json
settingspyc = os.path.join(self.projectdir, "settings.pyc")
if os.path.exists(settingspyc):
try:
os.unlink(settingspyc)
except IOError, e:
raise DeploymentException("Couldn't delete settings.pyc! {0}".format(e))
# Generate a new secret key for the settings. One may have been set at
# create time, but it should be considered public knowledge since the
# permissions hadn't been set yet.
self.config["SECRET_KEY"] = ''.join([random.choice(
'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)')
for _ in range(50)])
self.config.save()