def set_version_state(name, object=None, delete=False):
"""
Sets a simple 'state' on the server by creating a file with the
desired state's name + version and storing ``content`` as json
strings if supplied.
Returns the filename used to store state.
"""
if env.project_fullname:
state_name = '-'.join([env.project_fullname, name])
else:
state_name = name
with fab_settings(warn_only=True):
# Test for OS state.
if not exists('/var/local/woven', use_sudo=True):
sudo('mkdir /var/local/woven')
if not delete:
sudo('touch /var/local/woven/%s' % state_name)
if object != None:
fd, file_path = tempfile.mkstemp()
f = os.fdopen(fd, 'w')
f.write(json.dumps(object))
f.close()
put(file_path, '/tmp/%s' % state_name)
os.remove(file_path)
sudo('cp /tmp/%s /var/local/woven/%s' % (state_name, state_name))
else:
sudo('rm -f /var/local/woven/%s' % state_name)
return state_name
def vcs_upload():
"""
Uploads the project with the selected VCS tool.
"""
if env.deploy_tool == "git":
remote_path = "ssh://%s@%s%s" % (env.user, env.host_string,
env.repo_path)
if not exists(env.repo_path):
run("mkdir -p %s" % env.repo_path)
with cd(env.repo_path):
run("git init --bare")
local("git push -f %s master" % remote_path)
with cd(env.repo_path):
run("GIT_WORK_TREE=%s git checkout -f master" % env.proj_path)
run("GIT_WORK_TREE=%s git reset --hard" % env.proj_path)
elif env.deploy_tool == "hg":
remote_path = "ssh://%s@%s/%s" % (env.user, env.host_string,
env.repo_path)
with cd(env.repo_path):
if not exists("%s/.hg" % env.repo_path):
run("hg init")
print(env.repo_path)
with fab_settings(warn_only=True):
push = local("hg push -f %s" % remote_path)
if push.return_code == 255:
abort()
run("hg update")
def create_db_user():
"""Create the Postgres user."""
require('db_user', provided_by=('setup_env'))
with fab_settings(warn_only=True):
with cd('/tmp'):
# sudo('createuser -D -A -R %(db_user)s' % env, pty=True, user="******")
run('sudo -u postgres createuser -D -A -R %(db_user)s' % env, shell=False)
def version_state(name, prefix=False, no_content=False):
"""
If the server state exists return parsed json as a python object or True
prefix=True returns True if any files exist with ls [prefix]*
"""
if env.project_fullname:
full_name = '-'.join([env.project_fullname, name])
else:
full_name = name
current_state = False
state = State(full_name)
state_path = '/var/local/woven/%s' % full_name
if not prefix and not no_content and exists(state_path):
# Get size.
content = int(sudo('ls -s %s' % state_path).split()[0])
if content:
fd, file_path = tempfile.mkstemp()
os.close(fd)
get(state_path, file_path)
with open(file_path, "r") as f:
content = f.read()
object = json.loads(content)
current_state = object
else:
current_state = True
elif not prefix and no_content and exists(state_path):
current_state = True
elif prefix:
with fab_settings(warn_only=True):
# Find any version.
current_state = sudo('ls /var/local/woven/*%s' % name)
if not current_state.failed:
current_state = True
return current_state
def bootstrap(deploy_level='staging'):
"""
Does nothing in this module.
"""
setup_env(deploy_level)
install_packages()
with fab_settings(warn_only=True): #in case the db is already initialized
_run_postgres_command(
'initdb') #initializes things for the first time.
_fix_ident()
_run_postgres_command(
'restart') #start the server so that we can create some db_user and db
# _set_postgres_passwd()
#
# create_db_user()
# create_db()
if env.move_db_now:
print red(
'MOVING THE DB! THIS IS A ONE TIME COMMAND, DO NOT RUN BOOTSTRAP OR MOVE AGAIN OR YOU WILL LOSE ALL YOUR DATA'
)
_relocate_db()
else:
print 'not moving db now??'
print 'env.move_db_now == %s' % env.move_db_now
def server_state(name, no_content=False):
"""
If the server state exists return parsed json as a python object or True
prefix=True returns True if any files exist with ls [prefix]*
"""
with fab_settings(project_fullname=''):
return version_state(name, no_content=no_content)
def version_state(name, prefix=False, no_content=False):
"""
If the server state exists return parsed json as a python object or True
prefix=True returns True if any files exist with ls [prefix]*
"""
if env.project_fullname: full_name = '-'.join([env.project_fullname, name])
else: full_name = name
current_state = False
state = State(full_name)
state_path = '/var/local/woven/%s' % full_name
if not prefix and not no_content and exists(state_path):
content = int(sudo('ls -s %s' % state_path).split()[0]) #get size
if content:
fd, file_path = tempfile.mkstemp()
os.close(fd)
get(state_path, file_path)
with open(file_path, "r") as f:
content = f.read()
object = json.loads(content)
current_state = object
else:
current_state = True
elif not prefix and no_content and exists(state_path):
current_state = True
elif prefix:
with fab_settings(warn_only=True): #find any version
current_state = sudo('ls /var/local/woven/*%s' % name)
if not current_state.failed: current_state = True
return current_state
def server_state(name, no_content=False):
"""
If the server state exists return parsed json as a python object or True
prefix=True returns True if any files exist with ls [prefix]*
"""
with fab_settings(project_fullname=''):
return version_state(name, no_content=no_content)
def set_version_state(name, object=None, delete=False):
"""
Sets a simple 'state' on the server by creating a file
with the desired state's name + version and storing ``content`` as json strings if supplied
returns the filename used to store state
"""
if env.project_fullname:
state_name = '-'.join([env.project_fullname, name])
else:
state_name = name
with fab_settings(warn_only=True):
#Test for os state
if not exists('/var/local/woven', use_sudo=True):
sudo('mkdir /var/local/woven')
if not delete:
sudo('touch /var/local/woven/%s' % state_name)
if object <> None:
fd, file_path = tempfile.mkstemp()
f = os.fdopen(fd, 'w')
f.write(json.dumps(object))
f.close()
put(file_path, '/tmp/%s' % state_name)
os.remove(file_path)
sudo('cp /tmp/%s /var/local/woven/%s' % (state_name, state_name))
else:
sudo('rm -f /var/local/woven/%s' % state_name)
return state_name
def create_db_user():
"""Create the Postgres user."""
require('db_user', provided_by=('setup_env'))
with fab_settings(warn_only=True):
with cd('/tmp'):
# sudo('createuser -D -A -R %(db_user)s' % env, pty=True, user="******")
run('sudo -u postgres createuser -D -A -R %(db_user)s' % env,
shell=False)
def set_server_state(name,object=None,delete=False):
"""
Sets a simple 'state' on the server by creating a file
with the desired state's name and storing ``content`` as json strings if supplied
returns the filename used to store state
"""
with fab_settings(project_fullname=''):
return set_version_state(name,object,delete)
def set_server_state(name, object=None, delete=False):
"""
Sets a simple 'state' on the server by creating a file
with the desired state's name and storing ``content`` as json strings if supplied
returns the filename used to store state
"""
with fab_settings(project_fullname=''):
return set_version_state(name, object, delete)
def run(self, command, capture=True):
self.logger.debug(command)
with fab_settings(warn_only=True):
with shell_env(**self.env.dict()):
result = local(command, capture=capture)
if not result.succeeded:
self.logger.error(result.stderr)
print result.stderr
return result
def run(self, command, capture=True):
self.logger.debug(command)
with fab_settings(warn_only=True):
with shell_env(**self.env.dict()):
result = local(command, capture=capture)
if not result.succeeded:
self.logger.error(result.stderr)
print result.stderr
return result
def fab_run_local(cmd):
logger.debug('fab_run_local cmd: %s' % cmd)
with fab_settings(warn_only=True):
# run command
r = local(cmd, capture=True)
# log output
logger.debug('fab_run_local output: %s' % r)
return r
def deploy():
""" --> [REMOTE] Deploy to target env """
with ensure_target():
# Avoid pushing dumps & other rubbish.
# Only push up to the last explicit commit
# local("git add --all .")
local("git add -f %s" % settings.STATIC_DIST_FILEPATH)
with fab_settings(hide('warnings'), warn_only=True):
local("git commit -m 'deploy %s/%s'" % (settings.PROJECT_NAME, env.target_stage))
local("git rm -r --cached %s" % settings.STATIC_DIST_FILEPATH)
local("git push %s %s -f" % (env.target_stage, get_current_branch()))
local("git commit --amend -a --no-edit --allow-empty")
def run(self, command='', warn_only=False, cd=None):
env.host_string = '%s@%s:%s' % (self.user, self.host, self.port)
if self.empty is None:
with hide('output'):
self.empty = str(run('echo'))
if cd is None:
cd = '/home/{}'.format(self.user)
with (fab_cd(cd)):
with fab_settings(warn_only=warn_only):
with hide('output'):
output = str(run(command))
text = output[len(self.empty)+1:].replace('\r', '').strip()
return text
def run(self, command='', warn_only=False, cd=None):
env.host_string = '%s@%s:%s' % (self.user, self.host, self.port)
if self.empty is None:
with hide('output'):
self.empty = str(run('echo'))
if cd is None:
cd = '/home/{}'.format(self.user)
with (fab_cd(cd)):
with fab_settings(warn_only=warn_only):
with hide('output'):
output = str(run(command))
text = output[len(self.empty) + 1:].replace('\r', '').strip()
return text
def deploy():
""" --> [REMOTE] Deploy to target env """
with ensure_target():
# Avoid pushing dumps & other rubbish.
# Only push up to the last explicit commit
# local("git add --all .")
local("git add -f %s" % settings.STATIC_DIST_FILEPATH)
with fab_settings(hide('warnings'), warn_only=True):
local("git commit -m 'deploy %s/%s'" %
(settings.PROJECT_NAME, env.target_stage))
local("git rm -r --cached %s" % settings.STATIC_DIST_FILEPATH)
local("git push %s %s -f" % (env.target_stage, get_current_branch()))
local("git commit --amend -a --no-edit --allow-empty")
def build_app_package(cp, server, project_dir, app, conf_dir):
app_dir = os.path.join(project_dir, app)
#app_conf_template_dir = os.path.join(conf_dir, app)
#def_conf_template_dir = os.path.join(settings.DEFAULT_CONF_DIR, app)
with fab_settings(warn_only=True, echo_stdin=False):
if os.path.exists(app_dir):
handlers = settings.ensure_build_handlers()
for handler in handlers:
handler(config_parser=cp, server=server, project_dir=project_dir, app_name=app)
#handle_configuration(cp, server, app_dir, app_conf_template_dir, def_conf_template_dir)
return _generate_package(app_dir)
else:
abort('app dir %s does not exist' % app_dir)
def pushdb():
"""
Backup the local database, upload it, and restore it remotely.
"""
prompt = ("This will delete your production database and copy the contents from "
"the development database. Continue?")
if not confirm(prompt, default=False):
abort("Aborting by user request.")
local_backup("%s_development.sql" % env.proj_name)
local("scp {2}_development.sql {0}@{1}:/home/{0}/".format(
env.user, env.host_string, env.proj_name))
with fab_settings(warn_only=True):
# This last part can output some errors, but the restoration goes well
restore("%s_development.sql" % env.proj_name)
def pushdb():
"""
Backup the local database, upload it, and restore it remotely.
"""
prompt = (
"This will delete your production database and copy the contents from " "the development database. Continue?"
)
if not confirm(prompt, default=False):
abort("Aborting by user request.")
local_backup("%s_development.sql" % env.proj_name)
local("scp {2}_development.sql {0}@{1}:/home/{0}/".format(env.user, env.host_string, env.proj_name))
with fab_settings(warn_only=True):
# This last part can output some errors, but the restoration goes well
restore("%s_development.sql" % env.proj_name)
def _generate_package(app_dir):
with lcd(app_dir):
with fab_settings(warn_only=True):
if not os.path.exists(os.path.join(app_dir, 'setup.py')):
abort('setup.py is not in %s ' % app_dir)
build_script_dir = os.path.join(app_dir, 'scripts/build.sh')
if os.path.exists(build_script_dir) and os.access(build_script_dir, os.X_OK):
local(build_script_dir)
else:
local("python setup.py -q sdist")
app_dist_dir = os.path.join(app_dir, 'dist')
filepath = os.listdir(app_dist_dir)[0]
local("mv -f dist/*.tar.gz %s/" % settings.DIST_TEMP)
filepath = os.path.join(settings.DIST_TEMP, filepath)
return filepath
def run_dservices(profile_names, host, remote_path, settings=None):
"""
Run the configuration script on the remote machine.
"""
# Create a list of variables (name, value) to pass to the script run
# on the remote machine.
defines = []
if settings:
for key, value in six.iteritems(settings):
defines += ['-D"%s"="%s"' % (key, value)]
profiles = []
for profile_name in profile_names:
profile = os.path.basename(profile_name)
if not profile.endswith('.xml'):
profile = profile + '.xml'
profiles += ['share/tero/profiles/' + profile]
with fab_settings(abort_on_prompts=True, host_string=host):
with fab.cd(remote_path):
cmdline = ['./bin/dservices'] + defines + profiles
fab.run(' '.join(cmdline))
def run_dservices(profile_names, host, remote_path, settings=None):
"""
Run the configuration script on the remote machine.
"""
# Create a list of variables (name, value) to pass to the script run
# on the remote machine.
defines = []
if settings:
for key, value in settings.iteritems():
defines += ['-D"%s"="%s"' % (key, value)]
profiles = []
for profile_name in profile_names:
profile = os.path.basename(profile_name)
if not profile.endswith('.xml'):
profile = profile + '.xml'
profiles += ['share/tero/profiles/' + profile]
with fab_settings(abort_on_prompts=True, host_string=host):
with fab.cd(remote_path):
cmdline = ['./bin/dservices'] + defines + profiles
fab.run(' '.join(cmdline))
def run_collector(collector, config): PACKAGE = 'matrix_package__%s__%s' % (collector, timestamp_pretty) PACKAGE_DIR = os.path.abspath(os.path.join(settings.DATA_DIR, 'tmp', PACKAGE)) for name,config in config.items(): dir_item = os.path.abspath(os.path.join(PACKAGE_DIR, config['name4'])) ensure_dir(dir_item) for command,outfile in getattr(settings, '%s_CLI_COMMANDS' % collector.upper()): if collector == 'cml': cmd = settings.CML_CLI_COMMAND_SYNTAX % (settings.CML_CLI_DIR, settings.CML_CLI_DIR, config['name4'], command, dir_item, outfile) elif collector == 'emc': cmd = settings.EMC_CLI_COMMAND_SYNTAX % (config['ip'], command, os.path.join(dir_item, outfile)) elif collector == 'brocade': cmd = settings.BROCADE_CLI_COMMAND_SYNTAX % (config['ssh-key'], config['username'], config['ip'], command, os.path.join(dir_item, outfile)) if settings.DEBUG: print cmd else: with fab_settings(warn_only=True): # run command local(cmd) # create tgz cmd = 'cd %s; tar -czf %s/%s.tgz %s' % (settings.TMP_DIR, settings.ARCHIVE_DIR, PACKAGE, PACKAGE) if settings.DEBUG: print cmd else: # run command local(cmd) # delete uncompressed shutil.rmtree(PACKAGE_DIR)
def upload_apache_conf():
"""
Upload and link Supervisor configuration from the template.
"""
require('environment', 'httpd_services_template_name', provided_by=('setup_env'))
env.tmp_destination = posixpath.join('/', 'tmp', env.httpd_services_template_name)
files.upload_template(env.httpd_local_template_path, env.tmp_destination, context=env.httpd_dict, use_sudo=True)
env.httpd_sudo_user = settings.SUDO_USER
sudo('chown -R %(httpd_sudo_user)s %(tmp_destination)s' % env)
sudo('chgrp -R %(httpd_user_group)s %(tmp_destination)s' % env)
sudo('chmod -R g+w %(tmp_destination)s' % env)
sudo('mv -f %(tmp_destination)s %(httpd_remote_services_template_path)s' % env)
if env.os == 'ubuntu':
sudo('a2enmod proxy')
sudo('a2enmod proxy_http')
#should already be enabled for redhat
elif env.os != 'redhat':
utils.abort('OS Not recognized in Web Module')
with(fab_settings(warn_only=True)):
sudo('rm %(httpd_remote_conf_root)s/%(project)s' % env)
sudo('ln -s %(httpd_remote_services_template_path)s %(httpd_remote_conf_root)s/%(project)s' % env) #symbolic link our apache conf to the 'sites-enabled' folder
def bootstrap(deploy_level='staging'):
"""
Does nothing in this module.
"""
setup_env(deploy_level)
install_packages()
with fab_settings(warn_only=True): #in case the db is already initialized
_run_postgres_command('initdb') #initializes things for the first time.
_fix_ident()
_run_postgres_command('restart') #start the server so that we can create some db_user and db
# _set_postgres_passwd()
#
# create_db_user()
# create_db()
if env.move_db_now:
print red('MOVING THE DB! THIS IS A ONE TIME COMMAND, DO NOT RUN BOOTSTRAP OR MOVE AGAIN OR YOU WILL LOSE ALL YOUR DATA')
_relocate_db()
else:
print 'not moving db now??'
print 'env.move_db_now == %s' % env.move_db_now
def deploy():
# Compress the build
tgz = tempfile.NamedTemporaryFile(suffix=".tar.gz")
local("git archive --prefix=geo/ HEAD -o '%s'" % tgz.name)
env.tgz_name = tgz.name
print green("Starting deployment")
# Put the build onto the server
print green("Pushing the build")
put(env.tgz_name, 'build.tar.gz')
run("rm -rf geo")
run("tar -zxf build.tar.gz")
run("rm build.tar.gz")
# Copy over settings that we don't want checked into the source.
print green("Copying settings")
put("geo/settings/deployment.py", "geo/geo/settings/local.py")
with cd('geo'):
# Make sure puppet is installed, and run it
print green("Provisioning")
sudo("apt-get -y install puppet")
sudo("puppet apply puppet/manifest.pp --modulepath=puppet/modules/ "
"--hiera_config puppet/hiera.yaml")
# Build python dependency libraries
print green("Building virtualenv")
run("sh reset_environment.sh", quiet=True)
# Bounce or start the supervisor processes
print green("Restarting supervisor")
with fab_settings(hide('running'), warn_only=True):
running = run("ps aux | grep [s]upervisord")
if running:
sudo("killall -1 supervisord")
else:
sudo("supervisord -c /etc/supervisord.conf")
print green("Finished deployment")
def setup_project():
upload_template_and_reload("elastic_search")
upload_template_and_reload("settings")
upload_template_and_reload("fuseki-acceptance")
upload_template_and_reload("fuseki-production")
with project():
if env.reqs_path:
with fab_settings(warn_only=False):
pip("-r %s/%s" % (env.proj_path, env.reqs_path))
pip("gunicorn setproctitle psycopg2 django-compressor python3-memcached")
manage("migrate")
python("import django;"
"django.setup();"
"from django.conf import settings;"
"from django.contrib.sites.models import Site;"
"site, _ = Site.objects.get_or_create(id=settings.SITE_ID);"
"site.domain = '" + env.live_host + "';"
"site.save();")
if env.admin_pass:
pw = env.admin_pass
user_py = ("import django;"
"django.setup();"
"from django.contrib.auth.models import User;"
"u, _ = User.objects.get_or_create(username='******');"
"u.is_staff = u.is_superuser = True;"
"u.set_password('%s');"
"u.save();" % pw)
python(user_py, show=False)
shadowed = "*" * len(pw)
print_command(user_py.replace("'%s'" % pw, "'%s'" % shadowed))
python("import django;"
"django.setup();"
"from django.conf import settings;"
"from rest_framework.authtoken.models import Token;"
"from django.contrib.auth.models import User;"
"u, _ = User.objects.get_or_create(username='******');"
"Token.objects.get_or_create(user=u, key='{}');".format(env.nave_auth_token)
)
return True
def prepare_source(source):
get_source_cmd = None
if svc.check_path(source):
get_source_cmd = svc.get_prepare_command(source, settings.PROJECT_NAME)
elif os.path.isdir(source):
get_source_cmd = "cp -rf %s %s" % (source, settings.PROJECT_NAME)
else:
raise ValueError('The given source "%s" is unrecognizable.')
project_dir = os.path.join(settings.BUILD_TEMP, settings.PROJECT_NAME)
with fab_settings(warn_only=True):
if not os.path.exists(settings.TEMP_DIR):
local("mkdir %s" % settings.TEMP_DIR)
if not os.path.exists(settings.BUILD_TEMP):
local("mkdir %s" % settings.BUILD_TEMP)
if os.path.exists(project_dir):
local("rm -rf %s" % project_dir)
with lcd(settings.BUILD_TEMP):
local(get_source_cmd)
if not os.path.exists(settings.DIST_TEMP):
local("mkdir %s" % settings.DIST_TEMP)
else:
local("rm -rf %s/*" % settings.DIST_TEMP)
return project_dir
def set_env(settings=None, setup_dir=""):
"""
Used in management commands or at the module level of a fabfile to
integrate woven project django.conf settings into fabric, and set the local current
working directory to the distribution root (where setup.py lives).
``settings`` is your django settings module to pass in
if you want to call this from a fabric script.
``setup_dir`` is an optional path to the directory containing setup.py
This would be used in instances where setup.py was not above the cwd
This function is used to set the environment for all hosts
"""
# switch the working directory to the distribution root where setup.py is
if hasattr(env, "setup_path") and env.setup_path:
setup_path = env.setup_path
else:
with fab_settings(fabfile="setup.py"):
if setup_dir:
setup_path = os.path.join(setup_dir, "setup.py")
else:
setup_path = find_fabfile()
if not setup_path:
print "Error: You must have a setup.py file in the current or a parent folder"
sys.exit(1)
local_working_dir = os.path.split(setup_path)[0]
os.chdir(local_working_dir)
setup = run_setup("setup.py", stop_after="init")
if setup.get_name() == "UNKNOWN" or setup.get_version() == "0.0.0" or not setup.packages:
print "ERROR: You must define a minimum of name, version and packages in your setup.py"
sys.exit(1)
# project env variables for deployment
env.project_name = setup.get_name() # project_name()
env.project_full_version = setup.get_version() # local('python setup.py --version').rstrip()
env.project_version = _parse_project_version(env.project_full_version)
env.project_fullname = "-".join([env.project_name, env.project_version])
env.project_package_name = setup.packages[0]
env.patch = False
# django settings are passed in by the command
# We'll assume that if the settings aren't passed in we're running from a fabfile
if not settings:
sys.path.insert(0, local_working_dir)
# import global settings
project_settings = import_module(env.project_name + ".settings")
else:
project_settings = settings
# If sqlite is used we can manage the database on first deployment
env.DEFAULT_DATABASE_ENGINE = project_settings.DATABASES["default"]["ENGINE"]
env.DEFAULT_DATABASE_NAME = project_settings.DATABASES["default"]["NAME"]
# overwrite with main sitesettings module
# just for MEDIA_URL, ADMIN_MEDIA_PREFIX, and STATIC_URL
# if this settings file exists
try:
site_settings = import_module(".".join([env.project_name, "sitesettings.settings"]))
project_settings.MEDIA_URL = site_settings.MEDIA_URL
project_settings.ADMIN_MEDIA_PREFIX = site_settings.ADMIN_MEDIA_PREFIX
project_settings.DATABASES = site_settings.DATABASES
if hasattr(site_settings, "STATIC_URL"):
project_settings.STATIC_URL = site_settings.STATIC_URL
else:
project_settings.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX
except ImportError:
pass
# update woven_env from project_settings
local_settings = dir(project_settings)
# only get settings that woven uses
for setting in local_settings:
if setting.isupper() and hasattr(woven_env, setting):
s = getattr(project_settings, setting, "")
woven_env[setting] = s
# upate the fabric env with all the woven settings
env.update(woven_env)
# set any user/password defaults if they are not supplied
# Fabric would get the user from the options by default as the system user
# We will overwrite that
if woven_env.HOST_USER:
env.user = woven_env.HOST_USER
env.password = woven_env.HOST_PASSWORD
# set the hosts if they aren't already
if not env.hosts:
env.hosts = woven_env.HOSTS
if not env.roledefs:
env.roledefs = woven_env.ROLEDEFS
# reverse_lookup hosts to roles
role_lookup = {}
for role in env.roles:
r_hosts = env.roledefs[role]
for host in r_hosts:
# since port is not handled by fabric.main.normalize we'll do it ourselves
role_lookup["%s:%s" % (host, str(woven_env.HOST_SSH_PORT))] = role
# now add any hosts that aren't already defined in roles
for host in env.hosts:
host_string = "%s:%s" % (host, str(woven_env.HOST_SSH_PORT))
if host_string not in role_lookup.keys():
role_lookup[host_string] = ""
env.role_lookup = role_lookup
env.hosts = role_lookup.keys()
# remove any unneeded db adaptors - except sqlite
remove_backends = ["postgresql_psycopg2", "mysql"]
for db in project_settings.DATABASES:
engine = project_settings.DATABASES[db]["ENGINE"].split(".")[-1]
if engine in remove_backends:
remove_backends.remove(engine)
for backend in remove_backends:
if backend == "postgresql_psycopg2":
rm = "python-psycopg2"
elif backend == "mysql":
rm = "python-mysqldb"
env.HOST_BASE_PACKAGES.remove(rm)
# packages can be just the base + extra packages
# or role dependent we need to just map out the packages to hosts and roles here
packages = {}
all_packages = set([])
for role in env.roles:
packages[role] = env.ROLE_PACKAGES.get(role, [])
if not packages[role]:
packages[role] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
all_packages = set(packages[role]) | all_packages
# no role
packages[""] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
all_packages = set(packages[""]) | all_packages
# conveniently add gunicorn ppa
if "gunicorn" in all_packages:
if "ppa:bchesneau/gunicorn" not in env.LINUX_PACKAGE_REPOSITORIES:
env.LINUX_PACKAGE_REPOSITORIES.append("ppa:bchesneau/gunicorn")
env.packages = packages
# sanity check for unwanted combinations in the empty role
u = set(packages[""])
wsgi = u & set(["gunicorn", "uwsgi"])
if wsgi and "apache2" in u:
u = u - set(["apache2", "libapache2-mod-wsgi"])
# Used to detect certain apps eg South, static_builder
env.INSTALLED_APPS = project_settings.INSTALLED_APPS
env.packages[""] = list(u)
# per host
env.installed_packages = {}
env.uninstalled_packages = {}
# UFW firewall rules
firewall_rules = {}
for role in env.roles:
firewall_rules[role] = env.ROLE_UFW_RULES.get(role, [])
firewall_rules[""] = env.UFW_RULES
env.firewall_rules = firewall_rules
# Now update the env with any settings that are not defined by woven but may
# be used by woven or fabric
env.MEDIA_ROOT = project_settings.MEDIA_ROOT
env.MEDIA_URL = project_settings.MEDIA_URL
env.ADMIN_MEDIA_PREFIX = project_settings.ADMIN_MEDIA_PREFIX
if not env.STATIC_URL:
env.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX
env.TEMPLATE_DIRS = project_settings.TEMPLATE_DIRS
# Set the server /etc/timezone
env.TIME_ZONE = project_settings.TIME_ZONE
# Used to detect certain apps eg South, static_builder
env.INSTALLED_APPS = project_settings.INSTALLED_APPS
# SSH key
if env.SSH_KEY_FILENAME:
env.KEY_FILENAME = env.SSH_KEY_FILENAME
else:
env.KEY_FILENAME = ""
# noinput
if not hasattr(env, "INTERACTIVE"):
env.INTERACTIVE = True
if not hasattr(env, "verbosity"):
env.verbosity = 1
# overwrite existing settings
if not hasattr(env, "overwrite"):
env.overwrite = False
# South integration defaults
env.nomigration = False
env.manualmigration = False
env.migration = ""
env.root_disabled = False
# Sites
env.sites = {}
env.shell = "/bin/bash --noprofile -l -c"
def set_env(settings=None, setup_dir=''):
"""
Used in management commands or at the module level of a fabfile to
integrate woven project django.conf settings into fabric, and set the local current
working directory to the distribution root (where setup.py lives).
``settings`` is your django settings module to pass in
if you want to call this from a fabric script.
``setup_dir`` is an optional path to the directory containing setup.py
This would be used in instances where setup.py was not above the cwd
This function is used to set the environment for all hosts
"""
#switch the working directory to the distribution root where setup.py is
if hasattr(env, 'setup_path') and env.setup_path:
setup_path = env.setup_path
else:
with fab_settings(fabfile='setup.py'):
if setup_dir:
setup_path = os.path.join(setup_dir, 'setup.py')
else:
setup_path = find_fabfile()
if not setup_path:
print 'Error: You must have a setup.py file in the current or a parent folder'
sys.exit(1)
local_working_dir = os.path.split(setup_path)[0]
os.chdir(local_working_dir)
setup = run_setup('setup.py', stop_after="init")
if setup.get_name() == 'UNKNOWN' or setup.get_version(
) == '0.0.0' or not setup.packages:
print "ERROR: You must define a minimum of name, version and packages in your setup.py"
sys.exit(1)
#project env variables for deployment
env.project_name = setup.get_name() #project_name()
env.project_full_version = setup.get_version(
) #local('python setup.py --version').rstrip()
env.project_version = _parse_project_version(env.project_full_version)
env.project_fullname = '-'.join([env.project_name, env.project_version])
env.project_package_name = setup.packages[0]
env.patch = False
#django settings are passed in by the command
#We'll assume that if the settings aren't passed in we're running from a fabfile
if not settings:
sys.path.insert(0, local_working_dir)
#import global settings
project_settings = import_module(env.project_name + '.settings')
else:
project_settings = settings
#If sqlite is used we can manage the database on first deployment
env.DEFAULT_DATABASE_ENGINE = project_settings.DATABASES['default'][
'ENGINE']
env.DEFAULT_DATABASE_NAME = project_settings.DATABASES['default']['NAME']
#overwrite with main sitesettings module
#just for MEDIA_URL, ADMIN_MEDIA_PREFIX, and STATIC_URL
#if this settings file exists
try:
site_settings = import_module('.'.join(
[env.project_name, 'sitesettings.settings']))
project_settings.MEDIA_URL = site_settings.MEDIA_URL
project_settings.ADMIN_MEDIA_PREFIX = site_settings.ADMIN_MEDIA_PREFIX
project_settings.DATABASES = site_settings.DATABASES
if hasattr(site_settings, 'STATIC_URL'):
project_settings.STATIC_URL = site_settings.STATIC_URL
else:
project_settings.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX
except ImportError:
pass
#update woven_env from project_settings
local_settings = dir(project_settings)
#only get settings that woven uses
for setting in local_settings:
if setting.isupper() and hasattr(woven_env, setting):
s = getattr(project_settings, setting, '')
woven_env[setting] = s
#upate the fabric env with all the woven settings
env.update(woven_env)
#set any user/password defaults if they are not supplied
#Fabric would get the user from the options by default as the system user
#We will overwrite that
if woven_env.HOST_USER:
env.user = woven_env.HOST_USER
env.password = woven_env.HOST_PASSWORD
#set the hosts if they aren't already
if not env.hosts: env.hosts = woven_env.HOSTS
if not env.roledefs: env.roledefs = woven_env.ROLEDEFS
#reverse_lookup hosts to roles
role_lookup = {}
for role in env.roles:
r_hosts = env.roledefs[role]
for host in r_hosts:
#since port is not handled by fabric.main.normalize we'll do it ourselves
role_lookup['%s:%s' % (host, str(woven_env.HOST_SSH_PORT))] = role
#now add any hosts that aren't already defined in roles
for host in env.hosts:
host_string = '%s:%s' % (host, str(woven_env.HOST_SSH_PORT))
if host_string not in role_lookup.keys():
role_lookup[host_string] = ''
env.role_lookup = role_lookup
env.hosts = role_lookup.keys()
#remove any unneeded db adaptors - except sqlite
remove_backends = ['postgresql_psycopg2', 'mysql']
for db in project_settings.DATABASES:
engine = project_settings.DATABASES[db]['ENGINE'].split('.')[-1]
if engine in remove_backends: remove_backends.remove(engine)
for backend in remove_backends:
if backend == 'postgresql_psycopg2': rm = 'python-psycopg2'
elif backend == 'mysql': rm = 'python-mysqldb'
env.HOST_BASE_PACKAGES.remove(rm)
#packages can be just the base + extra packages
#or role dependent we need to just map out the packages to hosts and roles here
packages = {}
all_packages = set([])
for role in env.roles:
packages[role] = env.ROLE_PACKAGES.get(role, [])
if not packages[role]:
packages[role] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
all_packages = set(packages[role]) | all_packages
#no role
packages[''] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
all_packages = set(packages['']) | all_packages
#conveniently add gunicorn ppa
if 'gunicorn' in all_packages:
if 'ppa:bchesneau/gunicorn' not in env.LINUX_PACKAGE_REPOSITORIES:
env.LINUX_PACKAGE_REPOSITORIES.append('ppa:bchesneau/gunicorn')
env.packages = packages
#sanity check for unwanted combinations in the empty role
u = set(packages[''])
wsgi = u & set(['gunicorn', 'uwsgi'])
if wsgi and 'apache2' in u:
u = u - set(['apache2', 'libapache2-mod-wsgi'])
#Used to detect certain apps eg South, static_builder
env.INSTALLED_APPS = project_settings.INSTALLED_APPS
env.packages[''] = list(u)
#per host
env.installed_packages = {}
env.uninstalled_packages = {}
#UFW firewall rules
firewall_rules = {}
for role in env.roles:
firewall_rules[role] = env.ROLE_UFW_RULES.get(role, [])
firewall_rules[''] = env.UFW_RULES
env.firewall_rules = firewall_rules
#Now update the env with any settings that are not defined by woven but may
#be used by woven or fabric
env.MEDIA_ROOT = project_settings.MEDIA_ROOT
env.MEDIA_URL = project_settings.MEDIA_URL
env.ADMIN_MEDIA_PREFIX = project_settings.ADMIN_MEDIA_PREFIX
if not env.STATIC_URL: env.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX
env.TEMPLATE_DIRS = project_settings.TEMPLATE_DIRS
#Set the server /etc/timezone
env.TIME_ZONE = project_settings.TIME_ZONE
#Used to detect certain apps eg South, static_builder
env.INSTALLED_APPS = project_settings.INSTALLED_APPS
#SSH key
if env.SSH_KEY_FILENAME: env.KEY_FILENAME = env.SSH_KEY_FILENAME
else: env.KEY_FILENAME = ''
#noinput
if not hasattr(env, 'INTERACTIVE'): env.INTERACTIVE = True
if not hasattr(env, 'verbosity'): env.verbosity = 1
#overwrite existing settings
if not hasattr(env, 'overwrite'): env.overwrite = False
#South integration defaults
env.nomigration = False
env.manualmigration = False
env.migration = ''
env.root_disabled = False
#Sites
env.sites = {}
env.shell = '/bin/bash --noprofile -l -c'
def create_db():
"""Create the Postgres database."""
require('db_name', 'db_user', provided_by=('setup_env'))
with fab_settings(warn_only=True):
run('sudo -u postgres createdb -O %(db_user)s %(db_name)s' % env, shell=False)
def create_db():
"""Create the Postgres database."""
require('db_name', 'db_user', provided_by=('setup_env'))
with fab_settings(warn_only=True):
run('sudo -u postgres createdb -O %(db_user)s %(db_name)s' % env,
shell=False)
def set_env(settings=None, setup_dir=''):
"""
Used in management commands or at the module level of a fabfile to
integrate woven project django.conf settings into fabric, and set
the local current working directory to the distribution root
(where setup.py lives).
``settings`` is your django settings module to pass in if you want
to call this from a fabric script.
``setup_dir`` is an optional path to the directory containing
setup.py This would be used in instances where setup.py was not
above the CWD.
This function is used to set the environment for all hosts.
"""
# Switch the working directory to the distribution root where
# setup.py is.
if hasattr(env, 'setup_path') and env.setup_path:
setup_path = env.setup_path
else:
with fab_settings(fabfile='setup.py'):
if setup_dir:
setup_path = os.path.join(setup_dir, 'setup.py')
else:
setup_path = find_fabfile()
if not setup_path:
print 'Error: You must have a setup.py file in ' \
'the current or a parent folder'
sys.exit(1)
local_working_dir = os.path.split(setup_path)[0]
os.chdir(local_working_dir)
setup = run_setup('setup.py', stop_after="init")
if (setup.get_name() == 'UNKNOWN' or
setup.get_version() == '0.0.0' or
not setup.packages):
print "ERROR: You must define a minimum of name, version " \
"and packages in your setup.py"
sys.exit(1)
# Project env variables for deployment.
# Project_name().
env.project_name = setup.get_name()
#local('python setup.py --version').rstrip()
env.project_full_version = setup.get_version()
env.project_version = _parse_project_version(env.project_full_version)
env.project_fullname = '-'.join([env.project_name, env.project_version])
env.project_package_name = setup.packages[0]
env.patch = False
# Django settings are passed in by the command. We'll assume that
# if the settings aren't passed in we're running from a fabfile.
if not settings:
sys.path.insert(0, local_working_dir)
# Import global settings.
project_settings = import_module(env.project_name + '.settings')
else:
project_settings = settings
# If SQLite is used we can manage the database on first
# deployment.
env.DEFAULT_DATABASE_ENGINE = \
project_settings.DATABASES['default']['ENGINE']
env.DEFAULT_DATABASE_NAME = \
project_settings.DATABASES['default']['NAME']
# Overwrite with main sitesettings module just for MEDIA_URL,
# ADMIN_MEDIA_PREFIX, and STATIC_URL if this settings file exists.
try:
site_settings = import_module(
'.'.join([env.project_name, 'sitesettings.settings']))
project_settings.MEDIA_URL = site_settings.MEDIA_URL
project_settings.ADMIN_MEDIA_PREFIX = site_settings.ADMIN_MEDIA_PREFIX
project_settings.DATABASES = site_settings.DATABASES
if hasattr(site_settings, 'STATIC_URL'):
project_settings.STATIC_URL = site_settings.STATIC_URL
else:
project_settings.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX
except ImportError:
pass
# Update woven_env from project_settings.
local_settings = dir(project_settings)
# Only get settings that woven uses.
for setting in local_settings:
if setting.isupper() and hasattr(woven_env, setting):
s = getattr(project_settings, setting, '')
woven_env[setting] = s
# Upate the fabric env with all the woven settings.
env.update(woven_env)
# Set any user/password defaults if they are not supplied.
# Fabric would get the user from the options by default as the
# system user. We will overwrite that.
if woven_env.HOST_USER:
env.user = woven_env.HOST_USER
env.password = woven_env.HOST_PASSWORD
# Set the hosts if they aren't already.
if not env.hosts:
env.hosts = woven_env.HOSTS
if not env.roledefs:
env.roledefs = woven_env.ROLEDEFS
# Reverse_lookup hosts to roles.
role_lookup = {}
for role in env.roles:
r_hosts = env.roledefs[role]
for host in r_hosts:
# Since port is not handled by fabric.main.normalize we'll
# do it ourselves.
role_lookup['%s:%s' % (host, str(woven_env.HOST_SSH_PORT))] = role
# Now add any hosts that aren't already defined in roles.
for host in env.hosts:
host_string = '%s:%s' % (host, str(woven_env.HOST_SSH_PORT))
if host_string not in role_lookup.keys():
role_lookup[host_string] = ''
env.role_lookup = role_lookup
env.hosts = role_lookup.keys()
# Remove any unneeded db adaptors - except sqlite.
remove_backends = ['postgresql_psycopg2', 'mysql']
for db in project_settings.DATABASES:
engine = project_settings.DATABASES[db]['ENGINE'].split('.')[-1]
if engine in remove_backends:
remove_backends.remove(engine)
for backend in remove_backends:
if backend == 'postgresql_psycopg2':
rm = 'python-psycopg2'
elif backend == 'mysql':
rm = 'python-mysqldb'
env.HOST_BASE_PACKAGES.remove(rm)
# Packages can be just the base + extra packages or role-
# dependent. We need to just map out the packages to hosts and
# roles here.
packages = {}
all_packages = set([])
for role in env.roles:
packages[role] = env.ROLE_PACKAGES.get(role, [])
if not packages[role]:
packages[role] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
all_packages = set(packages[role]) | all_packages
# No role.
packages[''] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
all_packages = set(packages['']) | all_packages
# Conveniently add gunicorn ppa.
if 'gunicorn' in all_packages:
if 'ppa:bchesneau/gunicorn' not in env.LINUX_PACKAGE_REPOSITORIES:
env.LINUX_PACKAGE_REPOSITORIES.append('ppa:bchesneau/gunicorn')
env.packages = packages
# Sanity-check for unwanted combinations in the empty role.
u = set(packages[''])
wsgi = u & set(['gunicorn', 'uwsgi'])
if wsgi and 'apache2' in u:
u = u - set(['apache2', 'libapache2-mod-wsgi'])
# Used to detect certain apps eg. South, static_builder.
env.INSTALLED_APPS = project_settings.INSTALLED_APPS
env.packages[''] = list(u)
# Per host.
env.installed_packages = {}
env.uninstalled_packages = {}
# UFW firewall rules.
firewall_rules = {}
for role in env.roles:
firewall_rules[role] = env.ROLE_UFW_RULES.get(role, [])
firewall_rules[''] = env.UFW_RULES
env.firewall_rules = firewall_rules
# Now update the env with any settings that are not defined by
# woven but may be used by woven or fabric.
env.MEDIA_ROOT = project_settings.MEDIA_ROOT
env.MEDIA_URL = project_settings.MEDIA_URL
try:
env.ADMIN_MEDIA_PREFIX = project_settings.ADMIN_MEDIA_PREFIX
except AttributeError:
env.ADMIN_MEDIA_PREFIX = ''
if not env.STATIC_URL:
env.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX
env.TEMPLATE_DIRS = project_settings.TEMPLATE_DIRS
# Set the server /etc/timezone.
env.TIME_ZONE = project_settings.TIME_ZONE
# Used to detect certain apps eg. South, static_builder.
env.INSTALLED_APPS = project_settings.INSTALLED_APPS
# SSH key.
if (not hasattr(env, 'key_filename') and
not env.key_filename and
env.SSH_KEY_FILENAME):
env.key_filename = env.SSH_KEY_FILENAME
elif not hasattr(env, 'key_filename'):
env.key_filename = None
# Noinput.
if not hasattr(env, 'INTERACTIVE'):
env.INTERACTIVE = True
if not hasattr(env, 'verbosity'):
env.verbosity = 1
# Overwrite existing settings.
if not hasattr(env, 'overwrite'):
env.overwrite = False
# South integration defaults.
env.nomigration = False
env.manualmigration = False
env.migration = ''
env.root_disabled = False
# Sites.
env.sites = {}
env.shell = '/bin/bash --noprofile -l -c'
