def provision(update=False):
"""Uploads an install script to /project_name/scripts and runs it.
The script will not download solr if '/tmp/{project_name}/solr.zip' exists,
nor it will attempt an install (eg. unpack and copy) if the following file
exists: '{supervisor_dir}/solr/fabric_solr_install_success' (root of where
solr is installed).
Use update=True is as an override.
"""
# upload the script to {project_dir}/scripts/setup_solr.sh
user = cget("user")
solr_dir = cset('solr_dir', pjoin(cget("service_dir"), 'solr'))
script_name = "setup_solr.sh"
source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name)
dest_scripts = cget("script_dir")
create_target_directories([dest_scripts, solr_dir], "700", user)
context = dict(env['ctx'])
destination = pjoin(dest_scripts, script_name)
upload_template_with_perms(source, destination, context, mode="644")
# run the script
show(yellow("Installing solr with update=%s." % update))
with settings(sudo_prefix=SUDO_PREFIX, warn_only=True):
script = destination
# the script will copy files into: MTURK/solr
ret = sudo("MTURK={home} && UPDATE={update} && . {script}".format(
home=cget('service_dir'),
script=script,
update='true' if update else 'false'))
if ret.return_code != 0:
show(yellow("Error while installing sorl."))
def setup_ssh():
"""Uploads ssh from the local folder specified in config."""
user = cget("user")
ssh_target_dir = cget("ssh_target")
# Ensure SSH directory is created with proper perms.
create_target_directories([ssh_target_dir], "700", user)
# Upload SSH config and keys.
# TODO: Add copying files from remote folder (upload_ssh_keys_from_local).
if cget('upload_ssh_keys_from_local') or True:
files = cget('ssh_files')
if not files:
show(yellow("No SSH files to upload."))
return
show(yellow("Uploading SSH configuration and keys"))
for name in files:
show(u"File: {0}".format(name))
local_path = pjoin(local_files_dir("ssh"), name)
remote_path = pjoin(ssh_target_dir, name)
put_file_with_perms(local_path, remote_path, "600", user, user)
else:
# Remoty copying of files
raise Exception('Not implemented!'
' Please set upload_ssh_keys_from_local to True!')
def fetch_project_code():
"""Fetches project code from Github.
If specified, resets state to selected branch or commit (from context).
"""
branch = cget("branch")
commit = cget("commit")
project_dir = cget("project_dir")
repo_dir = pjoin(project_dir, "code")
url = cget("source_url")
user = cget("user")
if commit:
rev = commit
else:
rev = "origin/%s" % (branch or "master")
with settings(sudo_prefix=SUDO_PREFIX):
if not dir_exists(pjoin(repo_dir, ".git")):
show(yellow("Cloning repository following: %s"), rev)
sudo("git clone %s %s" % (url, repo_dir), user=user)
with cd(repo_dir):
sudo("git reset --hard %s" % rev, user=user)
else:
show(yellow("Updating repository following: %s"), rev)
with cd(repo_dir):
sudo("git fetch origin", user=user) # Prefetch changes.
sudo("git clean -f", user=user) # Clean local files.
sudo("git reset --hard %s" % rev, user=user)
def provision(update=False):
"""Uploads an install script to /project_name/scripts and runs it.
The script will not download solr if '/tmp/{project_name}/solr.zip' exists,
nor it will attempt an install (eg. unpack and copy) if the following file
exists: '{supervisor_dir}/solr/fabric_solr_install_success' (root of where
solr is installed).
Use update=True is as an override.
"""
# upload the script to {project_dir}/scripts/setup_solr.sh
user = cget("user")
solr_dir = cset('solr_dir', pjoin(cget("service_dir"), 'solr'))
script_name = "setup_solr.sh"
source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name)
dest_scripts = cget("script_dir")
create_target_directories([dest_scripts, solr_dir], "700", user)
context = dict(env['ctx'])
destination = pjoin(dest_scripts, script_name)
upload_template_with_perms(source, destination, context, mode="644")
# run the script
show(yellow("Installing solr with update=%s." % update))
with settings(sudo_prefix=SUDO_PREFIX, warn_only=True):
script = destination
# the script will copy files into: MTURK/solr
ret = sudo("MTURK={home} && UPDATE={update} && . {script}".format(
home=cget('service_dir'), script=script,
update='true' if update else 'false'))
if ret.return_code != 0:
show(yellow("Error while installing sorl."))
def configure():
"""Creates all neccessary folders and uploads settings."""
user = cget("user")
sdir = pjoin(cget('service_dir'), 'nginx')
logdir = pjoin(cget('log_dir'), 'nginx')
create_target_directories([sdir, logdir], "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("nginx")
dest_dir = "/etc/nginx"
confs = cget("nginx_files") or [local_dir]
show(yellow("Uploading nginx configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir,
context, mode="644", directories_mode="700")
else:
upload_template_with_perms(
source, destination, context, mode="644")
enabled = cget("nginx_sites_enabled")
with settings(hide("running", "stderr", "stdout"), sudo_prefix=SUDO_PREFIX,
warn_only=True):
show("Enabling sites: %s." % enabled)
for s in enabled:
available = '/etc/nginx/sites-available'
enabled = '/etc/nginx/sites-enabled'
ret = sudo("ln -s {available}/{site} {enabled}/{site}".format(
available=available, enabled=enabled, site=s))
if ret.failed:
show(red("Error enabling site: {}: {}.".format(s, ret)))
def fetch_project_code():
"""Fetches project code from Github.
If specified, resets state to selected branch or commit (from context).
"""
branch = cget("branch")
commit = cget("commit")
project_dir = cget("project_dir")
repo_dir = pjoin(project_dir, "code")
url = cget("source_url")
user = cget("user")
if commit:
rev = commit
else:
rev = "origin/%s" % (branch or "master")
with settings(sudo_prefix=SUDO_PREFIX):
if not dir_exists(pjoin(repo_dir, ".git")):
show(yellow("Cloning repository following: %s"), rev)
sudo("git clone %s %s" % (url, repo_dir), user=user)
with cd(repo_dir):
sudo("git reset --hard %s" % rev, user=user)
else:
show(yellow("Updating repository following: %s"), rev)
with cd(repo_dir):
sudo("git fetch origin", user=user) # Prefetch changes.
sudo("git clean -f", user=user) # Clean local files.
sudo("git reset --hard %s" % rev, user=user)
def configure():
"""Creates all neccessary folders and uploads settings.
Keep in mind that the base for filenames is /etc, because the files reside
in /etc/crond.d/ etc. Thus those files/folders must be specified explictly.
Additionally this will format and upload
manage_py_exec and
manage_py_exec_silent
scripts.
"""
user = cget("user")
logdir = pjoin(cget('log_dir'), 'cron')
create_target_directories([logdir], "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("cron")
dest_dir = "/etc"
confs = cget("cron_files")
show(yellow("Uploading cron configuration files: %s." % confs))
if not confs or len(confs) == 0:
show(red("No files to upload for cron."))
return
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir,
context, mode="644", user='******', group='root',
directories_mode="700")
else:
upload_template_with_perms(
source, destination, context, mode="644", user='******',
group='root')
# format and upload command execution script used by cron
scripts = ['manage_py_exec', 'manage_py_exec_silent']
for script_name in scripts:
source = pjoin(cget("local_root"), 'deployment', 'scripts',
script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
show(yellow("Reloading cron"))
with settings(hide("stderr"), warn_only=True):
res = service("cron", "reload")
if res.return_code == 2:
show(red("Error reloading cron!"))
def configure():
"""Creates all neccessary folders and uploads settings.
Keep in mind that the base for filenames is /etc, because the files reside
in /etc/crond.d/ etc. Thus those files/folders must be specified explictly.
Additionally this will format and upload
manage_py_exec and
manage_py_exec_silent
scripts.
"""
user = cget("user")
logdir = pjoin(cget('log_dir'), 'cron')
create_target_directories([logdir], "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("cron")
dest_dir = "/etc"
confs = cget("cron_files")
show(yellow("Uploading cron configuration files: %s." % confs))
if not confs or len(confs) == 0:
show(red("No files to upload for cron."))
return
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir,
context, mode="644", user='******', group='root',
directories_mode="700")
else:
upload_template_with_perms(
source, destination, context, mode="644", user='******', group='root')
# format and upload command execution script used by cron
scripts = ['manage_py_exec', 'manage_py_exec_silent']
for script_name in scripts:
source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
show(yellow("Reloading cron"))
with settings(hide("stderr"), sudo_prefix=SUDO_PREFIX, warn_only=True):
res = sudo("service cron reload")
if res.return_code == 2:
show(red("Error reloading cron!"))
def prepare_target_env():
"""Prepare all things needed before source code deployment."""
user = cget("user")
project_dir = cget("project_dir")
# Ensure proper directory structure.
create_target_directories([project_dir], '755', user)
dirs = ["code", "logs", "logs/old", "misc", "virtualenv",
"media", "static", "scripts", "services"]
dirs = [pjoin(project_dir, d) for d in dirs]
create_target_directories(dirs, "755", user)
# Create Virtualenv if not present.
create_virtualenv()
def configure():
"""Uploads solr configuration files."""
context = dict(env["ctx"])
local_dir = local_files_dir("solr")
dest_dir = pjoin(cget('service_dir'), 'solr')
confs = cget("solr_files") or [local_dir]
show(yellow("Uploading solr configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir,
context, mode="644", directories_mode="700")
else:
upload_template_with_perms(
source, destination, context, mode="644")
def __reload_services(setup=False):
"""Reloads previously configured services"""
nginx.reload()
supervisor.reload()
if setup:
services_conf = pjoin(cget('service_dir'), 'supervisor', 'config',
'supervisor-services.conf')
supervisor.reload(conf=services_conf)
def prepare_global_env():
"""Ensure global settings - one time only."""
prep_apt_get()
install_system_requirements()
setup_ssh()
setup_virtualenv()
nginx.provision()
solr.provision(update=cget("setup_environment"))
def __reload_services(setup=False):
"""Reloads previously configured services"""
nginx.reload()
supervisor.reload()
if setup:
services_conf = pjoin(cget('service_dir'), 'supervisor', 'config',
'supervisor-services.conf')
supervisor.reload(conf=services_conf)
def prepare_target_env():
"""Prepare all things needed before source code deployment."""
user = cget("user")
project_dir = cget("project_dir")
# Ensure proper directory structure.
create_target_directories([project_dir], '755', user)
dirs = [
"code", "logs", "logs/old", "misc", "virtualenv", "media", "static",
"scripts", "services"
]
dirs = [pjoin(project_dir, d) for d in dirs]
create_target_directories(dirs, "755", user)
# Create Virtualenv if not present.
create_virtualenv()
def upload_settings_files():
"""Uploads target specific (templated) settings files.
If specified also uploads user supplied locals.py.
*Warning*: Settings are uploaded from your local template file.
Make sure to have proper branch/revision checked out.
"""
base_dir = cget("base_dir")
user = cget("user")
locals_path = cget("locals_path")
show(yellow("Uploading Django settings files."))
# This is Template context not the deployment context.
# A split should happen some time.
context = dict(env["ctx"])
context
# Upload main settings and ensure permissions.
source = pjoin(local_files_dir("django"), "settings_template.py")
destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name"))
upload_template_with_perms(source, destination, context, mode="644",
user=user, group=user)
# We could be deploying from different directory.
# Try our best to find correct path.
# First - direct, absolute match.
if not os.path.isfile(locals_path):
# Try relative to deployment directory.
this_dir = os.path.dirname(os.path.abspath(__file__))
locals_path = pjoin(this_dir, locals_path)
if not os.path.isfile(locals_path): # :((
msg = u"Warning: Specified local settings path is incorrect: {0}."
show(red(msg.format(locals_path)))
confirm_or_abort(red("\nDo you want to continue?"))
locals_path = None
# Upload user supplied locals if present.
if locals_path:
show(yellow("Uploading your custom local settings files."))
destination = pjoin(base_dir, "settings", "local.py")
put_file_with_perms(locals_path, destination, mode="644", user=user,
group=user)
def setup_ssh():
"""Uploads ssh from the local folder specified in config."""
user = cget("user")
ssh_target_dir = cget("ssh_target")
# Ensure SSH directory is created with proper perms.
create_target_directories([ssh_target_dir], "700", user)
# Upload SSH config and keys.
# TODO: Add copying files from remote folder (upload_ssh_keys_from_local).
if cget('upload_ssh_keys_from_local') or True:
files = cget('ssh_files')
show(yellow("Uploading SSH configuration and keys"))
for name in files:
show(u"File: {0}".format(name))
local_path = pjoin(local_files_dir("ssh"), name)
remote_path = pjoin(ssh_target_dir, name)
put_file_with_perms(local_path, remote_path, "600", user, user)
else:
# Remoty copying of files
raise Exception('Not implemented!'
' Please set upload_ssh_keys_from_local to True!')
def configure():
"""Creates all neccessary folders and uploads settings."""
user = cget("user")
sdir = pjoin(cget('service_dir'), 'nginx')
logdir = pjoin(cget('log_dir'), 'nginx')
create_target_directories([sdir, logdir], "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("nginx")
dest_dir = "/etc/nginx"
confs = cget("nginx_files") or [local_dir]
show(yellow("Uploading nginx configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source,
local_dir,
dest_dir,
context,
mode="644",
directories_mode="700")
else:
upload_template_with_perms(source,
destination,
context,
mode="644")
enabled = cget("nginx_sites_enabled")
with settings(hide("running", "stderr", "stdout"),
sudo_prefix=SUDO_PREFIX,
warn_only=True):
show("Enabling sites: %s." % enabled)
for s in enabled:
available = '/etc/nginx/sites-available'
enabled = '/etc/nginx/sites-enabled'
ret = sudo("ln -s {available}/{site} {enabled}/{site}".format(
available=available, enabled=enabled, site=s))
if ret.failed:
show(red("Error enabling site: {}: {}.".format(s, ret)))
def install_system_requirements():
"""Installs packages included in system_requirements.txt.
This is done before fetch, thus the file is taken from *local* storage.
"""
reqs = cget('system_requirements', [])
for req in reqs:
requirements = pjoin(local_files_dir("requirements"), req)
show(yellow("Processing system requirements file: %s" %
requirements))
with open(requirements) as f:
r = ' '.join([f.strip() for f in f.readlines()])
name = 'requirements: {0}'.format(r)
with settings(sudo_prefix=SUDO_PREFIX):
install_without_prompt(r, name, silent=False)
def configure():
"""Uploads postgresql configuration files."""
context = dict(env["ctx"])
local_dir = local_files_dir("postgresql")
dest_dir = "/etc/postgresql"
confs = cget("postgresql_files") or [local_dir]
show(yellow("Uploading postgresql configuration files: {}.".format(confs)))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir,
context, mode="644", directories_mode="700")
else:
upload_template_with_perms(
source, destination, context, mode="644")
def install_system_requirements():
"""Installs packages included in system_requirements.txt.
This is done before fetch, thus the file is taken from *local* storage.
"""
reqs = cget('system_requirements')
if reqs:
for req in reqs:
requirements = pjoin(local_files_dir("requirements"), req)
show(
yellow("Processing system requirements file: %s" %
requirements))
with open(requirements) as f:
r = ' '.join([f.strip() for f in f.readlines()])
name = 'requirements: {0}'.format(r)
with settings(sudo_prefix=SUDO_PREFIX):
install_without_prompt(r, name, silent=False)
def configure():
"""Uploads postgresql configuration files."""
context = dict(env["ctx"])
local_dir = local_files_dir("postgresql")
dest_dir = "/etc/postgresql"
confs = cget("postgresql_files") or [local_dir]
show(yellow("Uploading postgresql configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source,
local_dir,
dest_dir,
context,
mode="644",
directories_mode="700")
else:
upload_template_with_perms(source,
destination,
context,
mode="644")
def set_instance_conf():
"""Compute all instance specific paths and settings.
Put *all* settings computation login here.
"""
# Use to escape % characters in config files
cset("p", '%')
cset("percent_escape_key", "p")
# System
cset("user", env["user"])
# Common project settings.
cset("prefix", cget("default_prefix"))
cset("project_name", "%s-%s" % (cget("prefix"), cget("instance")))
cset(
'settings_full_name', '.'.join(
[cget('django_project_name'), 'settings',
cget('settings_name')]))
# Host directories
cset("project_dir", pjoin(cget("projects_dir"), cget("project_name")))
cset("virtualenv_dir", pjoin(cget("project_dir"), "virtualenv"))
cset("deployment_files",
pjoin(cget("project_dir"), "code", "deployment", "files"))
cset('service_dir', pjoin(cget("project_dir"), "services"))
cset("script_dir", pjoin(cget("project_dir"), "scripts"))
# Local directories
this_dir = os.path.dirname(os.path.abspath(__file__))
deployment_dir = os.path.abspath(pjoin(this_dir, os.path.pardir))
cset('local_root', deployment_dir)
# Directory with manage.py script.
if not os.path.isabs(cget('manage_py_dir')):
mpy_dir = pjoin(cget("project_dir"), "code", cget('manage_py_dir'))
cset("manage_py_dir", mpy_dir, force=True)
cset("base_dir", pjoin(cget("project_dir"), "code", cget("project_inner")))
cset("log_dir", pjoin(cget("project_dir"), "logs"))
cset("doc_dir", pjoin(cget("project_dir"), "doc"))
# Database settings
cset("db_name", "%s_%s" % (cget("prefix"), cget("instance")))
cset("db_user", cget("db_name"))
cset("db_password", cget("db_user"))
cset("db_port", "")
cset("db_host", "localhost")
# SSH
user = cget("user")
cget("ssh_target") or cset("ssh_target", "/home/%s/.ssh" % user)
def upload_settings_files():
"""Uploads target specific (templated) settings files.
If specified also uploads user supplied locals.py.
*Warning*: Settings are uploaded from your local template file.
Make sure to have proper branch/revision checked out.
"""
base_dir = cget("base_dir")
user = cget("user")
locals_path = cget("locals_path")
show(yellow("Uploading Django settings files."))
# This is Template context not the deployment context.
# A split should happen some time.
context = dict(env["ctx"])
context
# Upload main settings and ensure permissions.
# source = pjoin(local_files_dir("django"), "settings_template.py")
# destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name"))
# upload_template_with_perms(source, destination, context, mode="644",
# user=user, group=user)
# We could be deploying from different directory.
# Try our best to find correct path.
# First - direct, absolute match.
if not locals_path:
return
if not os.path.isfile(locals_path):
# Try relative to deployment directory.
this_dir = os.path.dirname(os.path.abspath(__file__))
locals_path = pjoin(this_dir, locals_path)
if not os.path.isfile(locals_path): # :((
msg = u"Warning: Specified local settings path is incorrect: {0}."
show(red(msg.format(locals_path)))
confirm_or_abort(red("\nDo you want to continue?"))
locals_path = None
# Upload user supplied locals if present.
if locals_path:
show(yellow("Uploading your custom local settings files."))
destination = pjoin(base_dir, "settings", "local.py")
put_file_with_perms(locals_path,
destination,
mode="644",
user=user,
group=user)
# Upload Google Prediction credentials
this_dir = os.path.dirname(os.path.abspath(__file__))
cred_file = pjoin(this_dir, '..', 'prediction.dat')
base_dir = cget('project_dir')
if os.path.isfile(cred_file):
show(yellow("Uploading Google Prediction credentials storage."))
destination = pjoin(base_dir, 'code', 'prediction.dat')
# Main project directory
show(yellow("Uploading to %s." % destination))
put_file_with_perms(cred_file,
destination,
mode="644",
user=user,
group=user)
# Classification module
destination = pjoin(base_dir, 'code', 'urlannotator', 'classification',
'prediction.dat')
show(yellow("Uploading to %s." % destination))
put_file_with_perms(cred_file,
destination,
mode="644",
user=user,
group=user)
def prepare_global_env():
"""Ensure global settings - one time only."""
system.install_requirements()
setup_ssh()
solr.provision(update=cget("setup_environment"))
def clear_pyc():
"""Removes all pyc file to get rid of stale modules."""
with settings(sudo_prefix=SUDO_PREFIX):
show(yellow("Clearing *.pyc files."))
with cd(cget("project_dir")):
run('find . -type f -name "*.pyc" -delete')
def setup_database():
# Ensure we have database and user set up.
show(yellow("Setting database up"))
ensure_user(cget("db_user"), cget("db_password"))
ensure_database(cget("db_name"), cget("db_user"))
ensure_language(cget("db_name"), 'plpgsql')
def set_instance_conf():
"""Compute all instance specific paths and settings.
Put *all* settings computation login here.
"""
# Use to escape % characters in config files
cset("p", '%')
cset("percent_escape_key", "p")
# System
cset("user", env["user"])
# Common project settings.
cset("prefix", cget("default_prefix"))
cset("project_name", "%s-%s" % (cget("prefix"), cget("instance")))
cset('settings_full_name', '.'.join([cget('django_project_name'),
'settings', cget('settings_name')]))
# Host directories
cset("project_dir", pjoin(cget("projects_dir"), cget("project_name")))
cset("virtualenv_dir", pjoin(cget("project_dir"), "virtualenv"))
cset("deployment_files", pjoin(cget("project_dir"), "code", "deployment",
"files"))
cset('service_dir', pjoin(cget("project_dir"), "services"))
cset("script_dir", pjoin(cget("project_dir"), "scripts"))
# Local directories
this_dir = os.path.dirname(os.path.abspath(__file__))
deployment_dir = os.path.abspath(pjoin(this_dir, os.path.pardir))
cset('local_root', deployment_dir)
# Directory with manage.py script.
if not os.path.isabs(cget('manage_py_dir')):
mpy_dir = pjoin(cget("project_dir"), "code", cget('manage_py_dir'))
cset("manage_py_dir", mpy_dir, force=True)
cset("base_dir", pjoin(cget("project_dir"), "code", cget("project_inner")))
cset("log_dir", pjoin(cget("project_dir"), "logs"))
cset("doc_dir", pjoin(cget("project_dir"), "doc"))
# Database settings
cset("db_name", "%s_%s" % (cget("prefix"), cget("instance")))
cset("db_user", cget("db_name"))
cset("db_password", cget("db_user"))
cset("db_port", "")
cset("db_host", "localhost")
# SSH
user = cget("user")
cget("ssh_target") or cset("ssh_target", "/home/%s/.ssh" % user)
