def start_supervisor():
"""Start supervisor process."""
conf = pjoin(cget('service_dir'), 'supervisor', 'config',
'supervisord.conf')
pname = cget('supervisor_process_id')
show(yellow("Starting supervisor with id: %s." % pname))
return sudo('supervisord --configuration="%s"' % conf)
def started():
code_dir = pjoin(cget('project_dir'), 'code')
with cd(code_dir):
code_branch = cget('branch', 'master')
run('git clean -f')
run('git fetch origin')
run('git reset --hard origin/{}'.format(code_branch))
def start_supervisor(conf=None):
"""Start supervisor process."""
if not conf:
conf = pjoin(cget("service_dir"), "supervisor", "config", "supervisord.conf")
pname = cget("supervisor_process_id")
show(yellow("Starting supervisor with id: %s." % pname))
return sudo('supervisord --configuration="%s"' % conf)
def start_supervisor(conf=None):
"""Start supervisor process."""
if not conf:
conf = pjoin(cget('service_dir'), 'supervisor', 'config',
'supervisord.conf')
pname = cget('supervisor_process_id')
show(yellow("Starting supervisor with id: %s." % pname))
return sudo('supervisord --configuration="%s"' % conf)
def create_virtualenv():
"""Creates the virtualenv."""
user = cget("user")
ve_dir = cget("virtualenv_dir")
bin_path = pjoin(ve_dir, "bin")
if not dir_exists(bin_path) or not exists(pjoin(bin_path, "activate")):
show(yellow("Setting up new Virtualenv in: %s"), ve_dir)
with settings(hide("stdout", "running"), sudo_prefix=SUDO_PREFIX):
sudo("virtualenv --distribute %s" % ve_dir, user=user)
def create_virtualenv():
"""Creates the virtualenv."""
user = cget("user")
ve_dir = cget("virtualenv_dir")
bin_path = pjoin(ve_dir, "bin")
if not dir_exists(bin_path) or not exists(pjoin(bin_path, "activate")):
show(yellow("Setting up new Virtualenv in: %s"), ve_dir)
with settings(hide("stdout", "running"), sudo_prefix=SUDO_PREFIX):
sudo("virtualenv --distribute %s" % ve_dir, user=user)
def copy_to_virtualenv(package):
"""
Copies package from global Python, to local virtualenv.
"""
global_packages = '%s/lib/python2.7/dist-packages' % cget('global_python')
virtual_packages = '%s/lib/python2.7/site-packages' % cget('virtualenv_dir')
with settings(warn_only=True, sudo_prefix=SUDO_PREFIX):
sudo("ln -s %s/%s %s/%s" % (global_packages, package, virtual_packages,
package))
def copy_to_virtualenv(package):
"""
Copies package from global Python, to local virtualenv.
"""
global_packages = '%s/lib/python2.7/dist-packages' % cget('global_python')
virtual_packages = '%s/lib/python2.7/site-packages' % cget(
'virtualenv_dir')
with settings(warn_only=True, sudo_prefix=SUDO_PREFIX):
sudo("ln -s %s/%s %s/%s" %
(global_packages, package, virtual_packages, package))
def upload_project_configuration():
source_file = pjoin(cget(u'deploy_files'), u'gunicorn',
'gunicorn_start')
target_file = pjoin(cget(u'project_dir'), u'bin',
u'gunicorn_start')
print yellow('Uploading gunicorn starting script')
upload_template(source_file, target_file, config)
print yellow('Upload done!')
with cd(cget('project_dir')):
run('mkdir -p run')
run('chmod u+x bin/gunicorn_start')
print yellow('Starting gunicorn!')
run('./bin/gunicorn_start')
def configure():
"""Upload supervisor configuration files."""
user = cget('user')
# settings directories
sdir = cset('supervisor_dir', pjoin(cget('service_dir'), 'supervisor'))
slogdir = cset('supervisor_log_dir', pjoin(cget('log_dir'), 'supervisor'))
cset("supervisor_process_base", cget('project_name').replace('-', '_'))
cset("supervisor_process_id",
'%s%s' % (cget('supervisor_process_base'), '_supervisor'))
# create all dirs and log dirs
dirs = ['', 'config', cget('project_name')]
dirs = [pjoin(sdir, l) for l in dirs]
log_dirs = ['', cget('project_name'), 'child_auto', 'solr']
log_dirs = [pjoin(slogdir, l) for l in log_dirs]
create_target_directories(dirs + log_dirs, "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("supervisor")
dest_dir = pjoin(sdir, 'config')
confs = cget("supervisor_files")
show(yellow("Uploading service configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source,
local_dir,
dest_dir,
context,
mode="644",
directories_mode="700")
else:
upload_template_with_perms(source,
destination,
context,
mode="644")
scripts = [
'supervisorctl.sh', 'supervisord.sh', 'rabbitmq.sh',
'celery-worker.sh', 'supervisord-services.sh',
'supervisorctl-services.sh'
]
for script_name in scripts:
source = pjoin(cget("local_root"), 'deployment', 'scripts',
script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
def run_supevisordctl(command, conf=None):
"""Start supervisor process."""
if not conf:
conf = pjoin(cget('service_dir'), 'supervisor', 'config',
'supervisord.conf')
show(yellow("Running supervisorctrl: %s." % command))
return sudo('supervisorctl --configuration="%s" %s' % (conf, command))
def update_virtualenv():
"""Updates virtual Python environment."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
cache = cget("pip_cache")
show(yellow("Updating Python virtual environment."))
show(green("Be patient. It may take a while."))
for req in cget('pip_requirements'):
requirements = pjoin(remote_files_dir('requirements'), req)
show(yellow("Processing requirements file: %s" % requirements))
with settings(warn_only=True):
with prefix("source %s" % activate):
run("pip install --no-input --download-cache=%s"
" --requirement %s --log=/tmp/pip.log" %
(cache, requirements))
def upload_project_configuration():
source_file = pjoin(cget('deploy_files'), 'nginx',
'project.conf')
target_file = pjoin(cget('services_dir'), 'nginx',
'sites-available', cget('nginx_server_name'))
# make sure that there is such directory
run('mkdir -p {}'.format(pjoin(cget('services_dir'), 'nginx',
'sites-available')))
run('mkdir -p {}'.format(pjoin(cget('project_dir'), 'logs')))
print yellow('Uploading site configuration.')
upload_template(source_file, target_file, config)
print yellow('Testing configuration...')
# test nginx congiguration
# I wonder if using just sudo wouldn't be better here...
run('sudo nginx -t')
def build():
"""Creates the documentation files and adds them to the database.
The first step is to create automatic module documentation using apidoc.
Next the documentation is built and added to the database using
updatedoc management command from django-sphinxdoc.
See doc/readme.rst for more details.
Apidoc generating script resides in deployment/files/doc/make_apidoc.sh.
Note: This script is project-specific.
"""
show(yellow("Bulding documentation"))
apidoc_script = pjoin(cget("script_dir"), "make_apidoc.sh")
with settings(hide("running", "stdout"), warn_only=True):
run(apidoc_script)
run_django_cmd('updatedoc', args='-b ' + cget('project_display_name_slug'))
def configure():
"""Configures the doc module.
Creates doc_dir and copies documentation sources there.
Formats and uploads scripts for building the documentation.
Finally, loads django-sphinxdoc project from a fixture.
Project fixture can be found in deployment/files/doc/fixture.json.
"""
# Add extra context variables used by sphinx-doc and apidoc
excluded = cget('autodoc_excluded_apps')
excluded = ' '.join(excluded) if excluded else ''
cset("autodoc_excluded_apps", excluded, force=True)
cset("project_display_name_slug", slugify(cget('project_display_name')))
# Asure the doc folder exists
user = cget('user')
ddir = cget('doc_dir')
source_doc_dir = pjoin(cget("project_dir"), "code", "doc")
create_target_directories([ddir], "755", user)
# Delete the content of the folder if exist
with settings(hide("running", "stdout", "stderr")):
output = sudo('ls -A {doc_dir}'.format(doc_dir=ddir))
if len(output) > 0:
sudo("rm -r {doc_dir}/*".format(doc_dir=ddir))
# Copy files to doc dir
with settings(hide("running", "stdout")):
run("cp -r {source}/* {dest}".format(source=source_doc_dir, dest=ddir))
ensure_permissions(ddir, user=user, group=user, recursive=True)
context = dict(env["ctx"])
# Upload formatted build script
scripts = ['make_apidoc.sh']
local_dir = local_files_dir("doc")
show(yellow("Uploading doc scripts: {0}.".format(' '.join(scripts))))
for script_name in scripts:
source = pjoin(local_dir, script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
# Upload formatted conf.py file
show(yellow("Uploading formatted conf.py file."))
conf_file = "conf_formatted.py"
source = pjoin(cget("local_root"), "doc", 'source', conf_file)
destination = pjoin(ddir, 'source', conf_file)
upload_template_with_perms(source, destination, context, mode="755")
# Add Project to the database using formatted fixture and loaddata
show(yellow("Adding django-sphinxdoc database models."))
fixture_name = "fixture.json"
source = pjoin(local_dir, fixture_name)
destination = pjoin(ddir, fixture_name)
upload_template_with_perms(source, destination, context, mode="755")
run_django_cmd('loaddata', args=destination)
def shutdown():
"""Requests supervisor process and all controlled services shutdown."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
show(yellow("Shutting supervisor down."))
with prefix("source %s" % activate):
with settings(hide("stderr", "stdout", "running"), warn_only=True):
res = run_supevisordctl("shutdown all")
if res.return_code != 2:
msg = "Could not shutdown supervisor, process does not exists."
show(yellow(msg))
def shutdown():
"""Requests supervisor process and all controlled services shutdown."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
show(yellow("Shutting supervisor down."))
with prefix("source %s" % activate):
with settings(hide("stderr", "stdout", "running"), warn_only=True):
res = run_supevisordctl('shutdown all')
if res.return_code != 2:
msg = "Could not shutdown supervisor, process does not exists."
show(yellow(msg))
def configure():
"""Upload supervisor configuration files."""
user = cget("user")
# settings directories
sdir = cset("supervisor_dir", pjoin(cget("service_dir"), "supervisor"))
slogdir = cset("supervisor_log_dir", pjoin(cget("log_dir"), "supervisor"))
cset("supervisor_process_base", cget("project_name").replace("-", "_"))
cset("supervisor_process_id", "%s%s" % (cget("supervisor_process_base"), "_supervisor"))
# create all dirs and log dirs
dirs = ["", "config", cget("project_name")]
dirs = [pjoin(sdir, l) for l in dirs]
log_dirs = ["", cget("project_name"), "child_auto", "solr"]
log_dirs = [pjoin(slogdir, l) for l in log_dirs]
create_target_directories(dirs + log_dirs, "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("supervisor")
dest_dir = pjoin(sdir, "config")
confs = cget("supervisor_files")
show(yellow("Uploading service configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700")
else:
upload_template_with_perms(source, destination, context, mode="644")
scripts = [
"supervisorctl.sh",
"supervisord.sh",
"rabbitmq.sh",
"celery-worker.sh",
"supervisord-services.sh",
"supervisorctl-services.sh",
]
for script_name in scripts:
source = pjoin(cget("local_root"), "deployment", "scripts", script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
def reload(conf=None):
"""Start or restart supervisor process."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
show(yellow("Reloading supervisor."))
with prefix("source %s" % activate):
with settings(hide("stderr", "stdout", "running"), warn_only=True):
res = run_supevisordctl("reload", conf=conf)
if res.return_code != 0:
show(yellow("Supervisor unavailable, starting new process."))
res = start_supervisor(conf=conf)
if res.return_code != 0:
show(red("Error starting supervisor!."))
def update_virtualenv():
"""Updates virtual Python environment."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
user = cget("user")
cache = cget("pip_cache")
show(yellow("Updating Python virtual environment."))
show(green("Be patient. It may take a while."))
for req in cget('pip_requirements'):
requirements = pjoin(remote_files_dir('requirements'), req)
show(yellow("Processing requirements file: %s" % requirements))
with settings(warn_only=True, sudo_prefix=SUDO_PREFIX):
with prefix("source %s" % activate):
sudo("pip install --no-input --download-cache=%s"
" --requirement %s --log=/tmp/pip.log" % (
cache, requirements), user=user)
show(yellow("Linking python-q4 to local virtualenv."))
for package in ['PyQt4', 'sip.so']:
copy_to_virtualenv(package)
def install_git():
version = cget('git_version', 'v1.9.1')
with cd('/tmp'):
dirname = 'git-{}'.format(version[1:])
tarname = '{}.tar.gz'.format(version)
run('wget -O {0} https://github.com/git/git/archive/{0}'
.format(tarname))
run('tar xf {}'.format(tarname))
with cd(dirname):
run('make configure')
run('./configure --prefix={}'.format(INSTALL_PREFIX))
run('make all')
sudo('make install')
def reload(conf=None):
"""Start or restart supervisor process."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
show(yellow("Reloading supervisor."))
with prefix("source %s" % activate):
with settings(hide("stderr", "stdout", "running"), warn_only=True):
res = run_supevisordctl('reload', conf=conf)
if res.return_code != 0:
show(yellow("Supervisor unavailable, starting new process."))
res = start_supervisor(conf=conf)
if res.return_code != 0:
show(red("Error starting supervisor!."))
def update_virtualenv():
"""Updates virtual Python environment."""
ve_dir = cget("virtualenv_dir")
activate = pjoin(ve_dir, "bin", "activate")
user = cget("user")
cache = cget("pip_cache")
show(yellow("Updating Python virtual environment."))
show(green("Be patient. It may take a while."))
for req in cget('pip_requirements'):
requirements = pjoin(remote_files_dir('requirements'), req)
show(yellow("Processing requirements file: %s" % requirements))
with settings(warn_only=True, sudo_prefix=SUDO_PREFIX):
with prefix("source %s" % activate):
sudo("pip install --no-input --download-cache=%s"
" --requirement %s --log=/tmp/pip.log" %
(cache, requirements),
user=user)
show(yellow("Linking python-q4 to local virtualenv."))
for package in ['PyQt4', 'sip.so']:
copy_to_virtualenv(package)
def configure():
"""Upload supervisor configuration files."""
user = cget('user')
# settings directories
sdir = cset('supervisor_dir', pjoin(cget('service_dir'), 'supervisor'))
slogdir = cset('supervisor_log_dir', pjoin(cget('log_dir'), 'supervisor'))
cset("supervisor_process_base", cget('project_name').replace('-', '_'))
cset("supervisor_process_id",
'%s%s' % (cget('supervisor_process_base'), '_supervisor'))
# create all dirs and log dirs
dirs = ['', 'config', cget('project_name')]
dirs = [pjoin(sdir, l) for l in dirs]
log_dirs = ['', cget('project_name'), 'child_auto', 'solr']
log_dirs = [pjoin(slogdir, l) for l in log_dirs]
create_target_directories(dirs + log_dirs, "700", user)
context = dict(env["ctx"])
local_dir = local_files_dir("supervisor")
dest_dir = pjoin(sdir, 'config')
confs = cget("supervisor_files")
show(yellow("Uploading service configuration files: %s." % confs))
for name in confs:
source = pjoin(local_dir, name)
destination = pjoin(dest_dir, name)
if isdir(source):
upload_templated_folder_with_perms(source, local_dir, dest_dir,
context, mode="644", directories_mode="700")
else:
upload_template_with_perms(
source, destination, context, mode="644")
scripts = ['supervisorctl.sh', 'supervisord.sh']
for script_name in scripts:
source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
def configure():
"""Configures the doc module.
Creates doc_dir and copies documentation sources there.
Formats and uploads scripts for building the documentation.
Finally, loads django-sphinxdoc project from a fixture.
Project fixture can be found in deployment/files/doc/fixture.json.
"""
# Add extra context variables used by sphinx-doc and apidoc
excluded = cget('autodoc_excluded_apps')
excluded = ' '.join(excluded) if excluded else ''
cset("autodoc_excluded_apps", excluded, force=True)
cset("project_display_name_slug", slugify(cget('project_display_name')))
# Asure the doc folder exists
user = cget('user')
ddir = cget('doc_dir')
source_doc_dir = pjoin(cget("project_dir"), "code", "docs")
create_target_directories([ddir], "755", user)
# Delete the content of the folder if exist
with settings(hide("running", "stdout", "stderr")):
output = sudo('ls -A {doc_dir}'.format(doc_dir=ddir))
if len(output) > 0:
sudo("rm -r {doc_dir}/*".format(doc_dir=ddir))
# Copy files to doc dir
with settings(hide("running", "stdout")):
run("cp -r {source}/* {dest}".format(source=source_doc_dir, dest=ddir))
ensure_permissions(ddir, user=user, group=user, recursive=True)
context = dict(env["ctx"])
# Upload formatted build script
scripts = ['make_apidoc.sh']
local_dir = local_files_dir("doc")
show(yellow("Uploading doc scripts: {0}.".format(' '.join(scripts))))
for script_name in scripts:
source = pjoin(local_dir, script_name)
destination = pjoin(cget("script_dir"), script_name)
upload_template_with_perms(source, destination, context, mode="755")
# Upload formatted conf.py file
show(yellow("Uploading formatted conf.py file."))
conf_file = "conf_formatted.py"
source = pjoin(cget("local_root"), "docs", 'source', conf_file)
destination = pjoin(ddir, 'source', conf_file)
upload_template_with_perms(source, destination, context, mode="755")
def build():
"""Creates the documentation files and adds them to the database.
The first step is to create automatic module documentation using apidoc.
Next the documentation is built and added to the database using
updatedoc management command from django-sphinxdoc.
See doc/readme.rst for more details.
Apidoc generating script resides in deployment/files/doc/make_apidoc.sh.
Note: This script is project-specific.
"""
show(yellow("Bulding documentation"))
apidoc_script = pjoin(cget("script_dir"), "make_apidoc.sh")
with settings(hide("running", "stdout"), warn_only=True):
run(apidoc_script)
def run_supevisordctl(command, conf=None):
"""Start supervisor process."""
if not conf:
conf = pjoin(cget("service_dir"), "supervisor", "config", "supervisord.conf")
show(yellow("Running supervisorctrl: %s." % command))
return sudo('supervisorctl --configuration="%s" %s' % (conf, command))
def get_module():
distro = cget('distro', 'ubuntu')
if distro not in MODULES:
abort(red('Unknown distro: %s' % distro))
return MODULES[distro]
def sync_database():
with virtualenv():
with cd(cget(u'code_dir')):
print yellow('Migrating database')
run('python manage.py syncdb --migrate')
def run_supevisordctl(command):
"""Start supervisor process."""
conf = pjoin(cget('service_dir'), 'supervisor', 'config',
'supervisord.conf')
show(yellow("Running supervisorctrl: %s." % command))
return sudo('supervisorctl --configuration="%s" %s' % (conf, command))