def install_site():
"Add the virtualhost config file to the webserver's config, activate logrotate"
require('release', provided_by=[deploy, setup])
with cd('%(prj_path)s/releases/%(release)s' % env):
with settings(user=env.adminuser, pty=True):
run('cp server-setup/%(webserver)s.conf /etc/%(webserver)s/sites-available/%(prj_name)s' % env)
if env.use_daemontools: # activate new service runner
run('cp server-setup/service-run.sh /etc/service/%(prj_name)s/run; chmod a+x /etc/service/%(prj_name)s/run;' % env)
else: # delete old service dir
run('echo; if [ -d /etc/service/%(prj_name)s ]; then rm -rf /etc/service/%(prj_name)s; fi' % env)
if env.use_supervisor: # activate new supervisor.conf
run('cp server-setup/supervisor.conf /etc/supervisor/conf.d/%(prj_name)s.conf' % env)
if env.use_celery:
run('cp server-setup/supervisor-celery.conf /etc/supervisor/conf.d/%(prj_name)s-celery.conf' % env)
else: # delete old config file
# if you set a process name in supervisor.ini, then you must add it like %(prj_name):appserver
run('echo; if [ -f /etc/supervisor/%(prj_name)s.ini ]; then supervisorctl %(prj_name)s stop rm /etc/supervisor/%(prj_name)s.ini; fi' % env)
run('echo; if [ -f /etc/supervisor/conf.d/%(prj_name)s.conf ]; then supervisorctl %(prj_name)s stop rm /etc/supervisor/conf.d/%(prj_name)s.conf; fi' % env)
if env.use_celery:
run('echo; if [ -f /etc/supervisor/%(prj_name)s-celery.ini ]; then supervisorctl celery celerybeat stop rm /etc/supervisor/%(prj_name)s-celery.ini; fi' % env)
run('echo; if [ -f /etc/supervisor/conf.d/%(prj_name)s-celery.conf ]; then supervisorctl celery celerybeat stop rm /etc/supervisor/conf.d/%(prj_name)s-celery.conf; fi' % env)
if env.use_celery and env.use_daemontools:
run('cp server-setup/service-run-celeryd.sh /etc/service/%(prj_name)s-celery/run; chmod a+x /etc/service/%(prj_name)s-celery/run;' % env)
# try logrotate
with settings(warn_only=True):
run('cp server-setup/logrotate.conf /etc/logrotate.d/website-%(prj_name)s' % env)
if env.use_celery:
run('cp server-setup/logrotate-celery.conf /etc/logrotate.d/celery' % env)
run('cp server-setup/letsencrypt.conf /etc/letsencrypt/configs/%(cryptdomain)s.conf' % env)
with settings(warn_only=True):
run('cd /etc/%(webserver)s/sites-enabled/; ln -s ../sites-available/%(prj_name)s %(prj_name)s' % env)
def rsync_media(upload=False, delete=False):
require('PROJECT')
local_dir = add_slash(Path(env.PROJECT.package, 'media'))
remote_dir = add_slash(env.PROJECT.media)
rsync_project(remote_dir, local_dir, delete=delete, upload=upload)
def set_site_domain():
""" Set the site domain dependent on env.host. """
status_update('Updating domain for default Django Site to %s' % env.host)
set_django_settings()
require('django_settings')
require('host')
if 'production' in env.host:
host = 'onepercentclub.com'
else:
host = env.host
sites_command = (
"from django.contrib.sites.models import Site; "
"site = Site.objects.get(pk=1); "
"site.domain = '%s'; "
"site.name = 'onepercentclub.com'; "
"site.save()"
) % host
with virtualenv():
run_web('echo "%s" | ./manage.py shell --plain --settings=%s' % (
sites_command, env.django_settings
))
def manage_run(command):
"""Run a Django management command on the remote server."""
require('environment')
manage_base = u"%(virtualenv_root)s/bin/django-admin.py " % env
if '--settings' not in command:
command = u"%s --settings=%s" % (command, env.settings)
project_run(u'%s %s' % (manage_base, command))
def run_web(*args, **kwargs):
""" Run a command as the web user. """
require('web_user')
kwargs.setdefault('user', env.web_user)
return sudo(*args, **kwargs)
def prompt_role():
"""
Query the user for the role (and thus host) to use and store results in env.
"""
require('roledefs')
if not env.host_string:
role_options = ', '.join([role for role in env.roledefs.keys()])
def validate(value):
if not value in env.roledefs.keys():
raise Exception('Role should be one of %s' % role_options)
return value
role = prompt('Please specify a role (one of %s): ' % role_options, validate=validate)
# Warning: this is a rather hackish solution - Fabric doesn't seem
# to like the hosts for a task to be changed in the middle of a
# process. This works ok, but limits us to one host per role.
assert len(env.roledefs[role]) == 1, \
'This is a hackish solution that does not work for a role with multiple hosts.'
env.host = env.roledefs[role][0]
env.host_string = env.host
def prepare_django():
""" Prepare a deployment. """
set_django_settings()
require('django_settings')
status_update('Preparing deployment.')
with virtualenv():
# TODO: Filter out the following messages:
# "Could not find a tag or branch '<commit_id>', assuming commit."
run('pip install -q --allow-all-external --allow-unverified django-admin-tools -r requirements/requirements.txt')
run('grunt compass:dist')
# Remove and compile the .pyc files.
run('find . -name \*.pyc -delete')
run('./manage.py compile_pyc --settings=%s' % env.django_settings)
# Prepare the translations.
run('./translations.sh compile %s' % env.django_settings)
# Disabled until the following problem is fixed:
# ERROR: test_site_profile_not_available (django.contrib.auth.tests.models.ProfileTestCase)
#run_web('./manage.py test -v 0')
# Make sure the web user can read and write the static media dir.
run('chmod a+rw static/media')
run_web('./manage.py syncdb --migrate --noinput --settings=%s' % env.django_settings)
run_web('./manage.py collectstatic --clear -l -v 0 --noinput --settings=%s' % env.django_settings)
def deploy(loglevel=DEFAULT_SALT_LOGLEVEL):
"""Deploy to a given environment by pushing the latest states and executing the highstate."""
require('environment')
sync()
target = "-G 'environment:{0}'".format(env.environment)
salt('saltutil.sync_all', target, loglevel)
highstate(target)
def encrypt(*args, **kwargs):
"""Encrypt a secret value for a given environment."""
require('environment')
# Convert ASCII key to binary
temp_key = '/tmp/tmp.key'
with hide('running', 'stdout', 'stderr'):
local('gpg --dearmor < {} > {}'.format(env.gpg_key, temp_key))
# Encrypt each file
for name in args:
local(
'gpg --no-default-keyring --keyring {} '
'--trust-model always -aer {}_salt_key {}'.format(
temp_key, env.environment, name))
# Encrypt each value
updates = {}
for name, value in kwargs.items():
updates[name] = '{}'.format(
local(
'echo -n "{}" | '
'gpg --no-default-keyring --keyring {} '
'--trust-model always -aer {}_salt_key'.format(
value, temp_key, env.environment), capture=True))
os.remove(temp_key)
if updates:
print(yaml.dump(updates, default_flow_style=False, default_style='|', indent=2))
def bdeploy():
require('hosts')
require('path')
print(green("\n#####Beginning %s build\n" % ('testing' if env.is_test else 'production')))
local('sencha app build %s' % ('testing' if env.is_test else 'production'))
qdeploy()
def deploy():
require('hosts')
require('path')
print(green("\n#####Beginning deployment to %s & %s on %s\n" % (env.mob_domain, env.stubs_domain, env.hosts[0])))
env.mob_release_label = None
env.stubs_release_label = None
if env.is_test:
create_tag = prompt('Tag this release? [y/N] ')
if create_tag.lower() == 'y':
print("\nShowing latest tags for reference\n")
local('git tag | tail -5')
env.refspec = prompt('Enter tag name [in format VX_X_X]? ') #Can't use .'s as seperaters as they cause import problems on the server
local('git tag %s -am "Tagging version %s in fabfile"' % (env.refspec, env.refspec))
local('git push --tags')
env.mob_release_label = mob_package_name + '-' + env.refspec
env.stubs_release_label = stubs_package_name + '-' + env.refspec
if not env.mob_release_label:
# An existing tag must be specified
local('git tag | tail -5')
env.refspec = prompt('Choose tag to build from: ')
local('git tag | grep "%s"' % env.refspec)
env.mob_release_label = mob_package_name + '-' + env.refspec
env.stubs_release_label = stubs_package_name + '-' + env.refspec
# import time
# env.release_label = package_name + '-' + time.strftime('%Y%m%d%H%M%S')
_upload_tar_from_git()
_install_site()
_symlink_current_release()
restart_webserver()
print(green("\n#####Deployment successful for %s & %s\n" % (env.mob_domain, env.stubs_domain)))
def bootstrap():
"""Sets up the remote environment"""
require('hosts', provided_by=[remote])
sub_add_repos()
sub_install_packages()
sub_install_shiny()
reload_server()
def push():
"""Pushes the current folder to the remote machine's Shiny apps folder"""
require('hosts', provided_by=[remote])
sudo('mkdir -p /www-shiny')
sudo('chown -R ubuntu:ubuntu /www-shiny')
sudo('rm -rf /www-shiny/*')
put('./', '/www-shiny/')
def provision():
require('deploy_type', provided_by=[dev, prod])
commands = (
'apt-get install --yes --no-upgrade %s' % ' '.join(PACKAGES),
)
if env.deploy_type == 'prod':
commands += (
'mkdir -p %s' % env.deploy_dir,
'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir),
)
if env.deploy_type == 'dev':
local('; '.join('sudo %s' % cmd for cmd in commands))
local('sudo apt-get build-dep python-numpy python-psycopg2 --no-upgrade')
local('sudo npm config set registry http://registry.npmjs.org/')
local('sudo npm -g install yuglify')
return
sudo('; '.join(commands))
sudo('apt-get build-dep python-numpy python-psycopg2 --no-upgrade')
sudo('npm config set registry http://registry.npmjs.org/')
sudo('npm -g install yuglify')
sudo('mv -f /etc/nginx/sites-enabled/default /etc/nginx/sites-available')
provision_api()
git_checkout()
def setup_vagrant():
"""Sets up the Vagrant environment"""
require('hosts', provided_by=[vagrant]) # Sets the environment for Fabric
sub_add_repos()
sub_install_packages()
sub_install_shiny()
reload_server()
def deploy_web():
"""
Installs the output of the build on the web instances.
"""
require("configuration")
if exists(env.deploy_dir):
run("rm -rf %s" % env.deploy_dir)
run("tar -xvzf %s" % env.build_archive)
run("mv %s deploy" % env.git_tag)
run("source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv")
env.SHELL_ENV = dict(
DJANGO_SETTINGS_MODULE=env.django_settings_module,
DJANGO_CONFIGURATION=env.django_configuration,
CONFIG_HTTP_PORT=env.config_http_port,
CONFIG_SERVER_NAME=env.config_server_name,
)
print env.SHELL_ENV
with cd(env.deploy_dir):
with prefix("source /usr/local/bin/virtualenvwrapper.sh && workon venv"), shell_env(**env.SHELL_ENV):
requirements_path = "/".join(["codalab", "requirements", "dev_azure_nix.txt"])
pip_cmd = "pip install -r {0}".format(requirements_path)
run(pip_cmd)
# additional requirements for bundle service
run("pip install SQLAlchemy simplejson")
with cd("codalab"):
run("python manage.py config_gen")
run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
run("python manage.py syncdb --migrate")
run("python scripts/initialize.py")
run("python manage.py collectstatic --noinput")
sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
def fetch_logs():
"""
Fetch logs from the web instances into ~/logs.
"""
require("configuration")
with cd(env.deploy_dir):
get("codalab/var/*.log", "~/logs/%(host)s/%(path)s")
def bootstrap():
""" initialize remote host environment (virtualenv, deploy, update) """
require('root', provided_by=env.deployments)
print magenta("Cloning Repository")
with cd(env.root):
run("git clone %s" % env.git_repository)
# some one time setup things
with cd(env.project_root):
if env.git_branch != 'master':
run('git checkout %s' % (env.git_branch,))
run('mkdir static')
run('mkdir media')
with cd(env.code_root):
run('ln -sf settings_%s.py settings.py' % env.environment)
# create virtualenv and install all the requirements
execute('create_virtualenv')
execute('update_requirements')
execute('create_database')
execute('syncdb')
execute('migrate')
print magenta("Load initial data")
with cd(env.project_root), prefix('source env/bin/activate'):
run('./manage.py loaddata allink_user.json')
# only compile messages if locale folder is present
if os.path.isdir('locale'):
execute('compilemessages')
execute('collectstatic')
def hosts():
"""List applicable hosts for the specified environment.
Use this to determine which hosts will be affected by an
environment-specific operation."""
require('settings', provided_by=[prd, stg])
for host in env.instances:
print env.hosts
def deploy():
"""Deploy to a given environment."""
# NOTE: chef will check every 30 minutes or so whether the
# repo has changed, and if so, redeploy. Or you can use this
# to make it run immediately.
require('environment')
sudo('chef-client')
def syncdb():
"""runs syncdb on the remote host"""
require('project_root', provided_by=env.deployments)
print magenta("Syncronize database")
with cd(env.project_root):
with prefix('source env/bin/activate'):
run('./manage.py syncdb --noinput')
def uninstall_crontab():
"""
Remove a previously install cron jobs script from cron.d
"""
require('settings', provided_by=[production, staging])
sudo('rm /etc/cron.d/%(PROJECT_FILENAME)s' % app_config.__dict__)
def user_sshkey():
"""
Upload an SSH key to the remote system for the current user.
:Example:
fab --config=config/local.conf local system.user_sshkey
"""
require('PUBLIC_SSH_KEY')
with open(env.PUBLIC_SSH_KEY) as reader:
key = reader.read()
remote_directory = '/home/{}/.ssh'.format(env.user)
remote_authkeys = '{}/authorized_keys'.format(remote_directory)
new_directory = False
if not files.exists(remote_directory):
new_directory = True
# Create the ".ssh" directory.
run('mkdir -p {}'.format(remote_directory))
# Add the key to "authorized keys".
files.append(remote_authkeys, key)
if new_directory:
# Set directory permission to "700".
run('chmod 700 {}'.format(remote_directory))
# Set file permission to "600".
run('chmod 600 {}'.format(remote_authkeys))
def list_releases():
"List the releases on the server"
require('releases_dir',
provided_by=['dev','prod'])
run("ls -t %(releases_dir)s"%env)
def setup_minion(*roles):
"""Setup a minion server with a set of roles."""
require('environment')
for r in roles:
if r not in VALID_ROLES:
abort('%s is not a valid server role for this project.' % r)
config = {
'master': 'localhost' if env.master == env.host else env.master,
'output': 'mixed',
'grains': {
'environment': env.environment,
'roles': list(roles),
},
'mine_functions': {
'network.interfaces': [],
'network.ip_addrs': []
},
}
_, path = tempfile.mkstemp()
with open(path, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
sudo("mkdir -p /etc/salt")
put(local_path=path, remote_path="/etc/salt/minion", use_sudo=True)
# install salt minion if it's not there already
install_salt(SALT_VERSION, master=False, minion=True, restart=True)
# queries server for its fully qualified domain name to get minion id
key_name = run('python -c "import socket; print socket.getfqdn()"')
execute(accept_key, key_name)
def ve_run(cmd):
"""
Helper function.
Runs a command using the virtualenv environment
"""
require('root')
return sshagent_run('source %s/bin/activate; %s' % (env.root, cmd))
def new_user(username, email):
"""Add new user function, pass username, email
:param username: string of new user
:param email: string of new email
"""
require('hosts', provided_by=[sample])
require('ini', provided_by=[sample])
parse_ini(env["ini_file"])
import transaction
from bookie.models import initialize_sql
initialize_sql(dict(env.ini.items('app:main')))
from bookie.models import DBSession
from bookie.models.auth import get_random_word, User
sess = DBSession()
u = User()
u.username = unicode(username)
passwd = get_random_word(8)
u.password = passwd
u.email = unicode(email)
u.activated = True
u.is_admin = False
u.api_key = User.gen_api_key()
print dict(u)
print passwd
sess.add(u)
sess.flush()
transaction.commit()
def install_crontab():
"""
Install cron jobs script into cron.d.
"""
require('settings', provided_by=[production, staging])
sudo('cp %(SERVER_REPOSITORY_PATH)s/crontab /etc/cron.d/%(PROJECT_FILENAME)s' % app_config.__dict__)
def deploy(branch=None):
"""Deploy to a given environment."""
require('environment')
if branch is not None:
env.branch = branch
requirements = False
migrations = False
# Fetch latest changes
with cd(env.code_root):
with settings(user=env.project_user):
run('git fetch origin')
# Look for new requirements or migrations
requirements = match_changes(env.branch, "'requirements\/'")
migrations = match_changes(env.branch, "'\/migrations\/'")
if requirements or migrations:
supervisor_command('stop %(environment)s:*' % env)
with settings(user=env.project_user):
run("git reset --hard origin/%(branch)s" % env)
upload_local_settings()
if requirements:
update_requirements()
# New requirements might need new tables/migrations
syncdb()
elif migrations:
syncdb()
collectstatic()
supervisor_command('restart %(environment)s:*' % env)
def deploy(remote='origin'):
"""
Deploy the latest app to S3 and, if configured, to our servers.
"""
require('settings', provided_by=[production, staging])
if app_config.DEPLOY_TO_SERVERS:
require('branch', provided_by=[stable, master, branch])
if (app_config.DEPLOYMENT_TARGET == 'production' and env.branch != 'stable'):
utils.confirm("You are trying to deploy the '%s' branch to production.\nYou should really only deploy a stable branch.\nDo you know what you're doing?" % env.branch)
if app_config.DEPLOY_TO_SERVERS:
checkout_latest(remote)
#fabcast('update_copy')
#fabcast('assets.sync')
#fabcast('update_data')
if app_config.DEPLOY_CRONTAB:
install_crontab()
if app_config.DEPLOY_SERVICES:
deploy_confs()
render()
_gzip('www', '.gzip')
_deploy_to_s3()
_cleanup_minified_includes()
def open_heroku():
require('environment')
local('heroku open --remote {}'.format(env.environment))
def _report_coverage():
require('venv_path')
with cd(env.site_path):
run(_venv_exec('coverage report'))
def _venv_exec(cmd):
require('venv_path')
return '%s/bin/%s' % (env.venv_path, cmd)
def check_settings():
"Run the (added in 1.6) check management command to validate settings"
require('site_path')
require('venv_path')
_manage('check')
def runserver(bind_to='0.0.0.0', port='12000'):
""" Run a python development server on the host """
require('site_path')
require('venv_path')
_manage('runserver %s:%s' % (bind_to, port))
def schemamigration(app_name, flag=' --auto'):
require('site_path')
require('venv_path')
_manage('schemamigration %s %s' % (app_name, flag))
def uptime():
require('target', provided_by=(
staging,
production,
))
run('uptime')
def create_superuser():
require('environment')
local('heroku run python {}/manage.py '
'createsuperuser --remote {}'.format(env.project_name,
env.environment))
def migrate():
"""migrates all apps on remote host"""
require('project_root', provided_by=env.deployments)
print magenta("Migrate database")
with cd(env.project_root), prefix('source env/bin/activate'):
run('./manage.py migrate')
def ps():
"""
Scales a web dyno on Heroku
"""
require('environment')
local('heroku ps:scale web=1:hobby --remote {}'.format(env.environment))
def restart_webapp():
""" touch wsgi file to trigger reload """
require('project_root', provided_by=env.deployments)
print magenta("Restart webapp")
with cd(env.project_root):
run('touch apache.wsgi')
def migrate():
require('environment')
local('heroku run python {}/manage.py migrate --remote {}'.format(
env.project_name, env.environment))
def all(branch='master'):
"""Perform all publish sub-tasks at once"""
require('environment', provided_by=[stage])
execute(push, branch=branch)
execute(pull, branch=branch)
def restart_celery():
"""restarts the celery worker"""
require('project_root', provided_by=env.deployments)
print magenta("Restart celery")
run('nine-supervisorctl restart %s' % env.celery_worker)
def clear_cache():
require('srvr', 'path', 'within_virtualenv', provided_by=env.servers)
with cd(env.path), prefix(env.within_virtualenv):
run('./manage.py clear_cache')
def create_virtualenv():
""" setup virtualenv on remote host """
require('project_root', provided_by=env.deployments)
run('virtualenv --setuptools --prompt="(%s)" %s' %
(env.project, os.path.join(env.project_root, 'env')))
def migrate(app=None):
require('srvr', 'path', 'within_virtualenv', provided_by=env.servers)
with cd(env.path), prefix(env.within_virtualenv):
run('./manage.py migrate {}'.format(app if app else ''))
def setup_environment(version=None):
require('srvr', 'path', 'within_virtualenv', provided_by=env.servers)
create_virtualenv()
clone_repo()
update(version)
install_requirements()
def install_mysql(choice='all'):
"""
Installs a local instance of MySQL of the web instance. This will only work
if the number of web instances is one.
choice: Indicates which assets to create/install:
'mysql' -> just install MySQL; don't create the databases
'site_db' -> just create the site database
'bundles_db' -> just create the bundle service database
'all' or '' -> install all three
"""
require('configuration')
if len(env.roledefs['web']) != 1:
raise Exception(
"Task install_mysql requires exactly one web instance.")
if choice == 'mysql':
choices = {'mysql'}
elif choice == 'site_db':
choices = {'site_db'}
elif choice == 'bundles_db':
choices = {'bundles_db'}
elif choice == 'all':
choices = {'mysql', 'site_db', 'bundles_db'}
else:
raise ValueError(
"Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." %
(choice))
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dba_password = configuration.getDatabaseAdminPassword()
if 'mysql' in choices:
sudo(
'DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server'
)
sudo('mysqladmin -u root password {0}'.format(dba_password))
if 'site_db' in choices:
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
cmds = [
"create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(
db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;"
.format(db_name, db_user)
]
run('mysql --user=root --password={0} --execute="{1}"'.format(
dba_password, " ".join(cmds)))
if 'bundles_db' in choices:
db_name = configuration.getBundleServiceDatabaseName()
db_user = configuration.getBundleServiceDatabaseUser()
db_password = configuration.getBundleServiceDatabasePassword()
cmds = [
"create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(
db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;"
.format(db_name, db_user)
]
run('mysql --user=root --password={0} --execute="{1}"'.format(
dba_password, " ".join(cmds)))
def update_index():
require('srvr', 'path', 'within_virtualenv', provided_by=env.servers)
with cd(env.path), prefix(env.within_virtualenv):
run('./manage.py update_index')
def _deploy():
# Update website
with cd(env.deploy_codalab_worksheets_dir):
run('git fetch')
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('./setup.sh')
# Update bundle service
with cd(env.deploy_codalab_cli_dir):
run('git fetch')
run('git checkout %s' % env.git_codalab_cli_tag)
run('git pull')
run('./setup.sh server')
# Create website-config.json
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
buf = StringIO()
json.dump(getWebsiteConfig(cfg),
buf,
sort_keys=True,
indent=4,
separators=(',', ': '))
buf.write('\n')
put(buf, '.codalab/website-config.json')
# Update the website configuration
with cd(env.deploy_codalab_worksheets_dir), cd('codalab'):
# Generate configuration files (nginx, supervisord)
run('./manage config_gen')
# Put configuration files in place.
sudo(
'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
)
sudo(
'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
)
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(
cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(),
cfg.getSslCertificateInstalledPath(),
use_sudo=True)
put(cfg.getSslCertificateKeyPath(),
cfg.getSslCertificateKeyInstalledPath(),
use_sudo=True)
else:
logger.info(
"Skipping certificate installation because both files are not specified."
)
# Configure the bundle server
with cd(env.deploy_codalab_cli_dir), cd('codalab'), cd('bin'):
# For generating the bundle_server_config.json file.
run('./cl config server/engine_url mysql://%s:%s@localhost:3306/%s' % ( \
cfg.getBundleServiceDatabaseUser(),
cfg.getBundleServiceDatabasePassword(),
cfg.getBundleServiceDatabaseName(),
))
# Send out emails from here (e.g., for password reset)
email_info = cfg.getEmailInfo()
run('./cl config email/host %s' % email_info['host'])
run('./cl config email/user %s' % email_info['user'])
run('./cl config email/password %s' % email_info['password'])
# Send notifications.
run('./cl config server/admin_email %s' % cfg.getAdminEmail())
run('./cl config server/support_email %s' % cfg.getSupportEmail())
run('./cl config server/instance_name %s' % cfg.label)
# Update database
with cd(env.deploy_codalab_cli_dir):
run('venv/bin/alembic upgrade head')
# Set up the bundles database.
with cd(env.deploy_codalab_cli_dir):
run('scripts/create-root-user.py %s' % cfg.getDatabaseAdminPassword())
def reinstall_requirement(which):
require('srvr', 'path', 'within_virtualenv', provided_by=env.servers)
with cd(env.path), prefix(env.within_virtualenv):
run('pip uninstall {0} && pip install --no-deps {0}'.format(which))
def _svn_checkout(svn_url,
dir_name,
username=None,
update_existing=True,
do_local=False):
"""
Check out the Subversion project from <svn_url> into directory <dir_name>.
This method makes one attempt to check out without specifying a password.
On subsequent attempts, it will prompt for the password, giving up after
three attempts.
If the project already exists and <update_existing> is True, then
"svn update" will be run in the project directory.
if <do_local> is True, the project will be checked out on the local
machine.
"""
require("svnpass", used_for="checking out Subversion projects")
if do_local:
homedir = os.environ["HOME"]
fexists = os.path.exists
frun = _capture_local
else:
with hide("running", "stdout", "stderr"):
homedir = run("echo $HOME", pty=False)
fexists = _exists
frun = run
path = os.path.join(homedir, dir_name)
if fexists(path):
if update_existing:
frun("cd %s && svn up" % path)
else:
attempts = -1
while True:
if attempts < 0 or env.svnpass is not None:
tmppass = env.svnpass
else:
if username is not None:
unm = username
else:
unm = env.user
prompt = "Enter Subversion password for %s" % unm
tmppass = _get_password(prompt)
with hide("running", "warnings", "stderr"):
with settings(warn_only=True):
if username is not None:
user_arg = "--username %s " % username
else:
user_arg = ""
print("svn co %s%s %s" % (user_arg, svn_url, path))
if tmppass is not None:
pass_arg = "--password %s " % tmppass
else:
pass_arg = ""
rtnval = frun("(echo; echo; echo; echo) |" +
" svn co %s%s%s %s" %
(user_arg, pass_arg, svn_url, path),
pty=False)
if not rtnval.failed:
if env.svnpass is None:
env.svnpass = tmppass
break
attempts += 1
if attempts > 3:
print("Giving up after %d attempts" % \
(attempts - 1), file=sys.stderr)
break
def test_connections():
"""
Verifies that we can connect to all instances.
"""
require('configuration')
sudo('hostname')
def run_django_command(command):
require('srvr', 'path', 'within_virtualenv', provided_by=env.servers)
with cd(env.path):
run('{} python manage.py {}'.format(env.within_virtualenv, command))
def _sudo(cmd):
require('webapp_user')
return sudo(cmd, user='******' % env)
def setup_environment(version=None):
require('srvr', provided_by=env.servers)
clone_repo()
update(version)
install_requirements()
def deploy(git_ref, upgrade=False):
"""Deploy project.
Deploy the code of the given git reference to the previously selected
environment.
Args:
upgrade(Optional[bool]):
Pass ``upgrade=True`` to upgrade the versions of the already
installed project requirements (with pip)
git_ref(str): name branch you make deploy.
Example:
>>>fab environment:vagrant deploy:devel.
"""
require('hosts', 'user', 'group', 'site_dir', 'django_settings')
# Retrives git reference metadata and creates a temp directory with the
# contents resulting of applying a ``git archive`` command.
message = white('Creating git archive from {0}'.format(git_ref), bold=True)
with cmd_msg(message):
repo = ulocal(
'basename `git rev-parse --show-toplevel`', capture=True)
commit = ulocal(
'git rev-parse --short {0}'.format(git_ref), capture=True)
branch = ulocal(
'git rev-parse --abbrev-ref HEAD', capture=True)
tmp_dir = '/tmp/blob-{0}-{1}/'.format(repo, commit)
ulocal('rm -fr {0}'.format(tmp_dir))
ulocal('mkdir {0}'.format(tmp_dir))
ulocal('git archive {0} ./src | tar -xC {1} --strip 1'.format(
commit, tmp_dir))
# Uploads the code of the temp directory to the host with rsync telling
# that it must delete old files in the server, upload deltas by checking
# file checksums recursivelly in a zipped way; changing the file
# permissions to allow read, write and execution to the owner, read and
# execution to the group and no permissions for any other user.
with cmd_msg(white('Uploading code to server...', bold=True)):
ursync_project(
local_dir=tmp_dir,
remote_dir=env.site_dir,
delete=True,
default_opts='-chrtvzP',
extra_opts='--chmod=750',
exclude=["*.pyc", "env/", "cover/"]
)
# Performs the deployment task, i.e. Install/upgrade project
# requirements, syncronize and migrate the database changes, collect
# static files, reload the webserver, etc.
message = white('Running deployment tasks', bold=True)
with cmd_msg(message, grouped=True):
with virtualenv():
message = white('Installing Python requirements with pip')
with cmd_msg(message, spaces=2):
run('pip install -{0}r ./requirements/production.txt'.format(
'U' if upgrade else ''))
message = white('Migrating database')
with cmd_msg(message, spaces=2):
run('python manage.py migrate --noinput')
message = white('Collecting static files')
with cmd_msg(message, spaces=2):
run('python manage.py collectstatic --noinput')
message = white('Setting file permissions')
with cmd_msg(message, spaces=2):
run('chgrp -R {0} .'.format(env.group))
run('chgrp -R {0} ../media'.format(env.group))
message = white('Restarting webserver')
with cmd_msg(message, spaces=2):
run('touch ../reload')
message = white('Registering deployment')
with cmd_msg(message, spaces=2):
register_deployment(commit, branch)
# Clean the temporary snapshot files that was just deployed to the host
message = white('Cleaning up...', bold=True)
with cmd_msg(message):
ulocal('rm -fr {0}'.format(tmp_dir))
puts(green(SUCCESS_ART), show_prefix=False)
puts(white('Code from {0} was succesfully deployed to host {1}'.format(
git_ref, ', '.join(env.hosts)), bold=True), show_prefix=False)
def manage_run(command):
require('environment')
project_name = get_project_name()
manage_sh = u'/var/www/%s/manage.sh ' % project_name
with settings(host_string=hostnames_for_role('web')[0]):
sudo(manage_sh + command, user=project_name)
def check_deploy():
require('srvr', provided_by=env.servers)
if env.srvr in ['stg', 'liv']:
run_django_command('check --deploy')
