def deploy():
maintenance("begin")
supervisor_stop()
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd("deploy/codalab"):
run("git pull")
run("pip install -r requirements/dev_azure_nix.txt")
run("python manage.py syncdb --migrate")
run("python manage.py collectstatic --noinput")
# Generate config
run("python manage.py config_gen")
run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
# run('python scripts/initialize.py') # maybe not needed
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())
# Setup bundle service for worksheets
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd("deploy/bundles"):
run("git pull")
run("alembic upgrade head")
supervisor()
maintenance("end")
def deploy():
maintenance("begin")
supervisor_stop()
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd('deploy/codalab'):
run('git pull')
run('pip install -r requirements/dev_azure_nix.txt')
run('python manage.py syncdb --migrate')
run('python manage.py collectstatic --noinput')
# Generate config
run('python manage.py config_gen')
run('mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json')
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# run('python scripts/initialize.py') # maybe not needed
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Setup bundle service for worksheets
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd('deploy/bundles'):
run('git pull')
run('alembic upgrade head')
supervisor()
maintenance("end")
def install_config():
'''
Install configuration files (do multiple times).
'''
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
run('python manage.py config_gen')
run('mkdir -p ~/.codalab && python scripts/set-oauth-key.py ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def _deploy():
# Update competition website
# Pull branch and run requirements file, for info about requirments look into dev_setp.sh
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
# local.py is generated here. For more info about content look into deploy/__.init__.py
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab',
'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
# For more info look into https://github.com/greyside/django-config-gen
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Put nginx and supervisor configuration files in place, ln creates symbolic links
sudo(
'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
)
sudo(
'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
)
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' %
cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(
cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(),
cfg.getSslCertificateInstalledPath(),
use_sudo=True)
put(cfg.getSslCertificateKeyPath(),
cfg.getSslCertificateKeyInstalledPath(),
use_sudo=True)
else:
logger.info(
"Skipping certificate installation because both files are not specified."
)
def _deploy():
# Update website
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Update bundle service
with cd(env.deploy_codalab_cli_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_cli_tag)
run('./setup.sh')
run('venv/bin/pip install MySQL-Python')
run('venv/bin/alembic upgrade head')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Create a superuser for OAuth
run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
# Allow bundle service to connect to website for OAuth
run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
# Put nginx and supervisor configuration files in place
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def _deploy():
# Update website
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('./dev_setup.sh')
# Update bundle service
with cd(env.deploy_codalab_cli_dir):
run('git checkout %s' % env.git_codalab_cli_tag)
run('git pull')
run('./setup.sh')
run('venv/bin/pip install MySQL-Python')
run('venv/bin/alembic upgrade head')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Create a superuser for OAuth
run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
# Allow bundle service to connect to website for OAuth
run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
# Put nginx and supervisor configuration files in place
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def deploy_web():
"""
Installs the output of the build on the web instances.
"""
require('configuration')
if exists(env.deploy_dir):
run('rm -rf %s' % env.deploy_dir)
run('tar -xvzf %s' % env.build_archive)
run('mv %s deploy' % env.git_tag)
run('source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv')
env.SHELL_ENV = dict(DJANGO_SETTINGS_MODULE=env.django_settings_module,
DJANGO_CONFIGURATION=env.django_configuration,
CONFIG_HTTP_PORT=env.config_http_port,
CONFIG_SERVER_NAME=env.config_server_name)
print env.SHELL_ENV
with cd(env.deploy_dir):
with prefix('source /usr/local/bin/virtualenvwrapper.sh && workon venv'
), shell_env(**env.SHELL_ENV):
requirements_path = "/".join(
['codalab', 'requirements', 'dev_azure_nix.txt'])
pip_cmd = 'pip install -r {0}'.format(requirements_path)
run(pip_cmd)
# additional requirements for bundle service
run('pip install SQLAlchemy simplejson')
with cd('codalab'):
run('python manage.py config_gen')
run('mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json'
)
run('python manage.py syncdb --migrate')
run('python scripts/initialize.py')
run('python manage.py collectstatic --noinput')
sudo(
'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
)
sudo(
'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
)
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run('newrelic-admin generate-config %s newrelic.ini' %
cfg.getNewRelicKey())
def _deploy():
# Update competition website
# Pull branch and run requirements file, for info about requirments look into dev_setp.sh
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
# local.py is generated here. For more info about content look into deploy/__.init__.py
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
# For more info look into https://github.com/greyside/django-config-gen
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Put nginx and supervisor configuration files in place, ln creates symbolic links
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def deploy_web():
"""
Installs the output of the build on the web instances.
"""
require("configuration")
if exists(env.deploy_dir):
run("rm -rf %s" % env.deploy_dir)
run("tar -xvzf %s" % env.build_archive)
run("mv %s deploy" % env.git_tag)
run("source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv")
env.SHELL_ENV = dict(
DJANGO_SETTINGS_MODULE=env.django_settings_module,
DJANGO_CONFIGURATION=env.django_configuration,
CONFIG_HTTP_PORT=env.config_http_port,
CONFIG_SERVER_NAME=env.config_server_name,
)
print env.SHELL_ENV
with cd(env.deploy_dir):
with prefix("source /usr/local/bin/virtualenvwrapper.sh && workon venv"), shell_env(**env.SHELL_ENV):
requirements_path = "/".join(["codalab", "requirements", "dev_azure_nix.txt"])
pip_cmd = "pip install -r {0}".format(requirements_path)
run(pip_cmd)
# additional requirements for bundle service
run("pip install SQLAlchemy simplejson")
with cd("codalab"):
run("python manage.py config_gen")
run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
run("python manage.py syncdb --migrate")
run("python scripts/initialize.py")
run("python manage.py collectstatic --noinput")
sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())
