def enable_cors():
"""
Enable cross-origin resource sharing for a Windows Azure storage service.
"""
require('configuration')
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
dep.ensureStorageHasCorsConfiguration()
def _deploy():
# Update competition website
# Pull branch and run requirements file, for info about requirments look into dev_setp.sh
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
# local.py is generated here. For more info about content look into deploy/__.init__.py
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab',
'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
# For more info look into https://github.com/greyside/django-config-gen
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Put nginx and supervisor configuration files in place, ln creates symbolic links
sudo(
'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
)
sudo(
'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
)
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' %
cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(
cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(),
cfg.getSslCertificateInstalledPath(),
use_sudo=True)
put(cfg.getSslCertificateKeyPath(),
cfg.getSslCertificateKeyInstalledPath(),
use_sudo=True)
else:
logger.info(
"Skipping certificate installation because both files are not specified."
)
def _deploy():
# Update website
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('./dev_setup.sh')
# Update bundle service
with cd(env.deploy_codalab_cli_dir):
run('git checkout %s' % env.git_codalab_cli_tag)
run('git pull')
run('./setup.sh')
run('venv/bin/pip install MySQL-Python')
run('venv/bin/alembic upgrade head')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Create a superuser for OAuth
run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
# Allow bundle service to connect to website for OAuth
run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
# Put nginx and supervisor configuration files in place
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(
env.label,
env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
)
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
put(local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(
current_directory),
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True)
put(local_path='{}/configs/upstart/codalab-monitor.conf'.format(
current_directory),
remote_path='/etc/init/codalab-monitor.conf',
use_sudo=True)
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor")
sudo("start codalab-compute-worker")
sudo("start codalab-monitor")
def teardown(choice):
"""
Deletes specified assets in the deployment. Be careful: there is no undoing!
choice: Indicates which assets to delete:
'build' -> provision the build machine
'web' -> provision the web instances
'all' -> provision everything
"""
assets = _validate_asset_choice(choice)
require('configuration')
logger.info("Teardown begins: %s.", assets)
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(configuration)
dep.Teardown(assets)
logger.info("Teardown is complete.")
def build():
"""
Builds artifacts to install on the deployment instances.
"""
require('configuration')
# Assemble source and configurations for the web site
build_dir = "/".join(['builds', env.git_user, env.git_repo])
src_dir = "/".join([build_dir, env.git_tag])
if exists(src_dir):
run('rm -rf %s' % (src_dir.rstrip('/')))
with settings(warn_only=True):
run('mkdir -p %s' % src_dir)
with cd(src_dir):
# TODO: why do we have the --branch and --single-branch tags here, this causes problems
run('git clone --depth=1 --branch %s --single-branch %s .' %
(env.git_tag, env.git_repo_url))
# Generate settings file (local.py)
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(configuration)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = "/".join(
['codalab', 'codalab', 'settings', 'local.py'])
put(buf, settings_file)
# Assemble source and configurations for the bundle service
if len(env.git_bundles_repo_url) > 0:
build_dir_b = "/".join(
['builds', env.git_bundles_user, env.git_bundles_repo])
src_dir_b = "/".join([build_dir_b, env.git_bundles_tag])
if exists(src_dir_b):
run('rm -rf %s' % (src_dir_b.rstrip('/')))
with settings(warn_only=True):
run('mkdir -p %s' % src_dir_b)
with cd(src_dir_b):
# TODO: why do we have the --branch and --single-branch tags here, this causes problems
run('git clone --depth=1 --branch %s --single-branch %s .' %
(env.git_bundles_tag, env.git_bundles_repo_url))
# Replace current bundles dir in main CodaLab other bundles repo.
bundles_dir = "/".join([src_dir, 'bundles'])
run('rm -rf %s' % (bundles_dir.rstrip('/')))
run('mv %s %s' % (src_dir_b, bundles_dir))
# Package everything
with cd(build_dir):
run('rm -f %s' % env.build_archive)
run('tar -cvf - %s | gzip -9 -c > %s' %
(env.git_tag, env.build_archive))
def provision(choice):
"""
Provisions specified assets in the deployment.
choice: Indicates which assets to provision:
'build' -> provision the build machine
'web' -> provision the web instances
'all' -> provision everything
"""
assets = _validate_asset_choice(choice)
require('configuration')
logger.info("Provisioning begins: %s.", assets)
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(configuration)
dep.Deploy(assets)
if 'build' in assets:
logger.info("Installing sofware on the build machine.")
execute(provision_build)
if 'web' in assets:
logger.info("Installing sofware on web instances.")
execute(provision_web)
logger.info("Provisioning is complete.")
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(
env.label,
env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
# make sure we remove old version, added 2/20/17 can remove a while after that
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip uninstall django-storages && pip uninstall django-storages-redux',
warn_only=True)
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install --upgrade pip && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
)
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
# Adding compute worker upstart config
broker_url = cfg.get_broker_url()
compute_worker_conf_template = open(
'{}/configs/upstart/codalab-compute-worker.conf'.format(
current_directory)).read()
compute_worker_conf = compute_worker_conf_template.format(
broker_url=broker_url)
compute_worker_conf_buf = StringIO()
compute_worker_conf_buf.write(compute_worker_conf)
put(
local_path=
compute_worker_conf_buf, # Path can be a file-like object, in this case our processed template
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True)
# Adding codalab monitor upstart config ### No longer needed!
# put(
# local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
# remote_path='/etc/init/codalab-monitor.conf',
# use_sudo=True
# )
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor"
) # just in case it's left from a previous install
sudo("start codalab-compute-worker")