def deploy():
maintenance("begin")
supervisor_stop()
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd("deploy/codalab"):
run("git pull")
run("pip install -r requirements/dev_azure_nix.txt")
run("python manage.py syncdb --migrate")
run("python manage.py collectstatic --noinput")
# Generate config
run("python manage.py config_gen")
run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
# run('python scripts/initialize.py') # maybe not needed
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())
# Setup bundle service for worksheets
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd("deploy/bundles"):
run("git pull")
run("alembic upgrade head")
supervisor()
maintenance("end")
def _deploy():
# Update competition website
# Pull branch and run requirements file, for info about requirments look into dev_setp.sh
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
# local.py is generated here. For more info about content look into deploy/__.init__.py
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab',
'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
# For more info look into https://github.com/greyside/django-config-gen
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Put nginx and supervisor configuration files in place, ln creates symbolic links
sudo(
'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
)
sudo(
'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
)
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' %
cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(
cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(),
cfg.getSslCertificateInstalledPath(),
use_sudo=True)
put(cfg.getSslCertificateKeyPath(),
cfg.getSslCertificateKeyInstalledPath(),
use_sudo=True)
else:
logger.info(
"Skipping certificate installation because both files are not specified."
)
def config(label):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup (e.g., prod, test, etc.)
"""
env.cfg_label = label
print "Deployment label is:", env.cfg_label
print "Loading configuration from:", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s",
configuration.getFilename())
env.roledefs = {'web': configuration.getWebHostnames()}
# Credentials
env.user = configuration.getVirtualMachineLogonUsername()
# Repository
env.git_codalab_tag = configuration.getGitTag()
env.deploy_codalab_dir = 'codalab-competitions' # Directory for codalab competitions
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration(
) # Prod or Test
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(
configuration.getServiceName())
print "Deployment configuration is for:", env.config_server_name
env.configuration = True
env.SHELL_ENV = {}
def _deploy():
# Update website
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Update bundle service
with cd(env.deploy_codalab_cli_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_cli_tag)
run('./setup.sh')
run('venv/bin/pip install MySQL-Python')
run('venv/bin/alembic upgrade head')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Create a superuser for OAuth
run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
# Allow bundle service to connect to website for OAuth
run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
# Put nginx and supervisor configuration files in place
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def install_config():
'''
Install configuration files (do multiple times).
'''
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
run('python manage.py config_gen')
run('mkdir -p ~/.codalab && python scripts/set-oauth-key.py ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def put_mysql_dump_to_new_database():
'''Puts dubmped database to new location'''
require('configuration')
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
db_host = "localhost"
db_database = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
backup_directory = os.path.dirname(os.path.realpath(__file__))
put(local_path='{}/competitiondump.sql.gz'.format(backup_directory),
remote_path='/home/azureuser/db_dump.sql.gz',
use_sudo=True)
def config(label):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup (e.g., prod, test, etc.)
"""
env.cfg_label = label
print "Deployment label is:", env.cfg_label
print "Loading configuration from:", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.roledefs = {'web': configuration.getWebHostnames()}
# Credentials
env.user = configuration.getVirtualMachineLogonUsername()
# Repository
env.git_codalab_tag = configuration.getGitTag()
env.deploy_codalab_dir = 'codalab-competitions' # Directory for codalab competitions
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration() # Prod or Test
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
print "Deployment configuration is for:", env.config_server_name
env.configuration = True
env.SHELL_ENV = {}
def put_mysql_dump_to_new_database():
'''Puts dubmped database to new location'''
require('configuration')
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
db_host = "localhost"
db_database = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
backup_directory = os.path.dirname(os.path.realpath(__file__))
put(local_path='{}/competitiondump.sql.gz'.format(backup_directory),
remote_path='/home/azureuser/db_dump.sql.gz',
use_sudo=True)
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(
env.label,
env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
)
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
put(local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(
current_directory),
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True)
put(local_path='{}/configs/upstart/codalab-monitor.conf'.format(
current_directory),
remote_path='/etc/init/codalab-monitor.conf',
use_sudo=True)
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor")
sudo("start codalab-compute-worker")
sudo("start codalab-monitor")
def get_database_dump():
'''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz'''
require('configuration')
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
db_host = "localhost"
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
dump_file_name = 'competitiondump.sql.gz'
run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s'
% (db_host, db_user, db_password, db_name, dump_file_name))
backup_directory = os.path.dirname(os.path.realpath(__file__))
get('%s' % dump_file_name, backup_directory)
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt')
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
put(
local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(current_directory),
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True
)
put(
local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
remote_path='/etc/init/codalab-monitor.conf',
use_sudo=True
)
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor")
sudo("start codalab-compute-worker")
sudo("start codalab-monitor")
def get_database_dump():
'''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz'''
require('configuration')
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
db_host = "localhost"
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
dump_file_name = 'launchdump-%s.sql.gz' % datetime.datetime.now().strftime(
'%Y-%m-%d-%H-%M-%S')
run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s'
% (db_host, db_user, db_password, db_name, dump_file_name))
backup_dir = os.environ.get("CODALAB_MYSQL_BACKUP_DIR", "")
get('/tmp/%s' % dump_file_name, backup_dir)
def enable_cors():
"""
Enable cross-origin resource sharing for a Windows Azure storage service.
"""
require('configuration')
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
dep.ensureStorageHasCorsConfiguration()
def install_mysql(choice='all'):
"""
Installs a local instance of MySQL of the web instance. This will only work
if the number of web instances is one.
choice: Indicates which assets to create/install:
'mysql' -> just install MySQL; don't create the databases
'website_db' -> just create the website database
'bundles_db' -> just create the bundle service database
'all' or '' -> install everything
"""
require('configuration')
if len(env.roledefs['web']) != 1:
raise Exception("Task install_mysql requires exactly one web instance.")
if choice == 'mysql':
choices = {'mysql'}
elif choice == 'website_db':
choices = {'website_db'}
elif choice == 'bundles_db':
choices = {'bundles_db'}
elif choice == 'all':
choices = {'mysql', 'website_db', 'bundles_db'}
else:
raise ValueError("Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." % (choice))
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dba_password = configuration.getDatabaseAdminPassword()
if 'mysql' in choices:
sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server')
sudo('mysqladmin -u root password {0}'.format(dba_password))
if 'website_db' in choices:
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
cmds = ["create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user)]
run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
if 'bundles_db' in choices:
db_name = configuration.getBundleServiceDatabaseName()
db_user = configuration.getBundleServiceDatabaseUser()
db_password = configuration.getBundleServiceDatabasePassword()
cmds = ["create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user)]
run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
def get_database_dump():
"""Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz"""
require("configuration")
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
db_host = "localhost"
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
dump_file_name = "launchdump-%s.sql.gz" % datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
run(
"mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s"
% (db_host, db_user, db_password, db_name, dump_file_name)
)
backup_dir = os.environ.get("CODALAB_MYSQL_BACKUP_DIR", "")
get("/tmp/%s" % dump_file_name, backup_dir)
def install_mysql(choice="all"):
"""
Installs a local instance of MySQL of the web instance. This will only work
if the number of web instances is one.
choice: Indicates which assets to create/install:
'mysql' -> just install MySQL; don't create the databases
'site_db' -> just create the site database
'bundles_db' -> just create the bundle service database
'all' or '' -> install all three
"""
require("configuration")
if len(env.roledefs["web"]) != 1:
raise Exception("Task install_mysql requires exactly one web instance.")
if choice == "mysql":
choices = {"mysql"}
elif choice == "site_db":
choices = {"site_db"}
elif choice == "bundles_db":
choices = {"bundles_db"}
elif choice == "all":
choices = {"mysql", "site_db", "bundles_db"}
else:
raise ValueError("Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." % (choice))
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dba_password = configuration.getDatabaseAdminPassword()
if "mysql" in choices:
sudo("DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server")
sudo("mysqladmin -u root password {0}".format(dba_password))
if "site_db" in choices:
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
cmds = [
"create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user),
]
run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
if "bundles_db" in choices:
db_name = configuration.getBundleServiceDatabaseName()
db_user = configuration.getBundleServiceDatabaseUser()
db_password = configuration.getBundleServiceDatabasePassword()
cmds = [
"create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user),
]
run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
def install_ssl_certificates():
"""
Installs SSL certificates on the web instance.
"""
require('configuration')
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def deploy_web():
"""
Installs the output of the build on the web instances.
"""
require('configuration')
if exists(env.deploy_dir):
run('rm -rf %s' % env.deploy_dir)
run('tar -xvzf %s' % env.build_archive)
run('mv %s deploy' % env.git_tag)
run('source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv')
env.SHELL_ENV = dict(DJANGO_SETTINGS_MODULE=env.django_settings_module,
DJANGO_CONFIGURATION=env.django_configuration,
CONFIG_HTTP_PORT=env.config_http_port,
CONFIG_SERVER_NAME=env.config_server_name)
print env.SHELL_ENV
with cd(env.deploy_dir):
with prefix('source /usr/local/bin/virtualenvwrapper.sh && workon venv'
), shell_env(**env.SHELL_ENV):
requirements_path = "/".join(
['codalab', 'requirements', 'dev_azure_nix.txt'])
pip_cmd = 'pip install -r {0}'.format(requirements_path)
run(pip_cmd)
# additional requirements for bundle service
run('pip install SQLAlchemy simplejson')
with cd('codalab'):
run('python manage.py config_gen')
run('mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json'
)
run('python manage.py syncdb --migrate')
run('python scripts/initialize.py')
run('python manage.py collectstatic --noinput')
sudo(
'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
)
sudo(
'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
)
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run('newrelic-admin generate-config %s newrelic.ini' %
cfg.getNewRelicKey())
def get_database_dump():
'''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz'''
require('configuration')
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
db_host = "localhost"
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
dump_file_name = 'competitiondump.sql.gz'
run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s' % (
db_host,
db_user,
db_password,
db_name,
dump_file_name)
)
backup_directory = os.path.dirname(os.path.realpath(__file__))
get('%s' % dump_file_name, backup_directory)
def install_ssl_certificates():
"""
Installs SSL certificates on the web instance.
"""
require("configuration")
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def install_mysql():
"""
Installs a local instance of MySQL of the web instance. This will only work
if the number of web instances is one.
"""
require('configuration')
if len(env.roledefs['web']) <> 1:
raise(Exception("Task install_mysql requires exactly one web instance."))
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dba_password = configuration.getDatabaseAdminPassword()
db_name = configuration.getDatabaseName()
db_user = configuration.getDatabaseUser()
db_password = configuration.getDatabasePassword()
sudo('DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server')
sudo('mysqladmin -u root password {0}'.format(dba_password))
cmds = ["create database {0};".format(db_name),
"create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
"GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user) ]
run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
def _deploy():
# Update competition website
# Pull branch and run requirements file, for info about requirments look into dev_setp.sh
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git pull')
run('git checkout %s' % env.git_codalab_tag)
run('./dev_setup.sh')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
# local.py is generated here. For more info about content look into deploy/__.init__.py
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
# For more info look into https://github.com/greyside/django-config-gen
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Put nginx and supervisor configuration files in place, ln creates symbolic links
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def deploy_web():
"""
Installs the output of the build on the web instances.
"""
require("configuration")
if exists(env.deploy_dir):
run("rm -rf %s" % env.deploy_dir)
run("tar -xvzf %s" % env.build_archive)
run("mv %s deploy" % env.git_tag)
run("source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv")
env.SHELL_ENV = dict(
DJANGO_SETTINGS_MODULE=env.django_settings_module,
DJANGO_CONFIGURATION=env.django_configuration,
CONFIG_HTTP_PORT=env.config_http_port,
CONFIG_SERVER_NAME=env.config_server_name,
)
print env.SHELL_ENV
with cd(env.deploy_dir):
with prefix("source /usr/local/bin/virtualenvwrapper.sh && workon venv"), shell_env(**env.SHELL_ENV):
requirements_path = "/".join(["codalab", "requirements", "dev_azure_nix.txt"])
pip_cmd = "pip install -r {0}".format(requirements_path)
run(pip_cmd)
# additional requirements for bundle service
run("pip install SQLAlchemy simplejson")
with cd("codalab"):
run("python manage.py config_gen")
run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
run("python manage.py syncdb --migrate")
run("python scripts/initialize.py")
run("python manage.py collectstatic --noinput")
sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
# Setup new relic
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())
def teardown(choice):
"""
Deletes specified assets in the deployment. Be careful: there is no undoing!
choice: Indicates which assets to delete:
'build' -> provision the build machine
'web' -> provision the web instances
'all' -> provision everything
"""
assets = _validate_asset_choice(choice)
require('configuration')
logger.info("Teardown begins: %s.", assets)
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(configuration)
dep.Teardown(assets)
logger.info("Teardown is complete.")
def build():
"""
Builds artifacts to install on the deployment instances.
"""
require('configuration')
# Assemble source and configurations for the web site
build_dir = "/".join(['builds', env.git_user, env.git_repo])
src_dir = "/".join([build_dir, env.git_tag])
if exists(src_dir):
run('rm -rf %s' % (src_dir.rstrip('/')))
with settings(warn_only=True):
run('mkdir -p %s' % src_dir)
with cd(src_dir):
# TODO: why do we have the --branch and --single-branch tags here, this causes problems
run('git clone --depth=1 --branch %s --single-branch %s .' %
(env.git_tag, env.git_repo_url))
# Generate settings file (local.py)
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(configuration)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = "/".join(
['codalab', 'codalab', 'settings', 'local.py'])
put(buf, settings_file)
# Assemble source and configurations for the bundle service
if len(env.git_bundles_repo_url) > 0:
build_dir_b = "/".join(
['builds', env.git_bundles_user, env.git_bundles_repo])
src_dir_b = "/".join([build_dir_b, env.git_bundles_tag])
if exists(src_dir_b):
run('rm -rf %s' % (src_dir_b.rstrip('/')))
with settings(warn_only=True):
run('mkdir -p %s' % src_dir_b)
with cd(src_dir_b):
# TODO: why do we have the --branch and --single-branch tags here, this causes problems
run('git clone --depth=1 --branch %s --single-branch %s .' %
(env.git_bundles_tag, env.git_bundles_repo_url))
# Replace current bundles dir in main CodaLab other bundles repo.
bundles_dir = "/".join([src_dir, 'bundles'])
run('rm -rf %s' % (bundles_dir.rstrip('/')))
run('mv %s %s' % (src_dir_b, bundles_dir))
# Package everything
with cd(build_dir):
run('rm -f %s' % env.build_archive)
run('tar -cvf - %s | gzip -9 -c > %s' %
(env.git_tag, env.build_archive))
def provision(choice):
"""
Provisions specified assets in the deployment.
choice: Indicates which assets to provision:
'build' -> provision the build machine
'web' -> provision the web instances
'all' -> provision everything
"""
assets = _validate_asset_choice(choice)
require('configuration')
logger.info("Provisioning begins: %s.", assets)
configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(configuration)
dep.Deploy(assets)
if 'build' in assets:
logger.info("Installing sofware on the build machine.")
execute(provision_build)
if 'web' in assets:
logger.info("Installing sofware on web instances.")
execute(provision_web)
logger.info("Provisioning is complete.")
def _deploy():
# Update website
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('./dev_setup.sh')
# Update bundle service
with cd(env.deploy_codalab_cli_dir):
run('git checkout %s' % env.git_codalab_cli_tag)
run('git pull')
run('./setup.sh')
run('venv/bin/pip install MySQL-Python')
run('venv/bin/alembic upgrade head')
# Create local.py
cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.getSettingsFileContent())
settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
put(buf, settings_file)
# Update the website configuration
env_prefix, env_shell = setup_env()
with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
# Generate configuration files (bundle_server_config, nginx, etc.)
run('python manage.py config_gen')
# Migrate database
run('python manage.py syncdb --migrate')
# Create static pages
run('python manage.py collectstatic --noinput')
# For sending email, have the right domain name.
run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
# Create a superuser for OAuth
run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
# Allow bundle service to connect to website for OAuth
run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
# Put nginx and supervisor configuration files in place
sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
# Setup new relic
run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
# Install SSL certficates (/etc/ssl/certs/)
require('configuration')
if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
else:
logger.info("Skipping certificate installation because both files are not specified.")
def config(label=None):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup.
"""
env.cfg_label = label
print "Deployment label is: ", env.cfg_label
filename = ".codalabconfig"
if 'cfg_path' not in env:
env.cfg_path = os.path.join(os.getcwd(), filename)
if os.path.exists(env.cfg_path) == False:
env.cfg_path = os.path.join(os.path.expanduser("~"), filename)
print "Loading configuration from: ", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
env.roledefs = {'build' : [configuration.getBuildHostname()]}
if label is not None:
env.roledefs.update({'web' : configuration.getWebHostnames()})
# Information about main CodaLab repo
env.git_user = configuration.getGitUser()
env.git_repo = configuration.getGitRepo()
env.git_tag = configuration.getGitTag()
env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo)
# Information about Bundles repo
env.git_bundles_user = configuration.getBundleServiceGitUser()
env.git_bundles_repo = configuration.getBundleServiceGitRepo()
env.git_bundles_tag = configuration.getBundleServiceGitTag()
if len(configuration.getBundleServiceUrl()) > 0:
env.git_bundles_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_bundles_user, env.git_bundles_repo)
else:
env.git_bundles_repo_url = ''
env.deploy_dir = 'deploy'
env.build_archive = '{0}.tar.gz'.format(env.git_tag)
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration()
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
env.configuration = True
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(
env.label,
env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
# make sure we remove old version, added 2/20/17 can remove a while after that
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip uninstall django-storages && pip uninstall django-storages-redux',
warn_only=True)
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install --upgrade pip && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
)
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
# Adding compute worker upstart config
broker_url = cfg.get_broker_url()
compute_worker_conf_template = open(
'{}/configs/upstart/codalab-compute-worker.conf'.format(
current_directory)).read()
compute_worker_conf = compute_worker_conf_template.format(
broker_url=broker_url)
compute_worker_conf_buf = StringIO()
compute_worker_conf_buf.write(compute_worker_conf)
put(
local_path=
compute_worker_conf_buf, # Path can be a file-like object, in this case our processed template
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True)
# Adding codalab monitor upstart config ### No longer needed!
# put(
# local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
# remote_path='/etc/init/codalab-monitor.conf',
# use_sudo=True
# )
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor"
) # just in case it's left from a previous install
sudo("start codalab-compute-worker")
def config(label=None):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup.
"""
env.cfg_label = label
print "Deployment label is: ", env.cfg_label
filename = ".codalabconfig"
if "cfg_path" not in env:
env.cfg_path = os.path.join(os.getcwd(), filename)
if os.path.exists(env.cfg_path) == False:
env.cfg_path = os.path.join(os.path.expanduser("~"), filename)
print "Loading configuration from: ", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
env.roledefs = {"build": [configuration.getBuildHostname()]}
if label is not None:
env.roledefs.update({"web": configuration.getWebHostnames()})
# Information about main CodaLab repo
env.git_user = configuration.getGitUser()
env.git_repo = configuration.getGitRepo()
env.git_tag = configuration.getGitTag()
env.git_repo_url = "https://github.com/{0}/{1}.git".format(env.git_user, env.git_repo)
# Information about Bundles repo
env.git_bundles_user = configuration.getBundleServiceGitUser()
env.git_bundles_repo = configuration.getBundleServiceGitRepo()
env.git_bundles_tag = configuration.getBundleServiceGitTag()
if len(configuration.getBundleServiceUrl()) > 0:
env.git_bundles_repo_url = "https://github.com/{0}/{1}.git".format(
env.git_bundles_user, env.git_bundles_repo
)
else:
env.git_bundles_repo_url = ""
env.deploy_dir = "deploy"
env.build_archive = "{0}.tar.gz".format(env.git_tag)
env.django_settings_module = "codalab.settings"
env.django_configuration = configuration.getDjangoConfiguration()
env.config_http_port = "80"
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
env.configuration = True
def config(label=None):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup.
"""
env.cfg_label = label
print "Deployment label is: ", env.cfg_label
if 'cfg_path' not in env:
env.cfg_path = os.path.join(os.environ['HOME'], '.codalabconfig')
print "Loading configuration from: ", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
env.roledefs = { 'build' : [ configuration.getBuildHostname() ] }
if label is not None:
env.roledefs.update({ 'web' : configuration.getWebHostnames() })
env.git_user = configuration.getGitUser()
env.git_repo = configuration.getGitRepo()
env.git_tag = configuration.getGitTag()
env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo)
env.deploy_dir = 'deploy'
env.build_archive = '{0}.tar.gz'.format(env.git_tag)
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration()
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
env.configuration = True
