def deploy(): maintenance("begin") supervisor_stop() env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd("deploy/codalab"): run("git pull") run("pip install -r requirements/dev_azure_nix.txt") run("python manage.py syncdb --migrate") run("python manage.py collectstatic --noinput") # Generate config run("python manage.py config_gen") run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json") sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf") sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf") # run('python scripts/initialize.py') # maybe not needed # Setup new relic cfg = DeploymentConfig(env.cfg_label, env.cfg_path) run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey()) # Setup bundle service for worksheets env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd("deploy/bundles"): run("git pull") run("alembic upgrade head") supervisor() maintenance("end")
def _deploy(): # Update competition website # Pull branch and run requirements file, for info about requirments look into dev_setp.sh env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git pull') run('git checkout %s' % env.git_codalab_tag) run('./dev_setup.sh') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) # local.py is generated here. For more info about content look into deploy/__.init__.py settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) # For more info look into https://github.com/greyside/django-config-gen run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Put nginx and supervisor configuration files in place, ln creates symbolic links sudo( 'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf' ) sudo( 'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf' ) # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len( cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info( "Skipping certificate installation because both files are not specified." )
def config(label): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup (e.g., prod, test, etc.) """ env.cfg_label = label print "Deployment label is:", env.cfg_label print "Loading configuration from:", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.roledefs = {'web': configuration.getWebHostnames()} # Credentials env.user = configuration.getVirtualMachineLogonUsername() # Repository env.git_codalab_tag = configuration.getGitTag() env.deploy_codalab_dir = 'codalab-competitions' # Directory for codalab competitions env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration( ) # Prod or Test env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format( configuration.getServiceName()) print "Deployment configuration is for:", env.config_server_name env.configuration = True env.SHELL_ENV = {}
def _deploy(): # Update website env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git pull') run('git checkout %s' % env.git_codalab_tag) run('./dev_setup.sh') # Update bundle service with cd(env.deploy_codalab_cli_dir): run('git pull') run('git checkout %s' % env.git_codalab_cli_tag) run('./setup.sh') run('venv/bin/pip install MySQL-Python') run('venv/bin/alembic upgrade head') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Create a superuser for OAuth run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword()) # Allow bundle service to connect to website for OAuth run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json') # Put nginx and supervisor configuration files in place sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def install_config(): ''' Install configuration files (do multiple times). ''' # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): run('python manage.py config_gen') run('mkdir -p ~/.codalab && python scripts/set-oauth-key.py ./config/generated/bundle_server_config.json > ~/.codalab/config.json') sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def put_mysql_dump_to_new_database(): '''Puts dubmped database to new location''' require('configuration') configuration = DeploymentConfig(env.cfg_label, env.cfg_path) db_host = "localhost" db_database = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() backup_directory = os.path.dirname(os.path.realpath(__file__)) put(local_path='{}/competitiondump.sql.gz'.format(backup_directory), remote_path='/home/azureuser/db_dump.sql.gz', use_sudo=True)
def config(label): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup (e.g., prod, test, etc.) """ env.cfg_label = label print "Deployment label is:", env.cfg_label print "Loading configuration from:", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.roledefs = {'web': configuration.getWebHostnames()} # Credentials env.user = configuration.getVirtualMachineLogonUsername() # Repository env.git_codalab_tag = configuration.getGitTag() env.deploy_codalab_dir = 'codalab-competitions' # Directory for codalab competitions env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration() # Prod or Test env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) print "Deployment configuration is for:", env.config_server_name env.configuration = True env.SHELL_ENV = {}
def deploy_compute_worker(label): ''' Deploy/update compute workers. For monitoring make sure the azure instance has the port 8000 forwarded :param label: Either test or prod ''' env.deploy_codalab_dir = 'codalab-competitions' # Create .codalabconfig within home directory env.label = label cfg = DeploymentConfig( env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig') dep = Deployment(cfg) buf = StringIO() buf.write(dep.get_compute_workers_file_content()) settings_file = os.path.join('~', '.codalabconfig') put(buf, settings_file) env.git_codalab_tag = cfg.getGitTag() env.logs_password = cfg.get_compute_worker_logs_password() # Initial setup with cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt' ) # run('./dev_setup.sh') # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8") current_directory = os.path.dirname(os.path.realpath(__file__)) put(local_path='{}/configs/upstart/codalab-compute-worker.conf'.format( current_directory), remote_path='/etc/init/codalab-compute-worker.conf', use_sudo=True) put(local_path='{}/configs/upstart/codalab-monitor.conf'.format( current_directory), remote_path='/etc/init/codalab-monitor.conf', use_sudo=True) # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password) with settings(warn_only=True): sudo("stop codalab-compute-worker") sudo("stop codalab-monitor") sudo("start codalab-compute-worker") sudo("start codalab-monitor")
def get_database_dump(): '''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz''' require('configuration') configuration = DeploymentConfig(env.cfg_label, env.cfg_path) db_host = "localhost" db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() dump_file_name = 'competitiondump.sql.gz' run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s' % (db_host, db_user, db_password, db_name, dump_file_name)) backup_directory = os.path.dirname(os.path.realpath(__file__)) get('%s' % dump_file_name, backup_directory)
def deploy_compute_worker(label): ''' Deploy/update compute workers. For monitoring make sure the azure instance has the port 8000 forwarded :param label: Either test or prod ''' env.deploy_codalab_dir = 'codalab-competitions' # Create .codalabconfig within home directory env.label = label cfg = DeploymentConfig(env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig') dep = Deployment(cfg) buf = StringIO() buf.write(dep.get_compute_workers_file_content()) settings_file = os.path.join('~', '.codalabconfig') put(buf, settings_file) env.git_codalab_tag = cfg.getGitTag() env.logs_password = cfg.get_compute_worker_logs_password() # Initial setup with cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt') # run('./dev_setup.sh') # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8") current_directory = os.path.dirname(os.path.realpath(__file__)) put( local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(current_directory), remote_path='/etc/init/codalab-compute-worker.conf', use_sudo=True ) put( local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory), remote_path='/etc/init/codalab-monitor.conf', use_sudo=True ) # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password) with settings(warn_only=True): sudo("stop codalab-compute-worker") sudo("stop codalab-monitor") sudo("start codalab-compute-worker") sudo("start codalab-monitor")
def get_database_dump(): '''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz''' require('configuration') configuration = DeploymentConfig(env.cfg_label, env.cfg_path) db_host = "localhost" db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() dump_file_name = 'launchdump-%s.sql.gz' % datetime.datetime.now().strftime( '%Y-%m-%d-%H-%M-%S') run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s' % (db_host, db_user, db_password, db_name, dump_file_name)) backup_dir = os.environ.get("CODALAB_MYSQL_BACKUP_DIR", "") get('/tmp/%s' % dump_file_name, backup_dir)
def enable_cors(): """ Enable cross-origin resource sharing for a Windows Azure storage service. """ require('configuration') cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) dep.ensureStorageHasCorsConfiguration()
def install_mysql(choice='all'): """ Installs a local instance of MySQL of the web instance. This will only work if the number of web instances is one. choice: Indicates which assets to create/install: 'mysql' -> just install MySQL; don't create the databases 'website_db' -> just create the website database 'bundles_db' -> just create the bundle service database 'all' or '' -> install everything """ require('configuration') if len(env.roledefs['web']) != 1: raise Exception("Task install_mysql requires exactly one web instance.") if choice == 'mysql': choices = {'mysql'} elif choice == 'website_db': choices = {'website_db'} elif choice == 'bundles_db': choices = {'bundles_db'} elif choice == 'all': choices = {'mysql', 'website_db', 'bundles_db'} else: raise ValueError("Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." % (choice)) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dba_password = configuration.getDatabaseAdminPassword() if 'mysql' in choices: sudo('DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server') sudo('mysqladmin -u root password {0}'.format(dba_password)) if 'website_db' in choices: db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() cmds = ["create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user)] run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds))) if 'bundles_db' in choices: db_name = configuration.getBundleServiceDatabaseName() db_user = configuration.getBundleServiceDatabaseUser() db_password = configuration.getBundleServiceDatabasePassword() cmds = ["create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user)] run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
def get_database_dump(): """Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz""" require("configuration") configuration = DeploymentConfig(env.cfg_label, env.cfg_path) db_host = "localhost" db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() dump_file_name = "launchdump-%s.sql.gz" % datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") run( "mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s" % (db_host, db_user, db_password, db_name, dump_file_name) ) backup_dir = os.environ.get("CODALAB_MYSQL_BACKUP_DIR", "") get("/tmp/%s" % dump_file_name, backup_dir)
def install_mysql(choice="all"): """ Installs a local instance of MySQL of the web instance. This will only work if the number of web instances is one. choice: Indicates which assets to create/install: 'mysql' -> just install MySQL; don't create the databases 'site_db' -> just create the site database 'bundles_db' -> just create the bundle service database 'all' or '' -> install all three """ require("configuration") if len(env.roledefs["web"]) != 1: raise Exception("Task install_mysql requires exactly one web instance.") if choice == "mysql": choices = {"mysql"} elif choice == "site_db": choices = {"site_db"} elif choice == "bundles_db": choices = {"bundles_db"} elif choice == "all": choices = {"mysql", "site_db", "bundles_db"} else: raise ValueError("Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." % (choice)) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dba_password = configuration.getDatabaseAdminPassword() if "mysql" in choices: sudo("DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server") sudo("mysqladmin -u root password {0}".format(dba_password)) if "site_db" in choices: db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() cmds = [ "create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user), ] run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds))) if "bundles_db" in choices: db_name = configuration.getBundleServiceDatabaseName() db_user = configuration.getBundleServiceDatabaseUser() db_password = configuration.getBundleServiceDatabasePassword() cmds = [ "create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user), ] run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
def install_ssl_certificates(): """ Installs SSL certificates on the web instance. """ require('configuration') cfg = DeploymentConfig(env.cfg_label, env.cfg_path) if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def deploy_web(): """ Installs the output of the build on the web instances. """ require('configuration') if exists(env.deploy_dir): run('rm -rf %s' % env.deploy_dir) run('tar -xvzf %s' % env.build_archive) run('mv %s deploy' % env.git_tag) run('source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv') env.SHELL_ENV = dict(DJANGO_SETTINGS_MODULE=env.django_settings_module, DJANGO_CONFIGURATION=env.django_configuration, CONFIG_HTTP_PORT=env.config_http_port, CONFIG_SERVER_NAME=env.config_server_name) print env.SHELL_ENV with cd(env.deploy_dir): with prefix('source /usr/local/bin/virtualenvwrapper.sh && workon venv' ), shell_env(**env.SHELL_ENV): requirements_path = "/".join( ['codalab', 'requirements', 'dev_azure_nix.txt']) pip_cmd = 'pip install -r {0}'.format(requirements_path) run(pip_cmd) # additional requirements for bundle service run('pip install SQLAlchemy simplejson') with cd('codalab'): run('python manage.py config_gen') run('mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json' ) run('python manage.py syncdb --migrate') run('python scripts/initialize.py') run('python manage.py collectstatic --noinput') sudo( 'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf' ) sudo( 'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf' ) # Setup new relic cfg = DeploymentConfig(env.cfg_label, env.cfg_path) run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())
def get_database_dump(): '''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz''' require('configuration') configuration = DeploymentConfig(env.cfg_label, env.cfg_path) db_host = "localhost" db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() dump_file_name = 'competitiondump.sql.gz' run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s' % ( db_host, db_user, db_password, db_name, dump_file_name) ) backup_directory = os.path.dirname(os.path.realpath(__file__)) get('%s' % dump_file_name, backup_directory)
def install_ssl_certificates(): """ Installs SSL certificates on the web instance. """ require("configuration") cfg = DeploymentConfig(env.cfg_label, env.cfg_path) if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def install_mysql(): """ Installs a local instance of MySQL of the web instance. This will only work if the number of web instances is one. """ require('configuration') if len(env.roledefs['web']) <> 1: raise(Exception("Task install_mysql requires exactly one web instance.")) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dba_password = configuration.getDatabaseAdminPassword() db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() sudo('DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server') sudo('mysqladmin -u root password {0}'.format(dba_password)) cmds = ["create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user) ] run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))
def _deploy(): # Update competition website # Pull branch and run requirements file, for info about requirments look into dev_setp.sh env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git pull') run('git checkout %s' % env.git_codalab_tag) run('./dev_setup.sh') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) # local.py is generated here. For more info about content look into deploy/__.init__.py settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) # For more info look into https://github.com/greyside/django-config-gen run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Put nginx and supervisor configuration files in place, ln creates symbolic links sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def deploy_web(): """ Installs the output of the build on the web instances. """ require("configuration") if exists(env.deploy_dir): run("rm -rf %s" % env.deploy_dir) run("tar -xvzf %s" % env.build_archive) run("mv %s deploy" % env.git_tag) run("source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv") env.SHELL_ENV = dict( DJANGO_SETTINGS_MODULE=env.django_settings_module, DJANGO_CONFIGURATION=env.django_configuration, CONFIG_HTTP_PORT=env.config_http_port, CONFIG_SERVER_NAME=env.config_server_name, ) print env.SHELL_ENV with cd(env.deploy_dir): with prefix("source /usr/local/bin/virtualenvwrapper.sh && workon venv"), shell_env(**env.SHELL_ENV): requirements_path = "/".join(["codalab", "requirements", "dev_azure_nix.txt"]) pip_cmd = "pip install -r {0}".format(requirements_path) run(pip_cmd) # additional requirements for bundle service run("pip install SQLAlchemy simplejson") with cd("codalab"): run("python manage.py config_gen") run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json") run("python manage.py syncdb --migrate") run("python scripts/initialize.py") run("python manage.py collectstatic --noinput") sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf") sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf") # Setup new relic cfg = DeploymentConfig(env.cfg_label, env.cfg_path) run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())
def teardown(choice): """ Deletes specified assets in the deployment. Be careful: there is no undoing! choice: Indicates which assets to delete: 'build' -> provision the build machine 'web' -> provision the web instances 'all' -> provision everything """ assets = _validate_asset_choice(choice) require('configuration') logger.info("Teardown begins: %s.", assets) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) dep.Teardown(assets) logger.info("Teardown is complete.")
def build(): """ Builds artifacts to install on the deployment instances. """ require('configuration') # Assemble source and configurations for the web site build_dir = "/".join(['builds', env.git_user, env.git_repo]) src_dir = "/".join([build_dir, env.git_tag]) if exists(src_dir): run('rm -rf %s' % (src_dir.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir) with cd(src_dir): # TODO: why do we have the --branch and --single-branch tags here, this causes problems run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_tag, env.git_repo_url)) # Generate settings file (local.py) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = "/".join( ['codalab', 'codalab', 'settings', 'local.py']) put(buf, settings_file) # Assemble source and configurations for the bundle service if len(env.git_bundles_repo_url) > 0: build_dir_b = "/".join( ['builds', env.git_bundles_user, env.git_bundles_repo]) src_dir_b = "/".join([build_dir_b, env.git_bundles_tag]) if exists(src_dir_b): run('rm -rf %s' % (src_dir_b.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir_b) with cd(src_dir_b): # TODO: why do we have the --branch and --single-branch tags here, this causes problems run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_bundles_tag, env.git_bundles_repo_url)) # Replace current bundles dir in main CodaLab other bundles repo. bundles_dir = "/".join([src_dir, 'bundles']) run('rm -rf %s' % (bundles_dir.rstrip('/'))) run('mv %s %s' % (src_dir_b, bundles_dir)) # Package everything with cd(build_dir): run('rm -f %s' % env.build_archive) run('tar -cvf - %s | gzip -9 -c > %s' % (env.git_tag, env.build_archive))
def provision(choice): """ Provisions specified assets in the deployment. choice: Indicates which assets to provision: 'build' -> provision the build machine 'web' -> provision the web instances 'all' -> provision everything """ assets = _validate_asset_choice(choice) require('configuration') logger.info("Provisioning begins: %s.", assets) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) dep.Deploy(assets) if 'build' in assets: logger.info("Installing sofware on the build machine.") execute(provision_build) if 'web' in assets: logger.info("Installing sofware on web instances.") execute(provision_web) logger.info("Provisioning is complete.")
def _deploy(): # Update website env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') run('./dev_setup.sh') # Update bundle service with cd(env.deploy_codalab_cli_dir): run('git checkout %s' % env.git_codalab_cli_tag) run('git pull') run('./setup.sh') run('venv/bin/pip install MySQL-Python') run('venv/bin/alembic upgrade head') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Create a superuser for OAuth run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword()) # Allow bundle service to connect to website for OAuth run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json') # Put nginx and supervisor configuration files in place sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def config(label=None): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup. """ env.cfg_label = label print "Deployment label is: ", env.cfg_label filename = ".codalabconfig" if 'cfg_path' not in env: env.cfg_path = os.path.join(os.getcwd(), filename) if os.path.exists(env.cfg_path) == False: env.cfg_path = os.path.join(os.path.expanduser("~"), filename) print "Loading configuration from: ", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() env.roledefs = {'build' : [configuration.getBuildHostname()]} if label is not None: env.roledefs.update({'web' : configuration.getWebHostnames()}) # Information about main CodaLab repo env.git_user = configuration.getGitUser() env.git_repo = configuration.getGitRepo() env.git_tag = configuration.getGitTag() env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo) # Information about Bundles repo env.git_bundles_user = configuration.getBundleServiceGitUser() env.git_bundles_repo = configuration.getBundleServiceGitRepo() env.git_bundles_tag = configuration.getBundleServiceGitTag() if len(configuration.getBundleServiceUrl()) > 0: env.git_bundles_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_bundles_user, env.git_bundles_repo) else: env.git_bundles_repo_url = '' env.deploy_dir = 'deploy' env.build_archive = '{0}.tar.gz'.format(env.git_tag) env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration() env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) env.configuration = True
def deploy_compute_worker(label): ''' Deploy/update compute workers. For monitoring make sure the azure instance has the port 8000 forwarded :param label: Either test or prod ''' env.deploy_codalab_dir = 'codalab-competitions' # Create .codalabconfig within home directory env.label = label cfg = DeploymentConfig( env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig') dep = Deployment(cfg) buf = StringIO() buf.write(dep.get_compute_workers_file_content()) settings_file = os.path.join('~', '.codalabconfig') put(buf, settings_file) env.git_codalab_tag = cfg.getGitTag() env.logs_password = cfg.get_compute_worker_logs_password() # Initial setup with cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') # make sure we remove old version, added 2/20/17 can remove a while after that run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip uninstall django-storages && pip uninstall django-storages-redux', warn_only=True) run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install --upgrade pip && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt' ) # run('./dev_setup.sh') # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8") current_directory = os.path.dirname(os.path.realpath(__file__)) # Adding compute worker upstart config broker_url = cfg.get_broker_url() compute_worker_conf_template = open( '{}/configs/upstart/codalab-compute-worker.conf'.format( current_directory)).read() compute_worker_conf = compute_worker_conf_template.format( broker_url=broker_url) compute_worker_conf_buf = StringIO() compute_worker_conf_buf.write(compute_worker_conf) put( local_path= compute_worker_conf_buf, # Path can be a file-like object, in this case our processed template remote_path='/etc/init/codalab-compute-worker.conf', use_sudo=True) # Adding codalab monitor upstart config ### No longer needed! # put( # local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory), # remote_path='/etc/init/codalab-monitor.conf', # use_sudo=True # ) # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password) with settings(warn_only=True): sudo("stop codalab-compute-worker") sudo("stop codalab-monitor" ) # just in case it's left from a previous install sudo("start codalab-compute-worker")
def config(label=None): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup. """ env.cfg_label = label print "Deployment label is: ", env.cfg_label filename = ".codalabconfig" if "cfg_path" not in env: env.cfg_path = os.path.join(os.getcwd(), filename) if os.path.exists(env.cfg_path) == False: env.cfg_path = os.path.join(os.path.expanduser("~"), filename) print "Loading configuration from: ", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() env.roledefs = {"build": [configuration.getBuildHostname()]} if label is not None: env.roledefs.update({"web": configuration.getWebHostnames()}) # Information about main CodaLab repo env.git_user = configuration.getGitUser() env.git_repo = configuration.getGitRepo() env.git_tag = configuration.getGitTag() env.git_repo_url = "https://github.com/{0}/{1}.git".format(env.git_user, env.git_repo) # Information about Bundles repo env.git_bundles_user = configuration.getBundleServiceGitUser() env.git_bundles_repo = configuration.getBundleServiceGitRepo() env.git_bundles_tag = configuration.getBundleServiceGitTag() if len(configuration.getBundleServiceUrl()) > 0: env.git_bundles_repo_url = "https://github.com/{0}/{1}.git".format( env.git_bundles_user, env.git_bundles_repo ) else: env.git_bundles_repo_url = "" env.deploy_dir = "deploy" env.build_archive = "{0}.tar.gz".format(env.git_tag) env.django_settings_module = "codalab.settings" env.django_configuration = configuration.getDjangoConfiguration() env.config_http_port = "80" env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) env.configuration = True
def config(label=None): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup. """ env.cfg_label = label print "Deployment label is: ", env.cfg_label if 'cfg_path' not in env: env.cfg_path = os.path.join(os.environ['HOME'], '.codalabconfig') print "Loading configuration from: ", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() env.roledefs = { 'build' : [ configuration.getBuildHostname() ] } if label is not None: env.roledefs.update({ 'web' : configuration.getWebHostnames() }) env.git_user = configuration.getGitUser() env.git_repo = configuration.getGitRepo() env.git_tag = configuration.getGitTag() env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo) env.deploy_dir = 'deploy' env.build_archive = '{0}.tar.gz'.format(env.git_tag) env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration() env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) env.configuration = True