def install_config(): ''' Install configuration files (do multiple times). ''' # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): run('python manage.py config_gen') run('mkdir -p ~/.codalab && python scripts/set-oauth-key.py ./config/generated/bundle_server_config.json > ~/.codalab/config.json') sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def build(): """ Builds artifacts to install on the deployment instances. """ require('configuration') #pip_cache_dir = "/".join(['builds', 'pip_cache']) #if not exists(pip_cache_dir): # run('mkdir -p %s' % pip_cache_dir) #with cd(pip_cache_dir): # pip_download_cache = run('pwd') build_dir = "/".join(['builds', env.git_user, env.git_repo]) src_dir = "/".join([build_dir, env.git_tag]) if exists(src_dir): run('rm -rf %s' % (src_dir.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir) with cd(src_dir): run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_tag, env.git_repo_url)) # Generate settings file (local.py) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = "/".join(['codalab', 'codalab', 'settings', 'local.py']) put(buf, settings_file) wheel_dir = "/".join([src_dir, 'wheel_packages']) requirements_dir = "/".join([src_dir, 'codalab', 'requirements']) #run('pip wheel --download-cache %s --wheel-dir=%s -r %s/dev_azure_nix.txt' % (pip_download_cache, # wheel_dir, # requirements_dir)) with cd(build_dir): run('rm -f %s' % env.build_archive) run('tar -cvf - %s | gzip -9 -c > %s' % (env.git_tag, env.build_archive))
def _deploy(): # Update competition website # Pull branch and run requirements file, for info about requirments look into dev_setp.sh env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git pull') run('git checkout %s' % env.git_codalab_tag) run('./dev_setup.sh') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) # local.py is generated here. For more info about content look into deploy/__.init__.py settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) # For more info look into https://github.com/greyside/django-config-gen run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Put nginx and supervisor configuration files in place, ln creates symbolic links sudo( 'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf' ) sudo( 'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf' ) # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len( cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info( "Skipping certificate installation because both files are not specified." )
def _deploy(): # Update website env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git pull') run('git checkout %s' % env.git_codalab_tag) run('./dev_setup.sh') # Update bundle service with cd(env.deploy_codalab_cli_dir): run('git pull') run('git checkout %s' % env.git_codalab_cli_tag) run('./setup.sh') run('venv/bin/pip install MySQL-Python') run('venv/bin/alembic upgrade head') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Create a superuser for OAuth run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword()) # Allow bundle service to connect to website for OAuth run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json') # Put nginx and supervisor configuration files in place sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def _deploy(): # Update website env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') run('./dev_setup.sh') # Update bundle service with cd(env.deploy_codalab_cli_dir): run('git checkout %s' % env.git_codalab_cli_tag) run('git pull') run('./setup.sh') run('venv/bin/pip install MySQL-Python') run('venv/bin/alembic upgrade head') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Create a superuser for OAuth run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword()) # Allow bundle service to connect to website for OAuth run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json') # Put nginx and supervisor configuration files in place sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def build(): """ Builds artifacts to install on the deployment instances. """ require('configuration') # Assemble source and configurations for the web site build_dir = "/".join(['builds', env.git_user, env.git_repo]) src_dir = "/".join([build_dir, env.git_tag]) if exists(src_dir): run('rm -rf %s' % (src_dir.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir) with cd(src_dir): # TODO: why do we have the --branch and --single-branch tags here, this causes problems run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_tag, env.git_repo_url)) # Generate settings file (local.py) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = "/".join( ['codalab', 'codalab', 'settings', 'local.py']) put(buf, settings_file) # Assemble source and configurations for the bundle service if len(env.git_bundles_repo_url) > 0: build_dir_b = "/".join( ['builds', env.git_bundles_user, env.git_bundles_repo]) src_dir_b = "/".join([build_dir_b, env.git_bundles_tag]) if exists(src_dir_b): run('rm -rf %s' % (src_dir_b.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir_b) with cd(src_dir_b): # TODO: why do we have the --branch and --single-branch tags here, this causes problems run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_bundles_tag, env.git_bundles_repo_url)) # Replace current bundles dir in main CodaLab other bundles repo. bundles_dir = "/".join([src_dir, 'bundles']) run('rm -rf %s' % (bundles_dir.rstrip('/'))) run('mv %s %s' % (src_dir_b, bundles_dir)) # Package everything with cd(build_dir): run('rm -f %s' % env.build_archive) run('tar -cvf - %s | gzip -9 -c > %s' % (env.git_tag, env.build_archive))
def build(): """ Builds artifacts to install on the deployment instances. """ require("configuration") # Assemble source and configurations for the web site build_dir = "/".join(["builds", env.git_user, env.git_repo]) src_dir = "/".join([build_dir, env.git_tag]) if exists(src_dir): run("rm -rf %s" % (src_dir.rstrip("/"))) with settings(warn_only=True): run("mkdir -p %s" % src_dir) with cd(src_dir): # TODO: why do we have the --branch and --single-branch tags here, this causes problems run("git clone --depth=1 --branch %s --single-branch %s ." % (env.git_tag, env.git_repo_url)) # Generate settings file (local.py) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = "/".join(["codalab", "codalab", "settings", "local.py"]) put(buf, settings_file) # Assemble source and configurations for the bundle service if len(env.git_bundles_repo_url) > 0: build_dir_b = "/".join(["builds", env.git_bundles_user, env.git_bundles_repo]) src_dir_b = "/".join([build_dir_b, env.git_bundles_tag]) if exists(src_dir_b): run("rm -rf %s" % (src_dir_b.rstrip("/"))) with settings(warn_only=True): run("mkdir -p %s" % src_dir_b) with cd(src_dir_b): # TODO: why do we have the --branch and --single-branch tags here, this causes problems run( "git clone --depth=1 --branch %s --single-branch %s ." % (env.git_bundles_tag, env.git_bundles_repo_url) ) # Replace current bundles dir in main CodaLab other bundles repo. bundles_dir = "/".join([src_dir, "bundles"]) run("rm -rf %s" % (bundles_dir.rstrip("/"))) run("mv %s %s" % (src_dir_b, bundles_dir)) # Package everything with cd(build_dir): run("rm -f %s" % env.build_archive) run("tar -cvf - %s | gzip -9 -c > %s" % (env.git_tag, env.build_archive))
def _deploy(): # Update competition website # Pull branch and run requirements file, for info about requirments look into dev_setp.sh env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir): run('git pull') run('git checkout %s' % env.git_codalab_tag) run('./dev_setup.sh') # Create local.py cfg = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(cfg) buf = StringIO() buf.write(dep.getSettingsFileContent()) # local.py is generated here. For more info about content look into deploy/__.init__.py settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py') put(buf, settings_file) # Update the website configuration env_prefix, env_shell = setup_env() with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'): # Generate configuration files (bundle_server_config, nginx, etc.) # For more info look into https://github.com/greyside/django-config-gen run('python manage.py config_gen') # Migrate database run('python manage.py syncdb --migrate') # Create static pages run('python manage.py collectstatic --noinput') # For sending email, have the right domain name. run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0]) # Put nginx and supervisor configuration files in place, ln creates symbolic links sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf') sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf') # Setup new relic run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey()) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info("Skipping certificate installation because both files are not specified.")
def build(): """ Builds artifacts to install on the deployment instances. """ require('configuration') # Assemble source and configurations for the web site build_dir = "/".join(['builds', env.git_user, env.git_repo]) src_dir = "/".join([build_dir, env.git_tag]) if exists(src_dir): run('rm -rf %s' % (src_dir.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir) with cd(src_dir): run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_tag, env.git_repo_url)) # Generate settings file (local.py) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dep = Deployment(configuration) buf = StringIO() buf.write(dep.getSettingsFileContent()) settings_file = "/".join(['codalab', 'codalab', 'settings', 'local.py']) put(buf, settings_file) # Assemble source and configurations for the bundle service if len(env.git_bundles_repo_url) > 0: build_dir_b = "/".join(['builds', env.git_bundles_user, env.git_bundles_repo]) src_dir_b = "/".join([build_dir_b, env.git_bundles_tag]) if exists(src_dir_b): run('rm -rf %s' % (src_dir_b.rstrip('/'))) with settings(warn_only=True): run('mkdir -p %s' % src_dir_b) with cd(src_dir_b): run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_bundles_tag, env.git_bundles_repo_url)) # Replace current bundles dir in main CodaLab other bundles repo. bundles_dir = "/".join([src_dir, 'bundles']) run('rm -rf %s' % (bundles_dir.rstrip('/'))) run('mv %s %s' % (src_dir_b, bundles_dir)) # Package everything with cd(build_dir): run('rm -f %s' % env.build_archive) run('tar -cvf - %s | gzip -9 -c > %s' % (env.git_tag, env.build_archive))