예제 #1
0
def build():
    """
    Builds artifacts to install on the deployment instances.
    """
    require('configuration')

    #pip_cache_dir = "/".join(['builds', 'pip_cache'])
    #if not exists(pip_cache_dir):
    #    run('mkdir -p %s' % pip_cache_dir)
    #with cd(pip_cache_dir):
    #    pip_download_cache = run('pwd')

    build_dir = "/".join(['builds', env.git_user, env.git_repo])
    src_dir = "/".join([build_dir, env.git_tag])
    if exists(src_dir):
        run('rm -rf %s' % (src_dir.rstrip('/')))
    with settings(warn_only=True):
        run('mkdir -p %s' % src_dir)
    with cd(src_dir):
        run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_tag, env.git_repo_url))
        # Generate settings file (local.py)
        configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
        dep = Deployment(configuration)
        buf = StringIO()
        buf.write(dep.getSettingsFileContent())
        settings_file = "/".join(['codalab', 'codalab', 'settings', 'local.py'])
        put(buf, settings_file)
    wheel_dir =  "/".join([src_dir, 'wheel_packages'])
    requirements_dir = "/".join([src_dir, 'codalab', 'requirements'])
    #run('pip wheel --download-cache %s --wheel-dir=%s -r %s/dev_azure_nix.txt' % (pip_download_cache,
    #                                                                              wheel_dir,
    #                                                                              requirements_dir))
    with cd(build_dir):
        run('rm -f %s' % env.build_archive)
        run('tar -cvf - %s | gzip -9 -c > %s' % (env.git_tag, env.build_archive))
예제 #2
0
def install_config():
    '''
    Install configuration files (do multiple times).
    '''
    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
            run('python manage.py config_gen')
            run('mkdir -p ~/.codalab && python scripts/set-oauth-key.py ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
            sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
            sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
            # Setup new relic
            run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")
예제 #3
0
def enable_cors():
    """
    Enable cross-origin resource sharing for a Windows Azure storage service.
    """
    require("configuration")
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    dep.ensureStorageHasCorsConfiguration()
예제 #4
0
def enable_cors():
    """
    Enable cross-origin resource sharing for a Windows Azure storage service.
    """
    require('configuration')
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    dep.ensureStorageHasCorsConfiguration()
예제 #5
0
def _deploy():
    # Update competition website
    # Pull branch and run requirements file, for info about requirments look into dev_setp.sh
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_tag)
        run('./dev_setup.sh')

    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    # local.py is generated here. For more info about content look into deploy/__.init__.py
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab',
                                 'settings', 'local.py')
    put(buf, settings_file)

    # Update the website configuration
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
        # Generate configuration files (bundle_server_config, nginx, etc.)
        # For more info look into https://github.com/greyside/django-config-gen
        run('python manage.py config_gen')
        # Migrate database
        run('python manage.py syncdb --migrate')
        # Create static pages
        run('python manage.py collectstatic --noinput')
        # For sending email, have the right domain name.
        run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
        # Put nginx and supervisor configuration files in place, ln creates symbolic links
        sudo(
            'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf'
        )
        sudo(
            'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf'
        )
        # Setup new relic
        run('newrelic-admin generate-config %s newrelic.ini' %
            cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(
            cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(),
            cfg.getSslCertificateInstalledPath(),
            use_sudo=True)
        put(cfg.getSslCertificateKeyPath(),
            cfg.getSslCertificateKeyInstalledPath(),
            use_sudo=True)
    else:
        logger.info(
            "Skipping certificate installation because both files are not specified."
        )
def _deploy():
    # Update website
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_tag)
        run('./dev_setup.sh')

    # Update bundle service
    with cd(env.deploy_codalab_cli_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_cli_tag)
        run('./setup.sh')
        run('venv/bin/pip install MySQL-Python')
        run('venv/bin/alembic upgrade head')

    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    # Update the website configuration
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
        # Generate configuration files (bundle_server_config, nginx, etc.)
        run('python manage.py config_gen')
        # Migrate database
        run('python manage.py syncdb --migrate')
        # Create static pages
        run('python manage.py collectstatic --noinput')
        # For sending email, have the right domain name.
        run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
        # Create a superuser for OAuth
        run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
        # Allow bundle service to connect to website for OAuth
        run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
        # Put nginx and supervisor configuration files in place
        sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
        sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
        # Setup new relic
        run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")
예제 #7
0
def _deploy():
    # Update website
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir):
        run('git checkout %s' % env.git_codalab_tag)
        run('git pull')
        run('./dev_setup.sh')

    # Update bundle service
    with cd(env.deploy_codalab_cli_dir):
        run('git checkout %s' % env.git_codalab_cli_tag)
        run('git pull')
        run('./setup.sh')
        run('venv/bin/pip install MySQL-Python')
        run('venv/bin/alembic upgrade head')

    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    # Update the website configuration
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
        # Generate configuration files (bundle_server_config, nginx, etc.)
        run('python manage.py config_gen')
        # Migrate database
        run('python manage.py syncdb --migrate')
        # Create static pages
        run('python manage.py collectstatic --noinput')
        # For sending email, have the right domain name.
        run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
        # Create a superuser for OAuth
        run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
        # Allow bundle service to connect to website for OAuth
        run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
        # Put nginx and supervisor configuration files in place
        sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
        sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
        # Setup new relic
        run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")
예제 #8
0
def deploy_compute_worker(label):
    '''
    Deploy/update compute workers.
    For monitoring make sure the azure instance has the port 8000 forwarded

    :param label: Either test or prod
    '''
    env.deploy_codalab_dir = 'codalab-competitions'
    # Create .codalabconfig within home directory
    env.label = label
    cfg = DeploymentConfig(
        env.label,
        env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.get_compute_workers_file_content())
    settings_file = os.path.join('~', '.codalabconfig')
    put(buf, settings_file)
    env.git_codalab_tag = cfg.getGitTag()
    env.logs_password = cfg.get_compute_worker_logs_password()

    # Initial setup
    with cd(env.deploy_codalab_dir):
        run('git checkout %s' % env.git_codalab_tag)
        run('git pull')
        run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
            )
        # run('./dev_setup.sh')

    # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")

    current_directory = os.path.dirname(os.path.realpath(__file__))

    put(local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(
        current_directory),
        remote_path='/etc/init/codalab-compute-worker.conf',
        use_sudo=True)
    put(local_path='{}/configs/upstart/codalab-monitor.conf'.format(
        current_directory),
        remote_path='/etc/init/codalab-monitor.conf',
        use_sudo=True)
    # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)

    with settings(warn_only=True):
        sudo("stop codalab-compute-worker")
        sudo("stop codalab-monitor")
        sudo("start codalab-compute-worker")
        sudo("start codalab-monitor")
예제 #9
0
def teardown(choice):
    """
    Deletes specified assets in the deployment. Be careful: there is no undoing!

    choice: Indicates which assets to delete:
        'build' -> provision the build machine
        'web'   -> provision the web instances
        'all'   -> provision everything
    """
    assets = _validate_asset_choice(choice)
    require('configuration')
    logger.info("Teardown begins: %s.", assets)
    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(configuration)
    dep.Teardown(assets)
    logger.info("Teardown is complete.")
예제 #10
0
def build():
    """
    Builds artifacts to install on the deployment instances.
    """
    require('configuration')

    # Assemble source and configurations for the web site
    build_dir = "/".join(['builds', env.git_user, env.git_repo])
    src_dir = "/".join([build_dir, env.git_tag])
    if exists(src_dir):
        run('rm -rf %s' % (src_dir.rstrip('/')))
    with settings(warn_only=True):
        run('mkdir -p %s' % src_dir)
    with cd(src_dir):
        # TODO: why do we have the --branch and --single-branch tags here, this causes problems
        run('git clone --depth=1 --branch %s --single-branch %s .' %
            (env.git_tag, env.git_repo_url))
        # Generate settings file (local.py)
        configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
        dep = Deployment(configuration)
        buf = StringIO()
        buf.write(dep.getSettingsFileContent())
        settings_file = "/".join(
            ['codalab', 'codalab', 'settings', 'local.py'])
        put(buf, settings_file)
    # Assemble source and configurations for the bundle service
    if len(env.git_bundles_repo_url) > 0:
        build_dir_b = "/".join(
            ['builds', env.git_bundles_user, env.git_bundles_repo])
        src_dir_b = "/".join([build_dir_b, env.git_bundles_tag])
        if exists(src_dir_b):
            run('rm -rf %s' % (src_dir_b.rstrip('/')))
        with settings(warn_only=True):
            run('mkdir -p %s' % src_dir_b)
        with cd(src_dir_b):
            # TODO: why do we have the --branch and --single-branch tags here, this causes problems
            run('git clone --depth=1 --branch %s --single-branch %s .' %
                (env.git_bundles_tag, env.git_bundles_repo_url))
        # Replace current bundles dir in main CodaLab other bundles repo.
        bundles_dir = "/".join([src_dir, 'bundles'])
        run('rm -rf %s' % (bundles_dir.rstrip('/')))
        run('mv %s %s' % (src_dir_b, bundles_dir))
    # Package everything
    with cd(build_dir):
        run('rm -f %s' % env.build_archive)
        run('tar -cvf - %s | gzip -9 -c > %s' %
            (env.git_tag, env.build_archive))
예제 #11
0
파일: fabfile.py 프로젝트: xbaro/codalab
def deploy_compute_worker(label):
    '''
    Deploy/update compute workers.
    For monitoring make sure the azure instance has the port 8000 forwarded

    :param label: Either test or prod
    '''
    env.deploy_codalab_dir = 'codalab-competitions'
    # Create .codalabconfig within home directory
    env.label = label
    cfg = DeploymentConfig(env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.get_compute_workers_file_content())
    settings_file = os.path.join('~', '.codalabconfig')
    put(buf, settings_file)
    env.git_codalab_tag = cfg.getGitTag()
    env.logs_password = cfg.get_compute_worker_logs_password()

    # Initial setup
    with cd(env.deploy_codalab_dir):
        run('git checkout %s' % env.git_codalab_tag)
        run('git pull')
        run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt')
        # run('./dev_setup.sh')

    # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")

    current_directory = os.path.dirname(os.path.realpath(__file__))

    put(
        local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(current_directory),
        remote_path='/etc/init/codalab-compute-worker.conf',
        use_sudo=True
    )
    put(
        local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
        remote_path='/etc/init/codalab-monitor.conf',
        use_sudo=True
    )
    # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)

    with settings(warn_only=True):
        sudo("stop codalab-compute-worker")
        sudo("stop codalab-monitor")
        sudo("start codalab-compute-worker")
        sudo("start codalab-monitor")
예제 #12
0
def build():
    """
    Builds artifacts to install on the deployment instances.
    """
    require("configuration")

    # Assemble source and configurations for the web site
    build_dir = "/".join(["builds", env.git_user, env.git_repo])
    src_dir = "/".join([build_dir, env.git_tag])
    if exists(src_dir):
        run("rm -rf %s" % (src_dir.rstrip("/")))
    with settings(warn_only=True):
        run("mkdir -p %s" % src_dir)
    with cd(src_dir):
        # TODO: why do we have the --branch and --single-branch tags here, this causes problems
        run("git clone --depth=1 --branch %s --single-branch %s ." % (env.git_tag, env.git_repo_url))
        # Generate settings file (local.py)
        configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
        dep = Deployment(configuration)
        buf = StringIO()
        buf.write(dep.getSettingsFileContent())
        settings_file = "/".join(["codalab", "codalab", "settings", "local.py"])
        put(buf, settings_file)
    # Assemble source and configurations for the bundle service
    if len(env.git_bundles_repo_url) > 0:
        build_dir_b = "/".join(["builds", env.git_bundles_user, env.git_bundles_repo])
        src_dir_b = "/".join([build_dir_b, env.git_bundles_tag])
        if exists(src_dir_b):
            run("rm -rf %s" % (src_dir_b.rstrip("/")))
        with settings(warn_only=True):
            run("mkdir -p %s" % src_dir_b)
        with cd(src_dir_b):
            # TODO: why do we have the --branch and --single-branch tags here, this causes problems
            run(
                "git clone --depth=1 --branch %s --single-branch %s ." % (env.git_bundles_tag, env.git_bundles_repo_url)
            )
        # Replace current bundles dir in main CodaLab other bundles repo.
        bundles_dir = "/".join([src_dir, "bundles"])
        run("rm -rf %s" % (bundles_dir.rstrip("/")))
        run("mv %s %s" % (src_dir_b, bundles_dir))
    # Package everything
    with cd(build_dir):
        run("rm -f %s" % env.build_archive)
        run("tar -cvf - %s | gzip -9 -c > %s" % (env.git_tag, env.build_archive))
예제 #13
0
파일: fabfile.py 프로젝트: xbaro/codalab
def _deploy():
    # Update competition website
    # Pull branch and run requirements file, for info about requirments look into dev_setp.sh
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_tag)
        run('./dev_setup.sh')

    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    # local.py is generated here. For more info about content look into deploy/__.init__.py
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    # Update the website configuration
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
        # Generate configuration files (bundle_server_config, nginx, etc.)
        # For more info look into https://github.com/greyside/django-config-gen
        run('python manage.py config_gen')
        # Migrate database
        run('python manage.py syncdb --migrate')
        # Create static pages
        run('python manage.py collectstatic --noinput')
        # For sending email, have the right domain name.
        run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
        # Put nginx and supervisor configuration files in place, ln creates symbolic links
        sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
        sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
        # Setup new relic
        run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")
예제 #14
0
파일: fabfile.py 프로젝트: abmnv/codalab
def build():
    """
    Builds artifacts to install on the deployment instances.
    """
    require('configuration')

    # Assemble source and configurations for the web site
    build_dir = "/".join(['builds', env.git_user, env.git_repo])
    src_dir = "/".join([build_dir, env.git_tag])
    if exists(src_dir):
        run('rm -rf %s' % (src_dir.rstrip('/')))
    with settings(warn_only=True):
        run('mkdir -p %s' % src_dir)
    with cd(src_dir):
        run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_tag, env.git_repo_url))
        # Generate settings file (local.py)
        configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
        dep = Deployment(configuration)
        buf = StringIO()
        buf.write(dep.getSettingsFileContent())
        settings_file = "/".join(['codalab', 'codalab', 'settings', 'local.py'])
        put(buf, settings_file)
    # Assemble source and configurations for the bundle service
    if len(env.git_bundles_repo_url) > 0:
        build_dir_b = "/".join(['builds', env.git_bundles_user, env.git_bundles_repo])
        src_dir_b = "/".join([build_dir_b, env.git_bundles_tag])
        if exists(src_dir_b):
            run('rm -rf %s' % (src_dir_b.rstrip('/')))
        with settings(warn_only=True):
            run('mkdir -p %s' % src_dir_b)
        with cd(src_dir_b):
            run('git clone --depth=1 --branch %s --single-branch %s .' % (env.git_bundles_tag, env.git_bundles_repo_url))
        # Replace current bundles dir in main CodaLab other bundles repo.
        bundles_dir = "/".join([src_dir, 'bundles'])
        run('rm -rf %s' % (bundles_dir.rstrip('/')))
        run('mv %s %s' % (src_dir_b, bundles_dir))
    # Package everything
    with cd(build_dir):
        run('rm -f %s' % env.build_archive)
        run('tar -cvf - %s | gzip -9 -c > %s' % (env.git_tag, env.build_archive))
예제 #15
0
def provision(choice):
    """
    Provisions specified assets in the deployment.

    choice: Indicates which assets to provision:
        'build' -> provision the build machine
        'web'   -> provision the web instances
        'all'   -> provision everything
    """
    assets = _validate_asset_choice(choice)
    require('configuration')
    logger.info("Provisioning begins: %s.", assets)
    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(configuration)
    dep.Deploy(assets)
    if 'build' in assets:
        logger.info("Installing sofware on the build machine.")
        execute(provision_build)
    if 'web' in assets:
        logger.info("Installing sofware on web instances.")
        execute(provision_web)
    logger.info("Provisioning is complete.")
예제 #16
0
def deploy_compute_worker(label):
    '''
    Deploy/update compute workers.
    For monitoring make sure the azure instance has the port 8000 forwarded

    :param label: Either test or prod
    '''
    env.deploy_codalab_dir = 'codalab-competitions'
    # Create .codalabconfig within home directory
    env.label = label
    cfg = DeploymentConfig(
        env.label,
        env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.get_compute_workers_file_content())
    settings_file = os.path.join('~', '.codalabconfig')
    put(buf, settings_file)
    env.git_codalab_tag = cfg.getGitTag()
    env.logs_password = cfg.get_compute_worker_logs_password()

    # Initial setup
    with cd(env.deploy_codalab_dir):
        run('git checkout %s' % env.git_codalab_tag)
        run('git pull')

        # make sure we remove old version, added 2/20/17 can remove a while after that
        run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip uninstall django-storages && pip uninstall django-storages-redux',
            warn_only=True)

        run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install --upgrade pip && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
            )

        # run('./dev_setup.sh')

    # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")

    current_directory = os.path.dirname(os.path.realpath(__file__))

    # Adding compute worker upstart config
    broker_url = cfg.get_broker_url()
    compute_worker_conf_template = open(
        '{}/configs/upstart/codalab-compute-worker.conf'.format(
            current_directory)).read()
    compute_worker_conf = compute_worker_conf_template.format(
        broker_url=broker_url)
    compute_worker_conf_buf = StringIO()
    compute_worker_conf_buf.write(compute_worker_conf)
    put(
        local_path=
        compute_worker_conf_buf,  # Path can be a file-like object, in this case our processed template
        remote_path='/etc/init/codalab-compute-worker.conf',
        use_sudo=True)

    # Adding codalab monitor upstart config  ### No longer needed!
    # put(
    #     local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
    #     remote_path='/etc/init/codalab-monitor.conf',
    #     use_sudo=True
    # )
    # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)

    with settings(warn_only=True):
        sudo("stop codalab-compute-worker")
        sudo("stop codalab-monitor"
             )  # just in case it's left from a previous install
        sudo("start codalab-compute-worker")