def install_site(): "Add the virtualhost config file to the webserver's config, activate logrotate" require('release', provided_by=[deploy, setup]) with cd('%(prj_path)s/releases/%(release)s' % env): with settings(user=env.adminuser, pty=True): run('cp server-setup/%(webserver)s.conf /etc/%(webserver)s/sites-available/%(prj_name)s' % env) if env.use_daemontools: # activate new service runner run('cp server-setup/service-run.sh /etc/service/%(prj_name)s/run; chmod a+x /etc/service/%(prj_name)s/run;' % env) else: # delete old service dir run('echo; if [ -d /etc/service/%(prj_name)s ]; then rm -rf /etc/service/%(prj_name)s; fi' % env) if env.use_supervisor: # activate new supervisor.conf run('cp server-setup/supervisor.conf /etc/supervisor/conf.d/%(prj_name)s.conf' % env) if env.use_celery: run('cp server-setup/supervisor-celery.conf /etc/supervisor/conf.d/%(prj_name)s-celery.conf' % env) else: # delete old config file # if you set a process name in supervisor.ini, then you must add it like %(prj_name):appserver run('echo; if [ -f /etc/supervisor/%(prj_name)s.ini ]; then supervisorctl %(prj_name)s stop rm /etc/supervisor/%(prj_name)s.ini; fi' % env) run('echo; if [ -f /etc/supervisor/conf.d/%(prj_name)s.conf ]; then supervisorctl %(prj_name)s stop rm /etc/supervisor/conf.d/%(prj_name)s.conf; fi' % env) if env.use_celery: run('echo; if [ -f /etc/supervisor/%(prj_name)s-celery.ini ]; then supervisorctl celery celerybeat stop rm /etc/supervisor/%(prj_name)s-celery.ini; fi' % env) run('echo; if [ -f /etc/supervisor/conf.d/%(prj_name)s-celery.conf ]; then supervisorctl celery celerybeat stop rm /etc/supervisor/conf.d/%(prj_name)s-celery.conf; fi' % env) if env.use_celery and env.use_daemontools: run('cp server-setup/service-run-celeryd.sh /etc/service/%(prj_name)s-celery/run; chmod a+x /etc/service/%(prj_name)s-celery/run;' % env) # try logrotate with settings(warn_only=True): run('cp server-setup/logrotate.conf /etc/logrotate.d/website-%(prj_name)s' % env) if env.use_celery: run('cp server-setup/logrotate-celery.conf /etc/logrotate.d/celery' % env) run('cp server-setup/letsencrypt.conf /etc/letsencrypt/configs/%(cryptdomain)s.conf' % env) with settings(warn_only=True): run('cd /etc/%(webserver)s/sites-enabled/; ln -s ../sites-available/%(prj_name)s %(prj_name)s' % env)
def rsync_media(upload=False, delete=False): require('PROJECT') local_dir = add_slash(Path(env.PROJECT.package, 'media')) remote_dir = add_slash(env.PROJECT.media) rsync_project(remote_dir, local_dir, delete=delete, upload=upload)
def set_site_domain(): """ Set the site domain dependent on env.host. """ status_update('Updating domain for default Django Site to %s' % env.host) set_django_settings() require('django_settings') require('host') if 'production' in env.host: host = 'onepercentclub.com' else: host = env.host sites_command = ( "from django.contrib.sites.models import Site; " "site = Site.objects.get(pk=1); " "site.domain = '%s'; " "site.name = 'onepercentclub.com'; " "site.save()" ) % host with virtualenv(): run_web('echo "%s" | ./manage.py shell --plain --settings=%s' % ( sites_command, env.django_settings ))
def manage_run(command): """Run a Django management command on the remote server.""" require('environment') manage_base = u"%(virtualenv_root)s/bin/django-admin.py " % env if '--settings' not in command: command = u"%s --settings=%s" % (command, env.settings) project_run(u'%s %s' % (manage_base, command))
def run_web(*args, **kwargs): """ Run a command as the web user. """ require('web_user') kwargs.setdefault('user', env.web_user) return sudo(*args, **kwargs)
def prompt_role(): """ Query the user for the role (and thus host) to use and store results in env. """ require('roledefs') if not env.host_string: role_options = ', '.join([role for role in env.roledefs.keys()]) def validate(value): if not value in env.roledefs.keys(): raise Exception('Role should be one of %s' % role_options) return value role = prompt('Please specify a role (one of %s): ' % role_options, validate=validate) # Warning: this is a rather hackish solution - Fabric doesn't seem # to like the hosts for a task to be changed in the middle of a # process. This works ok, but limits us to one host per role. assert len(env.roledefs[role]) == 1, \ 'This is a hackish solution that does not work for a role with multiple hosts.' env.host = env.roledefs[role][0] env.host_string = env.host
def prepare_django(): """ Prepare a deployment. """ set_django_settings() require('django_settings') status_update('Preparing deployment.') with virtualenv(): # TODO: Filter out the following messages: # "Could not find a tag or branch '<commit_id>', assuming commit." run('pip install -q --allow-all-external --allow-unverified django-admin-tools -r requirements/requirements.txt') run('grunt compass:dist') # Remove and compile the .pyc files. run('find . -name \*.pyc -delete') run('./manage.py compile_pyc --settings=%s' % env.django_settings) # Prepare the translations. run('./translations.sh compile %s' % env.django_settings) # Disabled until the following problem is fixed: # ERROR: test_site_profile_not_available (django.contrib.auth.tests.models.ProfileTestCase) #run_web('./manage.py test -v 0') # Make sure the web user can read and write the static media dir. run('chmod a+rw static/media') run_web('./manage.py syncdb --migrate --noinput --settings=%s' % env.django_settings) run_web('./manage.py collectstatic --clear -l -v 0 --noinput --settings=%s' % env.django_settings)
def deploy(loglevel=DEFAULT_SALT_LOGLEVEL): """Deploy to a given environment by pushing the latest states and executing the highstate.""" require('environment') sync() target = "-G 'environment:{0}'".format(env.environment) salt('saltutil.sync_all', target, loglevel) highstate(target)
def encrypt(*args, **kwargs): """Encrypt a secret value for a given environment.""" require('environment') # Convert ASCII key to binary temp_key = '/tmp/tmp.key' with hide('running', 'stdout', 'stderr'): local('gpg --dearmor < {} > {}'.format(env.gpg_key, temp_key)) # Encrypt each file for name in args: local( 'gpg --no-default-keyring --keyring {} ' '--trust-model always -aer {}_salt_key {}'.format( temp_key, env.environment, name)) # Encrypt each value updates = {} for name, value in kwargs.items(): updates[name] = '{}'.format( local( 'echo -n "{}" | ' 'gpg --no-default-keyring --keyring {} ' '--trust-model always -aer {}_salt_key'.format( value, temp_key, env.environment), capture=True)) os.remove(temp_key) if updates: print(yaml.dump(updates, default_flow_style=False, default_style='|', indent=2))
def bdeploy(): require('hosts') require('path') print(green("\n#####Beginning %s build\n" % ('testing' if env.is_test else 'production'))) local('sencha app build %s' % ('testing' if env.is_test else 'production')) qdeploy()
def deploy(): require('hosts') require('path') print(green("\n#####Beginning deployment to %s & %s on %s\n" % (env.mob_domain, env.stubs_domain, env.hosts[0]))) env.mob_release_label = None env.stubs_release_label = None if env.is_test: create_tag = prompt('Tag this release? [y/N] ') if create_tag.lower() == 'y': print("\nShowing latest tags for reference\n") local('git tag | tail -5') env.refspec = prompt('Enter tag name [in format VX_X_X]? ') #Can't use .'s as seperaters as they cause import problems on the server local('git tag %s -am "Tagging version %s in fabfile"' % (env.refspec, env.refspec)) local('git push --tags') env.mob_release_label = mob_package_name + '-' + env.refspec env.stubs_release_label = stubs_package_name + '-' + env.refspec if not env.mob_release_label: # An existing tag must be specified local('git tag | tail -5') env.refspec = prompt('Choose tag to build from: ') local('git tag | grep "%s"' % env.refspec) env.mob_release_label = mob_package_name + '-' + env.refspec env.stubs_release_label = stubs_package_name + '-' + env.refspec # import time # env.release_label = package_name + '-' + time.strftime('%Y%m%d%H%M%S') _upload_tar_from_git() _install_site() _symlink_current_release() restart_webserver() print(green("\n#####Deployment successful for %s & %s\n" % (env.mob_domain, env.stubs_domain)))
def bootstrap(): """Sets up the remote environment""" require('hosts', provided_by=[remote]) sub_add_repos() sub_install_packages() sub_install_shiny() reload_server()
def push(): """Pushes the current folder to the remote machine's Shiny apps folder""" require('hosts', provided_by=[remote]) sudo('mkdir -p /www-shiny') sudo('chown -R ubuntu:ubuntu /www-shiny') sudo('rm -rf /www-shiny/*') put('./', '/www-shiny/')
def provision(): require('deploy_type', provided_by=[dev, prod]) commands = ( 'apt-get install --yes --no-upgrade %s' % ' '.join(PACKAGES), ) if env.deploy_type == 'prod': commands += ( 'mkdir -p %s' % env.deploy_dir, 'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir), ) if env.deploy_type == 'dev': local('; '.join('sudo %s' % cmd for cmd in commands)) local('sudo apt-get build-dep python-numpy python-psycopg2 --no-upgrade') local('sudo npm config set registry http://registry.npmjs.org/') local('sudo npm -g install yuglify') return sudo('; '.join(commands)) sudo('apt-get build-dep python-numpy python-psycopg2 --no-upgrade') sudo('npm config set registry http://registry.npmjs.org/') sudo('npm -g install yuglify') sudo('mv -f /etc/nginx/sites-enabled/default /etc/nginx/sites-available') provision_api() git_checkout()
def setup_vagrant(): """Sets up the Vagrant environment""" require('hosts', provided_by=[vagrant]) # Sets the environment for Fabric sub_add_repos() sub_install_packages() sub_install_shiny() reload_server()
def deploy_web(): """ Installs the output of the build on the web instances. """ require("configuration") if exists(env.deploy_dir): run("rm -rf %s" % env.deploy_dir) run("tar -xvzf %s" % env.build_archive) run("mv %s deploy" % env.git_tag) run("source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv") env.SHELL_ENV = dict( DJANGO_SETTINGS_MODULE=env.django_settings_module, DJANGO_CONFIGURATION=env.django_configuration, CONFIG_HTTP_PORT=env.config_http_port, CONFIG_SERVER_NAME=env.config_server_name, ) print env.SHELL_ENV with cd(env.deploy_dir): with prefix("source /usr/local/bin/virtualenvwrapper.sh && workon venv"), shell_env(**env.SHELL_ENV): requirements_path = "/".join(["codalab", "requirements", "dev_azure_nix.txt"]) pip_cmd = "pip install -r {0}".format(requirements_path) run(pip_cmd) # additional requirements for bundle service run("pip install SQLAlchemy simplejson") with cd("codalab"): run("python manage.py config_gen") run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json") run("python manage.py syncdb --migrate") run("python scripts/initialize.py") run("python manage.py collectstatic --noinput") sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf") sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
def fetch_logs(): """ Fetch logs from the web instances into ~/logs. """ require("configuration") with cd(env.deploy_dir): get("codalab/var/*.log", "~/logs/%(host)s/%(path)s")
def bootstrap(): """ initialize remote host environment (virtualenv, deploy, update) """ require('root', provided_by=env.deployments) print magenta("Cloning Repository") with cd(env.root): run("git clone %s" % env.git_repository) # some one time setup things with cd(env.project_root): if env.git_branch != 'master': run('git checkout %s' % (env.git_branch,)) run('mkdir static') run('mkdir media') with cd(env.code_root): run('ln -sf settings_%s.py settings.py' % env.environment) # create virtualenv and install all the requirements execute('create_virtualenv') execute('update_requirements') execute('create_database') execute('syncdb') execute('migrate') print magenta("Load initial data") with cd(env.project_root), prefix('source env/bin/activate'): run('./manage.py loaddata allink_user.json') # only compile messages if locale folder is present if os.path.isdir('locale'): execute('compilemessages') execute('collectstatic')
def hosts(): """List applicable hosts for the specified environment. Use this to determine which hosts will be affected by an environment-specific operation.""" require('settings', provided_by=[prd, stg]) for host in env.instances: print env.hosts
def deploy(): """Deploy to a given environment.""" # NOTE: chef will check every 30 minutes or so whether the # repo has changed, and if so, redeploy. Or you can use this # to make it run immediately. require('environment') sudo('chef-client')
def syncdb(): """runs syncdb on the remote host""" require('project_root', provided_by=env.deployments) print magenta("Syncronize database") with cd(env.project_root): with prefix('source env/bin/activate'): run('./manage.py syncdb --noinput')
def uninstall_crontab(): """ Remove a previously install cron jobs script from cron.d """ require('settings', provided_by=[production, staging]) sudo('rm /etc/cron.d/%(PROJECT_FILENAME)s' % app_config.__dict__)
def user_sshkey(): """ Upload an SSH key to the remote system for the current user. :Example: fab --config=config/local.conf local system.user_sshkey """ require('PUBLIC_SSH_KEY') with open(env.PUBLIC_SSH_KEY) as reader: key = reader.read() remote_directory = '/home/{}/.ssh'.format(env.user) remote_authkeys = '{}/authorized_keys'.format(remote_directory) new_directory = False if not files.exists(remote_directory): new_directory = True # Create the ".ssh" directory. run('mkdir -p {}'.format(remote_directory)) # Add the key to "authorized keys". files.append(remote_authkeys, key) if new_directory: # Set directory permission to "700". run('chmod 700 {}'.format(remote_directory)) # Set file permission to "600". run('chmod 600 {}'.format(remote_authkeys))
def list_releases(): "List the releases on the server" require('releases_dir', provided_by=['dev','prod']) run("ls -t %(releases_dir)s"%env)
def setup_minion(*roles): """Setup a minion server with a set of roles.""" require('environment') for r in roles: if r not in VALID_ROLES: abort('%s is not a valid server role for this project.' % r) config = { 'master': 'localhost' if env.master == env.host else env.master, 'output': 'mixed', 'grains': { 'environment': env.environment, 'roles': list(roles), }, 'mine_functions': { 'network.interfaces': [], 'network.ip_addrs': [] }, } _, path = tempfile.mkstemp() with open(path, 'w') as f: yaml.dump(config, f, default_flow_style=False) sudo("mkdir -p /etc/salt") put(local_path=path, remote_path="/etc/salt/minion", use_sudo=True) # install salt minion if it's not there already install_salt(SALT_VERSION, master=False, minion=True, restart=True) # queries server for its fully qualified domain name to get minion id key_name = run('python -c "import socket; print socket.getfqdn()"') execute(accept_key, key_name)
def ve_run(cmd): """ Helper function. Runs a command using the virtualenv environment """ require('root') return sshagent_run('source %s/bin/activate; %s' % (env.root, cmd))
def new_user(username, email): """Add new user function, pass username, email :param username: string of new user :param email: string of new email """ require('hosts', provided_by=[sample]) require('ini', provided_by=[sample]) parse_ini(env["ini_file"]) import transaction from bookie.models import initialize_sql initialize_sql(dict(env.ini.items('app:main'))) from bookie.models import DBSession from bookie.models.auth import get_random_word, User sess = DBSession() u = User() u.username = unicode(username) passwd = get_random_word(8) u.password = passwd u.email = unicode(email) u.activated = True u.is_admin = False u.api_key = User.gen_api_key() print dict(u) print passwd sess.add(u) sess.flush() transaction.commit()
def install_crontab(): """ Install cron jobs script into cron.d. """ require('settings', provided_by=[production, staging]) sudo('cp %(SERVER_REPOSITORY_PATH)s/crontab /etc/cron.d/%(PROJECT_FILENAME)s' % app_config.__dict__)
def deploy(branch=None): """Deploy to a given environment.""" require('environment') if branch is not None: env.branch = branch requirements = False migrations = False # Fetch latest changes with cd(env.code_root): with settings(user=env.project_user): run('git fetch origin') # Look for new requirements or migrations requirements = match_changes(env.branch, "'requirements\/'") migrations = match_changes(env.branch, "'\/migrations\/'") if requirements or migrations: supervisor_command('stop %(environment)s:*' % env) with settings(user=env.project_user): run("git reset --hard origin/%(branch)s" % env) upload_local_settings() if requirements: update_requirements() # New requirements might need new tables/migrations syncdb() elif migrations: syncdb() collectstatic() supervisor_command('restart %(environment)s:*' % env)
def deploy(remote='origin'): """ Deploy the latest app to S3 and, if configured, to our servers. """ require('settings', provided_by=[production, staging]) if app_config.DEPLOY_TO_SERVERS: require('branch', provided_by=[stable, master, branch]) if (app_config.DEPLOYMENT_TARGET == 'production' and env.branch != 'stable'): utils.confirm("You are trying to deploy the '%s' branch to production.\nYou should really only deploy a stable branch.\nDo you know what you're doing?" % env.branch) if app_config.DEPLOY_TO_SERVERS: checkout_latest(remote) #fabcast('update_copy') #fabcast('assets.sync') #fabcast('update_data') if app_config.DEPLOY_CRONTAB: install_crontab() if app_config.DEPLOY_SERVICES: deploy_confs() render() _gzip('www', '.gzip') _deploy_to_s3() _cleanup_minified_includes()
def open_heroku(): require('environment') local('heroku open --remote {}'.format(env.environment))
def _report_coverage(): require('venv_path') with cd(env.site_path): run(_venv_exec('coverage report'))
def _venv_exec(cmd): require('venv_path') return '%s/bin/%s' % (env.venv_path, cmd)
def check_settings(): "Run the (added in 1.6) check management command to validate settings" require('site_path') require('venv_path') _manage('check')
def runserver(bind_to='0.0.0.0', port='12000'): """ Run a python development server on the host """ require('site_path') require('venv_path') _manage('runserver %s:%s' % (bind_to, port))
def schemamigration(app_name, flag=' --auto'): require('site_path') require('venv_path') _manage('schemamigration %s %s' % (app_name, flag))
def uptime(): require('target', provided_by=( staging, production, )) run('uptime')
def create_superuser(): require('environment') local('heroku run python {}/manage.py ' 'createsuperuser --remote {}'.format(env.project_name, env.environment))
def migrate(): """migrates all apps on remote host""" require('project_root', provided_by=env.deployments) print magenta("Migrate database") with cd(env.project_root), prefix('source env/bin/activate'): run('./manage.py migrate')
def ps(): """ Scales a web dyno on Heroku """ require('environment') local('heroku ps:scale web=1:hobby --remote {}'.format(env.environment))
def restart_webapp(): """ touch wsgi file to trigger reload """ require('project_root', provided_by=env.deployments) print magenta("Restart webapp") with cd(env.project_root): run('touch apache.wsgi')
def migrate(): require('environment') local('heroku run python {}/manage.py migrate --remote {}'.format( env.project_name, env.environment))
def all(branch='master'): """Perform all publish sub-tasks at once""" require('environment', provided_by=[stage]) execute(push, branch=branch) execute(pull, branch=branch)
def restart_celery(): """restarts the celery worker""" require('project_root', provided_by=env.deployments) print magenta("Restart celery") run('nine-supervisorctl restart %s' % env.celery_worker)
def clear_cache(): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) with cd(env.path), prefix(env.within_virtualenv): run('./manage.py clear_cache')
def create_virtualenv(): """ setup virtualenv on remote host """ require('project_root', provided_by=env.deployments) run('virtualenv --setuptools --prompt="(%s)" %s' % (env.project, os.path.join(env.project_root, 'env')))
def migrate(app=None): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) with cd(env.path), prefix(env.within_virtualenv): run('./manage.py migrate {}'.format(app if app else ''))
def setup_environment(version=None): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) create_virtualenv() clone_repo() update(version) install_requirements()
def install_mysql(choice='all'): """ Installs a local instance of MySQL of the web instance. This will only work if the number of web instances is one. choice: Indicates which assets to create/install: 'mysql' -> just install MySQL; don't create the databases 'site_db' -> just create the site database 'bundles_db' -> just create the bundle service database 'all' or '' -> install all three """ require('configuration') if len(env.roledefs['web']) != 1: raise Exception( "Task install_mysql requires exactly one web instance.") if choice == 'mysql': choices = {'mysql'} elif choice == 'site_db': choices = {'site_db'} elif choice == 'bundles_db': choices = {'bundles_db'} elif choice == 'all': choices = {'mysql', 'site_db', 'bundles_db'} else: raise ValueError( "Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." % (choice)) configuration = DeploymentConfig(env.cfg_label, env.cfg_path) dba_password = configuration.getDatabaseAdminPassword() if 'mysql' in choices: sudo( 'DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server' ) sudo('mysqladmin -u root password {0}'.format(dba_password)) if 'site_db' in choices: db_name = configuration.getDatabaseName() db_user = configuration.getDatabaseUser() db_password = configuration.getDatabasePassword() cmds = [ "create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format( db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;" .format(db_name, db_user) ] run('mysql --user=root --password={0} --execute="{1}"'.format( dba_password, " ".join(cmds))) if 'bundles_db' in choices: db_name = configuration.getBundleServiceDatabaseName() db_user = configuration.getBundleServiceDatabaseUser() db_password = configuration.getBundleServiceDatabasePassword() cmds = [ "create database {0};".format(db_name), "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format( db_user, db_password), "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;" .format(db_name, db_user) ] run('mysql --user=root --password={0} --execute="{1}"'.format( dba_password, " ".join(cmds)))
def update_index(): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) with cd(env.path), prefix(env.within_virtualenv): run('./manage.py update_index')
def _deploy(): # Update website with cd(env.deploy_codalab_worksheets_dir): run('git fetch') run('git checkout %s' % env.git_codalab_tag) run('git pull') run('./setup.sh') # Update bundle service with cd(env.deploy_codalab_cli_dir): run('git fetch') run('git checkout %s' % env.git_codalab_cli_tag) run('git pull') run('./setup.sh server') # Create website-config.json cfg = DeploymentConfig(env.cfg_label, env.cfg_path) buf = StringIO() json.dump(getWebsiteConfig(cfg), buf, sort_keys=True, indent=4, separators=(',', ': ')) buf.write('\n') put(buf, '.codalab/website-config.json') # Update the website configuration with cd(env.deploy_codalab_worksheets_dir), cd('codalab'): # Generate configuration files (nginx, supervisord) run('./manage config_gen') # Put configuration files in place. sudo( 'ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf' ) sudo( 'ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf' ) # Install SSL certficates (/etc/ssl/certs/) require('configuration') if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len( cfg.getSslCertificateKeyInstalledPath()) > 0): put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True) put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True) else: logger.info( "Skipping certificate installation because both files are not specified." ) # Configure the bundle server with cd(env.deploy_codalab_cli_dir), cd('codalab'), cd('bin'): # For generating the bundle_server_config.json file. run('./cl config server/engine_url mysql://%s:%s@localhost:3306/%s' % ( \ cfg.getBundleServiceDatabaseUser(), cfg.getBundleServiceDatabasePassword(), cfg.getBundleServiceDatabaseName(), )) # Send out emails from here (e.g., for password reset) email_info = cfg.getEmailInfo() run('./cl config email/host %s' % email_info['host']) run('./cl config email/user %s' % email_info['user']) run('./cl config email/password %s' % email_info['password']) # Send notifications. run('./cl config server/admin_email %s' % cfg.getAdminEmail()) run('./cl config server/support_email %s' % cfg.getSupportEmail()) run('./cl config server/instance_name %s' % cfg.label) # Update database with cd(env.deploy_codalab_cli_dir): run('venv/bin/alembic upgrade head') # Set up the bundles database. with cd(env.deploy_codalab_cli_dir): run('scripts/create-root-user.py %s' % cfg.getDatabaseAdminPassword())
def reinstall_requirement(which): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) with cd(env.path), prefix(env.within_virtualenv): run('pip uninstall {0} && pip install --no-deps {0}'.format(which))
def _svn_checkout(svn_url, dir_name, username=None, update_existing=True, do_local=False): """ Check out the Subversion project from <svn_url> into directory <dir_name>. This method makes one attempt to check out without specifying a password. On subsequent attempts, it will prompt for the password, giving up after three attempts. If the project already exists and <update_existing> is True, then "svn update" will be run in the project directory. if <do_local> is True, the project will be checked out on the local machine. """ require("svnpass", used_for="checking out Subversion projects") if do_local: homedir = os.environ["HOME"] fexists = os.path.exists frun = _capture_local else: with hide("running", "stdout", "stderr"): homedir = run("echo $HOME", pty=False) fexists = _exists frun = run path = os.path.join(homedir, dir_name) if fexists(path): if update_existing: frun("cd %s && svn up" % path) else: attempts = -1 while True: if attempts < 0 or env.svnpass is not None: tmppass = env.svnpass else: if username is not None: unm = username else: unm = env.user prompt = "Enter Subversion password for %s" % unm tmppass = _get_password(prompt) with hide("running", "warnings", "stderr"): with settings(warn_only=True): if username is not None: user_arg = "--username %s " % username else: user_arg = "" print("svn co %s%s %s" % (user_arg, svn_url, path)) if tmppass is not None: pass_arg = "--password %s " % tmppass else: pass_arg = "" rtnval = frun("(echo; echo; echo; echo) |" + " svn co %s%s%s %s" % (user_arg, pass_arg, svn_url, path), pty=False) if not rtnval.failed: if env.svnpass is None: env.svnpass = tmppass break attempts += 1 if attempts > 3: print("Giving up after %d attempts" % \ (attempts - 1), file=sys.stderr) break
def test_connections(): """ Verifies that we can connect to all instances. """ require('configuration') sudo('hostname')
def run_django_command(command): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) with cd(env.path): run('{} python manage.py {}'.format(env.within_virtualenv, command))
def _sudo(cmd): require('webapp_user') return sudo(cmd, user='******' % env)
def setup_environment(version=None): require('srvr', provided_by=env.servers) clone_repo() update(version) install_requirements()
def deploy(git_ref, upgrade=False): """Deploy project. Deploy the code of the given git reference to the previously selected environment. Args: upgrade(Optional[bool]): Pass ``upgrade=True`` to upgrade the versions of the already installed project requirements (with pip) git_ref(str): name branch you make deploy. Example: >>>fab environment:vagrant deploy:devel. """ require('hosts', 'user', 'group', 'site_dir', 'django_settings') # Retrives git reference metadata and creates a temp directory with the # contents resulting of applying a ``git archive`` command. message = white('Creating git archive from {0}'.format(git_ref), bold=True) with cmd_msg(message): repo = ulocal( 'basename `git rev-parse --show-toplevel`', capture=True) commit = ulocal( 'git rev-parse --short {0}'.format(git_ref), capture=True) branch = ulocal( 'git rev-parse --abbrev-ref HEAD', capture=True) tmp_dir = '/tmp/blob-{0}-{1}/'.format(repo, commit) ulocal('rm -fr {0}'.format(tmp_dir)) ulocal('mkdir {0}'.format(tmp_dir)) ulocal('git archive {0} ./src | tar -xC {1} --strip 1'.format( commit, tmp_dir)) # Uploads the code of the temp directory to the host with rsync telling # that it must delete old files in the server, upload deltas by checking # file checksums recursivelly in a zipped way; changing the file # permissions to allow read, write and execution to the owner, read and # execution to the group and no permissions for any other user. with cmd_msg(white('Uploading code to server...', bold=True)): ursync_project( local_dir=tmp_dir, remote_dir=env.site_dir, delete=True, default_opts='-chrtvzP', extra_opts='--chmod=750', exclude=["*.pyc", "env/", "cover/"] ) # Performs the deployment task, i.e. Install/upgrade project # requirements, syncronize and migrate the database changes, collect # static files, reload the webserver, etc. message = white('Running deployment tasks', bold=True) with cmd_msg(message, grouped=True): with virtualenv(): message = white('Installing Python requirements with pip') with cmd_msg(message, spaces=2): run('pip install -{0}r ./requirements/production.txt'.format( 'U' if upgrade else '')) message = white('Migrating database') with cmd_msg(message, spaces=2): run('python manage.py migrate --noinput') message = white('Collecting static files') with cmd_msg(message, spaces=2): run('python manage.py collectstatic --noinput') message = white('Setting file permissions') with cmd_msg(message, spaces=2): run('chgrp -R {0} .'.format(env.group)) run('chgrp -R {0} ../media'.format(env.group)) message = white('Restarting webserver') with cmd_msg(message, spaces=2): run('touch ../reload') message = white('Registering deployment') with cmd_msg(message, spaces=2): register_deployment(commit, branch) # Clean the temporary snapshot files that was just deployed to the host message = white('Cleaning up...', bold=True) with cmd_msg(message): ulocal('rm -fr {0}'.format(tmp_dir)) puts(green(SUCCESS_ART), show_prefix=False) puts(white('Code from {0} was succesfully deployed to host {1}'.format( git_ref, ', '.join(env.hosts)), bold=True), show_prefix=False)
def manage_run(command): require('environment') project_name = get_project_name() manage_sh = u'/var/www/%s/manage.sh ' % project_name with settings(host_string=hostnames_for_role('web')[0]): sudo(manage_sh + command, user=project_name)
def check_deploy(): require('srvr', provided_by=env.servers) if env.srvr in ['stg', 'liv']: run_django_command('check --deploy')