def deploy_www_root(): site_root = join(home_directory(SITE_USER), 'site') site_home = home_directory(SITE_USER) bindir = join(site_root, 'bin') with cd(home_directory(SITE_USER)): su('mkdir -p venvs site') with cd(site_root): su('mkdir -p logs bin env static') upload_template('runserver.sh', join(bindir, 'runserver.sh'), context={ 'site_version': 'veyepar', 'site_home': site_home, 'site_name': SITE_USER, 'django_dir': join(SITE_NAME, 'dj'), 'wsgi_module': 'dj.wsgi', 'django_settings_module': 'dj.settings', }, use_jinja=True, use_sudo=True, chown=True, user=SITE_USER) with cd(bindir): su('chmod +x runserver.sh')
def migrate(app): """ run south migration on specified app """ environment = join(home_directory(SITE_USER), 'site/bin/environment.sh') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'companionpages') with cd(djangodir): vsu('source %s; ./manage.py migrate %s' % (environment, app))
def collectstatic(site_version): vbin = join(home_directory(SITE_USER), 'venvs', site_version, 'bin') staticdir = join(home_directory(SITE_USER), 'site', 'static') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'dj') with cd(djangodir): su('source %s/activate; ./manage.py collectstatic --noinput --clear' % vbin) su('chmod -R a+rx %s' % staticdir)
def collectstatic(): environment = join(home_directory(SITE_USER), 'site/bin/environment.sh') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'companionpages') # ignoring logs and media objects in our s3 container # this shouldn't be necessary but is a consequence of using django-storages with the boto backend # and as of now, django-storages doesn't support separate containers for static and media. with cd(djangodir): vsu('source %s; ./manage.py collectstatic --noinput --clear --ignore *results* --ignore *log* --ignore *materials* --ignore *articles*' % environment)
def install_local_settings(): settings_dir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'dj', 'dj') staticdir = join(home_directory(SITE_USER), 'site', 'static') with cd(settings_dir): upload_template('local_settings.py', 'local_settings.py', context={ 'staticdir': staticdir, 'secret_key': randomstring(32), }, use_jinja=True, use_sudo=True, chown=True, user=SITE_USER)
def crontab_update_index(): site_root = join(home_directory(SITE_USER), 'site') bindir = join(site_root, 'bin') job = join(home_directory(SITE_USER), 'update_index') upload_template('update_index', job, context={ 'command': join(bindir, 'update_index.sh'), 'logfile': join(site_root, 'logs', 'cron_update_index.log'), }, use_jinja=True, use_sudo=True, template_dir=TEMPLATE_DIR) sudo('chown %s:%s %s' % (SITE_USER, SITE_USER, job)) su('crontab %s' % job)
def setup_env(environment, app): ''' Setup environment ''' if app is None: list_apps() raise EnvNotSetup('Please specify app') abort('') env.app = app env.environment = environment env.app_path_local = join(env.projects_path, env.app) infra_file = join(env.app_path_local, DATA_FILE) env.infra_data = json.loads(open(infra_file).read()) host = env.infra_data['hosts'][env.environment] env.user = host.get('user', 'root') ssh_port = host.get('ssh_port', 22) first_host = host['domains'][0] env.app_user = env.user env.host_string = '%s@%s:%s' % (env.user, first_host, ssh_port) env.home_path = user.home_directory(env.user) env.var_static_app = join(env.home_path, 'static', env.app) env.apps_path = join(env.home_path, 'apps') env.logs_path = join(env.home_path, 'logs') env.app_logs_path = join(env.logs_path, env.app) env.pipenv_path = join(env.home_path, '.local/bin/pipenv') env.app_path = join(env.apps_path, env.app) env.log_path = join(env.logs_path, env.app) setup_shell_envs()
def virtualenv_name(commit=None): if commit is None: repodir = join(home_directory(SITE_USER), 'site', SITE_NAME) with cd(repodir): commit = run('git rev-parse HEAD').strip() timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') return '%s-%s' % (timestamp, commit.replace('/', '_'))
def virtualenv_name(commit=None): if commit is None: repodir = join(home_directory(SITE_USER), 'site', SITE_NAME) with cd(repodir): commit = run('git rev-parse HEAD').strip() timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') return '%s-%s' % (timestamp, commit)
def setup_site_root(): site_root = join(home_directory(SITE_USER), 'site') bindir = join(site_root, 'bin') with cd(home_directory(SITE_USER)): su('mkdir -p venvs site') with cd(site_root): su('mkdir -p logs bin env media static') put(template_path('runserver.sh'), bindir, use_sudo=True) put(template_path('celeryworker.sh'), bindir, use_sudo=True) put(template_path('check_downloads.sh'), bindir, use_sudo=True) sudo('chown -R %s:%s %s' % (SITE_USER, SITE_USER, site_root)) with cd(bindir): su('chmod +x runserver.sh celeryworker.sh check_downloads.sh')
def papertrail(port): """install rsyslog and setup papertrail logs """ fabric.api.require('site', 'available', 'hosts', 'site_environment', provided_by=('dev', 'prod', 'vagrant')) su('curl -sSL https://get.rvm.io | bash -s stable --ruby') su('source %s' % join(home_directory(SITE_USER), '.rvm', 'scripts', 'rvm'))
def install_site_requirements(virtualenv): vbin = join(home_directory(SITE_USER), 'venvs', virtualenv, 'bin') su('%s/pip install django' % vbin) su('%s/pip install djangorestframework' % vbin) su('%s/pip install django_extensions' % vbin) su('%s/pip install gunicorn' % vbin) su('%s/pip install psycopg2' % vbin)
def setup_supervisor(): site_root = join(home_directory(SITE_USER), 'site') upload_template('researchcompendia.conf', '/etc/supervisor/conf.d/researchcompendia_web.conf', context={ 'command': join(site_root, 'bin', 'runserver.sh'), 'user': SITE_USER, 'group': SITE_GROUP, 'logfile': join(site_root, 'logs', 'gunicorn_supervisor.log'), }, use_jinja=True, use_sudo=True, template_dir=TEMPLATE_DIR) upload_template('celeryd.conf', '/etc/supervisor/conf.d/celeryd.conf', context={ 'command': join(site_root, 'bin', 'celeryworker.sh'), 'user': SITE_USER, 'group': SITE_GROUP, 'logfile': join(site_root, 'logs', 'celery_worker.log'), }, use_jinja=True, use_sudo=True, template_dir=TEMPLATE_DIR) supervisor.update_config()
def delete_admin(username): """Remove a project admin's system account and home directory""" if not user.exists(username): raise UserDoesNotExistError("User does not exist: " + username) user_home = user.home_directory(username) sudo('deluser ' + username) sudo('rm -Rf ' + user_home)
def setup_rabbitmq(user=None): """ installs official rabbitmq package and sets up user """ deb.add_apt_key( url='http://www.rabbitmq.com/rabbitmq-signing-key-public.asc') require.deb.source('rabbitmq-server', 'http://www.rabbitmq.com/debian/', 'testing', 'main') require.deb.uptodate_index(max_age={'hour': 1}) require.deb.packages([ 'rabbitmq-server', ]) if user is None: user = SITE_USER envfile = join(home_directory(SITE_USER), 'site/bin/environment.sh') secret = randomstring(64) sudo('rabbitmqctl delete_user guest') sudo('rabbitmqctl add_user %s "%s"' % (user, secret)) sudo('rabbitmqctl set_permissions -p / %s ".*" ".*" ".*"' % user) amqp_url = 'amqp://%s:%s@localhost:5672//' % (SITE_USER, secret) sed(envfile, 'DJANGO_BROKER_URL=".*"', 'DJANGO_BROKER_URL="%s"' % amqp_url, use_sudo=True)
def vsu(cmd, virtualenv=None, user=None): if virtualenv is None: virtualenv = get_site_version() if user is None: user = SITE_USER home = home_directory(user) venvdir = join(home, 'venvs', virtualenv, 'bin/activate') sudo("su %s -c 'source %s; %s'" % (user, venvdir, cmd))
def update(commit=None): site_root = join(home_directory(SITE_USER), 'site') repodir = join(site_root, SITE_NAME) if not files.is_dir(repodir): with cd(site_root): su('git clone %s %s' % (SITE_REPO, SITE_NAME)) with cd(repodir): su('git fetch') if commit is None: commit = 'origin/master' su('git checkout %s' % commit)
def checkout_repo(commit=None): site_root = join(home_directory(SITE_USER), 'site') repodir = join(site_root, SITE_NAME) if not files.is_dir(repodir): with cd(site_root): su('git clone %s %s' % (SITE_REPO, SITE_NAME)) with cd(repodir): su('git fetch') if commit is None: commit = 'origin/master' su('git checkout %s' % commit)
def setup_supervisor(): site_root = join(home_directory(SITE_USER), 'site') upload_template('veyepar.conf', '/etc/supervisor/conf.d/veyepar.conf', context={ 'command': join(site_root, 'bin', 'runserver.sh'), 'user': SITE_USER, 'group': SITE_GROUP, 'logfile': join(site_root, 'logs', 'gunicorn_supervisor.log'), }, use_jinja=True, use_sudo=True) supervisor.update_config()
def setup_nginx(): site_root = join(home_directory(SITE_USER), 'site') upload_template('veyepar_nginx', '/etc/nginx/sites-available/veyepar', context={ 'access_log': join(site_root, 'logs', 'access.log'), 'error_log': join(site_root, 'logs', 'error.log'), 'static_location': join(site_root, 'static/'), 'media_location': join(site_root, 'media/'), }, use_jinja=True, use_sudo=True) require.nginx.enabled('veyepar') require.nginx.disabled('default')
def setup_nginx(): site_root = join(home_directory(SITE_USER), 'site') upload_template('researchcompendia_nginx', '/etc/nginx/sites-available/researchcompendia', context={ 'server_name': env.site, 'access_log': join(site_root, 'logs', 'access.log'), 'error_log': join(site_root, 'logs', 'error.log'), 'static_location': join(site_root, 'static/'), 'media_location': join(site_root, 'media/'), }, use_jinja=True, use_sudo=True, template_dir=TEMPLATE_DIR) require.nginx.enabled('researchcompendia') require.nginx.disabled('default') put(template_path('maintenance_nginx'), '/etc/nginx/sites-available/maintenance', use_sudo=True) put(template_path('maintenance_index.html'), '/usr/share/nginx/www/index.html', use_sudo=True)
def setup_rabbitmq(user=None): """ installs official rabbitmq package and sets up user """ deb.add_apt_key(url='http://www.rabbitmq.com/rabbitmq-signing-key-public.asc') require.deb.source('rabbitmq-server', 'http://www.rabbitmq.com/debian/', 'testing', 'main') require.deb.uptodate_index(max_age={'hour': 1}) require.deb.packages(['rabbitmq-server',]) if user is None: user = SITE_USER envfile = join(home_directory(SITE_USER), 'site/bin/environment.sh') secret = randomstring(64) sudo('rabbitmqctl delete_user guest') sudo('rabbitmqctl add_user %s "%s"' % (user, secret)) sudo('rabbitmqctl set_permissions -p / %s ".*" ".*" ".*"' % user) amqp_url = 'amqp://%s:%s@localhost:5672//' % (SITE_USER, secret) sed(envfile, 'DJANGO_BROKER_URL=".*"', 'DJANGO_BROKER_URL="%s"' % amqp_url, use_sudo=True)
def sync_auth_keys(): ''' Add multiple public keys to the user's authorized SSH keys from GitHub. ''' if env.user == 'vagrant': return error('Did not run sync_auth_keys on vagrant!!! Bad Idea.') ssh_dir = join(user.home_directory(env.user), '.ssh') require.files.directory(ssh_dir, mode='700') authorized_keys_filename = join(ssh_dir, 'authorized_keys') require.files.file(authorized_keys_filename, mode='600') run('cat /dev/null > %s' % quote(authorized_keys_filename)) info('Fetching public keys from GitHub') for gh_user in SSH_USERS: r = requests.get('https://api.github.com/users/%s/keys' % gh_user) for key in r.json(): run('echo %s >> %s' % (quote(key['key']), quote(authorized_keys_filename))) success('Public keys synced')
def setup_supervisor(): # hacky workaround to ??? sudo('touch /var/run/supervisor.sock') sudo('chmod 777 /var/run/supervisor.sock') sudo('service supervisor restart') site_root = join(home_directory(SITE_USER), 'site') upload_template('veyepar.conf', '/etc/supervisor/conf.d/veyepar.conf', context={ 'command': join(site_root, 'bin', 'runserver.sh'), 'user': SITE_USER, 'group': SITE_GROUP, 'logfile': join(site_root, 'logs', 'gunicorn_supervisor.log'), }, use_jinja=True, use_sudo=True) supervisor.update_config()
def sync_auth_keys(): """ Add multiple public keys to the user's authorized SSH keys from GitHub. """ ssh_dir = posixpath.join(user.home_directory(env.user), '.ssh') require.files.directory(ssh_dir, mode='700', owner=env.user, use_sudo=True) authorized_keys_filename = posixpath.join(ssh_dir, 'authorized_keys') require.files.file( authorized_keys_filename, mode='600', owner=env.user, use_sudo=True) sudo('cat /dev/null > %s' % quote(authorized_keys_filename)) for gh_user in SSH_USERS: r = requests.get("https://api.github.com/users/%s/keys" % gh_user) for key in r.json(): sudo( "echo %s >> %s" % (quote(key["key"]), quote(authorized_keys_filename)))
def setup_envvars(): """ copies secrets from env files that are not checked in to the deployment repo """ env_template_dir = join(FAB_HOME, 'env') secret = randomstring(64) site_root = join(home_directory(SITE_USER), 'site') bindir = join(site_root, 'bin') static_root = join(site_root, 'static') media_root = join(site_root, 'media') destination_envfile = join(bindir, 'environment.sh') with cd(bindir): upload_template(env.site_environment, destination_envfile, context={ 'secret_key': secret, 'static_root': static_root, 'media_root': media_root, }, template_dir=env_template_dir, use_jinja=True, use_sudo=True, chown=True, user=SITE_USER)
def update_site_version(site_version): envfile = join(home_directory(SITE_USER), 'site/bin/environment.sh') sed(envfile, 'SITE_VERSION=".*"', 'SITE_VERSION="%s"' % site_version, use_sudo=True)
def mkvirtualenv(virtualenv): with cd(join(home_directory(SITE_USER), 'venvs')): su('virtualenv %s' % virtualenv)
def get_site_version(): site_line = run('grep SITE_VERSION %s' % join(home_directory(SITE_USER), 'site/bin/environment.sh')) match = re.search(r'export SITE_VERSION="(?P<version>[^"]+)', site_line) g = match.groupdict() return g['version']
def syncdb(): environment = join(home_directory(SITE_USER), 'site/bin/environment.sh') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'companionpages') with cd(djangodir): vsu('source %s; ./manage.py syncdb --noinput --migrate' % environment)
def install_site_requirements(virtualenv): home = home_directory(SITE_USER) with cd(join(home, 'site', SITE_NAME)): vsu('pip install -r requirements/production.txt', virtualenv=virtualenv)
def install_dabo(virtualenv): vbin = join(home_directory(SITE_USER), 'venvs', virtualenv, 'bin') su('source {}/activate; '.format(vbin) + \ 'cd $(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"); ' 'git clone https://github.com/dabodev/dabo.git dabo-master; ' 'ln -sf dabo-master/dabo' )
def syncdb(site_version): vbin = join(home_directory(SITE_USER), 'venvs', site_version, 'bin') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'dj') with cd(djangodir): su('source %s/activate; ./manage.py migrate --noinput' % vbin)
def mkvirtualenv(virtualenv): with cd(join(home_directory(SITE_USER), 'venvs')): su('virtualenv --python=python3.5 --system-site-packages %s' % virtualenv)
def load_fixtures(): environment = join(home_directory(SITE_USER), 'site/bin/environment.sh') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'companionpages') with cd(djangodir): vsu('source %s; ./manage.py loaddata fixtures/*' % environment)
def install_dabo(virtualenv): vbin = join(home_directory(SITE_USER), 'venvs', virtualenv, 'bin') su('source {}/activate; '.format(vbin) + \ 'cd $(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"); ' 'git clone https://github.com/dabodev/dabo.git dabo-master; ' 'ln -s dabo-master/dabo' )
def syncdb(site_version): vbin = join(home_directory(SITE_USER), 'venvs', site_version, 'bin') djangodir = join(home_directory(SITE_USER), 'site', SITE_NAME, 'dj') with cd(djangodir): su('source %s/activate; ./manage.py syncdb --noinput' % vbin)
def mkvirtualenv(virtualenv): with cd(join(home_directory(SITE_USER), 'venvs')): su('virtualenv --system-site-packages %s' % virtualenv)