def restart_nginx():
print(green("Restarting Nginx"))
if service.is_running('nginx'):
#service.restart('nginx')
service.reload('nginx')
else:
service.start('nginx')
def uninstall(app): # pragma: no cover
for file_ in [app.supervisor, app.nginx_location, app.nginx_site]:
file_ = str(file_)
if exists(file_):
sudo('rm %s' % file_)
service.reload('nginx')
sudo('supervisorctl stop %s' % app.name)
def stop(app, maintenance_hours=1):
"""pause app by changing the supervisord config
create a maintenance page giving a date when we expect the service will be back
:param maintenance_hours: Number of hours we expect the downtime to last.
"""
if maintenance_hours is not None:
require.directory(str(app.www_dir), use_sudo=True)
timestamp = helpers.strfnow(add_hours=maintenance_hours)
sudo_upload_template('503.html', dest=str(app.www_dir / '503.html'),
app_name=app.name, timestamp=timestamp)
require_supervisor(app.supervisor, app, pause=True)
supervisor.update_config()
service.reload('nginx')
def install():
"""
Before use it, must do
apt-get install sudo
adduser username sudo
add new network card
dhclient eth1
"""
# Management card address
mgtmacaddress = 'a6:88:fe:f0:d5:2d'
vswitchname = "vswitch0"
vswitchport = "eth0"
# Check if we use the management interface
(user, ip) = env.host_string.split("@")
ismanagement = False
print network
for interface in network.interfaces():
if interface != 'lo':
if network.mac(interface) == mgtmacaddress:
ismanagement = True
break
# Require docker
require_docker()
if not ismanagement:
raise Exception(
'Please use the management network card with %(mgtmacaddress)s MAC address'
% locals())
# Configure networks
require.file('/etc/network/interfaces', source='interfaces', use_sudo=True)
service.reload('networking')
# Add open vSwitch
bridges = ovswitch.bridges()
if vswitchname not in bridges:
ovswitch.add_bridge(vswitchname)
ovswitch.add_port2bridge(vswitchport, vswitchname)
print(red("Plese reboot your system"))
def install():
"""
Before use it, must do
apt-get install sudo
adduser username sudo
add new network card
dhclient eth1
"""
# Management card address
mgtmacaddress = 'a6:88:fe:f0:d5:2d'
vswitchname = "vswitch0"
vswitchport = "eth0"
# Check if we use the management interface
(user, ip) = env.host_string.split("@")
ismanagement = False
print network
for interface in network.interfaces():
if interface != 'lo':
if network.mac(interface) == mgtmacaddress:
ismanagement = True
break
# Require docker
require_docker()
if not ismanagement:
raise Exception('Please use the management network card with %(mgtmacaddress)s MAC address' % locals())
# Configure networks
require.file('/etc/network/interfaces', source='interfaces', use_sudo=True)
service.reload('networking')
# Add open vSwitch
bridges = ovswitch.bridges()
if vswitchname not in bridges:
ovswitch.add_bridge(vswitchname)
ovswitch.add_port2bridge(vswitchport,vswitchname)
print(red("Plese reboot your system"))
def uninstall(app): # pragma: no cover
"""uninstall the app"""
for path in (app.nginx_location, app.nginx_site, app.venv_dir):
if exists(str(path)):
if path == app.nginx_site:
nginx.disable(app.nginx_site.name)
files.remove(str(path), recursive=True, use_sudo=True)
stop.execute_inner(app)
if user.exists(app.name):
if app.stack != 'soundcomparisons':
sudo('dropdb --if-exists %s' % app.name, user='******')
else:
sudo('echo "drop database {0};" | mysql'.format(app.name))
sudo('userdel -rf %s' % app.name)
if exists(str(app.supervisor)):
files.remove(str(app.supervisor), recursive=True, use_sudo=True)
supervisor.update_config()
service.reload('nginx')
systemd.uninstall(app, pathlib.Path(os.getcwd()) / 'systemd')
def cache(app):
"""require an app to be put behind varnish
"""
require.deb.package('varnish')
create_file_as_root('/etc/default/varnish', DEFAULT)
create_file_as_root('/etc/varnish/main.vcl', MAIN_VCL)
sites_vcl = '/etc/varnish/sites.vcl'
site_config = path('/etc/varnish/sites/{app.name}.vcl'.format(app=app))
include = 'include "%s";' % site_config
if exists(sites_vcl):
append(sites_vcl, include, use_sudo=True)
else:
create_file_as_root(sites_vcl, include + '\n')
require.files.directory(str(site_config.dirname()), use_sudo=True)
create_file_as_root(site_config, SITE_VCL_TEMPLATE.format(app=app))
service.restart('varnish')
create_file_as_root(
app.nginx_site,
SITE_TEMPLATE.format(**get_template_variables(App(app.name, 6081, domain=app.domain))))
service.reload('nginx')
def cache(app): # pragma: no cover
"""require an app to be put behind varnish
"""
require.deb.package('varnish')
create_file_as_root('/etc/default/varnish', DEFAULT)
create_file_as_root('/etc/varnish/main.vcl', MAIN_VCL)
sites_vcl = '/etc/varnish/sites.vcl'
site_config_dir = '/etc/varnish/sites'
site_config = '/'.join(site_config_dir, '{app.name}.vcl'.format(app=app))
include = 'include "%s";' % site_config
if exists(sites_vcl):
append(sites_vcl, include, use_sudo=True)
else:
create_file_as_root(sites_vcl, include + '\n')
require.files.directory(site_config_dir, use_sudo=True)
create_file_as_root(site_config, SITE_VCL_TEMPLATE.format(app=app))
service.restart('varnish')
template_vars = get_template_variables(App(app.name, 6081, domain=app.domain))
template_vars['SITE'] = True
upload_template_as_root(app.nginx_site, 'nginx-app.conf', template_vars)
service.reload('nginx')
def cache(app):
"""require an app to be put behind varnish
"""
require.deb.package('varnish')
create_file_as_root('/etc/default/varnish', DEFAULT)
create_file_as_root('/etc/varnish/main.vcl', MAIN_VCL)
sites_vcl = '/etc/varnish/sites.vcl'
site_config = path('/etc/varnish/sites/{app.name}.vcl'.format(app=app))
include = 'include "%s";' % site_config
if exists(sites_vcl):
append(sites_vcl, include, use_sudo=True)
else:
create_file_as_root(sites_vcl, include + '\n')
require.files.directory(str(site_config.dirname()), use_sudo=True)
create_file_as_root(site_config, SITE_VCL_TEMPLATE.format(app=app))
service.restart('varnish')
create_file_as_root(
app.nginx_site,
SITE_TEMPLATE.format(
**get_template_variables(App(app.name, 6081, domain=app.domain))))
service.reload('nginx')
def reload_config():
service.reload(service_name)
def uncache(app): # pragma: no cover
tv = get_template_variables(app)
tv['auth'] = http_auth(app)
create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**tv))
service.reload('nginx')
def deploy(app, environment, with_alembic=False):
template_variables = get_template_variables(
app, 'true' if environment == 'production' else 'false')
require.users.user(app.name, shell='/bin/bash')
require.postfix.server(env['host'])
require.postgres.server()
for pkg in [
'libpq-dev',
'git',
'nginx',
'supervisor',
'openjdk-6-jre',
'make',
'sqlite3',
'curl',
]:
require.deb.package(pkg)
require.postgres.user(app.name, app.name)
require.postgres.database(app.name, app.name)
require.files.directory(app.venv, use_sudo=True)
require.python.virtualenv(app.venv, use_sudo=True)
require.files.directory(app.logs, use_sudo=True)
with virtualenv(app.venv):
sudo('pip install -U pip')
require.python.package('gunicorn', use_sudo=True)
install_repos('clld')
install_repos(app.name)
sudo('webassets -m %s.assets build' % app.name)
require_bibutils(app)
#
# configure nginx:
#
require.files.directory(str(app.nginx_location.dirname()),
owner='root',
group='root',
use_sudo=True)
template_variables['auth'] = http_auth(app)
if environment == 'test':
create_file_as_root('/etc/nginx/sites-available/default', DEFAULT_SITE)
create_file_as_root(app.nginx_location,
LOCATION_TEMPLATE.format(**template_variables))
elif environment == 'production':
create_file_as_root(app.nginx_site,
SITE_TEMPLATE.format(**template_variables))
create_file_as_root('/etc/logrotate.d/{0}'.format(app.name),
LOGROTATE_TEMPLATE.format(**template_variables))
maintenance(app,
hours=app.deploy_duration,
template_variables=template_variables)
service.reload('nginx')
#
# TODO: replace with initialization of db from data repos!
#
if confirm('Copy files?', default=False):
execute(copy_files, app)
if not with_alembic and confirm('Recreate database?', default=False):
supervisor(app, 'pause', template_variables)
local('pg_dump -f /tmp/{0.name}.sql {0.name}'.format(app))
local('gzip -f /tmp/{0.name}.sql'.format(app))
require.files.file('/tmp/{0.name}.sql.gz'.format(app),
source="/tmp/{0.name}.sql.gz".format(app))
sudo('gunzip -f /tmp/{0.name}.sql.gz'.format(app))
if postgres.database_exists(app.name):
with cd('/var/lib/postgresql'):
sudo('sudo -u postgres dropdb %s' % app.name)
require.postgres.database(app.name, app.name)
sudo('sudo -u {0.name} psql -f /tmp/{0.name}.sql -d {0.name}'.format(
app))
else:
if exists(app.src.joinpath('alembic.ini')):
if confirm('Upgrade database?', default=False):
supervisor(app, 'pause', template_variables)
with virtualenv(app.venv):
with cd(app.src):
sudo('sudo -u {0.name} {1} -n production upgrade head'.
format(app, app.bin('alembic')))
create_file_as_root(
app.config, CONFIG_TEMPLATES[environment].format(**template_variables))
create_file_as_root(app.newrelic_config,
NEWRELIC_TEMPLATE.format(**template_variables))
supervisor(app, 'run', template_variables)
time.sleep(5)
res = run('curl http://localhost:%s/_ping' % app.port)
assert json.loads(res)['status'] == 'ok'
def reload_configuration():
""" Reloads SSH configuration. """
service.reload('ssh')
def reload(component):
service.reload(component)
def deploy(app, environment, with_alembic=False):
template_variables = get_template_variables(
app, 'true' if environment == 'production' else 'false')
require.users.user(app.name, shell='/bin/bash')
require.postfix.server(env['host'])
require.postgres.server()
for pkg in [
'libpq-dev',
'git',
'nginx',
'supervisor',
'openjdk-6-jre',
'make',
'sqlite3',
'curl',
]:
require.deb.package(pkg)
require.postgres.user(app.name, app.name)
require.postgres.database(app.name, app.name)
require.files.directory(app.venv, use_sudo=True)
require.python.virtualenv(app.venv, use_sudo=True)
require.files.directory(app.logs, use_sudo=True)
with virtualenv(app.venv):
sudo('pip install -U pip')
require.python.package('gunicorn', use_sudo=True)
install_repos('clld')
install_repos(app.name)
sudo('webassets -m %s.assets build' % app.name)
require_bibutils(app)
#
# configure nginx:
#
require.files.directory(
str(app.nginx_location.dirname()), owner='root', group='root', use_sudo=True)
template_variables['auth'] = http_auth(app)
if environment == 'test':
create_file_as_root('/etc/nginx/sites-available/default', DEFAULT_SITE)
create_file_as_root(
app.nginx_location, LOCATION_TEMPLATE.format(**template_variables))
elif environment == 'production':
create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**template_variables))
create_file_as_root(
'/etc/logrotate.d/{0}'.format(app.name),
LOGROTATE_TEMPLATE.format(**template_variables))
maintenance(app, hours=app.deploy_duration, template_variables=template_variables)
service.reload('nginx')
#
# TODO: replace with initialization of db from data repos!
#
if confirm('Copy files?', default=False):
execute(copy_files, app)
if not with_alembic and confirm('Recreate database?', default=False):
supervisor(app, 'pause', template_variables)
local('pg_dump -f /tmp/{0.name}.sql {0.name}'.format(app))
local('gzip -f /tmp/{0.name}.sql'.format(app))
require.files.file(
'/tmp/{0.name}.sql.gz'.format(app),
source="/tmp/{0.name}.sql.gz".format(app))
sudo('gunzip -f /tmp/{0.name}.sql.gz'.format(app))
if postgres.database_exists(app.name):
with cd('/var/lib/postgresql'):
sudo('sudo -u postgres dropdb %s' % app.name)
require.postgres.database(app.name, app.name)
sudo('sudo -u {0.name} psql -f /tmp/{0.name}.sql -d {0.name}'.format(app))
else:
if exists(app.src.joinpath('alembic.ini')):
if confirm('Upgrade database?', default=False):
supervisor(app, 'pause', template_variables)
with virtualenv(app.venv):
with cd(app.src):
sudo('sudo -u {0.name} {1} -n production upgrade head'.format(
app, app.bin('alembic')))
create_file_as_root(
app.config, CONFIG_TEMPLATES[environment].format(**template_variables))
create_file_as_root(
app.newrelic_config, NEWRELIC_TEMPLATE.format(**template_variables))
supervisor(app, 'run', template_variables)
time.sleep(5)
res = run('curl http://localhost:%s/_ping' % app.port)
assert json.loads(res)['status'] == 'ok'
def uninstall(app):
for file_ in [app.supervisor, app.nginx_location, app.nginx_site]:
if exists(file_):
sudo('rm %s' % file_)
service.reload('nginx')
sudo('/etc/init.d/supervisor restart')
def enable_gzip():
""" Enables gzip compression """
uncomment('/etc/nginx/nginx.conf', 'gzip_types', use_sudo=True)
service.reload('nginx')
def deploy(app, with_blog=None, with_alembic=False):
"""deploy the app"""
assert system.distrib_id() == 'Ubuntu'
lsb_codename = system.distrib_codename()
if lsb_codename != 'xenial':
raise ValueError('unsupported platform: %s' % lsb_codename)
# See whether the local appconfig clone is up-to-date with the remot master:
remote_repo = local('git ls-remote [email protected]:shh-dlce/appconfig.git HEAD | awk \'{ print $1}\'')
local_clone = local('git rev-parse HEAD')
if remote_repo != local_clone:
if confirm('Local appconfig clone is not up-to-date '
'with remote master, continue?', default=False):
print("Continuing deployment.")
else:
print("Deployment aborted.")
return
require.deb.packages(getattr(app, 'require_deb_%s' % lsb_codename) + app.require_deb)
require.users.user(app.name, create_home=True, shell='/bin/bash')
require.directory(str(app.www_dir), use_sudo=True)
require.directory(str(app.www_dir / 'files'), use_sudo=True)
require_logging(app.log_dir,
logrotate=app.logrotate,
access_log=app.access_log, error_log=app.error_log)
workers = 3 if app.workers > 3 and env.environment == 'test' else app.workers
with_blog = with_blog if with_blog is not None else app.with_blog
if env.environment != 'staging':
# Test and production instances are publicly accessible over HTTPS.
letsencrypt.require_certbot()
letsencrypt.require_cert(env.host)
if env.environment == 'production':
letsencrypt.require_cert(app)
ctx = template_context(app, workers=workers, with_blog=with_blog)
if app.stack == 'soundcomparisons': # pragma: no cover
require.git.working_copy(
'https://github.com/{0}/{1}.git'.format(app.github_org, app.github_repos),
path=str(app.home_dir / app.name),
use_sudo=True,
user=app.name)
require_bower(app, d=app.home_dir / app.name / 'site' / 'js')
require_grunt(app, d=app.home_dir / app.name / 'site' / 'js')
require_php(app)
require_mysql(app)
with shell_env(SYSTEMD_PAGER=''):
require.nginx.server()
sudo_upload_template('nginx-php-fpm-app.conf', str(app.nginx_site), app=app, env=env)
nginx.enable(app.name)
if env.environment == 'production':
# We only enable systemd services when deploying to production, because we don't want
# to start and run things like backup to CDSTAR from non-production systems.
systemd.enable(app, pathlib.Path(os.getcwd()) / 'systemd')
service.reload('nginx')
return
#
# Create a virtualenv for the app and install the app package in development mode, i.e. with
# repository working copy in /usr/venvs/<APP>/src
#
require_venv(
app.venv_dir,
require_packages=[app.app_pkg] + app.require_pip,
assets_name=app.name if app.stack == 'clld' else None)
#
# If some of the static assets are managed via bower, update them.
#
require_bower(app)
require_grunt(app)
require_nginx(ctx)
if app.stack == 'clld':
require_bibutils()
require_postgres(app)
require_config(app.config, app, ctx)
# if gunicorn runs, make it gracefully reload the app by sending HUP
# TODO: consider 'supervisorctl signal HUP $name' instead (xenial+)
sudo('( [ -f {0} ] && kill -0 $(cat {0}) 2> /dev/null '
'&& kill -HUP $(cat {0}) ) || echo no reload '.format(app.gunicorn_pid))
if not with_alembic and confirm('Recreate database?', default=False):
stop.execute_inner(app)
upload_sqldump(app)
elif exists(str(app.src_dir / 'alembic.ini')) and confirm('Upgrade database?', default=False):
# Note: stopping the app is not strictly necessary, because
# the alembic revisions run in separate transactions!
stop.execute_inner(app, maintenance_hours=app.deploy_duration)
alembic_upgrade_head(app, ctx)
pip_freeze(app)
start.execute_inner(app)
check(app)
if env.environment == 'production':
systemd.enable(app, pathlib.Path(os.getcwd()) / 'systemd')
def _update_nginx(app, with_varnish=True, varnish_port=PORT):
if with_varnish:
app = app.replace(port=varnish_port)
ctx = deployment.template_context(app)
deployment.require_nginx(ctx)
service.reload('nginx')
def start(app):
"""start app by changing the supervisord config"""
require_supervisor(app.supervisor, app)
supervisor.update_config()
service.reload('nginx')
def reload_configuration():
""" Reloads SSH configuration. """
service.reload('ssh')
def deploy(app, environment, with_alembic=False, with_blog=False, with_files=True):
with settings(warn_only=True):
lsb_release = run('lsb_release -a')
for codename in ['trusty', 'precise']:
if codename in lsb_release:
lsb_release = codename
break
else:
if lsb_release != '{"status": "ok"}':
# if this were the case, we'd be in a test!
raise ValueError('unsupported platform: %s' % lsb_release)
if environment == 'test' and app.workers > 3:
app.workers = 3
template_variables = get_template_variables(
app,
monitor_mode='true' if environment == 'production' else 'false',
with_blog=with_blog)
require.users.user(app.name, shell='/bin/bash')
require.postfix.server(env['host'])
require.postgres.server()
require.deb.packages(app.require_deb)
require.postgres.user(app.name, app.name)
require.postgres.database(app.name, app.name)
require.files.directory(str(app.venv), use_sudo=True)
with_pg_collkey = getattr(app, 'pg_collkey', False)
if with_pg_collkey:
pg_version = '9.1' if lsb_release == 'precise' else '9.3'
if not exists('/usr/lib/postgresql/%s/lib/collkey_icu.so' % pg_version):
require.deb.packages(['postgresql-server-dev-%s' % pg_version, 'libicu-dev'])
upload_template_as_root(
'/tmp/Makefile', 'pg_collkey_Makefile', dict(pg_version=pg_version))
require.files.file(
'/tmp/collkey_icu.c',
source=os.path.join(
os.path.dirname(__file__), 'pg_collkey-v0.5', 'collkey_icu.c'))
with cd('/tmp'):
sudo('make')
sudo('make install')
init_pg_collkey(app)
if lsb_release == 'precise':
require.deb.package('python-dev')
require.python.virtualenv(str(app.venv), use_sudo=True)
else:
require.deb.package('python3-dev')
require.deb.package('python-virtualenv')
if not exists(str(app.venv.joinpath('bin'))):
sudo('virtualenv -q --python=python3 %s' % app.venv)
require.files.directory(str(app.logs), use_sudo=True)
if app.pages and not exists(str(app.pages)):
with cd(str(app.home)):
sudo('sudo -u {0} git clone https://github.com/clld/{0}-pages.git'.format(app.name))
with virtualenv(str(app.venv)):
require.python.pip('6.0.6')
sp = env['sudo_prefix']
env['sudo_prefix'] += ' -H' # set HOME for pip log/cache
require.python.packages(app.require_pip, use_sudo=True)
for name in [app.name] + getattr(app, 'dependencies', []):
pkg = '-e git+git://github.com/clld/%s.git#egg=%s' % (name, name)
require.python.package(pkg, use_sudo=True)
env['sudo_prefix'] = sp
sudo('webassets -m %s.assets build' % app.name)
require_bibutils(app)
#
# configure nginx:
#
require.files.directory(
os.path.dirname(str(app.nginx_location)),
owner='root', group='root', use_sudo=True)
restricted, auth = http_auth(app)
if restricted:
template_variables['auth'] = auth
template_variables['admin_auth'] = auth
if environment == 'test':
upload_template_as_root('/etc/nginx/sites-available/default', 'nginx-default.conf')
template_variables['SITE'] = False
upload_template_as_root(
app.nginx_location, 'nginx-app.conf', template_variables)
elif environment == 'production':
template_variables['SITE'] = True
upload_template_as_root(app.nginx_site, 'nginx-app.conf', template_variables)
upload_template_as_root(
'/etc/logrotate.d/{0}'.format(app.name), 'logrotate.conf', template_variables)
maintenance(app, hours=app.deploy_duration, template_variables=template_variables)
service.reload('nginx')
#
# TODO: replace with initialization of db from data repos!
#
if with_files:
if confirm('Copy files?', default=False):
execute(copy_files, app)
if not with_alembic and confirm('Recreate database?', default=False):
local('pg_dump -x -O -f /tmp/{0.name}.sql {0.name}'.format(app))
local('gzip -f /tmp/{0.name}.sql'.format(app))
require.files.file(
'/tmp/{0.name}.sql.gz'.format(app),
source="/tmp/{0.name}.sql.gz".format(app))
sudo('gunzip -f /tmp/{0.name}.sql.gz'.format(app))
supervisor(app, 'pause', template_variables)
if postgres.database_exists(app.name):
with cd('/var/lib/postgresql'):
sudo('sudo -u postgres dropdb %s' % app.name)
require.postgres.database(app.name, app.name)
if with_pg_collkey:
init_pg_collkey(app)
sudo('sudo -u {0.name} psql -f /tmp/{0.name}.sql -d {0.name}'.format(app))
else:
if exists(app.src.joinpath('alembic.ini')):
if confirm('Upgrade database?', default=False):
# Note: stopping the app is not strictly necessary, because the alembic
# revisions run in separate transactions!
supervisor(app, 'pause', template_variables)
with virtualenv(str(app.venv)):
with cd(str(app.src)):
sudo('sudo -u {0.name} {1} -n production upgrade head'.format(
app, app.bin('alembic')))
if confirm('Vacuum database?', default=False):
if confirm('VACUUM FULL?', default=False):
sudo('sudo -u postgres vacuumdb -f -z -d %s' % app.name)
else:
sudo('sudo -u postgres vacuumdb -z -d %s' % app.name)
template_variables['TEST'] = {'test': True, 'production': False}[environment]
upload_template_as_root(app.config, 'config.ini', template_variables)
upload_template_as_root(app.newrelic_config, 'newrelic.ini', template_variables)
supervisor(app, 'run', template_variables)
time.sleep(5)
res = run('curl http://localhost:%s/_ping' % app.port)
assert json.loads(res)['status'] == 'ok'
def uncache(app):
tv = get_template_variables(app)
tv['auth'] = http_auth(app)
create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**tv))
service.reload('nginx')
def enable_gzip():
""" Enables gzip compression """
uncomment('/etc/nginx/nginx.conf', 'gzip_types', use_sudo=True)
service.reload('nginx')
def deploy(base_dir):
sync(base_dir)
service.reload(APP_NAME)
def uninstall(app):
for file_ in [app.supervisor, app.nginx_location, app.nginx_site]:
if exists(file_):
sudo('rm %s' % file_)
service.reload('nginx')
sudo('/etc/init.d/supervisor restart')
