def restart_nginx(): print(green("Restarting Nginx")) if service.is_running('nginx'): #service.restart('nginx') service.reload('nginx') else: service.start('nginx')
def uninstall(app): # pragma: no cover for file_ in [app.supervisor, app.nginx_location, app.nginx_site]: file_ = str(file_) if exists(file_): sudo('rm %s' % file_) service.reload('nginx') sudo('supervisorctl stop %s' % app.name)
def stop(app, maintenance_hours=1): """pause app by changing the supervisord config create a maintenance page giving a date when we expect the service will be back :param maintenance_hours: Number of hours we expect the downtime to last. """ if maintenance_hours is not None: require.directory(str(app.www_dir), use_sudo=True) timestamp = helpers.strfnow(add_hours=maintenance_hours) sudo_upload_template('503.html', dest=str(app.www_dir / '503.html'), app_name=app.name, timestamp=timestamp) require_supervisor(app.supervisor, app, pause=True) supervisor.update_config() service.reload('nginx')
def install(): """ Before use it, must do apt-get install sudo adduser username sudo add new network card dhclient eth1 """ # Management card address mgtmacaddress = 'a6:88:fe:f0:d5:2d' vswitchname = "vswitch0" vswitchport = "eth0" # Check if we use the management interface (user, ip) = env.host_string.split("@") ismanagement = False print network for interface in network.interfaces(): if interface != 'lo': if network.mac(interface) == mgtmacaddress: ismanagement = True break # Require docker require_docker() if not ismanagement: raise Exception( 'Please use the management network card with %(mgtmacaddress)s MAC address' % locals()) # Configure networks require.file('/etc/network/interfaces', source='interfaces', use_sudo=True) service.reload('networking') # Add open vSwitch bridges = ovswitch.bridges() if vswitchname not in bridges: ovswitch.add_bridge(vswitchname) ovswitch.add_port2bridge(vswitchport, vswitchname) print(red("Plese reboot your system"))
def install(): """ Before use it, must do apt-get install sudo adduser username sudo add new network card dhclient eth1 """ # Management card address mgtmacaddress = 'a6:88:fe:f0:d5:2d' vswitchname = "vswitch0" vswitchport = "eth0" # Check if we use the management interface (user, ip) = env.host_string.split("@") ismanagement = False print network for interface in network.interfaces(): if interface != 'lo': if network.mac(interface) == mgtmacaddress: ismanagement = True break # Require docker require_docker() if not ismanagement: raise Exception('Please use the management network card with %(mgtmacaddress)s MAC address' % locals()) # Configure networks require.file('/etc/network/interfaces', source='interfaces', use_sudo=True) service.reload('networking') # Add open vSwitch bridges = ovswitch.bridges() if vswitchname not in bridges: ovswitch.add_bridge(vswitchname) ovswitch.add_port2bridge(vswitchport,vswitchname) print(red("Plese reboot your system"))
def uninstall(app): # pragma: no cover """uninstall the app""" for path in (app.nginx_location, app.nginx_site, app.venv_dir): if exists(str(path)): if path == app.nginx_site: nginx.disable(app.nginx_site.name) files.remove(str(path), recursive=True, use_sudo=True) stop.execute_inner(app) if user.exists(app.name): if app.stack != 'soundcomparisons': sudo('dropdb --if-exists %s' % app.name, user='******') else: sudo('echo "drop database {0};" | mysql'.format(app.name)) sudo('userdel -rf %s' % app.name) if exists(str(app.supervisor)): files.remove(str(app.supervisor), recursive=True, use_sudo=True) supervisor.update_config() service.reload('nginx') systemd.uninstall(app, pathlib.Path(os.getcwd()) / 'systemd')
def cache(app): """require an app to be put behind varnish """ require.deb.package('varnish') create_file_as_root('/etc/default/varnish', DEFAULT) create_file_as_root('/etc/varnish/main.vcl', MAIN_VCL) sites_vcl = '/etc/varnish/sites.vcl' site_config = path('/etc/varnish/sites/{app.name}.vcl'.format(app=app)) include = 'include "%s";' % site_config if exists(sites_vcl): append(sites_vcl, include, use_sudo=True) else: create_file_as_root(sites_vcl, include + '\n') require.files.directory(str(site_config.dirname()), use_sudo=True) create_file_as_root(site_config, SITE_VCL_TEMPLATE.format(app=app)) service.restart('varnish') create_file_as_root( app.nginx_site, SITE_TEMPLATE.format(**get_template_variables(App(app.name, 6081, domain=app.domain)))) service.reload('nginx')
def cache(app): # pragma: no cover """require an app to be put behind varnish """ require.deb.package('varnish') create_file_as_root('/etc/default/varnish', DEFAULT) create_file_as_root('/etc/varnish/main.vcl', MAIN_VCL) sites_vcl = '/etc/varnish/sites.vcl' site_config_dir = '/etc/varnish/sites' site_config = '/'.join(site_config_dir, '{app.name}.vcl'.format(app=app)) include = 'include "%s";' % site_config if exists(sites_vcl): append(sites_vcl, include, use_sudo=True) else: create_file_as_root(sites_vcl, include + '\n') require.files.directory(site_config_dir, use_sudo=True) create_file_as_root(site_config, SITE_VCL_TEMPLATE.format(app=app)) service.restart('varnish') template_vars = get_template_variables(App(app.name, 6081, domain=app.domain)) template_vars['SITE'] = True upload_template_as_root(app.nginx_site, 'nginx-app.conf', template_vars) service.reload('nginx')
def cache(app): """require an app to be put behind varnish """ require.deb.package('varnish') create_file_as_root('/etc/default/varnish', DEFAULT) create_file_as_root('/etc/varnish/main.vcl', MAIN_VCL) sites_vcl = '/etc/varnish/sites.vcl' site_config = path('/etc/varnish/sites/{app.name}.vcl'.format(app=app)) include = 'include "%s";' % site_config if exists(sites_vcl): append(sites_vcl, include, use_sudo=True) else: create_file_as_root(sites_vcl, include + '\n') require.files.directory(str(site_config.dirname()), use_sudo=True) create_file_as_root(site_config, SITE_VCL_TEMPLATE.format(app=app)) service.restart('varnish') create_file_as_root( app.nginx_site, SITE_TEMPLATE.format( **get_template_variables(App(app.name, 6081, domain=app.domain)))) service.reload('nginx')
def reload_config(): service.reload(service_name)
def uncache(app): # pragma: no cover tv = get_template_variables(app) tv['auth'] = http_auth(app) create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**tv)) service.reload('nginx')
def deploy(app, environment, with_alembic=False): template_variables = get_template_variables( app, 'true' if environment == 'production' else 'false') require.users.user(app.name, shell='/bin/bash') require.postfix.server(env['host']) require.postgres.server() for pkg in [ 'libpq-dev', 'git', 'nginx', 'supervisor', 'openjdk-6-jre', 'make', 'sqlite3', 'curl', ]: require.deb.package(pkg) require.postgres.user(app.name, app.name) require.postgres.database(app.name, app.name) require.files.directory(app.venv, use_sudo=True) require.python.virtualenv(app.venv, use_sudo=True) require.files.directory(app.logs, use_sudo=True) with virtualenv(app.venv): sudo('pip install -U pip') require.python.package('gunicorn', use_sudo=True) install_repos('clld') install_repos(app.name) sudo('webassets -m %s.assets build' % app.name) require_bibutils(app) # # configure nginx: # require.files.directory(str(app.nginx_location.dirname()), owner='root', group='root', use_sudo=True) template_variables['auth'] = http_auth(app) if environment == 'test': create_file_as_root('/etc/nginx/sites-available/default', DEFAULT_SITE) create_file_as_root(app.nginx_location, LOCATION_TEMPLATE.format(**template_variables)) elif environment == 'production': create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**template_variables)) create_file_as_root('/etc/logrotate.d/{0}'.format(app.name), LOGROTATE_TEMPLATE.format(**template_variables)) maintenance(app, hours=app.deploy_duration, template_variables=template_variables) service.reload('nginx') # # TODO: replace with initialization of db from data repos! # if confirm('Copy files?', default=False): execute(copy_files, app) if not with_alembic and confirm('Recreate database?', default=False): supervisor(app, 'pause', template_variables) local('pg_dump -f /tmp/{0.name}.sql {0.name}'.format(app)) local('gzip -f /tmp/{0.name}.sql'.format(app)) require.files.file('/tmp/{0.name}.sql.gz'.format(app), source="/tmp/{0.name}.sql.gz".format(app)) sudo('gunzip -f /tmp/{0.name}.sql.gz'.format(app)) if postgres.database_exists(app.name): with cd('/var/lib/postgresql'): sudo('sudo -u postgres dropdb %s' % app.name) require.postgres.database(app.name, app.name) sudo('sudo -u {0.name} psql -f /tmp/{0.name}.sql -d {0.name}'.format( app)) else: if exists(app.src.joinpath('alembic.ini')): if confirm('Upgrade database?', default=False): supervisor(app, 'pause', template_variables) with virtualenv(app.venv): with cd(app.src): sudo('sudo -u {0.name} {1} -n production upgrade head'. format(app, app.bin('alembic'))) create_file_as_root( app.config, CONFIG_TEMPLATES[environment].format(**template_variables)) create_file_as_root(app.newrelic_config, NEWRELIC_TEMPLATE.format(**template_variables)) supervisor(app, 'run', template_variables) time.sleep(5) res = run('curl http://localhost:%s/_ping' % app.port) assert json.loads(res)['status'] == 'ok'
def reload_configuration(): """ Reloads SSH configuration. """ service.reload('ssh')
def reload(component): service.reload(component)
def deploy(app, environment, with_alembic=False): template_variables = get_template_variables( app, 'true' if environment == 'production' else 'false') require.users.user(app.name, shell='/bin/bash') require.postfix.server(env['host']) require.postgres.server() for pkg in [ 'libpq-dev', 'git', 'nginx', 'supervisor', 'openjdk-6-jre', 'make', 'sqlite3', 'curl', ]: require.deb.package(pkg) require.postgres.user(app.name, app.name) require.postgres.database(app.name, app.name) require.files.directory(app.venv, use_sudo=True) require.python.virtualenv(app.venv, use_sudo=True) require.files.directory(app.logs, use_sudo=True) with virtualenv(app.venv): sudo('pip install -U pip') require.python.package('gunicorn', use_sudo=True) install_repos('clld') install_repos(app.name) sudo('webassets -m %s.assets build' % app.name) require_bibutils(app) # # configure nginx: # require.files.directory( str(app.nginx_location.dirname()), owner='root', group='root', use_sudo=True) template_variables['auth'] = http_auth(app) if environment == 'test': create_file_as_root('/etc/nginx/sites-available/default', DEFAULT_SITE) create_file_as_root( app.nginx_location, LOCATION_TEMPLATE.format(**template_variables)) elif environment == 'production': create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**template_variables)) create_file_as_root( '/etc/logrotate.d/{0}'.format(app.name), LOGROTATE_TEMPLATE.format(**template_variables)) maintenance(app, hours=app.deploy_duration, template_variables=template_variables) service.reload('nginx') # # TODO: replace with initialization of db from data repos! # if confirm('Copy files?', default=False): execute(copy_files, app) if not with_alembic and confirm('Recreate database?', default=False): supervisor(app, 'pause', template_variables) local('pg_dump -f /tmp/{0.name}.sql {0.name}'.format(app)) local('gzip -f /tmp/{0.name}.sql'.format(app)) require.files.file( '/tmp/{0.name}.sql.gz'.format(app), source="/tmp/{0.name}.sql.gz".format(app)) sudo('gunzip -f /tmp/{0.name}.sql.gz'.format(app)) if postgres.database_exists(app.name): with cd('/var/lib/postgresql'): sudo('sudo -u postgres dropdb %s' % app.name) require.postgres.database(app.name, app.name) sudo('sudo -u {0.name} psql -f /tmp/{0.name}.sql -d {0.name}'.format(app)) else: if exists(app.src.joinpath('alembic.ini')): if confirm('Upgrade database?', default=False): supervisor(app, 'pause', template_variables) with virtualenv(app.venv): with cd(app.src): sudo('sudo -u {0.name} {1} -n production upgrade head'.format( app, app.bin('alembic'))) create_file_as_root( app.config, CONFIG_TEMPLATES[environment].format(**template_variables)) create_file_as_root( app.newrelic_config, NEWRELIC_TEMPLATE.format(**template_variables)) supervisor(app, 'run', template_variables) time.sleep(5) res = run('curl http://localhost:%s/_ping' % app.port) assert json.loads(res)['status'] == 'ok'
def uninstall(app): for file_ in [app.supervisor, app.nginx_location, app.nginx_site]: if exists(file_): sudo('rm %s' % file_) service.reload('nginx') sudo('/etc/init.d/supervisor restart')
def enable_gzip(): """ Enables gzip compression """ uncomment('/etc/nginx/nginx.conf', 'gzip_types', use_sudo=True) service.reload('nginx')
def deploy(app, with_blog=None, with_alembic=False): """deploy the app""" assert system.distrib_id() == 'Ubuntu' lsb_codename = system.distrib_codename() if lsb_codename != 'xenial': raise ValueError('unsupported platform: %s' % lsb_codename) # See whether the local appconfig clone is up-to-date with the remot master: remote_repo = local('git ls-remote [email protected]:shh-dlce/appconfig.git HEAD | awk \'{ print $1}\'') local_clone = local('git rev-parse HEAD') if remote_repo != local_clone: if confirm('Local appconfig clone is not up-to-date ' 'with remote master, continue?', default=False): print("Continuing deployment.") else: print("Deployment aborted.") return require.deb.packages(getattr(app, 'require_deb_%s' % lsb_codename) + app.require_deb) require.users.user(app.name, create_home=True, shell='/bin/bash') require.directory(str(app.www_dir), use_sudo=True) require.directory(str(app.www_dir / 'files'), use_sudo=True) require_logging(app.log_dir, logrotate=app.logrotate, access_log=app.access_log, error_log=app.error_log) workers = 3 if app.workers > 3 and env.environment == 'test' else app.workers with_blog = with_blog if with_blog is not None else app.with_blog if env.environment != 'staging': # Test and production instances are publicly accessible over HTTPS. letsencrypt.require_certbot() letsencrypt.require_cert(env.host) if env.environment == 'production': letsencrypt.require_cert(app) ctx = template_context(app, workers=workers, with_blog=with_blog) if app.stack == 'soundcomparisons': # pragma: no cover require.git.working_copy( 'https://github.com/{0}/{1}.git'.format(app.github_org, app.github_repos), path=str(app.home_dir / app.name), use_sudo=True, user=app.name) require_bower(app, d=app.home_dir / app.name / 'site' / 'js') require_grunt(app, d=app.home_dir / app.name / 'site' / 'js') require_php(app) require_mysql(app) with shell_env(SYSTEMD_PAGER=''): require.nginx.server() sudo_upload_template('nginx-php-fpm-app.conf', str(app.nginx_site), app=app, env=env) nginx.enable(app.name) if env.environment == 'production': # We only enable systemd services when deploying to production, because we don't want # to start and run things like backup to CDSTAR from non-production systems. systemd.enable(app, pathlib.Path(os.getcwd()) / 'systemd') service.reload('nginx') return # # Create a virtualenv for the app and install the app package in development mode, i.e. with # repository working copy in /usr/venvs/<APP>/src # require_venv( app.venv_dir, require_packages=[app.app_pkg] + app.require_pip, assets_name=app.name if app.stack == 'clld' else None) # # If some of the static assets are managed via bower, update them. # require_bower(app) require_grunt(app) require_nginx(ctx) if app.stack == 'clld': require_bibutils() require_postgres(app) require_config(app.config, app, ctx) # if gunicorn runs, make it gracefully reload the app by sending HUP # TODO: consider 'supervisorctl signal HUP $name' instead (xenial+) sudo('( [ -f {0} ] && kill -0 $(cat {0}) 2> /dev/null ' '&& kill -HUP $(cat {0}) ) || echo no reload '.format(app.gunicorn_pid)) if not with_alembic and confirm('Recreate database?', default=False): stop.execute_inner(app) upload_sqldump(app) elif exists(str(app.src_dir / 'alembic.ini')) and confirm('Upgrade database?', default=False): # Note: stopping the app is not strictly necessary, because # the alembic revisions run in separate transactions! stop.execute_inner(app, maintenance_hours=app.deploy_duration) alembic_upgrade_head(app, ctx) pip_freeze(app) start.execute_inner(app) check(app) if env.environment == 'production': systemd.enable(app, pathlib.Path(os.getcwd()) / 'systemd')
def _update_nginx(app, with_varnish=True, varnish_port=PORT): if with_varnish: app = app.replace(port=varnish_port) ctx = deployment.template_context(app) deployment.require_nginx(ctx) service.reload('nginx')
def start(app): """start app by changing the supervisord config""" require_supervisor(app.supervisor, app) supervisor.update_config() service.reload('nginx')
def deploy(app, environment, with_alembic=False, with_blog=False, with_files=True): with settings(warn_only=True): lsb_release = run('lsb_release -a') for codename in ['trusty', 'precise']: if codename in lsb_release: lsb_release = codename break else: if lsb_release != '{"status": "ok"}': # if this were the case, we'd be in a test! raise ValueError('unsupported platform: %s' % lsb_release) if environment == 'test' and app.workers > 3: app.workers = 3 template_variables = get_template_variables( app, monitor_mode='true' if environment == 'production' else 'false', with_blog=with_blog) require.users.user(app.name, shell='/bin/bash') require.postfix.server(env['host']) require.postgres.server() require.deb.packages(app.require_deb) require.postgres.user(app.name, app.name) require.postgres.database(app.name, app.name) require.files.directory(str(app.venv), use_sudo=True) with_pg_collkey = getattr(app, 'pg_collkey', False) if with_pg_collkey: pg_version = '9.1' if lsb_release == 'precise' else '9.3' if not exists('/usr/lib/postgresql/%s/lib/collkey_icu.so' % pg_version): require.deb.packages(['postgresql-server-dev-%s' % pg_version, 'libicu-dev']) upload_template_as_root( '/tmp/Makefile', 'pg_collkey_Makefile', dict(pg_version=pg_version)) require.files.file( '/tmp/collkey_icu.c', source=os.path.join( os.path.dirname(__file__), 'pg_collkey-v0.5', 'collkey_icu.c')) with cd('/tmp'): sudo('make') sudo('make install') init_pg_collkey(app) if lsb_release == 'precise': require.deb.package('python-dev') require.python.virtualenv(str(app.venv), use_sudo=True) else: require.deb.package('python3-dev') require.deb.package('python-virtualenv') if not exists(str(app.venv.joinpath('bin'))): sudo('virtualenv -q --python=python3 %s' % app.venv) require.files.directory(str(app.logs), use_sudo=True) if app.pages and not exists(str(app.pages)): with cd(str(app.home)): sudo('sudo -u {0} git clone https://github.com/clld/{0}-pages.git'.format(app.name)) with virtualenv(str(app.venv)): require.python.pip('6.0.6') sp = env['sudo_prefix'] env['sudo_prefix'] += ' -H' # set HOME for pip log/cache require.python.packages(app.require_pip, use_sudo=True) for name in [app.name] + getattr(app, 'dependencies', []): pkg = '-e git+git://github.com/clld/%s.git#egg=%s' % (name, name) require.python.package(pkg, use_sudo=True) env['sudo_prefix'] = sp sudo('webassets -m %s.assets build' % app.name) require_bibutils(app) # # configure nginx: # require.files.directory( os.path.dirname(str(app.nginx_location)), owner='root', group='root', use_sudo=True) restricted, auth = http_auth(app) if restricted: template_variables['auth'] = auth template_variables['admin_auth'] = auth if environment == 'test': upload_template_as_root('/etc/nginx/sites-available/default', 'nginx-default.conf') template_variables['SITE'] = False upload_template_as_root( app.nginx_location, 'nginx-app.conf', template_variables) elif environment == 'production': template_variables['SITE'] = True upload_template_as_root(app.nginx_site, 'nginx-app.conf', template_variables) upload_template_as_root( '/etc/logrotate.d/{0}'.format(app.name), 'logrotate.conf', template_variables) maintenance(app, hours=app.deploy_duration, template_variables=template_variables) service.reload('nginx') # # TODO: replace with initialization of db from data repos! # if with_files: if confirm('Copy files?', default=False): execute(copy_files, app) if not with_alembic and confirm('Recreate database?', default=False): local('pg_dump -x -O -f /tmp/{0.name}.sql {0.name}'.format(app)) local('gzip -f /tmp/{0.name}.sql'.format(app)) require.files.file( '/tmp/{0.name}.sql.gz'.format(app), source="/tmp/{0.name}.sql.gz".format(app)) sudo('gunzip -f /tmp/{0.name}.sql.gz'.format(app)) supervisor(app, 'pause', template_variables) if postgres.database_exists(app.name): with cd('/var/lib/postgresql'): sudo('sudo -u postgres dropdb %s' % app.name) require.postgres.database(app.name, app.name) if with_pg_collkey: init_pg_collkey(app) sudo('sudo -u {0.name} psql -f /tmp/{0.name}.sql -d {0.name}'.format(app)) else: if exists(app.src.joinpath('alembic.ini')): if confirm('Upgrade database?', default=False): # Note: stopping the app is not strictly necessary, because the alembic # revisions run in separate transactions! supervisor(app, 'pause', template_variables) with virtualenv(str(app.venv)): with cd(str(app.src)): sudo('sudo -u {0.name} {1} -n production upgrade head'.format( app, app.bin('alembic'))) if confirm('Vacuum database?', default=False): if confirm('VACUUM FULL?', default=False): sudo('sudo -u postgres vacuumdb -f -z -d %s' % app.name) else: sudo('sudo -u postgres vacuumdb -z -d %s' % app.name) template_variables['TEST'] = {'test': True, 'production': False}[environment] upload_template_as_root(app.config, 'config.ini', template_variables) upload_template_as_root(app.newrelic_config, 'newrelic.ini', template_variables) supervisor(app, 'run', template_variables) time.sleep(5) res = run('curl http://localhost:%s/_ping' % app.port) assert json.loads(res)['status'] == 'ok'
def uncache(app): tv = get_template_variables(app) tv['auth'] = http_auth(app) create_file_as_root(app.nginx_site, SITE_TEMPLATE.format(**tv)) service.reload('nginx')
def deploy(base_dir): sync(base_dir) service.reload(APP_NAME)