def configure_mod_tile(): with sudo(user='******'): mkdir('/srv/tilery/tmp/tiles') mkdir('/srv/tilery/renderd') with sudo(), cd('/etc/apache2/'): put('remote/tile.load', 'mods-available/tile.load') put('remote/tile.conf', 'mods-available/tile.conf') put('remote/apache.conf', 'sites-enabled/000-default.conf') put('remote/ports.conf', 'ports.conf') run('a2enmod tile')
def http(): conf = template('remote/gunicorn.conf', workers=config.workers) with sudo(): put(conf, '/srv/addok/gunicorn.conf') nginx_conf = template('remote/nginx.conf', domain=config.domain) with sudo(): put(nginx_conf, '/etc/nginx/sites-enabled/addok') # On LXC containers, somaxconn cannot be changed. This must be done on the # host machine. run(f'sudo sysctl -w net.core.somaxconn={config.connections} || exit 0') restart()
def venv(): """Setup the python virtualenv.""" path = '/srv/addok/venv/' if not exists(path): with sudo(user='******'): run(f'python3 -m venv {path}') pip('install pip -U')
def ssh_keys(): """Install ssh keys from remote urls.""" with sudo(): for name, url in config.get('ssh_key_urls', {}).items(): key = requests.get(url).text.replace('\n', '') run('grep -q -r "{key}" .ssh/authorized_keys || echo "{key}" ' '| tee --append .ssh/authorized_keys'.format(key=key))
def create_bar(): with sudo(user='******'), cd('/srv/'): if exists('/tmp/foo'): mkdir('/tmp/foo/bar') else: print('/tmp/foo does not exist') ls('tamer')
def put_file(): with sudo(user='******'): put('README.md', '/tmp/foobarbaz') ls('/tmp/foobarbaz') run('cat /tmp/foobarbaz') put(StringIO('foobarbazéé'), '/tmp/foobarbaz') run('cat /tmp/foobarbaz')
def connection(): with connect(hostname=os.environ.get('USINE_TEST_HOST')): with sudo(): run('useradd -N usinetest -d /srv/usine/ ' '|| echo "usinetest exists"') yield run('userdel usinetest')
def import_custom_data(): """Send and import boundary and city SQL.""" wget('http://nuage.yohanboniface.me/boundary.json', '/tmp/boundary.json') run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \ -lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/boundary.sql /tmp/boundary.json -sql \ \\'SELECT name,"name:en","name:fr","name:ar","name:es","name:de","name:ru",iso FROM boundary\\' -nln itl_boundary""") import_sql_file('/tmp/boundary.sql') wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/city.csv', '/tmp/city.csv') run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \ -lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/city.sql /tmp/city.csv \ -select name,'name:en','name:fr','name:ar',capital,type,prio,ldir \ -nln city -oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \ -oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""") import_sql_file('/tmp/city.sql') wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/country.csv', '/tmp/country.csv') run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \ -lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/country.sql /tmp/country.csv \ -select name,'name:en','name:fr','name:ar',prio,iso,sov -nln country \ -oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \ -oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""") import_sql_file('/tmp/country.sql') with sudo(user='******'): wget('http://nuage.yohanboniface.me/disputed.json', '/srv/tilery/pianoforte/data/disputed.json')
def render(map='piano', min=1, max=10, threads=8): """Run a render command.""" with sudo(user='******'), screen(name='render'): run(f'render_list --map {map} --all --force --num-threads {threads} ' f'--socket /var/run/renderd/renderd.sock ' f'--tile-dir /srv/tilery/tmp/tiles ' f' --min-zoom {min} --max-zoom {max}')
def addok(command): """Run a addok command on the remote server. :command: the addok command to run. """ with sudo(user='******'): run(f'/srv/addok/venv/bin/addok {command}')
def pip(command): """Run a pip command on the remote server. :command: the pip command to run. """ with sudo(user='******'): run(f'/srv/addok/venv/bin/pip {command}')
def import_sql_file(path): # Remove transaction management, as it does not cover the DROP TABLE; # we'll cover the transaction manually with "psql --single-transaction" run(f'sed -i.bak "/BEGIN;/d" {path}') run(f'sed -i.bak "/END;/d" {path}') run(f'sed -i.bak "/COMMIT;/d" {path}') with sudo(user='******'): run(f'psql --single-transaction -d tilery --file {path}')
def install_imposm(force=False, release='0.6.0-alpha.4'): """Install imposm from binary. :force: install even if the binary already exists. :release: optionnal release to install """ if exists('/usr/bin/imposm') and not force: print('imposm already installed') return # Cf https://github.com/omniscale/imposm3/issues/165#issuecomment-395993259 wget(f'https://github.com/omniscale/imposm3/releases/download/v{release}/imposm-{release}-linux-x86-64.tar.gz', # noqa '/tmp/imposm.tar.gz') run('tar -xzf /tmp/imposm.tar.gz --directory /srv/tilery/src') with sudo(): run(f'ln --symbolic --force ' f'/srv/tilery/src/imposm-{release}-linux-x86-64/imposm ' f'/usr/bin/imposm') with sudo(user='******'): mkdir('/srv/tilery/tmp/imposm')
def deploy(): """Send config files.""" with sudo(user='******'): mkdir('/srv/tilery/pianoforte/data') put(config.source_dir / 'mapping.yml', '/srv/tilery/mapping.yml') imposm_conf = template('remote/imposm.conf', **config) put(imposm_conf, '/srv/tilery/imposm.conf') put('remote/renderd.conf', '/etc/renderd.conf') put(config.source_dir / 'dist/', '/srv/tilery/pianoforte/') put(config.source_dir / 'fonts/', '/srv/tilery/pianoforte/fonts') put(config.source_dir / 'icon/', '/srv/tilery/pianoforte/icon')
def letsencrypt(): """Configure letsencrypt.""" with sudo(): run('add-apt-repository --yes ppa:certbot/certbot') run('apt update') run('apt install -y certbot') mkdir('/var/www/letsencrypt/.well-known/acme-challenge') domains = ','.join(list(config.piano_domains) + list(config.forte_domains)) certbot_conf = template('remote/certbot.ini', domains=domains) put(certbot_conf, '/var/www/certbot.ini') run('certbot certonly -c /var/www/certbot.ini --non-interactive ' '--agree-tos')
def db(): """Create the database and the needed extensions.""" dest = '/ssd/postgresql' if not exists(dest): src = '/var/lib/postgresql' mv(src, dest) run(f'ln --symbolic --force {dest} {src}') chown('postgres:postgres', src) with sudo(user='******'): conf = template('remote/postgresql.conf', **config) put(conf, f'/etc/postgresql/{config.psql_version}/main/postgresql.conf') run('createuser tilery || exit 0') run('createdb tilery -O tilery || exit 0') run('psql tilery -c "CREATE EXTENSION IF NOT EXISTS postgis"')
def configure_munin(): psql_plugins = [ 'postgres_autovacuum', 'postgres_bgwriter', 'postgres_checkpoints', 'postgres_connections_db', 'postgres_users', 'postgres_xlog', 'nginx_status', 'nginx_request'] with sudo(), cd('/etc/munin'): put('remote/munin.conf', 'munin.conf') for plugin in Path('remote/munin').glob('*'): put(plugin, f'plugins/{plugin.name}') run(f'chmod +x plugins/{plugin.name}') for name in psql_plugins: run(f'ln --symbolic --force /usr/share/munin/plugins/{name} ' f'plugins/{name}') run('ln --symbolic --force /usr/share/munin/plugins/postgres_size_ ' 'plugins/postgres_size_tilery') restart(services='munin-node')
def import_data(remove_backup=False, push_mapping=False, no_screen=False): """Import OSM data.""" with sudo(user='******'), env(PGHOST='/var/run/postgresql/'): if push_mapping: put('mapping.yml', '/srv/tilery/mapping.yml') run('ls --full-time --time-style locale /srv/tilery/mapping.yml') if remove_backup: run('imposm import -config /srv/tilery/imposm.conf -removebackup') cmd = ('imposm import -diff -config /srv/tilery/imposm.conf ' '-read /srv/tilery/tmp/data.osm.pbf -overwritecache ' '-write -deployproduction 2>&1 | tee /tmp/imposm.log') if no_screen: run(cmd) else: with screen(name='import'): run(cmd) run('tail /tmp/imposm.log')
def system(): """Install the system deps.""" # Not installed in minimized 18.04. run('which sudo || apt install sudo') with sudo(): run('apt update') run('apt install -y postgresql postgis gdal-bin screen nginx ' 'software-properties-common wget unzip autoconf libtool g++ ' 'libmapnik-dev libleveldb1v5 libgeos-dev goaccess ' 'libprotobuf-dev unifont curl zlib1g-dev uuid-dev python-psycopg2 ' 'munin-node libdbd-pg-perl libwww-perl apache2 apache2-dev') run('useradd -N tilery -d /srv/tilery/ || exit 0') mkdir('/srv/tilery/src') mkdir('/srv/tilery/tmp') mkdir('/srv/tilery/letsencrypt/.well-known/acme-challenge') chown('tilery:users', '/srv/tilery/') run('chsh -s /bin/bash tilery') install_imposm() install_mod_tile() configure_mod_tile() configure_munin() install_goaccess()
def install_mod_tile(force=False): if not exists('/usr/bin/renderd') or force: with sudo(): run('add-apt-repository --yes ppa:osmadmins/ppa') run('apt-get update') run('apt install --yes libapache2-mod-tile')
def test_sudo(patch_client): with usine.sudo(): assert usine.run('whoami') == \ "sudo --set-home --preserve-env sh -c $'whoami'"
def test_sudo_user(patch_client): with usine.sudo(user="******"): assert usine.run('whoami') == \ "sudo --set-home --preserve-env --user=me --login sh -c $'whoami'"
def psql(query): """Run a psql command.""" with sudo(user='******'): run(f'psql tilery -c "{query}"')
def restart(services=None): """Restart services.""" services = services or 'renderd apache2 imposm munin-node' with sudo(): systemctl(f'restart {services}')
def test_unsudo(patch_client): with usine.sudo(): with usine.unsudo(): assert usine.run('whoami') == "sh -c $'whoami'"
def pass_env(): with env(FOO='bar'): run('echo $FOO') with sudo(user='******'), env(FOO='baz'): run('echo $FOO')
def download_shapefile(name, url, force): datapath = '/srv/tilery/data/' if not exists(datapath + name) or force: with sudo(user='******'): wget(url, '/tmp/data.zip') run(f'unzip -n /tmp/data.zip -d {datapath}')
def service(name): with sudo(): put(f'remote/{name}.service', f'/etc/systemd/system/{name}.service') systemctl(f'enable {name}.service')