def configure_mod_tile():
with sudo(user='******'):
mkdir('/srv/tilery/tmp/tiles')
mkdir('/srv/tilery/renderd')
with sudo(), cd('/etc/apache2/'):
put('remote/tile.load', 'mods-available/tile.load')
put('remote/tile.conf', 'mods-available/tile.conf')
put('remote/apache.conf', 'sites-enabled/000-default.conf')
put('remote/ports.conf', 'ports.conf')
run('a2enmod tile')
def http():
conf = template('remote/gunicorn.conf', workers=config.workers)
with sudo():
put(conf, '/srv/addok/gunicorn.conf')
nginx_conf = template('remote/nginx.conf', domain=config.domain)
with sudo():
put(nginx_conf, '/etc/nginx/sites-enabled/addok')
# On LXC containers, somaxconn cannot be changed. This must be done on the
# host machine.
run(f'sudo sysctl -w net.core.somaxconn={config.connections} || exit 0')
restart()
def venv():
"""Setup the python virtualenv."""
path = '/srv/addok/venv/'
if not exists(path):
with sudo(user='******'):
run(f'python3 -m venv {path}')
pip('install pip -U')
def ssh_keys():
"""Install ssh keys from remote urls."""
with sudo():
for name, url in config.get('ssh_key_urls', {}).items():
key = requests.get(url).text.replace('\n', '')
run('grep -q -r "{key}" .ssh/authorized_keys || echo "{key}" '
'| tee --append .ssh/authorized_keys'.format(key=key))
def create_bar():
with sudo(user='******'), cd('/srv/'):
if exists('/tmp/foo'):
mkdir('/tmp/foo/bar')
else:
print('/tmp/foo does not exist')
ls('tamer')
def put_file():
with sudo(user='******'):
put('README.md', '/tmp/foobarbaz')
ls('/tmp/foobarbaz')
run('cat /tmp/foobarbaz')
put(StringIO('foobarbazéé'), '/tmp/foobarbaz')
run('cat /tmp/foobarbaz')
def connection():
with connect(hostname=os.environ.get('USINE_TEST_HOST')):
with sudo():
run('useradd -N usinetest -d /srv/usine/ '
'|| echo "usinetest exists"')
yield
run('userdel usinetest')
def import_custom_data():
"""Send and import boundary and city SQL."""
wget('http://nuage.yohanboniface.me/boundary.json', '/tmp/boundary.json')
run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \
-lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/boundary.sql /tmp/boundary.json -sql \
\\'SELECT name,"name:en","name:fr","name:ar","name:es","name:de","name:ru",iso FROM boundary\\' -nln itl_boundary""")
import_sql_file('/tmp/boundary.sql')
wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/city.csv',
'/tmp/city.csv')
run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \
-lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/city.sql /tmp/city.csv \
-select name,'name:en','name:fr','name:ar',capital,type,prio,ldir \
-nln city -oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \
-oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""")
import_sql_file('/tmp/city.sql')
wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/country.csv',
'/tmp/country.csv')
run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \
-lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/country.sql /tmp/country.csv \
-select name,'name:en','name:fr','name:ar',prio,iso,sov -nln country \
-oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \
-oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""")
import_sql_file('/tmp/country.sql')
with sudo(user='******'):
wget('http://nuage.yohanboniface.me/disputed.json',
'/srv/tilery/pianoforte/data/disputed.json')
def render(map='piano', min=1, max=10, threads=8):
"""Run a render command."""
with sudo(user='******'), screen(name='render'):
run(f'render_list --map {map} --all --force --num-threads {threads} '
f'--socket /var/run/renderd/renderd.sock '
f'--tile-dir /srv/tilery/tmp/tiles '
f' --min-zoom {min} --max-zoom {max}')
def addok(command):
"""Run a addok command on the remote server.
:command: the addok command to run.
"""
with sudo(user='******'):
run(f'/srv/addok/venv/bin/addok {command}')
def pip(command):
"""Run a pip command on the remote server.
:command: the pip command to run.
"""
with sudo(user='******'):
run(f'/srv/addok/venv/bin/pip {command}')
def import_sql_file(path):
# Remove transaction management, as it does not cover the DROP TABLE;
# we'll cover the transaction manually with "psql --single-transaction"
run(f'sed -i.bak "/BEGIN;/d" {path}')
run(f'sed -i.bak "/END;/d" {path}')
run(f'sed -i.bak "/COMMIT;/d" {path}')
with sudo(user='******'):
run(f'psql --single-transaction -d tilery --file {path}')
def install_imposm(force=False, release='0.6.0-alpha.4'):
"""Install imposm from binary.
:force: install even if the binary already exists.
:release: optionnal release to install
"""
if exists('/usr/bin/imposm') and not force:
print('imposm already installed')
return
# Cf https://github.com/omniscale/imposm3/issues/165#issuecomment-395993259
wget(f'https://github.com/omniscale/imposm3/releases/download/v{release}/imposm-{release}-linux-x86-64.tar.gz', # noqa
'/tmp/imposm.tar.gz')
run('tar -xzf /tmp/imposm.tar.gz --directory /srv/tilery/src')
with sudo():
run(f'ln --symbolic --force '
f'/srv/tilery/src/imposm-{release}-linux-x86-64/imposm '
f'/usr/bin/imposm')
with sudo(user='******'):
mkdir('/srv/tilery/tmp/imposm')
def deploy():
"""Send config files."""
with sudo(user='******'):
mkdir('/srv/tilery/pianoforte/data')
put(config.source_dir / 'mapping.yml', '/srv/tilery/mapping.yml')
imposm_conf = template('remote/imposm.conf', **config)
put(imposm_conf, '/srv/tilery/imposm.conf')
put('remote/renderd.conf', '/etc/renderd.conf')
put(config.source_dir / 'dist/', '/srv/tilery/pianoforte/')
put(config.source_dir / 'fonts/', '/srv/tilery/pianoforte/fonts')
put(config.source_dir / 'icon/', '/srv/tilery/pianoforte/icon')
def letsencrypt():
"""Configure letsencrypt."""
with sudo():
run('add-apt-repository --yes ppa:certbot/certbot')
run('apt update')
run('apt install -y certbot')
mkdir('/var/www/letsencrypt/.well-known/acme-challenge')
domains = ','.join(list(config.piano_domains) + list(config.forte_domains))
certbot_conf = template('remote/certbot.ini', domains=domains)
put(certbot_conf, '/var/www/certbot.ini')
run('certbot certonly -c /var/www/certbot.ini --non-interactive '
'--agree-tos')
def db():
"""Create the database and the needed extensions."""
dest = '/ssd/postgresql'
if not exists(dest):
src = '/var/lib/postgresql'
mv(src, dest)
run(f'ln --symbolic --force {dest} {src}')
chown('postgres:postgres', src)
with sudo(user='******'):
conf = template('remote/postgresql.conf', **config)
put(conf,
f'/etc/postgresql/{config.psql_version}/main/postgresql.conf')
run('createuser tilery || exit 0')
run('createdb tilery -O tilery || exit 0')
run('psql tilery -c "CREATE EXTENSION IF NOT EXISTS postgis"')
def configure_munin():
psql_plugins = [
'postgres_autovacuum', 'postgres_bgwriter', 'postgres_checkpoints',
'postgres_connections_db', 'postgres_users', 'postgres_xlog',
'nginx_status', 'nginx_request']
with sudo(), cd('/etc/munin'):
put('remote/munin.conf', 'munin.conf')
for plugin in Path('remote/munin').glob('*'):
put(plugin, f'plugins/{plugin.name}')
run(f'chmod +x plugins/{plugin.name}')
for name in psql_plugins:
run(f'ln --symbolic --force /usr/share/munin/plugins/{name} '
f'plugins/{name}')
run('ln --symbolic --force /usr/share/munin/plugins/postgres_size_ '
'plugins/postgres_size_tilery')
restart(services='munin-node')
def import_data(remove_backup=False, push_mapping=False, no_screen=False):
"""Import OSM data."""
with sudo(user='******'), env(PGHOST='/var/run/postgresql/'):
if push_mapping:
put('mapping.yml', '/srv/tilery/mapping.yml')
run('ls --full-time --time-style locale /srv/tilery/mapping.yml')
if remove_backup:
run('imposm import -config /srv/tilery/imposm.conf -removebackup')
cmd = ('imposm import -diff -config /srv/tilery/imposm.conf '
'-read /srv/tilery/tmp/data.osm.pbf -overwritecache '
'-write -deployproduction 2>&1 | tee /tmp/imposm.log')
if no_screen:
run(cmd)
else:
with screen(name='import'):
run(cmd)
run('tail /tmp/imposm.log')
def system():
"""Install the system deps."""
# Not installed in minimized 18.04.
run('which sudo || apt install sudo')
with sudo():
run('apt update')
run('apt install -y postgresql postgis gdal-bin screen nginx '
'software-properties-common wget unzip autoconf libtool g++ '
'libmapnik-dev libleveldb1v5 libgeos-dev goaccess '
'libprotobuf-dev unifont curl zlib1g-dev uuid-dev python-psycopg2 '
'munin-node libdbd-pg-perl libwww-perl apache2 apache2-dev')
run('useradd -N tilery -d /srv/tilery/ || exit 0')
mkdir('/srv/tilery/src')
mkdir('/srv/tilery/tmp')
mkdir('/srv/tilery/letsencrypt/.well-known/acme-challenge')
chown('tilery:users', '/srv/tilery/')
run('chsh -s /bin/bash tilery')
install_imposm()
install_mod_tile()
configure_mod_tile()
configure_munin()
install_goaccess()
def install_mod_tile(force=False):
if not exists('/usr/bin/renderd') or force:
with sudo():
run('add-apt-repository --yes ppa:osmadmins/ppa')
run('apt-get update')
run('apt install --yes libapache2-mod-tile')
def test_sudo(patch_client):
with usine.sudo():
assert usine.run('whoami') == \
"sudo --set-home --preserve-env sh -c $'whoami'"
def test_sudo_user(patch_client):
with usine.sudo(user="******"):
assert usine.run('whoami') == \
"sudo --set-home --preserve-env --user=me --login sh -c $'whoami'"
def psql(query):
"""Run a psql command."""
with sudo(user='******'):
run(f'psql tilery -c "{query}"')
def restart(services=None):
"""Restart services."""
services = services or 'renderd apache2 imposm munin-node'
with sudo():
systemctl(f'restart {services}')
def test_unsudo(patch_client):
with usine.sudo():
with usine.unsudo():
assert usine.run('whoami') == "sh -c $'whoami'"
def pass_env():
with env(FOO='bar'):
run('echo $FOO')
with sudo(user='******'), env(FOO='baz'):
run('echo $FOO')
def download_shapefile(name, url, force):
datapath = '/srv/tilery/data/'
if not exists(datapath + name) or force:
with sudo(user='******'):
wget(url, '/tmp/data.zip')
run(f'unzip -n /tmp/data.zip -d {datapath}')
def service(name):
with sudo():
put(f'remote/{name}.service',
f'/etc/systemd/system/{name}.service')
systemctl(f'enable {name}.service')
