def addok(command):
"""Run a addok command on the remote server.
:command: the addok command to run.
"""
with sudo(user='******'):
run(f'/srv/addok/venv/bin/addok {command}')
def render(map='piano', min=1, max=10, threads=8):
"""Run a render command."""
with sudo(user='******'), screen(name='render'):
run(f'render_list --map {map} --all --force --num-threads {threads} '
f'--socket /var/run/renderd/renderd.sock '
f'--tile-dir /srv/tilery/tmp/tiles '
f' --min-zoom {min} --max-zoom {max}')
def pip(command):
"""Run a pip command on the remote server.
:command: the pip command to run.
"""
with sudo(user='******'):
run(f'/srv/addok/venv/bin/pip {command}')
def put_file():
with sudo(user='******'):
put('README.md', '/tmp/foobarbaz')
ls('/tmp/foobarbaz')
run('cat /tmp/foobarbaz')
put(StringIO('foobarbazéé'), '/tmp/foobarbaz')
run('cat /tmp/foobarbaz')
def connection():
with connect(hostname=os.environ.get('USINE_TEST_HOST')):
with sudo():
run('useradd -N usinetest -d /srv/usine/ '
'|| echo "usinetest exists"')
yield
run('userdel usinetest')
def ssh_keys():
"""Install ssh keys from remote urls."""
with sudo():
for name, url in config.get('ssh_key_urls', {}).items():
key = requests.get(url).text.replace('\n', '')
run('grep -q -r "{key}" .ssh/authorized_keys || echo "{key}" '
'| tee --append .ssh/authorized_keys'.format(key=key))
def venv():
"""Setup the python virtualenv."""
path = '/srv/addok/venv/'
if not exists(path):
with sudo(user='******'):
run(f'python3 -m venv {path}')
pip('install pip -U')
def test_trying_to_create_existing_folder(connection):
path = '/tmp/usinetestfolder'
run(f'rmdir {path} || exit 0')
mkdir(path)
mkdir(path) # Should not fail
with pytest.raises(SystemExit):
mkdir(path, parents=False)
run(f'rmdir {path}')
def test_folder_creation_existence_and_deletion(connection):
path = '/tmp/usinetestfolder'
run(f'rmdir {path} || exit 0')
assert not exists(path)
mkdir(path)
assert exists(path)
run(f'rmdir {path}')
assert not exists(path)
def test_dry_run(connection):
path = '/tmp/usinetestfile'
run(f'rm {path} || exit 0')
assert not exists(path)
usine.client.dry_run = True
run(f'touch {path}')
usine.client.dry_run = False
assert not exists(path)
def test_quoting(connection):
res = run('echo "pouet"')
assert res.stdout == 'pouet\r\n'
res = run('''echo "pouet with \\'quotes\\'"''')
assert res.stdout == '''pouet with 'quotes'\r\n'''
res = run('echo "pouet with \\\'quotes\\\'"')
assert res.stdout == '''pouet with 'quotes'\r\n'''
res = run("echo \"pouet with \\'quotes\\'\"")
assert res.stdout == '''pouet with 'quotes'\r\n'''
def install_matomo():
token = os.environ.get('MATOMO_TOKEN')
if not token:
sys.exist('You need to provide $MATOMO_TOKEN env var')
wget('https://raw.githubusercontent.com/matomo-org/matomo-log-analytics'
'/master/import_logs.py',
'/srv/tilery/src/matomo.py')
cron = template('remote/run-matomo', matomo_token=token)
put(cron, '/etc/cron.daily/run-matomo')
run('chmod +x /etc/cron.daily/run-matomo')
def test_folder_mv(connection):
path = '/tmp/usinetestfolder'
newpath = '/tmp/usinenewtestfolder'
run(f'rmdir {newpath} || exit 0')
mkdir(path)
mv(path, newpath)
assert exists(newpath)
assert not exists(path)
run(f'rmdir {newpath}')
assert not exists(newpath)
def configure_mod_tile():
with sudo(user='******'):
mkdir('/srv/tilery/tmp/tiles')
mkdir('/srv/tilery/renderd')
with sudo(), cd('/etc/apache2/'):
put('remote/tile.load', 'mods-available/tile.load')
put('remote/tile.conf', 'mods-available/tile.conf')
put('remote/apache.conf', 'sites-enabled/000-default.conf')
put('remote/ports.conf', 'ports.conf')
run('a2enmod tile')
def http():
conf = template('remote/gunicorn.conf', workers=config.workers)
with sudo():
put(conf, '/srv/addok/gunicorn.conf')
nginx_conf = template('remote/nginx.conf', domain=config.domain)
with sudo():
put(nginx_conf, '/etc/nginx/sites-enabled/addok')
# On LXC containers, somaxconn cannot be changed. This must be done on the
# host machine.
run(f'sudo sysctl -w net.core.somaxconn={config.connections} || exit 0')
restart()
def import_sql_file(path):
# Remove transaction management, as it does not cover the DROP TABLE;
# we'll cover the transaction manually with "psql --single-transaction"
run(f'sed -i.bak "/BEGIN;/d" {path}')
run(f'sed -i.bak "/END;/d" {path}')
run(f'sed -i.bak "/COMMIT;/d" {path}')
with sudo(user='******'):
run(f'psql --single-transaction -d tilery --file {path}')
def import_data(remove_backup=False, push_mapping=False, no_screen=False):
"""Import OSM data."""
with sudo(user='******'), env(PGHOST='/var/run/postgresql/'):
if push_mapping:
put('mapping.yml', '/srv/tilery/mapping.yml')
run('ls --full-time --time-style locale /srv/tilery/mapping.yml')
if remove_backup:
run('imposm import -config /srv/tilery/imposm.conf -removebackup')
cmd = ('imposm import -diff -config /srv/tilery/imposm.conf '
'-read /srv/tilery/tmp/data.osm.pbf -overwritecache '
'-write -deployproduction 2>&1 | tee /tmp/imposm.log')
if no_screen:
run(cmd)
else:
with screen(name='import'):
run(cmd)
run('tail /tmp/imposm.log')
def test_file_cp(connection):
path = '/tmp/usinetestfile'
targetpath = '/tmp/usinetargettestfile'
run(f'rm {targetpath} || exit 0')
run(f'touch {path}')
cp(path, targetpath)
assert exists(targetpath)
assert exists(path)
run(f'rm {targetpath}')
assert not exists(targetpath)
run(f'rm {path}')
assert not exists(path)
def letsencrypt():
"""Configure letsencrypt."""
with sudo():
run('add-apt-repository --yes ppa:certbot/certbot')
run('apt update')
run('apt install -y certbot')
mkdir('/var/www/letsencrypt/.well-known/acme-challenge')
domains = ','.join(list(config.piano_domains) + list(config.forte_domains))
certbot_conf = template('remote/certbot.ini', domains=domains)
put(certbot_conf, '/var/www/certbot.ini')
run('certbot certonly -c /var/www/certbot.ini --non-interactive '
'--agree-tos')
def install_imposm(force=False, release='0.6.0-alpha.4'):
"""Install imposm from binary.
:force: install even if the binary already exists.
:release: optionnal release to install
"""
if exists('/usr/bin/imposm') and not force:
print('imposm already installed')
return
# Cf https://github.com/omniscale/imposm3/issues/165#issuecomment-395993259
wget(f'https://github.com/omniscale/imposm3/releases/download/v{release}/imposm-{release}-linux-x86-64.tar.gz', # noqa
'/tmp/imposm.tar.gz')
run('tar -xzf /tmp/imposm.tar.gz --directory /srv/tilery/src')
with sudo():
run(f'ln --symbolic --force '
f'/srv/tilery/src/imposm-{release}-linux-x86-64/imposm '
f'/usr/bin/imposm')
with sudo(user='******'):
mkdir('/srv/tilery/tmp/imposm')
def test_folder_creation_existence_and_deletion_with_subfolder(connection):
path = '/tmp/usinetestfolder'
subpath = f'{path}/subfolder'
run(f'rmdir {subpath} || exit 0')
run(f'rmdir {path} || exit 0')
assert not exists(path)
with pytest.raises(SystemExit):
mkdir(subpath, parents=False)
mkdir(subpath)
assert exists(subpath)
run(f'rmdir {subpath}')
run(f'rmdir {path}')
assert not exists(path)
def import_custom_data():
"""Send and import boundary and city SQL."""
wget('http://nuage.yohanboniface.me/boundary.json', '/tmp/boundary.json')
run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \
-lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/boundary.sql /tmp/boundary.json -sql \
\\'SELECT name,"name:en","name:fr","name:ar","name:es","name:de","name:ru",iso FROM boundary\\' -nln itl_boundary""")
import_sql_file('/tmp/boundary.sql')
wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/city.csv',
'/tmp/city.csv')
run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \
-lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/city.sql /tmp/city.csv \
-select name,'name:en','name:fr','name:ar',capital,type,prio,ldir \
-nln city -oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \
-oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""")
import_sql_file('/tmp/city.sql')
wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/country.csv',
'/tmp/country.csv')
run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \
-lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/country.sql /tmp/country.csv \
-select name,'name:en','name:fr','name:ar',prio,iso,sov -nln country \
-oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \
-oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""")
import_sql_file('/tmp/country.sql')
with sudo(user='******'):
wget('http://nuage.yohanboniface.me/disputed.json',
'/srv/tilery/pianoforte/data/disputed.json')
def system():
"""Install the system deps."""
# Not installed in minimized 18.04.
run('which sudo || apt install sudo')
with sudo():
run('apt update')
run('apt install -y postgresql postgis gdal-bin screen nginx '
'software-properties-common wget unzip autoconf libtool g++ '
'libmapnik-dev libleveldb1v5 libgeos-dev goaccess '
'libprotobuf-dev unifont curl zlib1g-dev uuid-dev python-psycopg2 '
'munin-node libdbd-pg-perl libwww-perl apache2 apache2-dev')
run('useradd -N tilery -d /srv/tilery/ || exit 0')
mkdir('/srv/tilery/src')
mkdir('/srv/tilery/tmp')
mkdir('/srv/tilery/letsencrypt/.well-known/acme-challenge')
chown('tilery:users', '/srv/tilery/')
run('chsh -s /bin/bash tilery')
install_imposm()
install_mod_tile()
configure_mod_tile()
configure_munin()
install_goaccess()
def db():
"""Create the database and the needed extensions."""
dest = '/ssd/postgresql'
if not exists(dest):
src = '/var/lib/postgresql'
mv(src, dest)
run(f'ln --symbolic --force {dest} {src}')
chown('postgres:postgres', src)
with sudo(user='******'):
conf = template('remote/postgresql.conf', **config)
put(conf,
f'/etc/postgresql/{config.psql_version}/main/postgresql.conf')
run('createuser tilery || exit 0')
run('createdb tilery -O tilery || exit 0')
run('psql tilery -c "CREATE EXTENSION IF NOT EXISTS postgis"')
def test_file_creation_existence_and_deletion(connection):
path = '/tmp/usinetestfile'
run(f'rm {path} || exit 0')
assert not exists(path)
run(f'touch {path}')
assert exists(path)
run(f'rm {path}')
assert not exists(path)
def test_file_mv(connection):
path = '/tmp/usinetestfile'
newpath = '/tmp/usinenewtestfile'
run(f'rm {newpath} || exit 0')
run(f'touch {path}')
mv(path, newpath)
assert exists(newpath)
assert not exists(path)
run(f'rm {newpath}')
assert not exists(newpath)
def test_folder_cp(connection):
path = '/tmp/usinetestfolder'
targetpath = '/tmp/usinetargettestfolder'
run(f'rmdir {targetpath} || exit 0')
mkdir(path)
cp(path, targetpath)
assert exists(targetpath)
assert exists(path)
run(f'rmdir {targetpath}')
assert not exists(targetpath)
run(f'rmdir {path}')
assert not exists(path)
def configure_munin():
psql_plugins = [
'postgres_autovacuum', 'postgres_bgwriter', 'postgres_checkpoints',
'postgres_connections_db', 'postgres_users', 'postgres_xlog',
'nginx_status', 'nginx_request']
with sudo(), cd('/etc/munin'):
put('remote/munin.conf', 'munin.conf')
for plugin in Path('remote/munin').glob('*'):
put(plugin, f'plugins/{plugin.name}')
run(f'chmod +x plugins/{plugin.name}')
for name in psql_plugins:
run(f'ln --symbolic --force /usr/share/munin/plugins/{name} '
f'plugins/{name}')
run('ln --symbolic --force /usr/share/munin/plugins/postgres_size_ '
'plugins/postgres_size_tilery')
restart(services='munin-node')
def test_simple_run(connection):
res = run('echo pouet')
assert res.stdout == 'pouet\r\n'
def test_cd():
with cd('/tmp'):
res = run('pwd')
assert res.stdout == '/tmp\r\n'
