def addok(command): """Run a addok command on the remote server. :command: the addok command to run. """ with sudo(user='******'): run(f'/srv/addok/venv/bin/addok {command}')
def render(map='piano', min=1, max=10, threads=8): """Run a render command.""" with sudo(user='******'), screen(name='render'): run(f'render_list --map {map} --all --force --num-threads {threads} ' f'--socket /var/run/renderd/renderd.sock ' f'--tile-dir /srv/tilery/tmp/tiles ' f' --min-zoom {min} --max-zoom {max}')
def pip(command): """Run a pip command on the remote server. :command: the pip command to run. """ with sudo(user='******'): run(f'/srv/addok/venv/bin/pip {command}')
def put_file(): with sudo(user='******'): put('README.md', '/tmp/foobarbaz') ls('/tmp/foobarbaz') run('cat /tmp/foobarbaz') put(StringIO('foobarbazéé'), '/tmp/foobarbaz') run('cat /tmp/foobarbaz')
def connection(): with connect(hostname=os.environ.get('USINE_TEST_HOST')): with sudo(): run('useradd -N usinetest -d /srv/usine/ ' '|| echo "usinetest exists"') yield run('userdel usinetest')
def ssh_keys(): """Install ssh keys from remote urls.""" with sudo(): for name, url in config.get('ssh_key_urls', {}).items(): key = requests.get(url).text.replace('\n', '') run('grep -q -r "{key}" .ssh/authorized_keys || echo "{key}" ' '| tee --append .ssh/authorized_keys'.format(key=key))
def venv(): """Setup the python virtualenv.""" path = '/srv/addok/venv/' if not exists(path): with sudo(user='******'): run(f'python3 -m venv {path}') pip('install pip -U')
def test_trying_to_create_existing_folder(connection): path = '/tmp/usinetestfolder' run(f'rmdir {path} || exit 0') mkdir(path) mkdir(path) # Should not fail with pytest.raises(SystemExit): mkdir(path, parents=False) run(f'rmdir {path}')
def test_folder_creation_existence_and_deletion(connection): path = '/tmp/usinetestfolder' run(f'rmdir {path} || exit 0') assert not exists(path) mkdir(path) assert exists(path) run(f'rmdir {path}') assert not exists(path)
def test_dry_run(connection): path = '/tmp/usinetestfile' run(f'rm {path} || exit 0') assert not exists(path) usine.client.dry_run = True run(f'touch {path}') usine.client.dry_run = False assert not exists(path)
def test_quoting(connection): res = run('echo "pouet"') assert res.stdout == 'pouet\r\n' res = run('''echo "pouet with \\'quotes\\'"''') assert res.stdout == '''pouet with 'quotes'\r\n''' res = run('echo "pouet with \\\'quotes\\\'"') assert res.stdout == '''pouet with 'quotes'\r\n''' res = run("echo \"pouet with \\'quotes\\'\"") assert res.stdout == '''pouet with 'quotes'\r\n'''
def install_matomo(): token = os.environ.get('MATOMO_TOKEN') if not token: sys.exist('You need to provide $MATOMO_TOKEN env var') wget('https://raw.githubusercontent.com/matomo-org/matomo-log-analytics' '/master/import_logs.py', '/srv/tilery/src/matomo.py') cron = template('remote/run-matomo', matomo_token=token) put(cron, '/etc/cron.daily/run-matomo') run('chmod +x /etc/cron.daily/run-matomo')
def test_folder_mv(connection): path = '/tmp/usinetestfolder' newpath = '/tmp/usinenewtestfolder' run(f'rmdir {newpath} || exit 0') mkdir(path) mv(path, newpath) assert exists(newpath) assert not exists(path) run(f'rmdir {newpath}') assert not exists(newpath)
def configure_mod_tile(): with sudo(user='******'): mkdir('/srv/tilery/tmp/tiles') mkdir('/srv/tilery/renderd') with sudo(), cd('/etc/apache2/'): put('remote/tile.load', 'mods-available/tile.load') put('remote/tile.conf', 'mods-available/tile.conf') put('remote/apache.conf', 'sites-enabled/000-default.conf') put('remote/ports.conf', 'ports.conf') run('a2enmod tile')
def http(): conf = template('remote/gunicorn.conf', workers=config.workers) with sudo(): put(conf, '/srv/addok/gunicorn.conf') nginx_conf = template('remote/nginx.conf', domain=config.domain) with sudo(): put(nginx_conf, '/etc/nginx/sites-enabled/addok') # On LXC containers, somaxconn cannot be changed. This must be done on the # host machine. run(f'sudo sysctl -w net.core.somaxconn={config.connections} || exit 0') restart()
def import_sql_file(path): # Remove transaction management, as it does not cover the DROP TABLE; # we'll cover the transaction manually with "psql --single-transaction" run(f'sed -i.bak "/BEGIN;/d" {path}') run(f'sed -i.bak "/END;/d" {path}') run(f'sed -i.bak "/COMMIT;/d" {path}') with sudo(user='******'): run(f'psql --single-transaction -d tilery --file {path}')
def import_data(remove_backup=False, push_mapping=False, no_screen=False): """Import OSM data.""" with sudo(user='******'), env(PGHOST='/var/run/postgresql/'): if push_mapping: put('mapping.yml', '/srv/tilery/mapping.yml') run('ls --full-time --time-style locale /srv/tilery/mapping.yml') if remove_backup: run('imposm import -config /srv/tilery/imposm.conf -removebackup') cmd = ('imposm import -diff -config /srv/tilery/imposm.conf ' '-read /srv/tilery/tmp/data.osm.pbf -overwritecache ' '-write -deployproduction 2>&1 | tee /tmp/imposm.log') if no_screen: run(cmd) else: with screen(name='import'): run(cmd) run('tail /tmp/imposm.log')
def test_file_cp(connection): path = '/tmp/usinetestfile' targetpath = '/tmp/usinetargettestfile' run(f'rm {targetpath} || exit 0') run(f'touch {path}') cp(path, targetpath) assert exists(targetpath) assert exists(path) run(f'rm {targetpath}') assert not exists(targetpath) run(f'rm {path}') assert not exists(path)
def letsencrypt(): """Configure letsencrypt.""" with sudo(): run('add-apt-repository --yes ppa:certbot/certbot') run('apt update') run('apt install -y certbot') mkdir('/var/www/letsencrypt/.well-known/acme-challenge') domains = ','.join(list(config.piano_domains) + list(config.forte_domains)) certbot_conf = template('remote/certbot.ini', domains=domains) put(certbot_conf, '/var/www/certbot.ini') run('certbot certonly -c /var/www/certbot.ini --non-interactive ' '--agree-tos')
def install_imposm(force=False, release='0.6.0-alpha.4'): """Install imposm from binary. :force: install even if the binary already exists. :release: optionnal release to install """ if exists('/usr/bin/imposm') and not force: print('imposm already installed') return # Cf https://github.com/omniscale/imposm3/issues/165#issuecomment-395993259 wget(f'https://github.com/omniscale/imposm3/releases/download/v{release}/imposm-{release}-linux-x86-64.tar.gz', # noqa '/tmp/imposm.tar.gz') run('tar -xzf /tmp/imposm.tar.gz --directory /srv/tilery/src') with sudo(): run(f'ln --symbolic --force ' f'/srv/tilery/src/imposm-{release}-linux-x86-64/imposm ' f'/usr/bin/imposm') with sudo(user='******'): mkdir('/srv/tilery/tmp/imposm')
def test_folder_creation_existence_and_deletion_with_subfolder(connection): path = '/tmp/usinetestfolder' subpath = f'{path}/subfolder' run(f'rmdir {subpath} || exit 0') run(f'rmdir {path} || exit 0') assert not exists(path) with pytest.raises(SystemExit): mkdir(subpath, parents=False) mkdir(subpath) assert exists(subpath) run(f'rmdir {subpath}') run(f'rmdir {path}') assert not exists(path)
def import_custom_data(): """Send and import boundary and city SQL.""" wget('http://nuage.yohanboniface.me/boundary.json', '/tmp/boundary.json') run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \ -lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/boundary.sql /tmp/boundary.json -sql \ \\'SELECT name,"name:en","name:fr","name:ar","name:es","name:de","name:ru",iso FROM boundary\\' -nln itl_boundary""") import_sql_file('/tmp/boundary.sql') wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/city.csv', '/tmp/city.csv') run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \ -lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/city.sql /tmp/city.csv \ -select name,'name:en','name:fr','name:ar',capital,type,prio,ldir \ -nln city -oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \ -oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""") import_sql_file('/tmp/city.sql') wget('https://raw.githubusercontent.com/tilery/mae-boundaries/master/country.csv', '/tmp/country.csv') run("""ogr2ogr --config PG_USE_COPY YES -lco GEOMETRY_NAME=geometry \ -lco DROP_TABLE=IF_EXISTS -f PGDump /tmp/country.sql /tmp/country.csv \ -select name,'name:en','name:fr','name:ar',prio,iso,sov -nln country \ -oo X_POSSIBLE_NAMES=Lon* -oo Y_POSSIBLE_NAMES=Lat* \ -oo KEEP_GEOM_COLUMNS=NO -a_srs EPSG:4326""") import_sql_file('/tmp/country.sql') with sudo(user='******'): wget('http://nuage.yohanboniface.me/disputed.json', '/srv/tilery/pianoforte/data/disputed.json')
def system(): """Install the system deps.""" # Not installed in minimized 18.04. run('which sudo || apt install sudo') with sudo(): run('apt update') run('apt install -y postgresql postgis gdal-bin screen nginx ' 'software-properties-common wget unzip autoconf libtool g++ ' 'libmapnik-dev libleveldb1v5 libgeos-dev goaccess ' 'libprotobuf-dev unifont curl zlib1g-dev uuid-dev python-psycopg2 ' 'munin-node libdbd-pg-perl libwww-perl apache2 apache2-dev') run('useradd -N tilery -d /srv/tilery/ || exit 0') mkdir('/srv/tilery/src') mkdir('/srv/tilery/tmp') mkdir('/srv/tilery/letsencrypt/.well-known/acme-challenge') chown('tilery:users', '/srv/tilery/') run('chsh -s /bin/bash tilery') install_imposm() install_mod_tile() configure_mod_tile() configure_munin() install_goaccess()
def db(): """Create the database and the needed extensions.""" dest = '/ssd/postgresql' if not exists(dest): src = '/var/lib/postgresql' mv(src, dest) run(f'ln --symbolic --force {dest} {src}') chown('postgres:postgres', src) with sudo(user='******'): conf = template('remote/postgresql.conf', **config) put(conf, f'/etc/postgresql/{config.psql_version}/main/postgresql.conf') run('createuser tilery || exit 0') run('createdb tilery -O tilery || exit 0') run('psql tilery -c "CREATE EXTENSION IF NOT EXISTS postgis"')
def test_file_creation_existence_and_deletion(connection): path = '/tmp/usinetestfile' run(f'rm {path} || exit 0') assert not exists(path) run(f'touch {path}') assert exists(path) run(f'rm {path}') assert not exists(path)
def test_file_mv(connection): path = '/tmp/usinetestfile' newpath = '/tmp/usinenewtestfile' run(f'rm {newpath} || exit 0') run(f'touch {path}') mv(path, newpath) assert exists(newpath) assert not exists(path) run(f'rm {newpath}') assert not exists(newpath)
def test_folder_cp(connection): path = '/tmp/usinetestfolder' targetpath = '/tmp/usinetargettestfolder' run(f'rmdir {targetpath} || exit 0') mkdir(path) cp(path, targetpath) assert exists(targetpath) assert exists(path) run(f'rmdir {targetpath}') assert not exists(targetpath) run(f'rmdir {path}') assert not exists(path)
def configure_munin(): psql_plugins = [ 'postgres_autovacuum', 'postgres_bgwriter', 'postgres_checkpoints', 'postgres_connections_db', 'postgres_users', 'postgres_xlog', 'nginx_status', 'nginx_request'] with sudo(), cd('/etc/munin'): put('remote/munin.conf', 'munin.conf') for plugin in Path('remote/munin').glob('*'): put(plugin, f'plugins/{plugin.name}') run(f'chmod +x plugins/{plugin.name}') for name in psql_plugins: run(f'ln --symbolic --force /usr/share/munin/plugins/{name} ' f'plugins/{name}') run('ln --symbolic --force /usr/share/munin/plugins/postgres_size_ ' 'plugins/postgres_size_tilery') restart(services='munin-node')
def test_simple_run(connection): res = run('echo pouet') assert res.stdout == 'pouet\r\n'
def test_cd(): with cd('/tmp'): res = run('pwd') assert res.stdout == '/tmp\r\n'