def default_gunicorn_sh(context, timeout=600, file_name='gunicorn.sh'):
num_workers = cpus()
file_name = os.path.join(context['ROOT_PATH'], file_name)
text = '''
#!/usr/bin/env bash
#!/bin/bash
NAME="{APP_NAME}" # Name of the application
USER={RUNING_USER} # the user to run as
GROUP={RUNING_USER} # the group to run as
NUM_WORKERS={num_workers} # how many worker processes should Gunicorn spawn
TIMEOUT={timeout}
AIOHTTP_APP={AIOHTTP_APP} # WSGI module name
echo "Starting $NAME as `whoami`"
exec pipenv run gunicorn $AIOHTTP_APP \\
--worker-class aiohttp.GunicornWebWorker \\
--name $NAME \\
--workers $NUM_WORKERS \\
--timeout $TIMEOUT \\
--user=$USER --group=$GROUP \\
--bind={GUNICORN_BIND} \\
--log-level=debug \\
--log-file=-
'''
text = text.format(num_workers=num_workers, timeout=timeout, **context)
return put(local_path=StringIO(text.decode('utf8')),
remote_path=file_name,
use_sudo=True,
mode=644,
temp_dir='/tmp')
def update_supervisord():
larva()
run("pip install supervisor")
num_cpus = cpus()
if env["host"] == "larva01.axiom":
num_cpus -= 4
env["system_cpus"] = num_cpus
upload_template('deploy/supervisord.conf', '/home/larva/supervisord.conf', context=copy(env), use_jinja=True, use_sudo=False, backup=False, mirror_local_mode=True, template_dir='.')
def deploy_particles():
stop_supervisord()
larva()
with cd(code_dir):
update_code()
update_libs()
update_supervisord()
start_supervisord()
for i in xrange(cpus()):
run("supervisorctl -c ~/supervisord.conf start particles:%s" % i)
def install_from_source(version=DEFAULT_VERSION):
"""
Install Node JS from source.
::
import fabtools
# Install Node.js
fabtools.nodejs.install_nodejs()
.. note:: This function may not work for old versions of Node.js.
"""
from fabtools.require.deb import packages as require_deb_packages
from fabtools.require.rpm import packages as require_rpm_packages
from fabtools.require import file as require_file
family = distrib_family()
if family == 'debian':
require_deb_packages([
'build-essential',
'libssl-dev',
'python',
])
elif family == 'redhat':
require_rpm_packages([
'gcc',
'gcc-c++',
'make',
'openssl-devel',
'python',
])
filename = 'node-v%s.tar.gz' % version
foldername = filename[0:-7]
require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % {
'version': version,
'filename': filename,
})
run('tar -xzf %s' % filename)
with cd(foldername):
run('./configure')
run('make -j%d' % (cpus() + 1))
run_as_root('make install')
run('rm -rf %(filename)s %(foldername)s' % locals())
def tiles():
require.deb.package("libapache2-mod-tile", update=True)
chown("/var/www/osm", owner="www-data", recursive=True)
chown("/var/run/renderd/renderd.sock", owner="www-data")
chown("/var/lib/mod_tile", owner="www-data", recursive=True)
pgconfig(for_import=True)
mapnik_db = "mapnikdb"
require.postgres.database(mapnik_db, config.GIS_USER)
sql_scripts = [
"/usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql",
"/usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql",
]
for script in sql_scripts:
sudo("psql -d %s -f %s" % (mapnik_db, script), user=config.GIS_USER)
processes = system.cpus()
cache = config.RAM_SIZE / 4 * 3
sudo(
"osm2pgsql --slim --number-processes %d -C %d -d %s %s"
" --cache-strategy sparse" % (processes, cache, mapnik_db, pbf_path()),
user=config.GIS_USER,
)
tables = ["planet_osm_line", "planet_osm_point", "planet_osm_polygon", "planet_osm_roads"]
tables_args = " ".join("-t " + table for table in tables)
with cd("/opt/osm"):
sudo("pg_dump -b -o %s %s | gzip > %s.gz" % (tables_args, mapnik_db, mapnik_db), user=config.GIS_USER)
sudo("gunzip -c %s.gz | psql %s" % (mapnik_db, config.GIS_DB), user=config.GIS_USER)
pgconfig(for_import=False)
require.file("/var/lib/mod_tile/planet-import-complete", use_sudo=True, owner="www-data")
context = {"db_name": config.GIS_DB, "db_user": config.GIS_USER, "db_password": config.GIS_PASSWORD}
require.files.template_file(
"/etc/mapnik-osm-data/inc/datasource-settings.xml.inc",
template_source="templates/datasource-settings.xml.inc",
context=context,
use_sudo=True,
owner="root",
)
require.apache.site_disabled("tileserver_site")
require.apache.site("100-tileserver_site.conf", template_source="templates/100-tileserver_site.conf")
require.service.restarted("renderd")
def deploy_workers():
stop_supervisord()
larva()
with cd(code_dir):
update_code()
update_libs()
update_supervisord()
start_supervisord()
run("supervisorctl -c ~/supervisord.conf start runs")
run("supervisorctl -c ~/supervisord.conf start datasets")
run("supervisorctl -c ~/supervisord.conf start gunicorn")
run("supervisorctl -c ~/supervisord.conf start shorelines")
num_cpus = cpus() - 4
for i in xrange(num_cpus):
run("supervisorctl -c ~/supervisord.conf start particles:%s" % i)
def osrm():
with cd('/opt/osm'):
pbf_base = pbf_path().partition('.')[0]
require.git.working_copy(
'https://github.com/DennisOSRM/Project-OSRM.git',
'osrm',
use_sudo=True,
user=config.GIS_USER)
require.directory('osrm/build', use_sudo=True, owner=config.GIS_USER)
with cd('osrm/build'):
sudo('cmake ..', user=config.GIS_USER)
sudo('make', user=config.GIS_USER)
sudo('ln -s ../profiles profiles', user=config.GIS_USER)
sudo('ln -s ../profile.lua profile.lua', user=config.GIS_USER)
require.files.template_file(
'.stxxl',
template_source='templates/.stxxl',
context={'disk': '/opt/osm/osrm/build/stxxl'},
use_sudo=True,
owner=config.GIS_USER)
sudo('./osrm-extract %s' % pbf_path(), user=config.GIS_USER)
sudo('./osrm-prepare %s.osrm' % pbf_base, user=config.GIS_USER)
context = {
'threads': system.cpus(),
'pbf_base': pbf_base,
}
require.files.template_file(
path='osrm-routed.ini',
template_source='templates/osrm-routed.ini',
context=context,
use_sudo=True,
owner=config.GIS_USER)
require.files.template_file(path='/etc/init.d/osrm-routed',
template_source='templates/osrm-routed',
context={'user': config.GIS_USER},
use_sudo=True,
owner='root')
sudo('chmod +x /etc/init.d/osrm-routed')
sudo('update-rc.d osrm-routed defaults')
require.service.started('osrm-routed')
def osrm():
with cd("/opt/osm"):
pbf_base = pbf_path().partition(".")[0]
require.git.working_copy(
"https://github.com/DennisOSRM/Project-OSRM.git", "osrm", use_sudo=True, user=config.GIS_USER
)
require.directory("osrm/build", use_sudo=True, owner=config.GIS_USER)
with cd("osrm/build"):
sudo("cmake ..", user=config.GIS_USER)
sudo("make", user=config.GIS_USER)
sudo("ln -s ../profiles profiles", user=config.GIS_USER)
sudo("ln -s ../profile.lua profile.lua", user=config.GIS_USER)
require.files.template_file(
".stxxl",
template_source="templates/.stxxl",
context={"disk": "/opt/osm/osrm/build/stxxl"},
use_sudo=True,
owner=config.GIS_USER,
)
sudo("./osrm-extract %s" % pbf_path(), user=config.GIS_USER)
sudo("./osrm-prepare %s.osrm" % pbf_base, user=config.GIS_USER)
context = {"threads": system.cpus(), "pbf_base": pbf_base}
require.files.template_file(
path="osrm-routed.ini",
template_source="templates/osrm-routed.ini",
context=context,
use_sudo=True,
owner=config.GIS_USER,
)
require.files.template_file(
path="/etc/init.d/osrm-routed",
template_source="templates/osrm-routed",
context={"user": config.GIS_USER},
use_sudo=True,
owner="root",
)
sudo("chmod +x /etc/init.d/osrm-routed")
sudo("update-rc.d osrm-routed defaults")
require.service.started("osrm-routed")
def install_from_source(version=DEFAULT_VERSION):
"""
Install Node JS from source.
::
import fabtools
# Install Node.js
fabtools.nodejs.install_nodejs()
.. note:: This function may not work for old versions of Node.js.
"""
from fabtools.require.deb import packages as require_deb_packages
from fabtools.require.rpm import packages as require_rpm_packages
from fabtools.require import file as require_file
family = distrib_family()
if family == "debian":
require_deb_packages(["build-essential", "libssl-dev", "python"])
elif family == "redhat":
require_rpm_packages(["gcc", "gcc-c++", "make", "openssl-devel", "python"])
filename = "node-v%s.tar.gz" % version
foldername = filename[0:-7]
require_file(url="http://nodejs.org/dist/v%(version)s/%(filename)s" % {"version": version, "filename": filename})
run("tar -xzf %s" % filename)
with cd(foldername):
run("./configure")
run("make -j%d" % (cpus() + 1))
run_as_root("make install")
run("rm -rf %(filename)s %(foldername)s" % locals())
def install_from_source(version=DEFAULT_VERSION, checkinstall=False):
"""
Install Node JS from source.
If *checkinstall* is ``True``, a distribution package will be built.
::
import fabtools
# Install Node.js
fabtools.nodejs.install_nodejs()
.. note:: This function may not work for old versions of Node.js.
"""
from fabtools.require.deb import packages as require_deb_packages
from fabtools.require.rpm import packages as require_rpm_packages
from fabtools.require import file as require_file
family = distrib_family()
if family == 'debian':
packages = [
'build-essential',
'libssl-dev',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_deb_packages(packages)
elif family == 'redhat':
packages = [
'gcc',
'gcc-c++',
'make',
'openssl-devel',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_rpm_packages(packages)
filename = 'node-v%s.tar.gz' % version
foldername = filename[0:-7]
require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % {
'version': version,
'filename': filename,
})
run('tar -xzf %s' % filename)
with cd(foldername):
run('./configure')
run('make -j%d' % (cpus() + 1))
if checkinstall:
run_as_root('checkinstall -y --pkgname=nodejs --pkgversion=%(version) '
'--showinstall=no make install' % locals())
else:
run_as_root('make install')
run('rm -rf %(filename)s %(foldername)s' % locals())
def install_from_source(version=DEFAULT_VERSION, checkinstall=False):
"""
Install Node JS from source.
If *checkinstall* is ``True``, a distribution package will be built.
::
import fabtools
# Install Node.js
fabtools.nodejs.install_nodejs()
.. note:: This function may not work for old versions of Node.js.
"""
from fabtools.require.deb import packages as require_deb_packages
from fabtools.require.rpm import packages as require_rpm_packages
from fabtools.require import file as require_file
family = distrib_family()
if family == 'debian':
packages = [
'build-essential',
'libssl-dev',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_deb_packages(packages)
elif family == 'redhat':
packages = [
'gcc',
'gcc-c++',
'make',
'openssl-devel',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_rpm_packages(packages)
filename = 'node-v%s.tar.gz' % version
foldername = filename[0:-7]
require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % {
'version': version,
'filename': filename,
})
run('tar -xzf %s' % filename)
with cd(foldername):
run('./configure')
run('make -j%d' % (cpus() + 1))
if checkinstall:
run_as_root(
'checkinstall -y --pkgname=nodejs --pkgversion=%(version) '
'--showinstall=no make install' % locals())
else:
run_as_root('make install')
run('rm -rf %(filename)s %(foldername)s' % locals())
def tiles():
require.deb.package('libapache2-mod-tile', update=True)
chown('/var/www/osm', owner='www-data', recursive=True)
chown('/var/run/renderd/renderd.sock', owner='www-data')
chown('/var/lib/mod_tile', owner='www-data', recursive=True)
pgconfig(for_import=True)
mapnik_db = 'mapnikdb'
require.postgres.database(mapnik_db, config.GIS_USER)
sql_scripts = [
'/usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql',
'/usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql',
]
for script in sql_scripts:
sudo('psql -d %s -f %s' % (mapnik_db, script), user=config.GIS_USER)
processes = system.cpus()
cache = config.RAM_SIZE / 4 * 3
sudo('osm2pgsql --slim --number-processes %d -C %d -d %s %s'
' --cache-strategy sparse' %
(processes, cache, mapnik_db, pbf_path()),
user=config.GIS_USER)
tables = [
'planet_osm_line',
'planet_osm_point',
'planet_osm_polygon',
'planet_osm_roads',
]
tables_args = ' '.join('-t ' + table for table in tables)
with cd('/opt/osm'):
sudo('pg_dump -b -o %s %s | gzip > %s.gz' %
(tables_args, mapnik_db, mapnik_db),
user=config.GIS_USER)
sudo('gunzip -c %s.gz | psql %s' % (mapnik_db, config.GIS_DB),
user=config.GIS_USER)
pgconfig(for_import=False)
require.file('/var/lib/mod_tile/planet-import-complete',
use_sudo=True,
owner='www-data')
context = {
'db_name': config.GIS_DB,
'db_user': config.GIS_USER,
'db_password': config.GIS_PASSWORD,
}
require.files.template_file(
'/etc/mapnik-osm-data/inc/datasource-settings.xml.inc',
template_source='templates/datasource-settings.xml.inc',
context=context,
use_sudo=True,
owner='root')
require.apache.site_disabled('tileserver_site')
require.apache.site('100-tileserver_site.conf',
template_source='templates/100-tileserver_site.conf')
require.service.restarted('renderd')