def default_gunicorn_sh(context, timeout=600, file_name='gunicorn.sh'): num_workers = cpus() file_name = os.path.join(context['ROOT_PATH'], file_name) text = ''' #!/usr/bin/env bash #!/bin/bash NAME="{APP_NAME}" # Name of the application USER={RUNING_USER} # the user to run as GROUP={RUNING_USER} # the group to run as NUM_WORKERS={num_workers} # how many worker processes should Gunicorn spawn TIMEOUT={timeout} AIOHTTP_APP={AIOHTTP_APP} # WSGI module name echo "Starting $NAME as `whoami`" exec pipenv run gunicorn $AIOHTTP_APP \\ --worker-class aiohttp.GunicornWebWorker \\ --name $NAME \\ --workers $NUM_WORKERS \\ --timeout $TIMEOUT \\ --user=$USER --group=$GROUP \\ --bind={GUNICORN_BIND} \\ --log-level=debug \\ --log-file=- ''' text = text.format(num_workers=num_workers, timeout=timeout, **context) return put(local_path=StringIO(text.decode('utf8')), remote_path=file_name, use_sudo=True, mode=644, temp_dir='/tmp')
def update_supervisord(): larva() run("pip install supervisor") num_cpus = cpus() if env["host"] == "larva01.axiom": num_cpus -= 4 env["system_cpus"] = num_cpus upload_template('deploy/supervisord.conf', '/home/larva/supervisord.conf', context=copy(env), use_jinja=True, use_sudo=False, backup=False, mirror_local_mode=True, template_dir='.')
def deploy_particles(): stop_supervisord() larva() with cd(code_dir): update_code() update_libs() update_supervisord() start_supervisord() for i in xrange(cpus()): run("supervisorctl -c ~/supervisord.conf start particles:%s" % i)
def install_from_source(version=DEFAULT_VERSION): """ Install Node JS from source. :: import fabtools # Install Node.js fabtools.nodejs.install_nodejs() .. note:: This function may not work for old versions of Node.js. """ from fabtools.require.deb import packages as require_deb_packages from fabtools.require.rpm import packages as require_rpm_packages from fabtools.require import file as require_file family = distrib_family() if family == 'debian': require_deb_packages([ 'build-essential', 'libssl-dev', 'python', ]) elif family == 'redhat': require_rpm_packages([ 'gcc', 'gcc-c++', 'make', 'openssl-devel', 'python', ]) filename = 'node-v%s.tar.gz' % version foldername = filename[0:-7] require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % { 'version': version, 'filename': filename, }) run('tar -xzf %s' % filename) with cd(foldername): run('./configure') run('make -j%d' % (cpus() + 1)) run_as_root('make install') run('rm -rf %(filename)s %(foldername)s' % locals())
def tiles(): require.deb.package("libapache2-mod-tile", update=True) chown("/var/www/osm", owner="www-data", recursive=True) chown("/var/run/renderd/renderd.sock", owner="www-data") chown("/var/lib/mod_tile", owner="www-data", recursive=True) pgconfig(for_import=True) mapnik_db = "mapnikdb" require.postgres.database(mapnik_db, config.GIS_USER) sql_scripts = [ "/usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql", "/usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql", ] for script in sql_scripts: sudo("psql -d %s -f %s" % (mapnik_db, script), user=config.GIS_USER) processes = system.cpus() cache = config.RAM_SIZE / 4 * 3 sudo( "osm2pgsql --slim --number-processes %d -C %d -d %s %s" " --cache-strategy sparse" % (processes, cache, mapnik_db, pbf_path()), user=config.GIS_USER, ) tables = ["planet_osm_line", "planet_osm_point", "planet_osm_polygon", "planet_osm_roads"] tables_args = " ".join("-t " + table for table in tables) with cd("/opt/osm"): sudo("pg_dump -b -o %s %s | gzip > %s.gz" % (tables_args, mapnik_db, mapnik_db), user=config.GIS_USER) sudo("gunzip -c %s.gz | psql %s" % (mapnik_db, config.GIS_DB), user=config.GIS_USER) pgconfig(for_import=False) require.file("/var/lib/mod_tile/planet-import-complete", use_sudo=True, owner="www-data") context = {"db_name": config.GIS_DB, "db_user": config.GIS_USER, "db_password": config.GIS_PASSWORD} require.files.template_file( "/etc/mapnik-osm-data/inc/datasource-settings.xml.inc", template_source="templates/datasource-settings.xml.inc", context=context, use_sudo=True, owner="root", ) require.apache.site_disabled("tileserver_site") require.apache.site("100-tileserver_site.conf", template_source="templates/100-tileserver_site.conf") require.service.restarted("renderd")
def deploy_workers(): stop_supervisord() larva() with cd(code_dir): update_code() update_libs() update_supervisord() start_supervisord() run("supervisorctl -c ~/supervisord.conf start runs") run("supervisorctl -c ~/supervisord.conf start datasets") run("supervisorctl -c ~/supervisord.conf start gunicorn") run("supervisorctl -c ~/supervisord.conf start shorelines") num_cpus = cpus() - 4 for i in xrange(num_cpus): run("supervisorctl -c ~/supervisord.conf start particles:%s" % i)
def osrm(): with cd('/opt/osm'): pbf_base = pbf_path().partition('.')[0] require.git.working_copy( 'https://github.com/DennisOSRM/Project-OSRM.git', 'osrm', use_sudo=True, user=config.GIS_USER) require.directory('osrm/build', use_sudo=True, owner=config.GIS_USER) with cd('osrm/build'): sudo('cmake ..', user=config.GIS_USER) sudo('make', user=config.GIS_USER) sudo('ln -s ../profiles profiles', user=config.GIS_USER) sudo('ln -s ../profile.lua profile.lua', user=config.GIS_USER) require.files.template_file( '.stxxl', template_source='templates/.stxxl', context={'disk': '/opt/osm/osrm/build/stxxl'}, use_sudo=True, owner=config.GIS_USER) sudo('./osrm-extract %s' % pbf_path(), user=config.GIS_USER) sudo('./osrm-prepare %s.osrm' % pbf_base, user=config.GIS_USER) context = { 'threads': system.cpus(), 'pbf_base': pbf_base, } require.files.template_file( path='osrm-routed.ini', template_source='templates/osrm-routed.ini', context=context, use_sudo=True, owner=config.GIS_USER) require.files.template_file(path='/etc/init.d/osrm-routed', template_source='templates/osrm-routed', context={'user': config.GIS_USER}, use_sudo=True, owner='root') sudo('chmod +x /etc/init.d/osrm-routed') sudo('update-rc.d osrm-routed defaults') require.service.started('osrm-routed')
def osrm(): with cd("/opt/osm"): pbf_base = pbf_path().partition(".")[0] require.git.working_copy( "https://github.com/DennisOSRM/Project-OSRM.git", "osrm", use_sudo=True, user=config.GIS_USER ) require.directory("osrm/build", use_sudo=True, owner=config.GIS_USER) with cd("osrm/build"): sudo("cmake ..", user=config.GIS_USER) sudo("make", user=config.GIS_USER) sudo("ln -s ../profiles profiles", user=config.GIS_USER) sudo("ln -s ../profile.lua profile.lua", user=config.GIS_USER) require.files.template_file( ".stxxl", template_source="templates/.stxxl", context={"disk": "/opt/osm/osrm/build/stxxl"}, use_sudo=True, owner=config.GIS_USER, ) sudo("./osrm-extract %s" % pbf_path(), user=config.GIS_USER) sudo("./osrm-prepare %s.osrm" % pbf_base, user=config.GIS_USER) context = {"threads": system.cpus(), "pbf_base": pbf_base} require.files.template_file( path="osrm-routed.ini", template_source="templates/osrm-routed.ini", context=context, use_sudo=True, owner=config.GIS_USER, ) require.files.template_file( path="/etc/init.d/osrm-routed", template_source="templates/osrm-routed", context={"user": config.GIS_USER}, use_sudo=True, owner="root", ) sudo("chmod +x /etc/init.d/osrm-routed") sudo("update-rc.d osrm-routed defaults") require.service.started("osrm-routed")
def install_from_source(version=DEFAULT_VERSION): """ Install Node JS from source. :: import fabtools # Install Node.js fabtools.nodejs.install_nodejs() .. note:: This function may not work for old versions of Node.js. """ from fabtools.require.deb import packages as require_deb_packages from fabtools.require.rpm import packages as require_rpm_packages from fabtools.require import file as require_file family = distrib_family() if family == "debian": require_deb_packages(["build-essential", "libssl-dev", "python"]) elif family == "redhat": require_rpm_packages(["gcc", "gcc-c++", "make", "openssl-devel", "python"]) filename = "node-v%s.tar.gz" % version foldername = filename[0:-7] require_file(url="http://nodejs.org/dist/v%(version)s/%(filename)s" % {"version": version, "filename": filename}) run("tar -xzf %s" % filename) with cd(foldername): run("./configure") run("make -j%d" % (cpus() + 1)) run_as_root("make install") run("rm -rf %(filename)s %(foldername)s" % locals())
def install_from_source(version=DEFAULT_VERSION, checkinstall=False): """ Install Node JS from source. If *checkinstall* is ``True``, a distribution package will be built. :: import fabtools # Install Node.js fabtools.nodejs.install_nodejs() .. note:: This function may not work for old versions of Node.js. """ from fabtools.require.deb import packages as require_deb_packages from fabtools.require.rpm import packages as require_rpm_packages from fabtools.require import file as require_file family = distrib_family() if family == 'debian': packages = [ 'build-essential', 'libssl-dev', 'python', ] if checkinstall: packages.append('checkinstall') require_deb_packages(packages) elif family == 'redhat': packages = [ 'gcc', 'gcc-c++', 'make', 'openssl-devel', 'python', ] if checkinstall: packages.append('checkinstall') require_rpm_packages(packages) filename = 'node-v%s.tar.gz' % version foldername = filename[0:-7] require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % { 'version': version, 'filename': filename, }) run('tar -xzf %s' % filename) with cd(foldername): run('./configure') run('make -j%d' % (cpus() + 1)) if checkinstall: run_as_root('checkinstall -y --pkgname=nodejs --pkgversion=%(version) ' '--showinstall=no make install' % locals()) else: run_as_root('make install') run('rm -rf %(filename)s %(foldername)s' % locals())
def install_from_source(version=DEFAULT_VERSION, checkinstall=False): """ Install Node JS from source. If *checkinstall* is ``True``, a distribution package will be built. :: import fabtools # Install Node.js fabtools.nodejs.install_nodejs() .. note:: This function may not work for old versions of Node.js. """ from fabtools.require.deb import packages as require_deb_packages from fabtools.require.rpm import packages as require_rpm_packages from fabtools.require import file as require_file family = distrib_family() if family == 'debian': packages = [ 'build-essential', 'libssl-dev', 'python', ] if checkinstall: packages.append('checkinstall') require_deb_packages(packages) elif family == 'redhat': packages = [ 'gcc', 'gcc-c++', 'make', 'openssl-devel', 'python', ] if checkinstall: packages.append('checkinstall') require_rpm_packages(packages) filename = 'node-v%s.tar.gz' % version foldername = filename[0:-7] require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % { 'version': version, 'filename': filename, }) run('tar -xzf %s' % filename) with cd(foldername): run('./configure') run('make -j%d' % (cpus() + 1)) if checkinstall: run_as_root( 'checkinstall -y --pkgname=nodejs --pkgversion=%(version) ' '--showinstall=no make install' % locals()) else: run_as_root('make install') run('rm -rf %(filename)s %(foldername)s' % locals())
def tiles(): require.deb.package('libapache2-mod-tile', update=True) chown('/var/www/osm', owner='www-data', recursive=True) chown('/var/run/renderd/renderd.sock', owner='www-data') chown('/var/lib/mod_tile', owner='www-data', recursive=True) pgconfig(for_import=True) mapnik_db = 'mapnikdb' require.postgres.database(mapnik_db, config.GIS_USER) sql_scripts = [ '/usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql', '/usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql', ] for script in sql_scripts: sudo('psql -d %s -f %s' % (mapnik_db, script), user=config.GIS_USER) processes = system.cpus() cache = config.RAM_SIZE / 4 * 3 sudo('osm2pgsql --slim --number-processes %d -C %d -d %s %s' ' --cache-strategy sparse' % (processes, cache, mapnik_db, pbf_path()), user=config.GIS_USER) tables = [ 'planet_osm_line', 'planet_osm_point', 'planet_osm_polygon', 'planet_osm_roads', ] tables_args = ' '.join('-t ' + table for table in tables) with cd('/opt/osm'): sudo('pg_dump -b -o %s %s | gzip > %s.gz' % (tables_args, mapnik_db, mapnik_db), user=config.GIS_USER) sudo('gunzip -c %s.gz | psql %s' % (mapnik_db, config.GIS_DB), user=config.GIS_USER) pgconfig(for_import=False) require.file('/var/lib/mod_tile/planet-import-complete', use_sudo=True, owner='www-data') context = { 'db_name': config.GIS_DB, 'db_user': config.GIS_USER, 'db_password': config.GIS_PASSWORD, } require.files.template_file( '/etc/mapnik-osm-data/inc/datasource-settings.xml.inc', template_source='templates/datasource-settings.xml.inc', context=context, use_sudo=True, owner='root') require.apache.site_disabled('tileserver_site') require.apache.site('100-tileserver_site.conf', template_source='templates/100-tileserver_site.conf') require.service.restarted('renderd')