def osm_tools():
apt.install('build-essential python-dev protobuf-compiler \
libprotobuf-dev libtokyocabinet-dev python-psycopg2 \
libgeos-c1 libgeos-dev python-pip\
libpq-dev libbz2-dev libprotobuf-c0 libprotobuf-c0-dev automake '
)
sudo('pip install imposm')
sudo('mkdir -p /usr/src/osmconvert')
with cd('/usr/src/osmconvert'):
run('wget -O - http://m.m.i24.cc/osmconvert.c | cc -x c - -lz -o osmconvert'
)
sudo(' cp osmconvert /usr/bin')
if files.exists('/usr/src/osm-bright'):
sudo('rm -rf /usr/src/osm-bright')
if files.exists('/usr/src/osm2pgsql'):
sudo('rm -rf /usr/src/osm2pgsql')
if files.exists('/usr/src/HighRoad'):
sudo('rm -rf /usr/src/HighRoad')
with cd('/usr/src'):
run('git clone https://github.com/mapbox/osm-bright.git')
run('svn co http://svn.openstreetmap.org/applications/utils/export/osm2pgsql'
)
run('git clone https://github.com/migurski/HighRoad.git')
with cd('/usr/src/osm2pgsql'):
run('./autogen.sh')
run('./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/ ')
run('make')
sudo('make install')
def install():
if not os.geteuid() == 0:
print("\033[31;1mMust be run as root\033[0m")
return 2
try:
print("\033[32mInstalling packages...\033[0m")
apt.update()
apt.upgrade()
apt.install(PACKAGES)
print("\033[32mAdding users and groups ...\033[0m")
user.add_groups(GROUPS)
user.add_users(USERS)
print("\033[32mInstalling directories...\033[0m")
files.install_dirs(DIRS)
print("\033[32mInstalling files...\033[0m")
files.install_files(FILES)
print("\033[32mSucceed!\033[0m")
except KeyboardInterrupt:
pass
except Exception:
print("\033[31;1mError:\033[0m")
raise
def install() :
debugOutput("Installing")
system.run("sudo easy_install -f http://www.nickblundell.org.uk/packages --upgrade wikidbase")
# Create a suitable settings file and a place for the db with www-data access
uploadsPath = os.path.join(WB_PATH, "files")
# XXX: pops up no module named wikidbase
wbSettings = getSettingsData()
settingsFile = os.path.join(WB_PATH, "settings.py")
databasePath = os.path.join(WB_PATH, "wbdata.db")
cacheTable = "wbcache"
# XXX: Adding these to end will cause settings.py logic to fail.
# Set settings.
wbSettings += """
DATABASE_ENGINE = "sqlite3"
DATABASE_NAME="%s"
CACHE_BACKEND="db://%s"
UPLOAD_FOLDER="%s"
""" % (databasePath, cacheTable, uploadsPath)
# Create folders and set permissions.
system.run("mkdir -p '%s'" % uploadsPath)
system.writeToFile(settingsFile, wbSettings)
system.run("chgrp -R www-data '%s'" % WB_PATH)
system.run("chmod -R g+rwx '%s'" % WB_PATH)
# Initialise app
sbdjango.initialiseApp("settings",DEFAULT_ADMIN_CREDENTIALS[0], DEFAULT_ADMIN_CREDENTIALS[1], DEFAULT_ADMIN_CREDENTIALS[2], pythonPath=WB_PATH, cacheTable=cacheTable)
# Do mod_python setup
apt.install("libapache2-mod-python")
def ubuntu_basic():
apt.update()
apt.upgrade()
apt.install(
'btrfs-tools subversion git python-dev curl unzip htop dstat screen \
sysstat iotop powertop latencytop screen chkconfig python-software-properties vim mercurial curl \
git-core subversion wget make multitail sysbench')
def install() :
if isInstalled() :
return
debugOutput("Installing utils")
for app in APPS :
interface.updateProgress(); apt.install(app)
def ubuntu_postgis():
sysctl.add('kernel.sem = 250 32000 100 128')
sysctl.add('kernel.shmall = 2097152')
sysctl.add('kernel.shmmax = 6656401410')
sysctl.add('kernel.shmmni = 4096')
sysctl.add('fs.file-max = 262140')
sysctl.load()
apt.install('postgresql-contrib-9.1 postgresql-9.1-postgis2')
def ubuntu_darkstat():
apt.install('darkstat')
put('config/darkstat/etc/*', '/etc/darkstat')
put('config/darkstat/nginx/*', '/etc/nginx/sites-available')
sudo(
'ln -sf /etc/nginx/sites-available/darkstat /etc/nginx/sites-enabled/darkstat'
)
sudo('service darkstat restart')
sudo('service nginx reload')
def install():
debugOutput("Installing")
if apt.isInstalled("mysql-server"):
debugOutput("mysql is installed")
interface.updateProgress("Installing mysql (this takes a while!)")
apt.install("mysql-server")
configure()
def install():
debugOutput("Installing")
if apt.isInstalled("php4"):
debugOutput("php4 is installed")
interface.updateProgress("Install php4")
apt.install("php4 libapache2-mod-php4 php4-mysql")
configure()
def install() :
debugOutput("Installing")
if apt.isInstalled("mysql-server") :
debugOutput("mysql is installed")
interface.updateProgress("Installing mysql (this takes a while!)")
apt.install("mysql-server")
configure()
def install() :
debugOutput("Installing")
if apt.isInstalled("php4") :
debugOutput("php4 is installed")
interface.updateProgress("Install php4")
apt.install("php4 libapache2-mod-php4 php4-mysql")
configure()
def install() :
debugOutput("Installing")
interface.updateProgress("Installing django")
system.downloadUncompressInstall("http://media.djangoproject.com/releases/0.96/Django-0.96.tar.gz","cd Django-0.96 && python setup.py install")
interface.updateProgress("")
apt.install("libapache2-mod-python")
interface.updateProgress("")
apt.install("python-mysqldb")
def install(domain):
if isInstalled():
return
debugOutput("Installing samba")
apt.install("samba samba-common libcupsys2 libkrb53 winbind smbclient")
interface.updateProgress()
sambaConfig = _createConfig(domain)
debugOutput(sambaConfig)
# XXX: Warning, what if samba file exists.
system.writeToConfigFile(SAMBA_CONFIG, sambaConfig)
system.run("""mkdir -p "%s" """ % SAMBA_HOME)
system.run("""mkdir -p "%s/netlogon" """ % SAMBA_HOME)
system.run("""mkdir -p "%s/profiles" """ % SAMBA_HOME)
system.run("""mkdir -p "%s/data" """ % SAMBA_HOME)
system.run("""mkdir -p "%s" """ % SAMBA_USERS_PATH)
system.run("""mkdir -p "%s" """ % SAMBA_SHARES_PATH)
system.run("""mkdir -p /var/spool/samba""", exitOnFail=False)
system.run("""chmod 777 /var/spool/samba/""")
system.run("""chmod 770 "%s/profiles" """ % SAMBA_HOME)
system.run("""chmod 770 "%s/netlogon" """ % SAMBA_HOME)
system.run("""chmod 775 "%s" """ % SAMBA_USERS_PATH)
system.run("""chown -R root:users "%s/" || chmod -R 771 "%s/" """ %
(SAMBA_HOME, SAMBA_HOME))
interface.updateProgress()
restart()
interface.updateProgress()
nsswitchString = system.readFromFile("/etc/nsswitch.conf")
onsswitchString = nsswitchString
nsswitchString = nsswitchString.replace("files dns", "files wins dns")
if nsswitchString != onsswitchString:
system.writeToConfigFile("/etc/nsswitch.conf", nsswitchString)
system.run("net groupmap add ntgroup='Domain Admins' unixgroup=root")
system.run("net groupmap add ntgroup='Domain Users' unixgroup=users")
system.run("net groupmap add ntgroup='Domain Guests' unixgroup=nogroup")
# TODO: Check this has not been done.
system.run("echo 'root = Administrator' > /etc/samba/smbusers")
interface.updateProgress()
# Add a default share that all users can access.
if not "Files" in getShares():
addShare("Files")
def install():
debugOutput("Installing")
interface.updateProgress("Installing django")
system.downloadUncompressInstall(
"http://media.djangoproject.com/releases/0.96/Django-0.96.tar.gz",
"cd Django-0.96 && python setup.py install")
interface.updateProgress("")
apt.install("libapache2-mod-python")
interface.updateProgress("")
apt.install("python-mysqldb")
def install(domain) :
if isInstalled() :
return
debugOutput("Installing samba")
apt.install("samba samba-common libcupsys2 libkrb53 winbind smbclient")
interface.updateProgress();
sambaConfig = _createConfig(domain)
debugOutput(sambaConfig)
# XXX: Warning, what if samba file exists.
system.writeToConfigFile(SAMBA_CONFIG, sambaConfig)
system.run("""mkdir -p "%s" """ % SAMBA_HOME)
system.run("""mkdir -p "%s/netlogon" """ % SAMBA_HOME)
system.run("""mkdir -p "%s/profiles" """ % SAMBA_HOME)
system.run("""mkdir -p "%s/data" """ % SAMBA_HOME)
system.run("""mkdir -p "%s" """ % SAMBA_USERS_PATH)
system.run("""mkdir -p "%s" """ % SAMBA_SHARES_PATH)
system.run("""mkdir -p /var/spool/samba""", exitOnFail=False)
system.run("""chmod 777 /var/spool/samba/""")
system.run("""chmod 770 "%s/profiles" """ % SAMBA_HOME)
system.run("""chmod 770 "%s/netlogon" """ % SAMBA_HOME)
system.run("""chmod 775 "%s" """ % SAMBA_USERS_PATH)
system.run("""chown -R root:users "%s/" || chmod -R 771 "%s/" """ % (SAMBA_HOME, SAMBA_HOME))
interface.updateProgress();
restart()
interface.updateProgress();
nsswitchString = system.readFromFile("/etc/nsswitch.conf")
onsswitchString = nsswitchString
nsswitchString = nsswitchString.replace("files dns","files wins dns")
if nsswitchString != onsswitchString :
system.writeToConfigFile("/etc/nsswitch.conf", nsswitchString)
system.run("net groupmap add ntgroup='Domain Admins' unixgroup=root")
system.run("net groupmap add ntgroup='Domain Users' unixgroup=users")
system.run("net groupmap add ntgroup='Domain Guests' unixgroup=nogroup")
# TODO: Check this has not been done.
system.run("echo 'root = Administrator' > /etc/samba/smbusers")
interface.updateProgress();
# Add a default share that all users can access.
if not "Files" in getShares() :
addShare("Files")
def ubuntu_geopython():
apt.install(
'python-mapnik python-imaging python-yaml libproj0 python-lxml python-shapely python-mongoengine python-mapnik mapnik-utils \
python-pyproj python-scipy python-scientific python-numpy \
python-psycopg2 python-pyke python-shapely python-pymongo python-gridfs ipython \
python-netcdf cython python-numexpr python-tables python-rpy2 python-lxml \
python-cairo-dev python-cairo python-imaging \
libxslt1.1 libxslt1-dev python-lxml python-pastedeploy python-paste python-pastescript python-webob'
)
sudo('ln -sf /usr/lib/x86_64-linux-gnu/libfreetype.so /usr/lib/')
sudo('ln -sf /usr/lib/x86_64-linux-gnu/libz.so /usr/lib/')
sudo('ln -sf /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib/')
sudo('pip install owslib')
def install():
debugOutput("Installing")
if apt.isInstalled("apache2"):
debugOutput("apache2 is installed")
interface.updateProgress("Installing apache2")
apt.install("apache2")
# Enable mod_proxy.
system.run("a2enmod proxy", exitOnFail=False)
system.run("a2enmod proxy_http", exitOnFail=False)
# Disable default apache site.
setSiteEnabled("default", False)
def install() :
debugOutput("Installing")
if apt.isInstalled("apache2") :
debugOutput("apache2 is installed")
interface.updateProgress("Installing apache2")
apt.install("apache2")
# Enable mod_proxy.
system.run("a2enmod proxy", exitOnFail=False)
system.run("a2enmod proxy_http", exitOnFail=False)
# Disable default apache site.
setSiteEnabled("default",False)
def ubuntu_munin():
# http://fak3r.com/2010/10/07/howto-monitor-tomcat-with-monit-and-munin-in-debian/
#http://serverfault.com/questions/339472/configuring-the-tomcat-plugin-of-munin
#apt.install('munin munin-node munin-plugins-extra libwww-perl libxml-simple-perl libdbd-pg-perl')
#apt.install('tomcat6-admin')
#apt.add_rep('ppa:zeppelinlg/munin')
#apt.update()
apt.install(
'munin munin-node munin-plugins-extra libwww-perl libxml-simple-perl libdbd-pg-perl'
)
sudo('pip install PyMunin')
put('config/munin/nginx/munin', '/etc/nginx/sites-available')
sudo(
'ln -sf /etc/nginx/sites-available/munin /etc/nginx/sites-enabled/munin'
)
#put('config/munin/etc/*','/etc/munin')
#put('config/munin/plugins/*','/usr/share/munin/plugins')
#put('config/munin/tomcat/*','/etc/tomcat6')
#with cd('/usr/share/munin/plugins'):
#sudo('chmod +x nginx_request')
#sudo('chmod +x nginx_status')
#sudo('chmod +x nginx_memory')
#sudo('ln -sf /usr/share/munin/plugins/nginx_request /etc/munin/plugins/nginx_request')
#sudo('ln -sf /usr/share/munin/plugins/nginx_status /etc/munin/plugins/nginx_status')
#sudo('ln -sf /usr/share/munin/plugins/nginx_memory /etc/munin/plugins/nginx_memory')
#sudo('ln -sf /usr/share/munin/plugins/netstat /etc/munin/plugins/netstat')
#sudo('ln -sf /usr/share/munin/plugins/tcp /etc/munin/plugins/tcp')
#sudo('ln -sf /usr/share/munin/plugins/tomcat_access /etc/munin/plugins/tomcat_access')
#sudo('ln -sf /usr/share/munin/plugins/tomcat_jvm /etc/munin/plugins/tomcat_jvm')
#sudo('ln -sf /usr/share/munin/plugins/tomcat_threads /etc/munin/plugins/tomcat_threads')
#sudo('ln -sf /usr/share/munin/plugins/tomcat_volume /etc/munin/plugins/tomcat_volume')
#sudo('ln -sf /usr/local/lib/python2.7/dist-packages/pymunin/plugins/pgstats.py')
#sudo ('service tomcat6 restart')
sudo('service munin-node restart')
sudo('sudo -u munin munin-cron')
sudo('service nginx restart')
def install(backupSources=None) :
debugOutput("Installing rsnapshot")
apt.install("rsnapshot")
# TODO: Sort out config file handling.
system.run("mkdir /.rsnapshot", exitOnFail=False)
system.run("chmod +rx /.rsnapshot")
rsnapshotConf = system.readFromFile("/etc/rsnapshot.conf")
rsnapshotConf = rsnapshotConf.replace("snapshot_root\t/var/cache/rsnapshot/","snapshot_root\t/.rsnapshot/")
#rsnapshotConf = rsnapshotConf.replace("#no_create_root\t1","no_create_root\t0")
rsnapshotConf = rsnapshotConf.replace("#interval","interval")
rsnapshotConf = rsnapshotConf.replace("\nbackup","\n#backup")
rsnapshotConf += "\n\n"
for backupSource in backupSources :
rsnapshotConf += "backup\t%s/\tbackups/\n" % backupSource.rstrip("/")
rsnapshotConf += "backup\t/home/\t.system/\n"
rsnapshotConf += "backup\t/etc/\t.system/\n"
debugOutput(rsnapshotConf)
system.writeToConfigFile("/etc/rsnapshot.conf",rsnapshotConf)
cronConf = system.readFromFile("/etc/cron.d/rsnapshot")
cronConf = cronConf.replace("# 0","0")
cronConf = cronConf.replace("# 30","30")
cronConf = cronConf.replace("*/4","*")
debugOutput(cronConf)
system.writeToConfigFile("/etc/cron.d/rsnapshot", cronConf)
interface.updateProgress()
system.run("rsnapshot hourly")
interface.updateProgress()
def install(backupSources=None):
debugOutput("Installing rsnapshot")
apt.install("rsnapshot")
# TODO: Sort out config file handling.
system.run("mkdir /.rsnapshot", exitOnFail=False)
system.run("chmod +rx /.rsnapshot")
rsnapshotConf = system.readFromFile("/etc/rsnapshot.conf")
rsnapshotConf = rsnapshotConf.replace(
"snapshot_root\t/var/cache/rsnapshot/", "snapshot_root\t/.rsnapshot/")
#rsnapshotConf = rsnapshotConf.replace("#no_create_root\t1","no_create_root\t0")
rsnapshotConf = rsnapshotConf.replace("#interval", "interval")
rsnapshotConf = rsnapshotConf.replace("\nbackup", "\n#backup")
rsnapshotConf += "\n\n"
for backupSource in backupSources:
rsnapshotConf += "backup\t%s/\tbackups/\n" % backupSource.rstrip("/")
rsnapshotConf += "backup\t/home/\t.system/\n"
rsnapshotConf += "backup\t/etc/\t.system/\n"
debugOutput(rsnapshotConf)
system.writeToConfigFile("/etc/rsnapshot.conf", rsnapshotConf)
cronConf = system.readFromFile("/etc/cron.d/rsnapshot")
cronConf = cronConf.replace("# 0", "0")
cronConf = cronConf.replace("# 30", "30")
cronConf = cronConf.replace("*/4", "*")
debugOutput(cronConf)
system.writeToConfigFile("/etc/cron.d/rsnapshot", cronConf)
interface.updateProgress()
system.run("rsnapshot hourly")
interface.updateProgress()
def install():
debugOutput("Installing")
system.run(
"sudo easy_install -f http://www.nickblundell.org.uk/packages --upgrade wikidbase"
)
# Create a suitable settings file and a place for the db with www-data access
uploadsPath = os.path.join(WB_PATH, "files")
# XXX: pops up no module named wikidbase
wbSettings = getSettingsData()
settingsFile = os.path.join(WB_PATH, "settings.py")
databasePath = os.path.join(WB_PATH, "wbdata.db")
cacheTable = "wbcache"
# XXX: Adding these to end will cause settings.py logic to fail.
# Set settings.
wbSettings += """
DATABASE_ENGINE = "sqlite3"
DATABASE_NAME="%s"
CACHE_BACKEND="db://%s"
UPLOAD_FOLDER="%s"
""" % (databasePath, cacheTable, uploadsPath)
# Create folders and set permissions.
system.run("mkdir -p '%s'" % uploadsPath)
system.writeToFile(settingsFile, wbSettings)
system.run("chgrp -R www-data '%s'" % WB_PATH)
system.run("chmod -R g+rwx '%s'" % WB_PATH)
# Initialise app
sbdjango.initialiseApp("settings",
DEFAULT_ADMIN_CREDENTIALS[0],
DEFAULT_ADMIN_CREDENTIALS[1],
DEFAULT_ADMIN_CREDENTIALS[2],
pythonPath=WB_PATH,
cacheTable=cacheTable)
# Do mod_python setup
apt.install("libapache2-mod-python")
def install(*packages):
"install packages"
print '-' * 10 + ' running apt install', ' '.join(packages)
packages = list(packages)
apt.install(packages)
return ('-' * 10 + ' Install complete', '')
def install(*packages):
"install packages"
print '-' *10 + ' running apt install', ' '.join(packages)
packages = list(packages)
apt.install(packages)
return('-' *10 + ' Install complete', '')
def install() :
debugOutput("Installing dialog.")
apt.install("dialog")
def ubuntu_nginx():
apt.install('nginx supervisor')
sudo('update-rc.d supervisor enable')
def install():
debugOutput("Installing dialog.")
apt.install("dialog")
def install():
if not os.geteuid() == 0:
print_red("Must be run as root")
return 2
try:
print_green("Installing packages...")
apt.update()
apt.upgrade()
apt.install(PACKAGES)
print_green("Adding users and groups...")
user.add_groups(GROUPS)
user.add_users(USERS)
print_green("Installing directories...")
files.install_dirs(DIRS)
print_green("Installing files...")
files.install_files(FILES)
print_green("Updating grub configs...")
command.run('update-grub')
print_green("Setting up MySQL...")
command.run_sudo_script("""
mysql -e "CREATE USER 'appbooster'@'localhost' IDENTIFIED BY 'appbooster';
CREATE DATABASE appdb;
GRANT ALL PRIVILEGES ON appdb.* TO 'appbooster'@'localhost';"
""", check=False)
print_green("Setting up Git server...")
command.run_sudo_script("""
set -e
if [ ! -f ~/.ssh/appbooster ]; then
ssh-keygen -q -f /home/appbooster/.ssh/appbooster -N ""
fi
git config --global user.name "appbooster"
git config --global user.email "appbooster@localhost"
""", user='******')
command.run_sudo_script("""
set -e
if [ ! -d /home/git/repositories/gitolite-admin.git ]; then
cd ~
gitolite setup -pk /home/appbooster/.ssh/appbooster.pub
fi
""", user='******')
print_green("Setting up appbooster host...")
command.run_sudo_script("""
set -e
cd /home/appbooster/host
virtualenv --no-site-packages ENV
source ENV/bin/activate
pip install -r requirements.txt
export ENVIRONMENT=prod
./manage.py migrate
deactivate
""", user='******')
command.run_sudo_script("""
mkdir -p ~/local/VREF
ln -sf /home/appbooster/host/scripts/update ~/local/VREF/deploy
""", user='******')
print_green("Setting up appbooster stats...")
command.run_sudo_script("""
set -e
cd /home/appbooster/stats
virtualenv --no-site-packages ENV
source ENV/bin/activate
pip install -r requirements.txt
deactivate
""", user='******')
print_green("Installing elasticsearch and logstash...")
command.run_script("""
ES_DEB_REPO="deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main"
LS_DEB_REPO="deb http://packages.elasticsearch.org/logstash/1.4/debian stable main"
wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -
if ! grep -Fxq "$ES_DEB_REPO" /etc/apt/sources.list; then
echo "$ES_DEB_REPO" >> /etc/apt/sources.list
fi
if ! grep -Fxq "$LS_DEB_REPO" /etc/apt/sources.list; then
echo "$LS_DEB_REPO" >> /etc/apt/sources.list
fi
apt-get update
apt-get -y install logstash elasticsearch
""")
print_green("Installing kibana...")
command.run_script("""
KI_NAME="kibana-4.0.0-BETA2"
if [[ ! -d /opt/$KI_NAME ]]; then
wget -O - https://download.elasticsearch.org/kibana/kibana/$KI_NAME.tar.gz | tar -C /opt -xzf -
fi
cat << EOF > /usr/local/lib/systemd/system/kibana.service
[Unit]
Description=$KI_NAME
After=elasticsearch.service
[Service]
Type=simple
ExecStart=/opt/$KI_NAME/bin/kibana
[Install]
WantedBy=multi-user.target
EOF
""")
print_green("Enabling systemd services...")
command.run_script("""
systemctl daemon-reload
systemctl enable docker-autostart.service
systemctl enable logstash
systemctl enable elasticsearch
systemctl enable kibana
systemctl stop logstash
systemctl stop elasticsearch
systemctl start logstash
systemctl start elasticsearch
systemctl stop kibana
systemctl start kibana
""")
print_green("Restarting services...")
command.run_sudo_script("""
sudo rm -f /etc/nginx/site-enabled/default
""")
command.run_script("""
systemctl reload uwsgi
systemctl reload nginx
""")
print_green("Succeed!")
except KeyboardInterrupt:
pass
except Exception:
print_red("Error:")
raise
def ubuntu_python():
apt.install('python-pkg-resources python-dev python-virtualenv')
sudo('pip install fabric')
def ubuntu_build_geo():
apt.install('gdal-bin proj g++ cpp libxml2 libxml2-dev \
lzma lzma-dev liblzma-dev libfreetype6 libfreetype6-dev \
ttf-unifont ttf-dejavu ttf-dejavu-core ttf-dejavu-extra \
libsqlite3-0 libsqlite3-dev spatialite-bin libspatialite3 libspatialite-dev \
libgeos-dev libgdal-dev libgeos++-dev libproj-dev libgeos-c1')
def ubuntu_build_essential():
apt.install(
'build-essential linux-headers-`uname -r` libpq-dev libxml2-dev libxslt-dev binutils'
)
def ubuntu_backup():
apt.install('rdiff rdiff-backup rdiff-backup-fs')
def ubuntu_tilemill():
apt.install('tilemill')
