def ensure():
git.ensure_git()
# Make sure the user exists
user_ensure('acme')
group_ensure('acme')
group_user_ensure('acme', 'acme')
# Allow the acme user to reload nginx when updating certs
files.append('/etc/sudoers',
'acme ALL=(root) NOPASSWD: {reload_nginx}'.format(
reload_nginx=_reload_nginx()),
use_sudo=True)
# Make sure acme.sh is installed for the acme user
# if not path.has('acme.sh', user='******'): Does not exist--it's imported from bash
if not files.exists('/home/acme/.acme.sh'):
git.ensure_clone_github('Neilpang/acme.sh',
'/home/acme/acme.sh',
user='******')
with cd("/home/acme/acme.sh"):
sudo("./acme.sh --install", user='******')
dir_ensure(well_known_base)
sudo(
"chown acme:acme {well_known} && chmod 755 {well_known}".format(
well_known=well_known_base)
) # Can't change attributes without sudo in a sticky (not write) directory... annoying
util.put_file("config/certs/lets-encrypt-x3-cross-signed.pem",
"/etc/ssl/certs/lets-encrypt-x3-cross-signed.pem",
user='******',
mode='0644')
def set_project_user_and_group(username, groupname):
""" Setup project user and group
@param username: system user for zope process
@param groupname: system process group for sites
"""
user_ensure(username)
group_ensure(groupname)
group_user_ensure(groupname, username)
def ensure_fcgiwrap(children=4):
select_package("apt")
package_ensure(["fcgiwrap"]) # On debian will automatically be enabled
# fcgi can't run status script because its default user (www-data) has no login shell--not sure why exactly but work around it by making a new user
user_ensure('fcgiwrap')
group_ensure('fcgiwrap')
group_user_ensure('fcgiwrap', 'fcgiwrap')
sudo('sed -i "s/www-data/fcgiwrap/" /lib/systemd/system/fcgiwrap.service')
sudo('echo "FCGI_CHILDREN={}" > /etc/default/fcgiwrap'.format(children))
sudo('systemctl daemon-reload')
sudo('/etc/init.d/fcgiwrap restart')
def _setup_users():
USERS = ['yuta1024', 'tyabuki', 'nhirokinet']
for user in USERS:
cuisine.user_ensure(user, shell='/bin/bash', passwd='yharima', encrypted_passwd=False)
cuisine.group_user_ensure('sudo', user)
# https://ubuntuforums.org/showthread.php?t=1318346
cuisine.group_user_ensure('adm', user)
with cuisine.mode_sudo():
cuisine.ssh_authorize(user, _get_public_key_from_github(user))
if not cuisine.is_ok(sudo('grep rsync /etc/sudoers && echo OK ; true')):
sudo('echo "%s ALL=(ALL) NOPASSWD: /usr/bin/rsync" | EDITOR="tee -a" visudo' % ','.join(USERS))
def ensure_email_contents(restore):
"""Restore the old email database if needed"""
user_ensure('vmail')
group_ensure('vmail')
group_user_ensure('vmail', 'vmail')
run("usermod -d /var/mail vmail")
#run('usermod -s /bin/false vmail')
if not dir_exists('/var/mail/vmail') and restore:
#with settings(user='******', host_string='burn'):
# get('/data/vmail', '/tmp')
put('/tmp/vmail', '/var/mail')
run('chown -R vmail:vmail /var/mail/vmail')
def ensure_email_contents(restore):
"""Restore the old email database if needed"""
user_ensure('vmail')
group_ensure('vmail')
group_user_ensure('vmail', 'vmail')
run("usermod -d /var/mail vmail")
#run('usermod -s /bin/false vmail')
if not dir_exists('/var/mail/vmail') and restore:
#with settings(user='******', host_string='burn'):
# get('/data/vmail', '/tmp')
put('/tmp/vmail', '/var/mail')
run('chown -R vmail:vmail /var/mail/vmail')
def add_user_to_docker_group(self):
"""
Make sure the ubuntu user is part of the docker group.
"""
log_green('adding the ubuntu user to the docker group')
data = load_state_from_disk()
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
user_ensure('ubuntu', home='/home/ubuntu', shell='/bin/bash')
group_ensure('docker', gid=55)
group_user_ensure('docker', 'ubuntu')
def ensure():
# Make sure the user exists
user_ensure('acme')
group_ensure('acme')
group_user_ensure('acme', 'acme')
# Make sure acme.sh is installed for the acme user
if not path.has('acme.sh', user='******'):
git.ensure_clone_github('Neilpang/acme.sh', '/home/acme/acme.sh', user='******')
with cd("/home/acme/acme.sh"):
sudo("./acme.sh --install", user='******')
dir_ensure(well_known_base)
sudo("chown acme:acme {well_known} && chmod 755 {well_known}".format(well_known=well_known_base)) # Can't change attributes without sudo in a sticky (not write) directory... annoying
def add_nagios_user():
'''
Adds Nagios user and groups to VMs and sets the right permissions.
'''
cuisine.group_ensure('nagios')
cuisine.user_ensure('nagios')
cuisine.group_user_ensure('nagios', 'nagios')
sudo('mkdir -p /usr/local/nagios')
sudo('mkdir -p /usr/local/nagios/libexec')
cuisine.dir_ensure('/usr/local/nagios')
cuisine.dir_ensure('/usr/local/nagios/libexec')
sudo('chown nagios.nagios /usr/local/nagios')
sudo('chown -R nagios.nagios /usr/local/nagios/libexec')
def add_nagios_user():
'''
Adds Nagios user and groups to VMs and sets the right permissions.
'''
cuisine.group_ensure('nagios')
cuisine.user_ensure('nagios')
cuisine.group_user_ensure('nagios', 'nagios')
sudo('mkdir -p /usr/local/nagios')
sudo('mkdir -p /usr/local/nagios/libexec')
cuisine.dir_ensure('/usr/local/nagios')
cuisine.dir_ensure('/usr/local/nagios/libexec')
sudo('chown nagios.nagios /usr/local/nagios')
sudo('chown -R nagios.nagios /usr/local/nagios/libexec')
def _setup_user():
for user in USERS:
keys = '\n'.join(_get_public_key_from_github(user))
cuisine.user_ensure(user, shell='/bin/bash')
cuisine.group_user_ensure('wheel', user)
sudo('echo "password" | passwd --stdin %s' % user)
sudo('mkdir -p /home/%s/.ssh' % user)
sudo('chown %s:%s /home/%s/.ssh' % (user, user, user))
sudo('chmod 700 /home/%s/.ssh' % user)
sudo('echo "%s" > /home/%s/.ssh/authorized_keys' % (keys, user))
sudo('chown %s:%s /home/%s/.ssh/authorized_keys' % (user, user, user))
sudo('chmod 600 /home/%s/.ssh/authorized_keys' % user)
def add_user_to_docker_group(distro):
""" make sure the user running jenkins is part of the docker group """
log_green('adding the user running jenkins into the docker group')
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
if 'centos' in distro.value:
user_ensure('centos', home='/home/centos', shell='/bin/bash')
group_ensure('docker', gid=55)
group_user_ensure('docker', 'centos')
if 'ubuntu' in distro.value:
user_ensure('ubuntu', home='/home/ubuntu', shell='/bin/bash')
group_ensure('docker', gid=55)
group_user_ensure('docker', 'ubuntu')
def initialize():
"""Log in to the server as root and create the initial user/group"""
env.user = '******'
mode_user()
group_ensure(env.remote_group)
user_ensure(env.remote_user, shell='/bin/bash')
group_user_ensure(env.remote_user, env.remote_group)
# copy local public key to user's authorized_keys for convenience
if os.path.exists('~/.ssh/id_rsa.pub'):
f = open('~/.ssh/id_rsa.pub', 'rb')
ssh_authorize(env.remote_user, f.read())
f.close()
file_append("/etc/sudoers", "%(remote_user)s ALL=(ALL) NOPASSWD:ALL\n" % env)
def ensure():
select_package("apt")
package_ensure(["znc"]) # On debian will automatically be enabled
user_ensure('znc')
group_ensure('znc')
group_user_ensure('znc', 'znc')
dir_ensure("/var/znc", mode='755')
dir_ensure("/var/znc/configs", mode='755')
run("chown znc:znc /var/znc")
util.put("/srv/znc.conf", "/var/znc/configs", user="******", mode="600")
util.put("config/keys/znc.pem", "/var/znc", user="******", mode="600")
util.put("config/znc/modules", "/var/znc", user="******", mode="755")
run("cp /var/znc/modules/*.so /usr/lib/znc")
systemd.add_unit("config/systemd/znc.service")
run("systemctl enable znc")
run("systemctl restart znc")
def ensure():
select_package("apt")
package_ensure(["znc"]) # On debian will automatically be enabled
user_ensure('znc')
group_ensure('znc')
group_user_ensure('znc', 'znc')
dir_ensure("/var/znc", mode='755')
dir_ensure("/var/znc/configs", mode='755')
run("chown znc:znc /var/znc")
util.put("/srv/znc.conf", "/var/znc/configs", user="******", mode="600")
util.put("config/keys/znc.pem", "/var/znc", user="******", mode="600")
util.put("config/znc/modules", "/var/znc", user="******", mode="755")
run("cp /var/znc/modules/*.so /usr/lib/znc")
systemd.add_unit("config/systemd/znc.service")
run("systemctl enable znc")
run("systemctl restart znc")
def ensure_fcgiwrap(children=4):
select_package("apt")
package_ensure(["fcgiwrap"]) # On debian will automatically be enabled
# fcgi can't run status script because its default user (www-data) has no login shell--not sure why exactly but work around it by making a new user
user_ensure('fcgiwrap', shell="/bin/sh")
group_ensure('fcgiwrap')
group_user_ensure('fcgiwrap', 'fcgiwrap')
# Needed because of Debian bug https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=792705. Fastcgi 1.1.0-6 (unstable as of writing) fixes this bug.
util.put_file("config/systemd/fcgiwrap.service",
"/etc/systemd/system/gcgiwrap.service",
mode='0644',
user='******')
# Not sure these two lines actually do anything
sudo('sed -i "s/www-data/fcgiwrap/" /lib/systemd/system/fcgiwrap.service')
sudo('echo "FCGI_CHILDREN={}" > /etc/default/fcgiwrap'.format(children))
sudo('systemctl daemon-reload')
sudo('/etc/init.d/fcgiwrap restart')
def install_sproutcore():
"""
get all Ruby rvm dependencies working alongside sproutcore
we want to do a multiuser install of rvm (hence why we pipe to `sudo bash` instead of just `bash`)
Also, we intentionally use the env.user (because he can sudo) and not env.deploy_user (that cannot sudo).
:return:
"""
# try:
# run('sproutcore')
# except:
try:
sudo('apt-get remove --purge ruby-rvm ruby --yes')
sudo('rm -rf /usr/share/ruby-rvm /etc/rmvrc /etc/profile.d/rvm.sh')
sudo('apt-get install build-essential --yes')
sudo('curl -L https://get.rvm.io | bash -s stable')
sudo('apt-get install -y build-essential openssl libreadline6 libreadline6-dev curl git-core zlib1g '
'zlib1g-dev libssl-dev libyaml-dev libxml2-dev libxslt-dev autoconf libc6-dev ncurses-dev automake libtool '
'bison subversion pkg-config sqlite3 libsqlite3-dev')
sudo('curl -L https://get.rvm.io | bash -s stable --ruby') # modified to install just to the calthorpe user
append('.bashrc', '''
[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" # This loads RVM into a shell session''')
#for local setup: [[ -s "/usr/local/rvm/scripts/rvm" ]] && source "/usr/local/rvm/scripts/rvm" # This loads RVM into a shell session
run('source /usr/local/rvm/scripts/rvm')
sudo("rvm install ruby-1.9.3")
sudo("rvm use 1.9.3")
sudo("rvm --default use 1.9.3")
sudo("gem install sproutcore")
cuisine.group_user_ensure('rvm', env.user)
cuisine.group_user_ensure('rvm', env.deploy_user)
except Exception, E:
print E
def add_nagios_user():
'''
Adds Nagios user and group and sets correct file permissions.
'''
cuisine.group_ensure('nagcmd')
cuisine.group_ensure('nagios')
cuisine.user_ensure('nagios')
cuisine.group_user_ensure('nagios', 'nagios')
cuisine.group_user_ensure('nagcmd', 'nagios')
cuisine.user_ensure('www-data')
cuisine.group_user_ensure('nagcmd', 'www-data')
sudo('mkdir -p /usr/local/nagios')
sudo('mkdir -p /usr/local/nagios/libexec')
cuisine.dir_ensure('/usr/local/nagios')
cuisine.dir_ensure('/usr/local/nagios/libexec')
sudo('chown nagios.nagcmd /usr/local/nagios')
sudo('chown -R nagios.nagcmd /usr/local/nagios/libexec')
def add_nagios_user():
'''
Adds Nagios user and group and sets correct file permissions.
'''
cuisine.group_ensure('nagcmd')
cuisine.group_ensure('nagios')
cuisine.user_ensure('nagios')
cuisine.group_user_ensure('nagios', 'nagios')
cuisine.group_user_ensure('nagcmd', 'nagios')
cuisine.user_ensure('www-data')
cuisine.group_user_ensure('nagcmd', 'www-data')
sudo('mkdir -p /usr/local/nagios')
sudo('mkdir -p /usr/local/nagios/libexec')
cuisine.dir_ensure('/usr/local/nagios')
cuisine.dir_ensure('/usr/local/nagios/libexec')
sudo('chown nagios.nagcmd /usr/local/nagios')
sudo('chown -R nagios.nagcmd /usr/local/nagios/libexec')
def setup_urbanfootprint(upgrade_env=True):
"""
Runs all the steps necessary to configure urbanfootprint
"""
set_paths()
print "ROOT = {0}\n".format(ROOT), \
"GIT_ROOT = {0}\n".format(GIT_ROOT), \
"BASE_PATH = {0}\n".format(BASE_PATH), \
"PROJECT_PATH: {0}\n".format(PROJ_ROOT), \
"WEBSOCKETS_ROOT: {0}\n".format(WEBSOCKETS_ROOT)
from fabfile.management import deploy
# Make sure deployment user exists and that the key is setup correctly
cuisine.user_ensure(env.deploy_user)
if env.user != env.deploy_user:
sudo('chsh -s /bin/bash {0}'.format(env.deploy_user))
sudo('mkdir -p ~{0}/.ssh/'.format(env.deploy_user), user=env.deploy_user)
sudo('cp ~/.ssh/id_rsa* ~{0}/.ssh/'.format(env.deploy_user))
sudo('chown {0}.{0} ~{0}/.ssh/id_rsa*'.format(env.deploy_user))
sudo('chmod 600 ~{0}/.ssh/id_rsa'.format(env.deploy_user), user=env.deploy_user)
# add UbuntuGIS repo
sudo('add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y')
sudo('add-apt-repository ppa:chris-lea/node.js -y')
sudo('add-apt-repository ppa:chris-lea/nginx-devel -y')
cuisine.package_update()
cuisine.package_upgrade()
# using oracle's jdk for good compatibility
# intentionally not install postgresql-9.1-postgis until we can upgrade to django 1.5.x and postgis 2.0
cuisine.package_ensure(
'build-essential openjdk-6-jre openjdk-6-jdk postgresql git python-software-properties proj libproj-dev '
'python-pip python-virtualenv python-dev virtualenvwrapper postgresql-server-dev-9.1 '
'gdal-bin libgdal1-dev nginx varnish supervisor redis-server curl python-gdal nodejs'
# 'libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev libboost-regex-dev '
# 'libboost-system-dev libboost-thread-dev
)
#install older postgis
create_template_postgis()
cuisine.group_user_ensure("www-data", env.deploy_user)
cuisine.group_user_ensure("sudo", env.deploy_user)
# setup deployment user git settings #TODO: make more sense of this... probably shouldn't be this for all setups
#sudo('su {0} -c "git config --global user.email \"[email protected]\""'.format(env.deploy_user))
#sudo('su {0} -c "git config --global user.name \"Craig Rindt\""'.format(env.deploy_user))
# create folders for calthorpe deployment
sudo('mkdir -p {git_root}'.format(git_root=GIT_ROOT))
sudo('chmod +t {git_root}'.format(git_root=GIT_ROOT))
sudo('chown -R {user}.www-data {git_root}'.format(git_root=GIT_ROOT, user=env.deploy_user))
sudo('mkdir -p /srv/calthorpe_static') # needs to match local_settings.py.deployment value
sudo('chmod +t /srv/calthorpe_static')
sudo('chown -R {0}.www-data /srv/calthorpe_static'.format(env.deploy_user))
sudo('mkdir -p /srv/calthorpe_media') # needs to match local_settings.py.deployment value
sudo('chmod +t /srv/calthorpe_media')
sudo('chown -R {0}.www-data /srv/calthorpe_media'.format(env.deploy_user))
#create virtualenv
if not cuisine.dir_exists(env.virtualenv_directory):
sudo("virtualenv {env}".format(env=env.virtualenv_directory))
sudo('chown -R {user}.www-data {env}'.format(user=env.deploy_user, env=env.virtualenv_directory))
install_mapnik()
install_osgeo()
# clone repo if needed
# if not cuisine.dir_exists(GIT_ROOT):
with cd(GIT_ROOT):
sudo('su {0} -c "git clone https://github.com/crindt/uf urbanfootprint"'.format(env.deploy_user))
sudo('chown -R {user}.www-data {BASE_PATH}/..'.format(user=env.deploy_user, BASE_PATH=BASE_PATH))
setup_databases()
with cd(PROJ_ROOT):
if not exists('local_settings.py'):
sudo('ln -s local_settings.py.{CLIENT} local_settings.py'.format(CLIENT=env.client))
with cd(WEBSOCKETS_ROOT):
# removes this
try:
sudo('rm -r carto')
except:
pass
sudo('npm install .')
sudo('git clone git://github.com/mapbox/carto.git')
sudo('npm install -g ./carto')
sudo('npm install -g millstone')
sudo('chown -R {0}.www-data node_modules/'.format(env.deploy_user))
# update varnish default port
#sed('/etc/default/varnish', r'^DAEMON_OPTS="-a :6081', 'DAEMON_OPTS="-a :80', use_sudo=True)
# soft link all configuration files
with cd('/etc/varnish'):
sudo('rm -f default.vcl')
#sudo('ln -s /srv/calthorpe/urbanfootprint/calthorpe/server/conf/etc/varnish/default.vcl.prod default.vcl')
nginx_configure()
with cd('/etc/supervisor/conf.d'):
sudo('rm -f calthorpe.conf')
# Link the appropriate supervisor config file. dev omits a web server, and the log files are different
supervisor_conf_ext = 'dev' if env.dev else 'prod'
link_supervisor_config_path = "ln -s {BASE_PATH}/conf/etc/supervisor/conf.d/calthorpe.supervisor.{supervisor_extension} calthorpe.conf"
sudo(link_supervisor_config_path.format(BASE_PATH=BASE_PATH, supervisor_extension=supervisor_conf_ext))
install_sproutcore()
# trigger deploy to update virtualenv and restart services
deploy(upgrade_env=upgrade_env)
patch_django_layermapping()
def create_user():
cuisine.user_ensure(
USER_NAME, home='/home/%s' % USER_NAME, shell='/bin/bash')
cuisine.group_user_ensure('www-data', USER_NAME)
def provision_user(admin_user, admin_group):
group_ensure(admin_group)
user_ensure(admin_user, shell='/bin/bash')
group_user_ensure(admin_group, admin_user)
append("/etc/sudoers", "%{group} ALL=(ALL) NOPASSWD:ALL".format(group=admin_group),
use_sudo=True)
def DoIt():
print green("DoIt started")
print yellow("Installing packages")
cuisine.package_update()
cuisine.package_ensure("linux-image-generic-lts-trusty")
cuisine.package_ensure("wget")
print yellow("Installing docker")
sudo("wget -qO- https://get.docker.com/ | sh")
print yellow("Adding ubuntu to the docker group")
try:
cuisine.group_user_ensure("docker", "ubuntu")
except:
pass
print yellow("Creating mldb storage")
sudo("install -d -o ubuntu -g ubuntu /mldb_data")
print yellow("Setting up the mldb start on boot")
confTmpl = cuisine.text_strip_margin(
"""
|# On the first boot, this will pull the latest mldb Container
|# subsequent executions will keep running the same (pulled) version
|description "MLDB Container"
|author "mldb.ai inc."
|start on filesystem and started docker
|stop on runlevel [!2345]
|respawn
|script
| /usr/bin/docker run -a stdin -a stdout -a stderr -v /mldb_data:/mldb_data -e MLDB_IDS="1000" -p 127.0.0.1:80:80 quay.io/datacratic/mldb:latest
|end script
|
"""):
cuisine.file_write("/etc/init/mldb.conf",
confTmpl, mode = "644",
owner = "root", group = "root",
sudo = True, check = True)
print yellow("Configuring banner")
confTmpl = cuisine.text_strip_margin(
"""
| The Virtual Appliance has booted, but you do not need to log into the
| Virtual Appliance to use MLDB.
|
| Once MLDB is up and running inside the Virtual Appliance, you will be
| able to connect to it via HTTP as per the documentation.
|
| Note that MLDB can take a few minutes to initialize the first time you
| launch the Virtual Appliance.
|
|
""")
cuisine.file_write("/etc/issue",
confTmpl, mode = "644",
owner = "root", group = "root",
sudo = True, check = True)
print yellow("Removing grub delay")
sudo('echo "GRUB_TIMEOUT=0\n" >> /etc/default/grub')
sudo("update-grub")
print yellow("Cleaning up...")
sudo("cat /dev/null > /home/ubuntu/.ssh/authorized_keys")
def deadtree():
"""Deadtree is the main services machine. It can be taken down at any time and rebuilt."""
apt.sudo_ensure() # cuisine.package_ensure is broken otherwise
# Set up /etc/skel
sudo("mkdir /etc/skel/.ssh || true")
sudo("chmod 700 /etc/skel/.ssh")
# Add a 'nobody' user
user_ensure('nobody')
group_ensure('nobody')
group_user_ensure('nobody', 'nobody')
sudo('usermod -s /bin/false nobody')
# Set up the firewall
put("config/firewalls/deadtree.sh", "/usr/local/bin", use_sudo=True)
sudo("sh /usr/local/bin/deadtree.sh")
# Set up nginx
already_installed = nginx.ensure()
nginx.ensure_site('config/nginx/default', cert='config/certs/za3k.com.pem', key='config/keys/blog.za3k.com.key')
nginx.ensure_fcgiwrap(children=4)
if not already_installed:
nginx.restart() # IPv[46] listener only changes on restart
# Set up letsencrypt
letsencrypt.ensure()
# Set up authorization to back up to the data server
#public_key = ssh.ensure_key('/root/.ssh/id_rsa')
#with settings(user='******', host_string='burn'):
# #put(public_key, '/home/zachary/test_authorized_keys')
# files.append('/home/deadtree/.ssh/authorized_keys', public_key)
# ddns.za3k.com (TCP port 80, web updater for DDNS)
# ns.za3k.com (UDP port 53, DNS server)
user_ensure('nsd')
group_ensure('nsd')
group_user_ensure('nsd', 'nsd')
with mode_sudo():
dir_ensure('/var/lib/nsd', mode='755')
sudo("chown nsd:nsd /var/lib/nsd")
package_ensure(["nsd"])
with cd("/var/lib/nsd"):
sudo("touch /var/lib/nsd/moreorcs.com.zone && chown nsd:nsd /var/lib/nsd/moreorcs.com.zone")
node.ensure()
put("config/ddns/moreorcs.com.zonetemplate", "/etc/nsd", mode='644', use_sudo=True)
supervisord.ensure()
git.ensure_clone_github('thingless/ddns', '/var/lib/nsd/ddns', user='******')
supervisord.ensure_config("config/supervisor/ddns.conf")
put("config/ddns/config.json", "/var/lib/nsd", mode='644', use_sudo=True)
sudo("chown nsd:nsd /var/lib/nsd/config.json")
# [Manual] Copy dnsDB.json from backup
sudo("cd /var/lib/nsd && ln -sf ddns/index.txt index.txt && chown nsd:nsd index.txt")
supervisord.update() # Run ddns
package_ensure(["nsd"])
put("config/ddns/nsd.conf", "/etc/nsd", mode='644', use_sudo=True)
sudo("systemctl restart nsd")
nginx.ensure_site('config/nginx/ddns.za3k.com', csr='config/certs/ddns.za3k.com.csr', key='config/keys/ddns.za3k.com.key', domain="ddns.za3k.com", letsencrypt=True, cert="config/certs/ddns.za3k.com.pem")
nginx.reload()
# blog.za3k.com
package_ensure(["php5-fpm", "mysql-server", "php5-mysql"])
nginx.ensure_site('config/nginx/blog.za3k.com', cert='config/certs/blog.za3k.com.pem', key='config/keys/blog.za3k.com.key', domain="blog.za3k.com", letsencrypt=True, csr="config/certs/blog.za3k.com.csr")
git.ensure_clone_za3k('za3k_blog', '/var/www/za3k_blog', user='******')
# TODO: Replace a database-specific password or make it more obvious it's not used? Currently we're using user ACLs and this gets ignored anyway, I think?
# [Manual] Load the blog database from backup at /srv/mysql -> /var/lib/mysql
sudo('systemctl restart mysql')
# etherpad.za3k.com
package_ensure(["sqlite3"])
user_ensure('etherpad')
group_ensure('etherpad')
group_user_ensure('etherpad', 'etherpad')
git.ensure_clone_github('ether/etherpad-lite', '/var/www/etherpad', commit='1.6.0', user='******')
nginx.ensure_site('config/nginx/etherpad.za3k.com', csr='config/certs/etherpad.za3k.com.csr', key='config/keys/etherpad.za3k.com.key', domain="etherpad.za3k.com", letsencrypt=True, cert="config/certs/etherpad.za3k.com.pem")
util.put("config/etherpad/APIKEY.txt", "/var/www/etherpad", user='******', mode='600')
util.put("config/etherpad/settings.json", "/var/www/etherpad", user='******', mode='644')
if not files.exists("/var/www/etherpad/var/sqlite.db"):
sudo("mkdir -p /var/www/etherpad/var", user='******')
with cd("/var/www/etherpad"):
sudo("npm install sqlite3")
sudo("rsync -av burn.za3k.com::etherpad --delete /var/www/etherpad/var", user='******')
supervisord.ensure()
supervisord.ensure_config("config/supervisor/etherpad.conf")
supervisord.update()
# forsale
nginx.ensure_site('config/nginx/forsale')
util.put('data/forsale', '/var/www', user='******', mode='755')
# gipc daily sync
# github personal backup
# github repo list
# -> updater
# irc.za3k.com -> irc
# -> webchat (qwebirc)
# jsfail.com
user_ensure('jsfail')
group_ensure('jsfail')
group_user_ensure('jsfail', 'jsfail')
nginx.ensure_site('config/nginx/jsfail.com')
util.put('data/jsfail', '/var/www', 'jsfail', mode='755')
# library.za3k.com -> website
# -> sync script
# -> card catalog
# MUST be user 2001 to match remote rsync
user_ensure('library', uid=2001)
group_ensure('library', gid=2001)
group_user_ensure('library', 'library')
with mode_sudo():
dir_ensure('/var/www/library', mode='755')
sudo("chown library:library /var/www/library")
put("config/library/library.sync", "/etc/cron.daily", mode='755', use_sudo=True)
sudo("/etc/cron.daily/library.sync")
nginx.ensure_site('config/nginx/library.za3k.com', csr='config/certs/library.za3k.com.csr', key='config/keys/library.za3k.com.key', domain="library.za3k.com", letsencrypt=True, cert="config/certs/library.za3k.com.pem")
# logs (nginx) and analysis (analog)
# mint sync
# moreorcs.com
user_ensure('moreorcs')
group_ensure('moreorcs')
group_user_ensure('moreorcs', 'moreorcs')
nginx.ensure_site('config/nginx/moreorcs.com', cert='config/certs/moreorcs.com.pem', key='config/keys/moreorcs.com.key', domain="moreorcs.com", letsencrypt=True, csr="config/certs/moreorcs.com.csr")
git.ensure_clone_github('za3k/moreorcs', '/var/www/moreorcs', user='******')
# nanowrimo.za3k.com
nginx.ensure_site('config/nginx/nanowrimo.za3k.com', csr='config/certs/nanowrimo.za3k.com.csr', key='config/keys/nanowrimo.za3k.com.key', domain="nanowrimo.za3k.com", letsencrypt=True, cert="config/certs/nanowrimo.za3k.com.pem")
util.put('data/nanowrimo', '/var/www', user='******', mode='755')
# nntp.za3k.com - Discontinued
# petchat.za3k.com
nginx.ensure_site('config/nginx/petchat.za3k.com')
if not files.exists('/var/www/petchat'):
git.ensure_clone_za3k('petchat', '/var/www/petchat', user='******')
# publishing.za3k.com
# thinkingtropes.com
nginx.ensure_site('config/nginx/thinkingtropes.com')
util.put('data/thinkingtropes', '/var/www', user='******', mode='755')
# thisisashell.com
nginx.ensure_site('config/nginx/thisisashell.com', csr='config/certs/thisisashell.com.csr', key='config/keys/thisisashell.com.key', domain="thisisashell.com", letsencrypt=True, cert="config/certs/thisisashell.com.pem")
# twitter archive
# za3k.com
user_ensure('za3k')
group_ensure('za3k')
group_user_ensure('za3k', 'za3k')
nginx.ensure_site('config/nginx/za3k.com', cert='config/certs/za3k.com.pem', key='config/keys/za3k.com.key', domain="za3k.com", letsencrypt=True, csr="config/certs/za3k.com.csr")
git.ensure_clone_za3k('za3k', '/var/www/za3k', user='******')
# Markdown .md
ruby.ensure()
ruby.ensure_gems(["redcarpet"])
# colony on the moon
sudo("rsync -av burn.za3k.com::colony --delete /var/www/colony", user='******')
# |-- status.za3k.com
sudo("mkdir -p /var/www/status && chmod 755 /var/www/status")
util.put("/srv/keys/backup_check", "/var/www/status", user='******', mode='600')
#util.put("/srv/keys/backup_check.pub", "/var/www/status", user='******', mode='644')
package_ensure(["parallel", "curl"])
nginx.reload()
def deadtree():
"""Deadtree is the main services machine. It can be taken down at any time and rebuilt."""
apt.sudo_ensure() # cuisine.package_ensure is broken otherwise
# Set up /etc/skel
sudo("mkdir /etc/skel/.ssh || true")
sudo("chmod 700 /etc/skel/.ssh")
# Add a 'nobody' user
user_ensure('nobody')
group_ensure('nobody')
group_user_ensure('nobody', 'nobody')
sudo('usermod -s /bin/false nobody')
# Set up logging
logs.setup()
# Set up the firewall
util.put_file("config/firewalls/deadtree.sh",
"/usr/local/bin/deadtree.sh",
mode='755',
user='******')
sudo("sh /usr/local/bin/deadtree.sh")
util.put_file("config/firewalls/iptables",
"/etc/network/if-pre-up.d/",
mode='755',
user='******')
# Set up authorization to back up to germinate
public_key = ssh.ensure_key('/var/local/germinate-backup', use_sudo=True)
with settings(user='******', host_string='germinate'):
files.append('/home/deadtree/.ssh/authorized_keys',
public_key,
use_sudo=True)
util.put_file("config/backup/sshconfig-deadtree",
"/root/.ssh/config",
user='******')
# Set up backup
package_ensure(["rsync"])
util.put_file("config/backup/generic-backup.sh",
"/var/local",
mode='755',
user='******')
util.put_file("config/backup/backup-exclude-base",
"/var/local/backup-exclude",
mode='644',
user='******')
util.put_file("config/backup/backup-deadtree.sh",
"/etc/cron.daily/backup-deadtree",
mode='755',
user='******')
# Set up nginx
already_installed = nginx.ensure()
nginx.ensure_site('config/nginx/default',
cert='config/certs/za3k.com.pem',
key='config/keys/blog.za3k.com.key')
nginx.ensure_fcgiwrap(children=4)
if not already_installed:
nginx.restart() # IPv[46] listener only changes on restart
# Set up letsencrypt
letsencrypt.ensure()
# Set up logging reports
package_ensure(["analog"])
with mode_sudo():
dir_ensure("/var/www/logs", mode='755')
util.put_file("config/logs/generate-logs",
"/etc/cron.daily/generate-logs",
mode='755',
user='******')
util.put_file("config/logs/analog.cfg",
"/etc/analog.cfg",
mode='644',
user='******')
# ddns.za3k.com (TCP port 80, web updater for DDNS)
# ns.za3k.com (UDP port 53, DNS server)
user_ensure('nsd')
group_ensure('nsd')
group_user_ensure('nsd', 'nsd')
with mode_sudo():
dir_ensure('/var/lib/nsd', mode='755')
sudo("chown nsd:nsd /var/lib/nsd")
package_ensure(["nsd"])
with cd("/var/lib/nsd"):
sudo(
"touch /var/lib/nsd/moreorcs.com.zone && chown nsd:nsd /var/lib/nsd/moreorcs.com.zone"
)
node.ensure()
util.put_file("config/ddns/moreorcs.com.zonetemplate",
"/etc/nsd/moreorcs.com.zonetemplate",
mode='644',
user='******')
supervisord.ensure()
git.ensure_clone_github('thingless/ddns', '/var/lib/nsd/ddns', user='******')
supervisord.ensure_config("config/supervisor/ddns.conf")
util.put_file("config/ddns/config.json",
"/var/lib/nsd/config.json",
mode='644',
user='******')
# [Manual] Copy dnsDB.json from backup
sudo(
"cd /var/lib/nsd && ln -sf ddns/index.txt index.txt && chown nsd:nsd index.txt"
)
supervisord.update() # Run ddns
package_ensure(["nsd"])
util.put_file("config/ddns/nsd.conf",
"/etc/nsd/nsd.conf",
mode='644',
user='******')
sudo("systemctl restart nsd")
nginx.ensure_site('config/nginx/ddns.za3k.com',
csr='config/certs/ddns.za3k.com.csr',
key='config/keys/ddns.za3k.com.key',
domain="ddns.za3k.com",
letsencrypt=True,
cert="config/certs/ddns.za3k.com.pem")
nginx.reload()
# blog.za3k.com
package_ensure(["php5-fpm", "mysql-server", "php5-mysql"])
nginx.ensure_site('config/nginx/blog.za3k.com',
cert='config/certs/blog.za3k.com.pem',
key='config/keys/blog.za3k.com.key',
domain="blog.za3k.com",
letsencrypt=True,
csr="config/certs/blog.za3k.com.csr")
git.ensure_clone_za3k('za3k_blog', '/var/www/za3k_blog', user='******')
# [Manual] Edit /etc/php5/fpm/php.ini
# upload_max_filesize = 20M
# post_max_size = 26M
# sudo("systemctl reload php5-fpm.service")
# Yes, www-data and not fcgiwrap
sudo("chown www-data:www-data -R /var/www/za3k_blog")
sudo("find . -type d -exec chmod 755 {} \;")
sudo("find . -type f -exec chmod 644 {} \;")
# TODO: Replace a database-specific password or make it more obvious it's not used? Currently we're using user ACLs and this gets ignored anyway, I think?
# [Manual] Load the blog database from backup at /srv/mysql -> /var/lib/mysql
sudo('systemctl restart mysql')
# deadtree.za3k.com
nginx.ensure_site('config/nginx/deadtree.za3k.com',
cert='config/certs/deadtree.za3k.com.pem',
key='config/keys/deadtree.za3k.com.key',
domain="deadtree.za3k.com",
letsencrypt=True,
csr="config/certs/deadtree.za3k.com.csr")
util.put_dir('data/deadtree/public',
'/var/www/public',
mode='755',
user='******')
# etherpad.za3k.com
package_ensure(["sqlite3"])
user_ensure('etherpad')
group_ensure('etherpad')
group_user_ensure('etherpad', 'etherpad')
git.ensure_clone_github('ether/etherpad-lite',
'/var/www/etherpad',
commit='1.6.0',
user='******')
nginx.ensure_site('config/nginx/etherpad.za3k.com',
csr='config/certs/etherpad.za3k.com.csr',
key='config/keys/etherpad.za3k.com.key',
domain="etherpad.za3k.com",
letsencrypt=True,
cert="config/certs/etherpad.za3k.com.pem")
util.put_file("config/etherpad/APIKEY.txt",
"/var/www/etherpad",
user='******',
mode='600')
util.put_file("config/etherpad/settings.json",
"/var/www/etherpad",
user='******',
mode='644')
if not files.exists("/var/www/etherpad/var/sqlite.db"):
sudo("mkdir -p /var/www/etherpad/var", user='******')
with cd("/var/www/etherpad"):
sudo("npm install sqlite3")
sudo(
"rsync -av germinate.za3k.com::etherpad --delete /var/www/etherpad/var",
user='******')
supervisord.ensure()
supervisord.ensure_config("config/supervisor/etherpad.conf")
supervisord.update()
# forsale
nginx.ensure_site('config/nginx/forsale')
util.put_dir('data/forsale', '/var/www/forsale', mode='755', user='******')
# gipc daily sync
# github personal backup
# github repo list
util.put_file("config/github/github-metadata-sync",
"/etc/cron.daily/github-metadata-sync",
mode='755',
user='******')
# -> updater
# irc.za3k.com -> irc
# -> webchat (qwebirc)
# jsfail.com
user_ensure('jsfail')
group_ensure('jsfail')
group_user_ensure('jsfail', 'jsfail')
nginx.ensure_site('config/nginx/jsfail.com')
util.put_dir('data/jsfail', '/var/www/jsfail', 'jsfail', mode='755')
# library.za3k.com -> website
# -> sync script
# -> card catalog
# MUST be user 2001 to match remote rsync
user_ensure('library', uid=2001)
group_ensure('library', gid=2001)
group_user_ensure('library', 'library')
with mode_sudo():
dir_ensure('/var/www/library', mode='755')
files.append('/etc/sudoers',
'za3k ALL=(root) NOPASSWD: /etc/cron.daily/library-sync',
use_sudo=True)
with settings(user='******', host_string='germinate'):
actual_key = ssh.get_public_key(
"/data/git/books.git/hooks/deadtree.library")
ssh_line = 'command="{command}",no-port-forwarding,no-x11-forwarding,no-agent-forwarding {key}'.format(
command="sudo /etc/cron.daily/library-sync", key=actual_key)
files.append('/home/za3k/.ssh/authorized_keys', ssh_line, use_sudo=True)
sudo("chown library:library /var/www/library")
util.put_file("config/library/library-sync",
"/etc/cron.daily/library-sync",
mode='755',
user='******')
sudo("/etc/cron.daily/library-sync")
nginx.ensure_site('config/nginx/library.za3k.com',
csr='config/certs/library.za3k.com.csr',
key='config/keys/library.za3k.com.key',
domain="library.za3k.com",
letsencrypt=True,
cert="config/certs/library.za3k.com.pem")
# logs (nginx) and analysis (analog)
# mint sync
# moreorcs.com
user_ensure('moreorcs')
group_ensure('moreorcs')
group_user_ensure('moreorcs', 'moreorcs')
nginx.ensure_site('config/nginx/moreorcs.com',
cert='config/certs/moreorcs.com.pem',
key='config/keys/moreorcs.com.key',
domain="moreorcs.com",
letsencrypt=True,
csr="config/certs/moreorcs.com.csr")
git.ensure_clone_github('za3k/moreorcs',
'/var/www/moreorcs',
user='******')
# nanowrimo.za3k.com
nginx.ensure_site('config/nginx/nanowrimo.za3k.com',
csr='config/certs/nanowrimo.za3k.com.csr',
key='config/keys/nanowrimo.za3k.com.key',
domain="nanowrimo.za3k.com",
letsencrypt=True,
cert="config/certs/nanowrimo.za3k.com.pem")
util.put_dir('data/nanowrimo',
'/var/www/nanowrimo',
user='******',
mode='755')
# nntp.za3k.com - Discontinued
# petchat.za3k.com
nginx.ensure_site('config/nginx/petchat.za3k.com')
if not files.exists('/var/www/petchat'):
git.ensure_clone_za3k('petchat', '/var/www/petchat', user='******')
# publishing.za3k.com
# thinkingtropes.com
nginx.ensure_site('config/nginx/thinkingtropes.com')
util.put_dir('data/thinkingtropes',
'/var/www/thinkingtropes',
user='******',
mode='755')
# thisisashell.com
nginx.ensure_site('config/nginx/thisisashell.com',
csr='config/certs/thisisashell.com.csr',
key='config/keys/thisisashell.com.key',
domain="thisisashell.com",
letsencrypt=True,
cert="config/certs/thisisashell.com.pem")
# isrickandmortyout.com
nginx.ensure_site('config/nginx/isrickandmortyout.com',
csr='config/certs/isrickandmortyout.com.csr',
key='config/keys/isrickandmortyout.com.key',
domain="isrickandmortyout.com",
letsencrypt=True,
cert="config/certs/isrickandmortyout.com.pem")
# twitter archive
# za3k.com
user_ensure('za3k')
group_ensure('za3k')
group_user_ensure('za3k', 'za3k')
nginx.ensure_site('config/nginx/za3k.com',
cert='config/certs/za3k.com.pem',
key='config/keys/za3k.com.key',
domain="za3k.com",
letsencrypt=True,
csr="config/certs/za3k.com.csr")
git.ensure_clone_za3k('za3k', '/var/www/za3k', user='******')
with settings(user='******', host_string='germinate'):
actual_key = ssh.get_public_key(
"/data/git/za3k.git/hooks/deadtree_key")
ssh_line = 'command="{command}",no-port-forwarding,no-x11-forwarding,no-agent-forwarding {key}'.format(
command="/usr/bin/git -C /var/www/za3k pull", key=actual_key)
files.append('/home/za3k/.ssh/authorized_keys', ssh_line, use_sudo=True)
# Markdown .md
ruby.ensure()
ruby.ensure_gems(["redcarpet"])
# Databases .view
package_ensure(["sqlite3"])
util.put_file("config/za3k/za3k-db-sync",
"/etc/cron.daily/za3k-db-sync",
mode='755',
user='******')
sudo("/etc/cron.daily/za3k-db-sync")
# colony on the moon
# disabled temp. because we're out of space
#sudo("rsync -av germinate.za3k.com::colony --delete /var/www/colony", user='******')
# .sc
package_ensure(["sc"])
nginx.reload()
def create_user(self):
user_ensure(self.user_name)
group_ensure(self.group_name)
group_user_ensure(self.user_name, self.group_name)
