def setup(): require( "directory", "venv_directory", "www_directory", "package_name", "repository", "project_name") # 1. clone project utils.clone_project() # 2. create virtualenv utils.create_virtualenv() # 3. install requirements utils.install_requirements() # 4. configure supervisor env.supervisor = { "command": supervisor.tornado_command() } supervisor.setup() supervisor.reload() # 5. configure nginx nginx.setup() nginx.reload()
def setup_code(sites): for site in sites: name = site['name'] path = base + '/' + app_name + '/' + name repo = site['repo'] type = site['type'] rr = name + '.' + app_name server_name = rr + '.' + domain aliyun.add_domain_record(domain, host, rr) deploy.pull_code(repo, path) nginx.add_server(server_name, path, type) nginx.reload()
def xenu(): """Xenu runs minecraft.""" # Set up the firewall util.put_file("config/firewalls/xenu.sh", "/usr/local/bin/xenu.sh", mode='755', user='******') sudo("sh /usr/local/bin/xenu.sh") util.put_file("config/firewalls/iptables", "/etc/network/if-pre-up.d/", mode='755', user='******') # Set up logging logs.setup() # Set up authorization to back up public_key = ssh.ensure_key('/var/local/germinate-backup', use_sudo=True) with settings(user='******', host_string='germinate'): files.append('/home/xenu-linux/.ssh/authorized_keys', public_key, use_sudo=True) sudo("mkdir -p /root/.ssh") util.put_file("config/backup/sshconfig-xenu", "/root/.ssh/config", user='******') # Set up backup package_ensure(["rsync"]) util.put_file("config/backup/generic-backup.sh", "/var/local", mode='755', user='******') util.put_file("config/backup/backup-exclude-xenu", "/var/local/backup-exclude", mode='644', user='******') util.put_file("config/backup/backup-xenu.sh", "/etc/cron.daily/backup-xenu", mode='755', user='******') # Minecraft prereqs package_ensure(["make", "tmux"]) # Set up java for minecraft if not path.has('java'): sudo( 'echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main" | tee /etc/apt/sources.list.d/webupd8team-java.list' ) sudo( 'echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list' ) sudo( 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886' ) sudo('apt-get update') sudo( 'echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections' ) package_ensure(["oracle-java8-installer"]) # fails # Set up nginx already_installed = nginx.ensure() if not already_installed: nginx.restart() # IPv[46] listener only changes on restart util.put_dir("data/minecraft-www", "/var/www/minecraft-www", user="******") nginx.ensure_site('config/nginx/minecraft.za3k.com') nginx.reload()
def deadtree(): """Deadtree is the main services machine. It can be taken down at any time and rebuilt.""" apt.sudo_ensure() # cuisine.package_ensure is broken otherwise # Set up /etc/skel sudo("mkdir /etc/skel/.ssh || true") sudo("chmod 700 /etc/skel/.ssh") # Add a 'nobody' user user_ensure('nobody') group_ensure('nobody') group_user_ensure('nobody', 'nobody') sudo('usermod -s /bin/false nobody') # Set up logging logs.setup() # Set up the firewall util.put_file("config/firewalls/deadtree.sh", "/usr/local/bin/deadtree.sh", mode='755', user='******') sudo("sh /usr/local/bin/deadtree.sh") util.put_file("config/firewalls/iptables", "/etc/network/if-pre-up.d/", mode='755', user='******') # Set up authorization to back up to germinate public_key = ssh.ensure_key('/var/local/germinate-backup', use_sudo=True) with settings(user='******', host_string='germinate'): files.append('/home/deadtree/.ssh/authorized_keys', public_key, use_sudo=True) util.put_file("config/backup/sshconfig-deadtree", "/root/.ssh/config", user='******') # Set up backup package_ensure(["rsync"]) util.put_file("config/backup/generic-backup.sh", "/var/local", mode='755', user='******') util.put_file("config/backup/backup-exclude-base", "/var/local/backup-exclude", mode='644', user='******') util.put_file("config/backup/backup-deadtree.sh", "/etc/cron.daily/backup-deadtree", mode='755', user='******') # Set up nginx already_installed = nginx.ensure() nginx.ensure_site('config/nginx/default', cert='config/certs/za3k.com.pem', key='config/keys/blog.za3k.com.key') nginx.ensure_fcgiwrap(children=4) if not already_installed: nginx.restart() # IPv[46] listener only changes on restart # Set up letsencrypt letsencrypt.ensure() # Set up logging reports package_ensure(["analog"]) with mode_sudo(): dir_ensure("/var/www/logs", mode='755') util.put_file("config/logs/generate-logs", "/etc/cron.daily/generate-logs", mode='755', user='******') util.put_file("config/logs/analog.cfg", "/etc/analog.cfg", mode='644', user='******') # ddns.za3k.com (TCP port 80, web updater for DDNS) # ns.za3k.com (UDP port 53, DNS server) user_ensure('nsd') group_ensure('nsd') group_user_ensure('nsd', 'nsd') with mode_sudo(): dir_ensure('/var/lib/nsd', mode='755') sudo("chown nsd:nsd /var/lib/nsd") package_ensure(["nsd"]) with cd("/var/lib/nsd"): sudo( "touch /var/lib/nsd/moreorcs.com.zone && chown nsd:nsd /var/lib/nsd/moreorcs.com.zone" ) node.ensure() util.put_file("config/ddns/moreorcs.com.zonetemplate", "/etc/nsd/moreorcs.com.zonetemplate", mode='644', user='******') supervisord.ensure() git.ensure_clone_github('thingless/ddns', '/var/lib/nsd/ddns', user='******') supervisord.ensure_config("config/supervisor/ddns.conf") util.put_file("config/ddns/config.json", "/var/lib/nsd/config.json", mode='644', user='******') # [Manual] Copy dnsDB.json from backup sudo( "cd /var/lib/nsd && ln -sf ddns/index.txt index.txt && chown nsd:nsd index.txt" ) supervisord.update() # Run ddns package_ensure(["nsd"]) util.put_file("config/ddns/nsd.conf", "/etc/nsd/nsd.conf", mode='644', user='******') sudo("systemctl restart nsd") nginx.ensure_site('config/nginx/ddns.za3k.com', csr='config/certs/ddns.za3k.com.csr', key='config/keys/ddns.za3k.com.key', domain="ddns.za3k.com", letsencrypt=True, cert="config/certs/ddns.za3k.com.pem") nginx.reload() # blog.za3k.com package_ensure(["php5-fpm", "mysql-server", "php5-mysql"]) nginx.ensure_site('config/nginx/blog.za3k.com', cert='config/certs/blog.za3k.com.pem', key='config/keys/blog.za3k.com.key', domain="blog.za3k.com", letsencrypt=True, csr="config/certs/blog.za3k.com.csr") git.ensure_clone_za3k('za3k_blog', '/var/www/za3k_blog', user='******') # [Manual] Edit /etc/php5/fpm/php.ini # upload_max_filesize = 20M # post_max_size = 26M # sudo("systemctl reload php5-fpm.service") # Yes, www-data and not fcgiwrap sudo("chown www-data:www-data -R /var/www/za3k_blog") sudo("find . -type d -exec chmod 755 {} \;") sudo("find . -type f -exec chmod 644 {} \;") # TODO: Replace a database-specific password or make it more obvious it's not used? Currently we're using user ACLs and this gets ignored anyway, I think? # [Manual] Load the blog database from backup at /srv/mysql -> /var/lib/mysql sudo('systemctl restart mysql') # deadtree.za3k.com nginx.ensure_site('config/nginx/deadtree.za3k.com', cert='config/certs/deadtree.za3k.com.pem', key='config/keys/deadtree.za3k.com.key', domain="deadtree.za3k.com", letsencrypt=True, csr="config/certs/deadtree.za3k.com.csr") util.put_dir('data/deadtree/public', '/var/www/public', mode='755', user='******') # etherpad.za3k.com package_ensure(["sqlite3"]) user_ensure('etherpad') group_ensure('etherpad') group_user_ensure('etherpad', 'etherpad') git.ensure_clone_github('ether/etherpad-lite', '/var/www/etherpad', commit='1.6.0', user='******') nginx.ensure_site('config/nginx/etherpad.za3k.com', csr='config/certs/etherpad.za3k.com.csr', key='config/keys/etherpad.za3k.com.key', domain="etherpad.za3k.com", letsencrypt=True, cert="config/certs/etherpad.za3k.com.pem") util.put_file("config/etherpad/APIKEY.txt", "/var/www/etherpad", user='******', mode='600') util.put_file("config/etherpad/settings.json", "/var/www/etherpad", user='******', mode='644') if not files.exists("/var/www/etherpad/var/sqlite.db"): sudo("mkdir -p /var/www/etherpad/var", user='******') with cd("/var/www/etherpad"): sudo("npm install sqlite3") sudo( "rsync -av germinate.za3k.com::etherpad --delete /var/www/etherpad/var", user='******') supervisord.ensure() supervisord.ensure_config("config/supervisor/etherpad.conf") supervisord.update() # forsale nginx.ensure_site('config/nginx/forsale') util.put_dir('data/forsale', '/var/www/forsale', mode='755', user='******') # gipc daily sync # github personal backup # github repo list util.put_file("config/github/github-metadata-sync", "/etc/cron.daily/github-metadata-sync", mode='755', user='******') # -> updater # irc.za3k.com -> irc # -> webchat (qwebirc) # jsfail.com user_ensure('jsfail') group_ensure('jsfail') group_user_ensure('jsfail', 'jsfail') nginx.ensure_site('config/nginx/jsfail.com') util.put_dir('data/jsfail', '/var/www/jsfail', 'jsfail', mode='755') # library.za3k.com -> website # -> sync script # -> card catalog # MUST be user 2001 to match remote rsync user_ensure('library', uid=2001) group_ensure('library', gid=2001) group_user_ensure('library', 'library') with mode_sudo(): dir_ensure('/var/www/library', mode='755') files.append('/etc/sudoers', 'za3k ALL=(root) NOPASSWD: /etc/cron.daily/library-sync', use_sudo=True) with settings(user='******', host_string='germinate'): actual_key = ssh.get_public_key( "/data/git/books.git/hooks/deadtree.library") ssh_line = 'command="{command}",no-port-forwarding,no-x11-forwarding,no-agent-forwarding {key}'.format( command="sudo /etc/cron.daily/library-sync", key=actual_key) files.append('/home/za3k/.ssh/authorized_keys', ssh_line, use_sudo=True) sudo("chown library:library /var/www/library") util.put_file("config/library/library-sync", "/etc/cron.daily/library-sync", mode='755', user='******') sudo("/etc/cron.daily/library-sync") nginx.ensure_site('config/nginx/library.za3k.com', csr='config/certs/library.za3k.com.csr', key='config/keys/library.za3k.com.key', domain="library.za3k.com", letsencrypt=True, cert="config/certs/library.za3k.com.pem") # logs (nginx) and analysis (analog) # mint sync # moreorcs.com user_ensure('moreorcs') group_ensure('moreorcs') group_user_ensure('moreorcs', 'moreorcs') nginx.ensure_site('config/nginx/moreorcs.com', cert='config/certs/moreorcs.com.pem', key='config/keys/moreorcs.com.key', domain="moreorcs.com", letsencrypt=True, csr="config/certs/moreorcs.com.csr") git.ensure_clone_github('za3k/moreorcs', '/var/www/moreorcs', user='******') # nanowrimo.za3k.com nginx.ensure_site('config/nginx/nanowrimo.za3k.com', csr='config/certs/nanowrimo.za3k.com.csr', key='config/keys/nanowrimo.za3k.com.key', domain="nanowrimo.za3k.com", letsencrypt=True, cert="config/certs/nanowrimo.za3k.com.pem") util.put_dir('data/nanowrimo', '/var/www/nanowrimo', user='******', mode='755') # nntp.za3k.com - Discontinued # petchat.za3k.com nginx.ensure_site('config/nginx/petchat.za3k.com') if not files.exists('/var/www/petchat'): git.ensure_clone_za3k('petchat', '/var/www/petchat', user='******') # publishing.za3k.com # thinkingtropes.com nginx.ensure_site('config/nginx/thinkingtropes.com') util.put_dir('data/thinkingtropes', '/var/www/thinkingtropes', user='******', mode='755') # thisisashell.com nginx.ensure_site('config/nginx/thisisashell.com', csr='config/certs/thisisashell.com.csr', key='config/keys/thisisashell.com.key', domain="thisisashell.com", letsencrypt=True, cert="config/certs/thisisashell.com.pem") # isrickandmortyout.com nginx.ensure_site('config/nginx/isrickandmortyout.com', csr='config/certs/isrickandmortyout.com.csr', key='config/keys/isrickandmortyout.com.key', domain="isrickandmortyout.com", letsencrypt=True, cert="config/certs/isrickandmortyout.com.pem") # twitter archive # za3k.com user_ensure('za3k') group_ensure('za3k') group_user_ensure('za3k', 'za3k') nginx.ensure_site('config/nginx/za3k.com', cert='config/certs/za3k.com.pem', key='config/keys/za3k.com.key', domain="za3k.com", letsencrypt=True, csr="config/certs/za3k.com.csr") git.ensure_clone_za3k('za3k', '/var/www/za3k', user='******') with settings(user='******', host_string='germinate'): actual_key = ssh.get_public_key( "/data/git/za3k.git/hooks/deadtree_key") ssh_line = 'command="{command}",no-port-forwarding,no-x11-forwarding,no-agent-forwarding {key}'.format( command="/usr/bin/git -C /var/www/za3k pull", key=actual_key) files.append('/home/za3k/.ssh/authorized_keys', ssh_line, use_sudo=True) # Markdown .md ruby.ensure() ruby.ensure_gems(["redcarpet"]) # Databases .view package_ensure(["sqlite3"]) util.put_file("config/za3k/za3k-db-sync", "/etc/cron.daily/za3k-db-sync", mode='755', user='******') sudo("/etc/cron.daily/za3k-db-sync") # colony on the moon # disabled temp. because we're out of space #sudo("rsync -av germinate.za3k.com::colony --delete /var/www/colony", user='******') # .sc package_ensure(["sc"]) nginx.reload()
def deadtree(): """Deadtree is the main services machine. It can be taken down at any time and rebuilt.""" apt.sudo_ensure() # cuisine.package_ensure is broken otherwise # Set up /etc/skel sudo("mkdir /etc/skel/.ssh || true") sudo("chmod 700 /etc/skel/.ssh") # Add a 'nobody' user user_ensure('nobody') group_ensure('nobody') group_user_ensure('nobody', 'nobody') sudo('usermod -s /bin/false nobody') # Set up the firewall put("config/firewalls/deadtree.sh", "/usr/local/bin", use_sudo=True) sudo("sh /usr/local/bin/deadtree.sh") # Set up nginx already_installed = nginx.ensure() nginx.ensure_site('config/nginx/default', cert='config/certs/za3k.com.pem', key='config/keys/blog.za3k.com.key') nginx.ensure_fcgiwrap(children=4) if not already_installed: nginx.restart() # IPv[46] listener only changes on restart # Set up letsencrypt letsencrypt.ensure() # Set up authorization to back up to the data server #public_key = ssh.ensure_key('/root/.ssh/id_rsa') #with settings(user='******', host_string='burn'): # #put(public_key, '/home/zachary/test_authorized_keys') # files.append('/home/deadtree/.ssh/authorized_keys', public_key) # ddns.za3k.com (TCP port 80, web updater for DDNS) # ns.za3k.com (UDP port 53, DNS server) user_ensure('nsd') group_ensure('nsd') group_user_ensure('nsd', 'nsd') with mode_sudo(): dir_ensure('/var/lib/nsd', mode='755') sudo("chown nsd:nsd /var/lib/nsd") package_ensure(["nsd"]) with cd("/var/lib/nsd"): sudo("touch /var/lib/nsd/moreorcs.com.zone && chown nsd:nsd /var/lib/nsd/moreorcs.com.zone") node.ensure() put("config/ddns/moreorcs.com.zonetemplate", "/etc/nsd", mode='644', use_sudo=True) supervisord.ensure() git.ensure_clone_github('thingless/ddns', '/var/lib/nsd/ddns', user='******') supervisord.ensure_config("config/supervisor/ddns.conf") put("config/ddns/config.json", "/var/lib/nsd", mode='644', use_sudo=True) sudo("chown nsd:nsd /var/lib/nsd/config.json") # [Manual] Copy dnsDB.json from backup sudo("cd /var/lib/nsd && ln -sf ddns/index.txt index.txt && chown nsd:nsd index.txt") supervisord.update() # Run ddns package_ensure(["nsd"]) put("config/ddns/nsd.conf", "/etc/nsd", mode='644', use_sudo=True) sudo("systemctl restart nsd") nginx.ensure_site('config/nginx/ddns.za3k.com', csr='config/certs/ddns.za3k.com.csr', key='config/keys/ddns.za3k.com.key', domain="ddns.za3k.com", letsencrypt=True, cert="config/certs/ddns.za3k.com.pem") nginx.reload() # blog.za3k.com package_ensure(["php5-fpm", "mysql-server", "php5-mysql"]) nginx.ensure_site('config/nginx/blog.za3k.com', cert='config/certs/blog.za3k.com.pem', key='config/keys/blog.za3k.com.key', domain="blog.za3k.com", letsencrypt=True, csr="config/certs/blog.za3k.com.csr") git.ensure_clone_za3k('za3k_blog', '/var/www/za3k_blog', user='******') # TODO: Replace a database-specific password or make it more obvious it's not used? Currently we're using user ACLs and this gets ignored anyway, I think? # [Manual] Load the blog database from backup at /srv/mysql -> /var/lib/mysql sudo('systemctl restart mysql') # etherpad.za3k.com package_ensure(["sqlite3"]) user_ensure('etherpad') group_ensure('etherpad') group_user_ensure('etherpad', 'etherpad') git.ensure_clone_github('ether/etherpad-lite', '/var/www/etherpad', commit='1.6.0', user='******') nginx.ensure_site('config/nginx/etherpad.za3k.com', csr='config/certs/etherpad.za3k.com.csr', key='config/keys/etherpad.za3k.com.key', domain="etherpad.za3k.com", letsencrypt=True, cert="config/certs/etherpad.za3k.com.pem") util.put("config/etherpad/APIKEY.txt", "/var/www/etherpad", user='******', mode='600') util.put("config/etherpad/settings.json", "/var/www/etherpad", user='******', mode='644') if not files.exists("/var/www/etherpad/var/sqlite.db"): sudo("mkdir -p /var/www/etherpad/var", user='******') with cd("/var/www/etherpad"): sudo("npm install sqlite3") sudo("rsync -av burn.za3k.com::etherpad --delete /var/www/etherpad/var", user='******') supervisord.ensure() supervisord.ensure_config("config/supervisor/etherpad.conf") supervisord.update() # forsale nginx.ensure_site('config/nginx/forsale') util.put('data/forsale', '/var/www', user='******', mode='755') # gipc daily sync # github personal backup # github repo list # -> updater # irc.za3k.com -> irc # -> webchat (qwebirc) # jsfail.com user_ensure('jsfail') group_ensure('jsfail') group_user_ensure('jsfail', 'jsfail') nginx.ensure_site('config/nginx/jsfail.com') util.put('data/jsfail', '/var/www', 'jsfail', mode='755') # library.za3k.com -> website # -> sync script # -> card catalog # MUST be user 2001 to match remote rsync user_ensure('library', uid=2001) group_ensure('library', gid=2001) group_user_ensure('library', 'library') with mode_sudo(): dir_ensure('/var/www/library', mode='755') sudo("chown library:library /var/www/library") put("config/library/library.sync", "/etc/cron.daily", mode='755', use_sudo=True) sudo("/etc/cron.daily/library.sync") nginx.ensure_site('config/nginx/library.za3k.com', csr='config/certs/library.za3k.com.csr', key='config/keys/library.za3k.com.key', domain="library.za3k.com", letsencrypt=True, cert="config/certs/library.za3k.com.pem") # logs (nginx) and analysis (analog) # mint sync # moreorcs.com user_ensure('moreorcs') group_ensure('moreorcs') group_user_ensure('moreorcs', 'moreorcs') nginx.ensure_site('config/nginx/moreorcs.com', cert='config/certs/moreorcs.com.pem', key='config/keys/moreorcs.com.key', domain="moreorcs.com", letsencrypt=True, csr="config/certs/moreorcs.com.csr") git.ensure_clone_github('za3k/moreorcs', '/var/www/moreorcs', user='******') # nanowrimo.za3k.com nginx.ensure_site('config/nginx/nanowrimo.za3k.com', csr='config/certs/nanowrimo.za3k.com.csr', key='config/keys/nanowrimo.za3k.com.key', domain="nanowrimo.za3k.com", letsencrypt=True, cert="config/certs/nanowrimo.za3k.com.pem") util.put('data/nanowrimo', '/var/www', user='******', mode='755') # nntp.za3k.com - Discontinued # petchat.za3k.com nginx.ensure_site('config/nginx/petchat.za3k.com') if not files.exists('/var/www/petchat'): git.ensure_clone_za3k('petchat', '/var/www/petchat', user='******') # publishing.za3k.com # thinkingtropes.com nginx.ensure_site('config/nginx/thinkingtropes.com') util.put('data/thinkingtropes', '/var/www', user='******', mode='755') # thisisashell.com nginx.ensure_site('config/nginx/thisisashell.com', csr='config/certs/thisisashell.com.csr', key='config/keys/thisisashell.com.key', domain="thisisashell.com", letsencrypt=True, cert="config/certs/thisisashell.com.pem") # twitter archive # za3k.com user_ensure('za3k') group_ensure('za3k') group_user_ensure('za3k', 'za3k') nginx.ensure_site('config/nginx/za3k.com', cert='config/certs/za3k.com.pem', key='config/keys/za3k.com.key', domain="za3k.com", letsencrypt=True, csr="config/certs/za3k.com.csr") git.ensure_clone_za3k('za3k', '/var/www/za3k', user='******') # Markdown .md ruby.ensure() ruby.ensure_gems(["redcarpet"]) # colony on the moon sudo("rsync -av burn.za3k.com::colony --delete /var/www/colony", user='******') # |-- status.za3k.com sudo("mkdir -p /var/www/status && chmod 755 /var/www/status") util.put("/srv/keys/backup_check", "/var/www/status", user='******', mode='600') #util.put("/srv/keys/backup_check.pub", "/var/www/status", user='******', mode='644') package_ensure(["parallel", "curl"]) nginx.reload()
base = x['base'] domain = x['domain'] sites = x['sites'] host = x['host'] dbi = env.get('ALI_RDS_INSTANCE') def setup_code(sites): for site in sites: name = site['name'] path = base + '/' + app_name + '/' + name repo = site['repo'] type = site['type'] rr = name + '.' + app_name server_name = rr + '.' + domain aliyun.add_domain_record(domain, host, rr) deploy.pull_code(repo, path) nginx.add_server(server_name, path, type) nginx.reload() aliyun.setup_db(dbi, app_name, app_password, app_title) print("------ Aliyun RDS setup complete ! --------") setup_code(sites) print("------ Code Deploy Complete ! --------") nginx.reload() end = time.time() duration = math.ceil(end - start) dingding.ding_md('Deploy Report, for' + 'easy_deploy:8', '### Deploy complete in *' + str(duration) + '* `s`. 🤯')