def custom_packages():
notify('Installing custom packages.')
notify('Installing PhantomJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo(
'wget -N https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 -O phantomjs.tar.bz2'
)
sudo('tar jxvf phantomjs.tar.bz2')
sudo('mv phantomjs-1.9.1-linux-x86_64 /etc/phantomjs')
cuisine.file_link('/etc/phantomjs/bin/phantomjs',
'/usr/bin/phantomjs',
symbolic=True)
notify('Installing CasperJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo(
'wget -N https://codeload.github.com/n1k0/casperjs/legacy.tar.gz/1.0.3 -O casperjs.tar.bz2'
)
sudo('tar xzvf casperjs.tar.bz2')
sudo('mv n1k0-casperjs-76fc831 /etc/casperjs')
cuisine.file_link('/etc/casperjs/bin/casperjs',
'/usr/bin/casperjs',
symbolic=True)
def ensure_production_settings():
notify('Configuring production settings.')
context = SENSITIVE
cuisine.mode_sudo()
content = cuisine.text_template(templates.production_settings, context)
cuisine.file_write(PROJECT['ROOT'] + '/moment/conf/production.py', content)
restart()
def hosts_conf():
notify('Writing hostname and hosts files.')
cuisine.mode_sudo()
sudo('echo "{NAME}" > /etc/hostname'.format(NAME=MACHINE['KEY']))
sudo('hostname -F /etc/hostname')
hosts = cuisine.text_template(templates.hosts, MACHINE)
cuisine.file_append('/etc/hosts', hosts)
def ensure_production_settings():
notify("Configuring production settings.")
context = SENSITIVE
cuisine.mode_sudo()
content = cuisine.text_template(templates.production_settings, context)
cuisine.file_write(PROJECT["ROOT"] + "/moment/conf/production.py", content)
restart()
def install_gunicorn(self):
self.run_virtualenv('pip -q install gunicorn')
with mode_sudo():
dir_ensure('/var/log/gunicorn/',
owner=self.user_name,
group=self.group_name,
)
if self.util.get_package_manager() == 'apt':
operations.put(
'gunicorn.conf',
'/etc/init/',
use_sudo=True, mode=644)
with mode_sudo():
file_attribs('/etc/init/gunicorn.conf',
mode=700,
owner='root',
group='root',
)
with settings(warn_only=True):
sed('/etc/init/gunicorn.conf',
'\{virtualenv\}',
self.virtualenv_dir,
use_sudo = True,
)
operations.put(
'gunicorn-launcher.sh',
self.virtualenv_dir + '/bin/',
use_sudo=True, mode=750)
with mode_sudo():
file_attribs(self.virtualenv_dir + '/bin/gunicorn-launcher.sh',
mode=700,
owner=self.user_name,
group=self.group_name,
)
with settings(warn_only=True):
sed(self.virtualenv_dir + '/bin/gunicorn-launcher.sh',
'\{virtualenv\}',
self.virtualenv_dir,
use_sudo = True,
)
sed(self.virtualenv_dir + '/bin/gunicorn-launcher.sh',
'\{project\}',
self.www_dir,
use_sudo = True,
)
# install gevent. This is non-critical and might fail so we go to
# warn-only mode
with settings(warn_only=True):
package_ensure('libevent-dev')
self.run_virtualenv('pip -q install gevent')
# # TODO: add -k gevent to gunicorn launcher script
# TODO
upstart_ensure('gunicorn')
def hosts_conf():
notify('Writing hostname and hosts files.')
cuisine.mode_sudo()
sudo('echo "{NAME}" > /etc/hostname'.format(NAME=MACHINE['KEY']))
sudo('hostname -F /etc/hostname')
hosts = cuisine.text_template(templates.hosts, MACHINE)
cuisine.file_append('/etc/hosts', hosts)
def configure_egg_cache():
"""Configure a system-wide egg-cache so we have a local cache
of eggs that we use in order to add speed and reduncancy to
zc.buildout."""
with mode_sudo():
dir_ensure("/etc/buildout/")
dir_ensure("/etc/buildout/downloads")
dir_ensure("/etc/buildout/eggs")
dir_ensure("/etc/buildout/extends")
if exists("/etc/buildout/default.cfg"):
sudo("rm -rf /etc/buildout/default.cfg")
sudo("touch /etc/buildout/default.cfg")
sudo('echo "[buildout]" >> /etc/buildout/default.cfg')
sudo('echo "eggs-directory = /etc/buildout/eggs" >> /etc/buildout/default.cfg')
sudo('echo "download-cache = /etc/buildout/downloads" >> /etc/buildout/default.cfg')
sudo('echo "extends-cache = /etc/buildout/extends" >> /etc/buildout/default.cfg')
# allow group `projects` to read/write in here
sudo("chown -R root:projects /etc/buildout/{eggs,downloads,extends}")
sudo("chmod -R 775 /etc/buildout/{eggs,downloads,extends}")
# force maintenance users to also use default.cfg (needed when running buildout via Fabric)
for user in env.admins:
with mode_sudo():
dir_ensure("/home/%s/.buildout" % user)
if exists("/home/%s/.buildout/default.cfg" % user):
sudo("rm -rf /home/%s/.buildout/default.cfg" % user)
sudo("ln -s /etc/buildout/default.cfg /home/%s/.buildout/default.cfg" % user)
sudo("chown -R %s /home/%s/.buildout" % (user, user))
def link_conf():
notify('Configuring necessary symlinks for our libraries.')
cuisine.mode_sudo()
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libjpeg.so', '/usr/lib/libjpeg.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libpng.so', '/usr/lib/libpng.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libz.so', '/usr/lib/libz.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libfreetype.so', '/usr/lib/libfreetype.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/liblcms.so', '/usr/lib/liblcms.so', symbolic=True)
def config():
"""Ensures the app configuration is in place."""
utilities.notify(u'Ensuring the app configuration settings.')
context = env
cuisine.mode_sudo()
content = cuisine.text_template(env.app_config_template, context)
cuisine.file_write(env.app_config_file, content)
execute(restart)
def _condition_ubuntu_network(self):
text = text_strip_margin("""
|# This file describes the network interfaces available on your system
|# and how to activate them. For more information, see interfaces(5).
|
|# The loopback network interface
|auto lo
|iface lo inet loopback
|""")
file_path = "/mnt/etc/network/interfaces"
with mode_sudo():
file_write(file_path, text)
for iface in self.interfaces['add']:
bootp = self.interfaces['add'][iface]['bootp']
if bootp == 'dhcp':
text = text_strip_margin("""
|# {iface}
|auto {iface}
|iface {iface} inet dhcp
|""".format(iface=iface))
with mode_sudo():
file_append(file_path, text)
elif bootp == 'static':
address = self.interfaces['add'][iface]['address']
netmask = self.interfaces['add'][iface]['netmask']
text = text_strip_margin("""
|# {iface}
|auto {iface}
|iface {iface} inet static
| address {addr}
| netmask {mask}
|""".format(iface=iface,
addr=address, mask=netmask))
with mode_sudo():
file_append(file_path, text)
try:
gateway = self.interfaces['add'][iface]['gateway']
except:
gateway = None
if gateway:
text = " gateway {g}\n".format(g=gateway)
with mode_sudo():
file_append(file_path, text)
try:
dnsserver = self.interfaces['add'][iface]['dnsserver']
except:
dnsserver = None
if dnsserver:
text = " dns-nameservers {d}\n".format(d=dnsserver)
with mode_sudo():
file_append(file_path, text)
else:
raise TypeError("network_config: {0} is not supported.\n".format(iface))
def management():
"""Ensures the app process management is in place."""
utilities.notify(u'Ensuring the app management settings.')
context = env
cuisine.mode_sudo()
content = cuisine.text_template(env.app_management_template, context)
cuisine.file_write(env.app_management_file, content)
execute(update)
execute(restart)
def dir_conf():
notify('Creating the working directory structure.')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_WORKSPACE'])
cuisine.dir_ensure(MACHINE['DIR_ENVIRONMENTS'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_PROJECTS'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_SSL'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_LOGS'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
def install_python_packages():
print white('--- install python packages ---', bold=True)
if not file_exists('/usr/bin/pip'):
run('wget https://bootstrap.pypa.io/get-pip.py')
with settings(mode_sudo()):
run('/usr/local/bin/python2.7 get-pip.py')
run('rm get-pip.py')
with settings(mode_sudo()):
run('ln -sf /usr/local/bin/pip /usr/bin/pip')
run('pip install ipython')
run('pip install virtualenv')
run('pip install Pygments')
def install_python_packages():
print white('--- install python packages ---', bold=True)
if not file_exists('/usr/bin/pip'):
run('wget https://bootstrap.pypa.io/get-pip.py')
with settings(mode_sudo()):
run('/usr/local/bin/python2.7 get-pip.py')
run('rm get-pip.py')
with settings(mode_sudo()):
run('ln -sf /usr/local/bin/pip /usr/bin/pip')
run('pip install ipython')
run('pip install virtualenv')
run('pip install Pygments')
def testModeSudo( self ): assert not cuisine.mode(cuisine.MODE_SUDO) cuisine.mode_sudo() assert cuisine.mode(cuisine.MODE_SUDO) cuisine.mode_user() assert not cuisine.mode(cuisine.MODE_SUDO) # We use the mode changer to switch to sudo temporarily with cuisine.mode_sudo(): assert cuisine.mode(cuisine.MODE_SUDO) assert cuisine.mode(cuisine.MODE_LOCAL) # We go into sudo from sudo with cuisine.mode_sudo(): assert cuisine.mode(cuisine.MODE_SUDO)
def testModeSudo(self):
assert not cuisine.mode(cuisine.MODE_SUDO)
cuisine.mode_sudo()
assert cuisine.mode(cuisine.MODE_SUDO)
cuisine.mode_user()
assert not cuisine.mode(cuisine.MODE_SUDO)
# We use the mode changer to switch to sudo temporarily
with cuisine.mode_sudo():
assert cuisine.mode(cuisine.MODE_SUDO)
assert cuisine.mode(cuisine.MODE_LOCAL)
# We go into sudo from sudo
with cuisine.mode_sudo():
assert cuisine.mode(cuisine.MODE_SUDO)
def add_user(username, password):
'''
Add user.
'''
mode_sudo()
if not user_check(username):
print(green("Creating user %s" % username))
user_create(username, shell='/bin/bash')
passwd(username, password)
sudo_group(username)
print(green("Added user %s to %s" % (username, env.host_string)))
else:
print(red("User already exists"))
def ensure_rq():
notify("Configuring RQ.")
context = {
"ACTION_DATE": MACHINE["ACTION_DATE"],
"NAME": PROJECT["NAME"],
"KEY": KEY,
"PROJECT_ROOT": PROJECT["ROOT"],
"PROJECT_ENV": PROJECT["ENV"],
}
cuisine.mode_sudo()
content = cuisine.text_template(templates.rq_supervisor, context)
cuisine.file_write("/etc/supervisor/conf.d/" + KEY + "-rq.conf", content)
restart()
def create_symlink(self):
(basedir, symlink_location) = self.www_dir.rsplit('/', 1)
with mode_sudo():
dir_ensure(basedir,
recursive=True,
)
with cd(basedir):
with mode_sudo():
file_link(
os.sep.join((
self.repo_dir,
self.repo_django_root
)),
symlink_location)
def ensure_rq():
notify('Configuring RQ.')
context = {
'ACTION_DATE': MACHINE['ACTION_DATE'],
'NAME': PROJECT['NAME'],
'KEY': KEY,
'PROJECT_ROOT': PROJECT['ROOT'],
'PROJECT_ENV': PROJECT['ENV'],
}
cuisine.mode_sudo()
content = cuisine.text_template(templates.rq_supervisor, context)
cuisine.file_write('/etc/supervisor/conf.d/' + KEY + '-rq.conf', content)
restart()
def _condition_centos_network(self):
for iface in self.interfaces['add']:
file_path = "/mnt/etc/sysconfig/network-scripts/ifcfg-" + iface
bootp = self.interfaces['add'][iface]['bootp']
if bootp == 'dhcp':
text = text_strip_margin("""
|#TEEFAA-BEGIN
|# The contents below are automatically generated by Teefaa. Do not modify.
|DEVICE="{iface}"
|BOOTPROTO="dhcp"
|NM_CONTROLLED="no"
|ONBOOT="yes"
|TYPE="Ethernet"
|#TEEFAA-END
|""".format(iface=iface))
with mode_sudo():
file_write(file_path, text)
elif bootp == 'static':
address = self.interfaces['add'][iface]['address']
netmask = self.interfaces['add'][iface]['netmask']
text = text_strip_margin("""
|# The contents below are automatically generated by Teefaa. Do not modify.
|NM_CONTROLLED=no
|BOOTPROTO=none
|ONBOOT=yes
|IPADDR={addr}
|NETMASK={mask}
|DEVICE={iface}
|PEERDNS=no
|""".format(iface=iface,
addr=address, mask=netmask))
with mode_sudo():
file_write(file_path, text)
try:
gateway = self.interfaces['add'][iface]['gateway']
text = "GATEWAY=" + gateway + '\n'
with mode_sudo():
file_append(file_path, text)
except:
pass
try:
dnsserver = self.interfaces['add'][iface]['dnsserver']
except:
dnsserver = None
if dnsserver:
text = "DNS1={d}\n".format(d=dnsserver)
with mode_sudo():
file_append(file_path, text)
def upload(site, tag='master'):
"""Upload project `site` files from tag or branch `master`."""
puts("Upload project {0}".format(site))
today = datetime.now().strftime('%Y%m%d-%H%M%S')
commit_id = str(local('git rev-parse {0}'.format(tag), True)).strip()
current = "%s-%s" % (today, commit_id[:8])
release_dir = os.path.join(INSTALL_DIR, site, 'releases')
with mode_sudo():
dir_ensure(release_dir)
local_temp_dir = mkdtemp()
archive = os.path.join(local_temp_dir, '{0}.tar.gz'.format(site))
git_arch_cmd = "git archive --format=tar.gz -o {archive} {commit_id}"
local(git_arch_cmd.format(archive=archive, commit_id=commit_id))
with lcd(local_temp_dir):
remote_temp_dir = run('mktemp -d')
put(archive, remote_temp_dir)
with cd(remote_temp_dir):
run("tar xzf {0}.tar.gz".format(site))
run("rm -f {0}.tar.gz".format(site))
sudo('chown -R root.root {0}'.format(remote_temp_dir))
sudo('chmod -R 755 {0}'.format(remote_temp_dir))
sudo("mv {tmp_dir} {release_dir}/{current}".format(release_dir=release_dir, current=current, tmp_dir=remote_temp_dir))
run('rm -rf {0}'.format(remote_temp_dir))
local('rm -rf {0}'.format(local_temp_dir))
return current
def provision_rabbitmq(admin_password):
append("/etc/apt/sources.list.d/rabbitmq.list",
"deb http://www.rabbitmq.com/debian/ testing main", use_sudo=True)
if not file_exists("/usr/sbin/rabbitmq-server"):
sudo("wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc")
sudo("apt-key add rabbitmq-signing-key-public.asc")
map(package_ensure_apt, ["rabbitmq-server", "rsync"])
dir_ensure("/etc/rabbitmq/rabbitmq.conf.d")
put("./conf/bunny/bunny.conf", "/etc/rabbitmq/rabbitmq.conf.d/", use_sudo=True)
sudo("chown -R rabbitmq.rabbitmq /srv/rabbitmq")
dir_ensure("/srv/rabbitmq/log", owner="rabbitmq", group="rabbitmq")
sudo("rm -rf /var/lib/rabbitmq")
sudo("rm -rf /var/log/rabbitmq")
with mode_sudo():
file_link("/srv/rabbitmq", "/var/lib/rabbitmq", owner="rabbitmq", group="rabbitmq")
file_link("/srv/rabbitmq/log", "/var/log/rabbitmq", owner="rabbitmq", group="rabbitmq")
sudo("service rabbitmq-server start")
sudo("rabbitmq-plugins enable rabbitmq_management")
sudo("rabbitmqctl add_user admin " + admin_password)
sudo("rabbitmqctl set_user_tags admin administrator")
sudo('rabbitmqctl set_permissions -p / admin ".*" ".*" ".*"')
sudo("rabbitmqctl delete_user guest")
sudo("service rabbitmq-server restart")
else:
print "skipped install, already have /usr/sbin/rabbitmq-server"
def setup_os():
with cuisine.mode_sudo():
cuisine.ssh_authorize( "matze" , cuisine.file_local_read("./cuisine_id.pub"))
for _ in PACKAGE_ENSURE: cuisine.package_ensure(_)
cuisine.run("a2enmod rewrite")
# TODO enable AllowOverride none => all for /var/www
cuisine.run("service apache2 restart")
def initialize_postgres():
"""Initialize the main database."""
version = sudo("psql --version | grep -ro '[8-9].[0-9]'")
conf_dir_prefix = "/etc/postgresql/%s/" % version
# temporarily allow root access from localhost
sudo('mv /etc/postgresql/%s/main/pg_hba.conf /etc/postgresql/%s/main/pg_hba.conf.bak' % (version, version))
sudo('echo "local all postgres ident" > /etc/postgresql/%s/main/pg_hba.conf' % version)
sudo('cat /etc/postgresql/%s/main/pg_hba.conf.bak >> /etc/postgresql/%s/main/pg_hba.conf' % (version, version))
sudo('service postgresql-%s restart || /etc/init.d/postgresql restart ' % version)
# set password
password = prompt('Enter a new database password for user `postgres`:')
sudo('psql template1 -c "ALTER USER postgres with encrypted password \'%s\';"' % password, user='******')
# configure daily dumps of all databases
with mode_sudo():
dir_ensure('/var/backups/postgresql', recursive=True)
sudo("echo 'localhost:*:*:postgres:%s' > /root/.pgpass" % password)
sudo('chmod 600 /root/.pgpass')
sudo("echo '0 7 * * * pg_dumpall --username postgres --file /var/backups/postgresql/postgresql_$(date +%%Y-%%m-%%d).dump' > /etc/cron.d/pg_dump")
# remove temporary root access
comment('/etc/postgresql/%s/main/pg_hba.conf' % version, 'local all postgres ident', use_sudo=True)
sudo('service postgresql%s restart || /etc/init.d/postgresql restart' % version)
def install_diff_highlight():
print white('--- install diff highlight ---', bold=True)
if not file_exists('/usr/local/bin/diff-highlight'):
run('wget https://raw.githubusercontent.com/git/git/master/contrib/diff-highlight/diff-highlight')
with settings(mode_sudo()):
run('chmod +x diff-highlight')
run('mv diff-highlight /usr/local/bin/diff-highlight')
def update_dir(update_dir_list):
with mode_sudo():
for dir in update_dir_list:
owner = update_dir_list[dir]['owner']
mode = update_dir_list[dir]['mode']
dir_ensure(dir, mode=mode, owner=owner)
def _condition_centos_fstab(self):
time.sleep(1)
file_path = "/mnt/etc/fstab"
label_type = self.disk_config['label_type']
device = self.disk_config['device']
system_format = self.disk_config['system']['format']
if label_type == 'mbr':
num = 0
elif label_type == 'gpt':
num = 1
text = text_strip_margin("""
|#TEEFAA-BEGIN
|# /etc/fstab
|# Created by Teefaa
|#
|# Accessible filesystems, by reference, are maintained under '/dev/disk'
|# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
|#
|{device}{system_num} / {system_format} defaults 1 1
|{device}{swap_num} swap swap defaults 0 0
|tmpfs /dev/shm tmpfs defaults 0 0
|devpts /dev/pts devpts gid=5,mode=620 0 0
|sysfs /sys sysfs defaults 0 0
|proc /proc proc defaults 0 0
|#TEEFAA-END
|""".format(device=device,
swap_num=num+1,
system_num=num+2,
system_format=system_format))
with mode_sudo():
file_write(file_path, text)
def _condition_ubuntu_mtab(self):
time.sleep(1)
file_path = "/mnt/etc/mtab"
label_type = self.disk_config['label_type']
device = self.disk_config['device']
system_format = self.disk_config['system']['format']
if label_type == 'mbr':
num = 0
elif label_type == 'gpt':
num = 1
text = text_strip_margin("""
|{device}{system_num} / {system_format} rw,errors=remount-ro 0 0
|proc /proc proc rw,noexec,nosuid,nodev 0 0
|sysfs /sys sysfs rw,noexec,nosuid,nodev 0 0
|none /sys/fs/fuse/connections fusectl rw 0 0
|none /sys/kernel/debug debugfs rw 0 0
|none /sys/kernel/security securityfs rw 0 0
|udev /dev devtmpfs rw,mode=0755 0 0
|devpts /dev/pts devpts rw,noexec,nosuid,gid=5,mode=0620 0 0
|tmpfs /run tmpfs rw,noexec,nosuid,size=10%,mode=0755 0 0
|none /run/lock tmpfs rw,noexec,nosuid,nodev,size=5242880 0 0
|none /run/shm tmpfs rw,nosuid,nodev 0 0
|rpc_pipefs /run/rpc_pipefs rpc_pipefs rw 0 0
|""".format(device=device,
system_num=num+2,
system_format=system_format))
with mode_sudo():
file_write(file_path, text)
def setup_packages():
cuisine.select_package('apt')
puts(green('Installing Ubuntu packages'))
with cuisine.mode_sudo():
cuisine.package_update()
cuisine.package_upgrade()
cuisine.package_ensure([
"aptitude",
"build-essential",
"curl",
"git",
"guake",
"haskell-platform",
"htop",
"ibux-mozc",
"libclang-dev",
'libncursesw5-dev',
"libssl-dev",
"paco",
"python3-dev",
"tmux",
"tree",
"wget",
"zip",
"zsh",
])
def _condition_ubuntu_fstab(self):
time.sleep(1)
file_path = "/mnt/etc/fstab"
label_type = self.disk_config['label_type']
device = self.disk_config['device']
system_format = self.disk_config['system']['format']
if label_type == 'mbr':
num = 0
elif label_type == 'gpt':
num = 1
text = text_strip_margin("""
|#TEEFAA-BEGIN
|# /etc/fstab: static file system information.
|#
|# Use 'blkid' to print the universally unique identifier for a
|# device; this may be used with UUID= as a more robust way to name devices
|# that works even if disks are added and removed. See fstab(5).
|#
|# <file system> <mount point> <type> <options> <dump> <pass>
|proc /proc proc nodev,noexec,nosuid 0 0
|{device}{swap_num} none swap sw 0 0
|{device}{system_num} / {system_format} errors=remount-ro 0 1
|#TEEFAA-END
|""".format(device=device,
swap_num=num+1,
system_num=num+2,
system_format=system_format))
with mode_sudo():
file_write(file_path, text)
def _create_user(self):
"""
Create admin user...
"""
print("Creating admin user...")
time.sleep(1)
root_dir = self.new_squashfs_dir
user = self.user
home_dir = '/home/' + user
# Create admin user
output = do_sudo(['grep', user, self.new_squashfs_dir + '/etc/passwd'],
warn_only=True)
if not user in output:
do_sudo(['chroot', root_dir, 'useradd', user, '-m',
'-s', '/bin/bash', '-d', home_dir])
# Copy ~/.ssh
ssh_dir = root_dir + home_dir + '/.ssh'
ssh_authorize = ssh_dir + '/authorized_keys'
with mode_sudo():
dir_ensure(ssh_dir, recursive=True, mode=700)
put(self.ssh_key + '.pub', ssh_authorize, mode=0644, use_sudo=True)
do_sudo(['chroot', root_dir, 'chown', '-R', self.user, home_dir + '/.ssh'])
# Copy /etc/ssh
text1 = do_sudo(['cat', '/etc/ssh/ssh_host_dsa_key'])
text2 = do_sudo(['cat', root_dir+'/etc/ssh/ssh_host_dsa_key'], warn_only=True)
if not text1 == text2:
do_sudo(['cp', '-rp', '/etc/ssh/*', root_dir + '/etc/ssh'])
# Enable sudo
config = root_dir + '/etc/sudoers'
text = user + " ALL=NOPASSWD:ALL"
output = do_sudo(['grep', '\"'+text+'\"', config], warn_only=True)
if not text in output:
append(config, text, use_sudo=True)
def utils_fix_hook():
"""
Fix permissions on the hook.log
"""
with cd(env.directory):
with cuisine.mode_sudo():
cuisine.file_attribs("{}/hook.log".format(env.virtualenvs), owner=env.user, group=env.user)
def put_csr(csr, user='******'):
with mode_sudo():
dir_ensure('/etc/ssl/csr', mode='1777')
csr_name = csr.split("/")[-1]
return util.put_file(csr,
'/etc/ssl/csr/' + csr_name,
mode='0644',
user=user)[0]
def solr_ensure(project_path, venv_path='.venv'):
with mode_sudo():
package_ensure('openjdk-7-jdk libxml2-dev libxslt1-dev python-dev')
dir_ensure('/usr/java')
file_link('/usr/lib/jvm/java-7-openjdk-amd64', '/usr/java/default')
package_ensure('solr-tomcat')
with virtualenv(project_path, venv_path):
python_package_ensure('pysolr lxml cssselect')
def install_diff_highlight():
print white('--- install diff highlight ---', bold=True)
if not file_exists('/usr/local/bin/diff-highlight'):
run('wget https://raw.githubusercontent.com/git/git/master/contrib/diff-highlight/diff-highlight'
)
with settings(mode_sudo()):
run('chmod +x diff-highlight')
run('mv diff-highlight /usr/local/bin/diff-highlight')
def put_key(key, user='******'):
with mode_sudo():
dir_ensure('/etc/ssl/private', mode='1777')
key_name = key.split("/")[-1]
return util.put_file(key,
'/etc/ssl/private/' + key_name,
mode='0640',
user=user)[0]
def _condition_centos_hostname(self):
file_path = "/mnt/etc/sysconfig/network"
text = text_strip_margin("""
|NETWORKING=yes
|HOSTNAME={h}
|""".format(h=self.hostname))
with mode_sudo():
file_write(file_path, text)
def hello():
with cs.cd('/usr/java') :
status = cs.run('java && echo OK ; true')
with cs.mode_sudo() :
if status.endswith('OK') :
cs.run('rpm -Uvh jre-7u7-linux-i586.rpm')
else :
cs.run('rpm -ivh jre-7u7-linux-i586.rpm')
cs.run(green('echo "Hello World"'))
def mongodb_ensure():
with mode_sudo():
if not run("cat /etc/apt/sources.list | grep '%s'" % (MONGO_REPO),
warn_only=True).succeeded:
run("apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10"
)
run("add-apt-repository '%s'" % (MONGO_REPO))
package_update()
package_ensure("mongodb-org")
def ubuntu_setup():
with cs.mode_sudo():
if not cs.package_ensure_apt('zsh',update=True) :
cs.run('chsh -s /bin/zsh')
if not cs.package_ensure_apt('git',update=True) :
pass
cs.package_ensure_apt('vim-enhansed',update=True)
cs.package_ensure_apt('gcc',update=True)
cs.package_ensure_apt('make',update=True)
def link_conf():
notify('Configuring necessary symlinks for our libraries.')
cuisine.mode_sudo()
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libjpeg.so',
'/usr/lib/libjpeg.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libpng.so',
'/usr/lib/libpng.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libz.so',
'/usr/lib/libz.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libfreetype.so',
'/usr/lib/libfreetype.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/liblcms.so',
'/usr/lib/liblcms.so',
symbolic=True)
def gunicorn_supervisor_ensure(project_path, project_name, template, key_env):
with mode_sudo(), cd(project_path):
config = '/etc/supervisor/conf.d/%s.conf' % project_name
package_ensure('supervisor')
python_package_ensure('setproctitle')
run("cp %s %s" % (template, config))
file_update(config, lambda x: text_template(x,key_env))
run("supervisorctl reread")
run("supervisorctl update")
run("supervisorctl restart %s" % (project_name))
def ensure_nginx():
notify('Configuring nginx.')
context = {
'ACTION_DATE': MACHINE['ACTION_DATE'],
'NAME': PROJECT['NAME'],
'KEY': KEY,
'APP_LOCATION': PROJECT['APP_LOCATION'],
'APP_PORT': PROJECT['APP_PORT'],
'LOCATION': MACHINE['LOCATION'],
'PORT': MACHINE['PORT'],
'PROJECT_ROOT': PROJECT['ROOT'],
'ACCESS_LOG': PROJECT['LOGS']['NGINX_ACCESS'],
'ERROR_LOG': PROJECT['LOGS']['NGINX_ERROR'],
'SERVER_NAMES': ' '.join(PROJECT['DOMAINS'])
}
cuisine.mode_sudo()
content = cuisine.text_template(templates.nginx, context)
cuisine.file_write('/etc/nginx/sites-enabled/' + KEY, content)
sudo('/etc/init.d/nginx restart')
def put_cert(cert, user='******'):
with mode_sudo():
dir_ensure('/etc/ssl/certs', mode='1777')
if user is None:
user = '******'
cert_name = cert.split("/")[-1]
return util.put_file(cert,
'/etc/ssl/certs/' + cert_name,
mode='0644',
user=user)[0]
def _setup_users():
USERS = ['yuta1024', 'tyabuki', 'nhirokinet']
for user in USERS:
cuisine.user_ensure(user, shell='/bin/bash', passwd='yharima', encrypted_passwd=False)
cuisine.group_user_ensure('sudo', user)
# https://ubuntuforums.org/showthread.php?t=1318346
cuisine.group_user_ensure('adm', user)
with cuisine.mode_sudo():
cuisine.ssh_authorize(user, _get_public_key_from_github(user))
if not cuisine.is_ok(sudo('grep rsync /etc/sudoers && echo OK ; true')):
sudo('echo "%s ALL=(ALL) NOPASSWD: /usr/bin/rsync" | EDITOR="tee -a" visudo' % ','.join(USERS))
def nginx_ensure(project_path, name, template, key_env):
with mode_sudo(), cd(project_path):
package_ensure('nginx')
run("cp %s /etc/nginx/sites-available/%s" % (template, name))
file_update('/etc/nginx/sites-available/%s' % name,
lambda x: text_template(x,key_env))
if not file_exists("/etc/nginx/sites-enabled/%s" % name):
run("ln -s -t /etc/nginx/sites-enabled /etc/nginx/sites-available/%s " % (
name))
file_unlink('/etc/nginx/sites-enabled/default')
run("service nginx restart")
def put_site_conf(nginx_file, context=None):
"""Install nginx config per site."""
version = get_config()['version']
if not os.path.exists(nginx_file):
fab.abort("Nginx conf {0} not found".format(nginx_file))
site_name = os.path.basename(nginx_file)
install_dir = os.path.join(_INSTALL_DIR, 'nginx', version)
conf_file = os.path.join(install_dir, 'conf', 'sites-enabled', site_name)
if context:
tpl_content = open(nginx_file, 'rb').read()
content = cuisine.text_template(tpl_content, context)
with cuisine.mode_sudo():
cuisine.file_write(conf_file, content)
else:
with cuisine.mode_sudo():
cuisine.file_upload(conf_file, nginx_file)
def configure_egg_cache():
"""Configure a system-wide egg-cache so we have a local cache
of eggs that we use in order to add speed and reduncancy to
zc.buildout."""
with mode_sudo():
dir_ensure('/etc/buildout/')
dir_ensure('/etc/buildout/downloads')
dir_ensure('/etc/buildout/eggs')
dir_ensure('/etc/buildout/extends')
if exists('/etc/buildout/default.cfg'):
sudo('rm -rf /etc/buildout/default.cfg')
sudo('touch /etc/buildout/default.cfg')
sudo('echo "[buildout]" >> /etc/buildout/default.cfg')
sudo(
'echo "eggs-directory = /etc/buildout/eggs" >> /etc/buildout/default.cfg'
)
sudo(
'echo "download-cache = /etc/buildout/downloads" >> /etc/buildout/default.cfg'
)
sudo(
'echo "extends-cache = /etc/buildout/extends" >> /etc/buildout/default.cfg'
)
# allow group `projects` to read/write in here
sudo('chown -R root:projects /etc/buildout/{eggs,downloads,extends}')
sudo('chmod -R 775 /etc/buildout/{eggs,downloads,extends}')
# force maintenance users to also use default.cfg (needed when running buildout via Fabric)
for user in env.admins:
with mode_sudo():
dir_ensure('/home/%s/.buildout' % user)
if exists('/home/%s/.buildout/default.cfg' % user):
sudo('rm -rf /home/%s/.buildout/default.cfg' % user)
sudo('ln -s /etc/buildout/default.cfg /home/%s/.buildout/default.cfg' %
user)
sudo('chown -R %s /home/%s/.buildout' % (user, user))
def main():
'''
Main function for the frycooker program.
'''
args = get_args()
settings = load_settings(args.settings, args.params)
enviro = load_enviro(args.environment)
try:
if args.sudo:
cuisine.mode_sudo()
tmp_dir = tempfile.mkdtemp(dir=settings["tmp_dir"])
settings["tmp_dir"] = tmp_dir
run_list, host_list, recipes, cookbooks = generate_run_list(
enviro, args)
if args.dryrun:
print("actions would be applied to the following hosts: %s" %
', '.join(host_list))
print(
"environment to be used: %s" % json.dumps(
enviro, sort_keys=True, indent=4, separators=(',', ': ')))
sys.exit(0)
output_pre_apply_messages(recipes, cookbooks, enviro, settings, args)
if not args.messages:
apply_recipes_cookbooks(enviro, settings, args, host_list,
run_list)
output_post_apply_messages(recipes, cookbooks, enviro, settings, args)
shutil.rmtree(tmp_dir)
print "actions completed successfully"
except Exception, e:
print "EXITING EARLY DUE TO AN EXCEPTION:"
print e
shutil.rmtree(tmp_dir)
sys.exit(2)
def install(force=False):
"""Install nginx HTTP server."""
version = get_config()['version']
cuisine.package_install(['build-essential', 'libpcre3-dev', 'zlib1g-dev'])
install_dir = os.path.join(_INSTALL_DIR, 'nginx', version)
nginx_bin = os.path.join(install_dir, 'sbin', 'nginx')
if cuisine.file_exists(nginx_bin):
if not force:
fab.puts("Nginx {0} found, skipping installation".format(version))
return
else:
fab.puts("Reinstalling nginx {0} found".format(version))
with cuisine.mode_sudo():
cuisine.dir_ensure(install_dir, True)
home_dir = os.path.join(install_dir, 'html')
cuisine.user_ensure(NGINX_USER, None, home_dir, shell='/sbin/nologin')
fab.sudo('passwd -l nginx')
download_url = _DOWNLOAD_URL.format(version=version)
src_dir = fab.run('mktemp -d')
with fab.cd(src_dir):
fab.puts("Downloading nginx {0}".format(version))
fab.run("wget -q '{0}' -O - | tar xz".format(download_url))
with fab.cd('nginx-{0}'.format(version)):
fab.puts("Compiling nginx {0}".format(version))
fab.run("./configure --prefix={0} --with-http_stub_status_module".format(install_dir))
fab.run("make")
fab.puts("Installing nginx {0}".format(version))
fab.sudo('make install')
with cuisine.mode_sudo():
cuisine.dir_ensure("{0}{1}".format(install_dir, '/conf/sites-enabled'))
fab.run("rm -rf '{0}'".format(src_dir))
def ensure_gunicorn():
notify('Configuring gunicorn.')
context = {
'ACTION_DATE': MACHINE['ACTION_DATE'],
'NAME': PROJECT['NAME'],
'KEY': KEY,
'APP_LOCATION': PROJECT['APP_LOCATION'],
'APP_PORT': PROJECT['APP_PORT'],
'APP_TIMEOUT': PROJECT['APP_TIMEOUT'],
'APP_WSGI': PROJECT['APP_WSGI'],
'APP_WORKERS': PROJECT['APP_WORKERS'],
'LOCATION': MACHINE['LOCATION'],
'PORT': MACHINE['PORT'],
'PROJECT_ROOT': PROJECT['ROOT'],
'PROJECT_ENV': PROJECT['ENV'],
'ACCESS_LOG': PROJECT['LOGS']['GUNICORN_ACCESS'],
'ERROR_LOG': PROJECT['LOGS']['GUNICORN_ERROR'],
}
cuisine.mode_sudo()
content = cuisine.text_template(templates.gunicorn_supervisor, context)
cuisine.file_write('/etc/supervisor/conf.d/' + KEY + '-gunicorn.conf',
content)
restart()
def dir_conf():
notify('Creating the working directory structure.')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_WORKSPACE'])
cuisine.dir_ensure(MACHINE['DIR_ENVIRONMENTS'],
recursive=True,
mode=MACHINE['DIR_MODE'],
owner=KEY,
group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_PROJECTS'],
recursive=True,
mode=MACHINE['DIR_MODE'],
owner=KEY,
group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_SSL'],
recursive=True,
mode=MACHINE['DIR_MODE'],
owner=KEY,
group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_LOGS'],
recursive=True,
mode=MACHINE['DIR_MODE'],
owner=KEY,
group=MACHINE['OWNER_GROUP'])
def create_virtualenv():
"""Create virtualenv for project."""
site = get_project_name()
version = get_config()['version']
virtualenv_dir = "{}/{}/virtualenv".format(SITES_DIR, site)
if cuisine.dir_exists(virtualenv_dir + "/bin"):
fab.puts("virtualenv for {0} already exists".format(site))
return
with cuisine.mode_sudo():
cuisine.dir_ensure(virtualenv_dir, recursive=True)
venv_bin = _python_bin_path(version, 'virtualenv')
fab.sudo("{venv_bin} {virtualenv_dir}".format(
venv_bin=venv_bin, virtualenv_dir=virtualenv_dir))
def install_bacula_master():
"""Install and configure Bacula Master."""
# Official repos only have version 5.0.1, we need 5.0.3
sudo('add-apt-repository ppa:mario-sitz/ppa')
sudo('apt-get update')
sudo(
'apt-get -yq install bacula-console bacula-director-pgsql bacula-sd-pgsql'
)
# folder and files that are expected to be there
with mode_sudo():
dir_ensure('/etc/bacula/clients/')
sudo('touch /etc/bacula/clients/remove_me_once_deployed.conf')
sudo('chown -R bacula /etc/bacula/clients/')
configure_bacula_master()
def install_upstart():
"""Install nginx upstart config."""
version = get_config()['version']
install_dir = os.path.join(_INSTALL_DIR, 'nginx', version)
nginx_bin = os.path.join(install_dir, 'sbin', 'nginx')
nginx_pid = os.path.join(install_dir, 'logs', 'nginx.pid')
context = {
'nginx_bin': nginx_bin,
'nginx_pid': nginx_pid,
}
nginx_tpl = os.path.join(ETC_DIR, 'init', 'nginx.conf')
tpl_content = cuisine.file_local_read(nginx_tpl)
content = cuisine.text_template(tpl_content, context)
with cuisine.mode_sudo():
cuisine.file_write('/etc/init/nginx.conf', content)
def initialize_postgres():
"""Initialize the main database."""
version = sudo("psql --version | grep -ro '[8-9].[0-9]'")
conf_dir_prefix = "/etc/postgresql/%s/" % version
# temporarily allow root access from localhost
sudo(
'mv /etc/postgresql/%s/main/pg_hba.conf /etc/postgresql/%s/main/pg_hba.conf.bak'
% (version, version))
sudo(
'echo "local all postgres ident" > /etc/postgresql/%s/main/pg_hba.conf'
% version)
sudo(
'cat /etc/postgresql/%s/main/pg_hba.conf.bak >> /etc/postgresql/%s/main/pg_hba.conf'
% (version, version))
sudo('service postgresql-%s restart || /etc/init.d/postgresql restart ' %
version)
# set password
password = prompt('Enter a new database password for user `postgres`:')
sudo(
'psql template1 -c "ALTER USER postgres with encrypted password \'%s\';"'
% password,
user='******')
# configure daily dumps of all databases
with mode_sudo():
dir_ensure('/var/backups/postgresql', recursive=True)
sudo("echo 'localhost:*:*:postgres:%s' > /root/.pgpass" % password)
sudo('chmod 600 /root/.pgpass')
sudo(
"echo '0 7 * * * pg_dumpall --username postgres --file /var/backups/postgresql/postgresql_$(date +%%Y-%%m-%%d).dump' > /etc/cron.d/pg_dump"
)
# remove temporary root access
comment('/etc/postgresql/%s/main/pg_hba.conf' % version,
'local all postgres ident',
use_sudo=True)
sudo('service postgresql%s restart || /etc/init.d/postgresql restart' %
version)
def do_upload(local_temp_dir, archive, current):
site = fab.env.fabix['_current_project']
release_dir = os.path.join(INSTALL_DIR, site, 'releases')
with cuisine.mode_sudo():
cuisine.dir_ensure(release_dir)
with fab.lcd(local_temp_dir):
remote_temp_dir = fab.run('mktemp -d')
fab.put(archive, remote_temp_dir)
with fab.cd(remote_temp_dir):
fab.run("tar xzf {0}.tar.gz".format(site))
fab.run("rm -f {0}.tar.gz".format(site))
fab.sudo('chown -R root.root {0}'.format(remote_temp_dir))
fab.sudo('chmod -R 755 {0}'.format(remote_temp_dir))
fab.sudo("mv {tmp_dir} {release_dir}/{current}".format(
release_dir=release_dir, current=current, tmp_dir=remote_temp_dir))
fab.run('rm -rf {0}'.format(remote_temp_dir))
fab.local('rm -rf {0}'.format(local_temp_dir))
return current
def put_conf(nginx_file):
"""Install global nginx config."""
version = get_config()['version']
install_dir = os.path.join(_INSTALL_DIR, 'nginx', version)
conf_file = os.path.join(install_dir, 'conf', 'nginx.conf')
if not os.path.exists(nginx_file):
fab.abort("Nginx conf {0} not found".format(nginx_file))
nginx_pid = os.path.join(install_dir, 'logs', 'nginx.pid')
context = {
'nginx_user': NGINX_USER,
'nginx_pid': nginx_pid,
}
tpl_content = open(nginx_file, 'rb').read()
content = cuisine.text_template(tpl_content, context)
with cuisine.mode_sudo():
cuisine.file_write(conf_file, content)
