def _create_user(self):
"""
Create admin user...
"""
print("Creating admin user...")
time.sleep(1)
root_dir = self.new_squashfs_dir
user = self.user
home_dir = '/home/' + user
# Create admin user
output = do_sudo(['grep', user, self.new_squashfs_dir + '/etc/passwd'],
warn_only=True)
if not user in output:
do_sudo(['chroot', root_dir, 'useradd', user, '-m',
'-s', '/bin/bash', '-d', home_dir])
# Copy ~/.ssh
ssh_dir = root_dir + home_dir + '/.ssh'
ssh_authorize = ssh_dir + '/authorized_keys'
with mode_sudo():
dir_ensure(ssh_dir, recursive=True, mode=700)
put(self.ssh_key + '.pub', ssh_authorize, mode=0644, use_sudo=True)
do_sudo(['chroot', root_dir, 'chown', '-R', self.user, home_dir + '/.ssh'])
# Copy /etc/ssh
text1 = do_sudo(['cat', '/etc/ssh/ssh_host_dsa_key'])
text2 = do_sudo(['cat', root_dir+'/etc/ssh/ssh_host_dsa_key'], warn_only=True)
if not text1 == text2:
do_sudo(['cp', '-rp', '/etc/ssh/*', root_dir + '/etc/ssh'])
# Enable sudo
config = root_dir + '/etc/sudoers'
text = user + " ALL=NOPASSWD:ALL"
output = do_sudo(['grep', '\"'+text+'\"', config], warn_only=True)
if not text in output:
append(config, text, use_sudo=True)
def update_dir(update_dir_list):
with mode_sudo():
for dir in update_dir_list:
owner = update_dir_list[dir]['owner']
mode = update_dir_list[dir]['mode']
dir_ensure(dir, mode=mode, owner=owner)
def custom_packages():
notify('Installing custom packages.')
notify('Installing PhantomJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo(
'wget -N https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 -O phantomjs.tar.bz2'
)
sudo('tar jxvf phantomjs.tar.bz2')
sudo('mv phantomjs-1.9.1-linux-x86_64 /etc/phantomjs')
cuisine.file_link('/etc/phantomjs/bin/phantomjs',
'/usr/bin/phantomjs',
symbolic=True)
notify('Installing CasperJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo(
'wget -N https://codeload.github.com/n1k0/casperjs/legacy.tar.gz/1.0.3 -O casperjs.tar.bz2'
)
sudo('tar xzvf casperjs.tar.bz2')
sudo('mv n1k0-casperjs-76fc831 /etc/casperjs')
cuisine.file_link('/etc/casperjs/bin/casperjs',
'/usr/bin/casperjs',
symbolic=True)
def configure_egg_cache():
"""Configure a system-wide egg-cache so we have a local cache
of eggs that we use in order to add speed and reduncancy to
zc.buildout."""
with mode_sudo():
dir_ensure("/etc/buildout/")
dir_ensure("/etc/buildout/downloads")
dir_ensure("/etc/buildout/eggs")
dir_ensure("/etc/buildout/extends")
if exists("/etc/buildout/default.cfg"):
sudo("rm -rf /etc/buildout/default.cfg")
sudo("touch /etc/buildout/default.cfg")
sudo('echo "[buildout]" >> /etc/buildout/default.cfg')
sudo('echo "eggs-directory = /etc/buildout/eggs" >> /etc/buildout/default.cfg')
sudo('echo "download-cache = /etc/buildout/downloads" >> /etc/buildout/default.cfg')
sudo('echo "extends-cache = /etc/buildout/extends" >> /etc/buildout/default.cfg')
# allow group `projects` to read/write in here
sudo("chown -R root:projects /etc/buildout/{eggs,downloads,extends}")
sudo("chmod -R 775 /etc/buildout/{eggs,downloads,extends}")
# force maintenance users to also use default.cfg (needed when running buildout via Fabric)
for user in env.admins:
with mode_sudo():
dir_ensure("/home/%s/.buildout" % user)
if exists("/home/%s/.buildout/default.cfg" % user):
sudo("rm -rf /home/%s/.buildout/default.cfg" % user)
sudo("ln -s /etc/buildout/default.cfg /home/%s/.buildout/default.cfg" % user)
sudo("chown -R %s /home/%s/.buildout" % (user, user))
def upload(site, tag='master'):
"""Upload project `site` files from tag or branch `master`."""
puts("Upload project {0}".format(site))
today = datetime.now().strftime('%Y%m%d-%H%M%S')
commit_id = str(local('git rev-parse {0}'.format(tag), True)).strip()
current = "%s-%s" % (today, commit_id[:8])
release_dir = os.path.join(INSTALL_DIR, site, 'releases')
with mode_sudo():
dir_ensure(release_dir)
local_temp_dir = mkdtemp()
archive = os.path.join(local_temp_dir, '{0}.tar.gz'.format(site))
git_arch_cmd = "git archive --format=tar.gz -o {archive} {commit_id}"
local(git_arch_cmd.format(archive=archive, commit_id=commit_id))
with lcd(local_temp_dir):
remote_temp_dir = run('mktemp -d')
put(archive, remote_temp_dir)
with cd(remote_temp_dir):
run("tar xzf {0}.tar.gz".format(site))
run("rm -f {0}.tar.gz".format(site))
sudo('chown -R root.root {0}'.format(remote_temp_dir))
sudo('chmod -R 755 {0}'.format(remote_temp_dir))
sudo("mv {tmp_dir} {release_dir}/{current}".format(release_dir=release_dir, current=current, tmp_dir=remote_temp_dir))
run('rm -rf {0}'.format(remote_temp_dir))
local('rm -rf {0}'.format(local_temp_dir))
return current
def gunicorn_ensure(path, template, config, venv_path='.venv'):
with virtualenv(path, venv_path):
python_package_ensure('gunicorn')
run("cp %s %s" % (template, config))
file_update(config, lambda x: text_template(x,env))
dir_ensure('%s/logs' % path)
dir_ensure('%s/run' % path)
def provision_rabbitmq(admin_password):
append("/etc/apt/sources.list.d/rabbitmq.list",
"deb http://www.rabbitmq.com/debian/ testing main", use_sudo=True)
if not file_exists("/usr/sbin/rabbitmq-server"):
sudo("wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc")
sudo("apt-key add rabbitmq-signing-key-public.asc")
map(package_ensure_apt, ["rabbitmq-server", "rsync"])
dir_ensure("/etc/rabbitmq/rabbitmq.conf.d")
put("./conf/bunny/bunny.conf", "/etc/rabbitmq/rabbitmq.conf.d/", use_sudo=True)
sudo("chown -R rabbitmq.rabbitmq /srv/rabbitmq")
dir_ensure("/srv/rabbitmq/log", owner="rabbitmq", group="rabbitmq")
sudo("rm -rf /var/lib/rabbitmq")
sudo("rm -rf /var/log/rabbitmq")
with mode_sudo():
file_link("/srv/rabbitmq", "/var/lib/rabbitmq", owner="rabbitmq", group="rabbitmq")
file_link("/srv/rabbitmq/log", "/var/log/rabbitmq", owner="rabbitmq", group="rabbitmq")
sudo("service rabbitmq-server start")
sudo("rabbitmq-plugins enable rabbitmq_management")
sudo("rabbitmqctl add_user admin " + admin_password)
sudo("rabbitmqctl set_user_tags admin administrator")
sudo('rabbitmqctl set_permissions -p / admin ".*" ".*" ".*"')
sudo("rabbitmqctl delete_user guest")
sudo("service rabbitmq-server restart")
else:
print "skipped install, already have /usr/sbin/rabbitmq-server"
def install_badvpn(path=None):
"""Installs and configures badvpn client and server"""
opts= dict(
cert_folder = "/etc/badvpn/nssdb",
path=path or env.get('path') or err('env.path must be set')
)
"""Install package"""
core.apt_get("badvpn","ppa:ambrop7/badvpn")
core.apt_get(["libnss3-tools"])
"""Install all configs"""
sudo("cp /etc/init.d/badvpn-server /etc/init.d/badvpn-client")
upload_template_jinja2("%(path)s/etc/init/badvpn-client" % opts,
"/etc/init/badvpn-client", use_sudo=True)
dir_ensure(opts["cert_folder"], recursive=True)
upload_template_jinja2("%(path)s/etc/badvpn/badvpn-client" % opts,
"/etc/badvpn/badvpn-client", use_sudo=True)
sudo("ln -s /etc/badvpn/badvpn-client /etc/default/badvpn-client")
upload_template_jinja2("%(path)s/etc/badvpn/badvpn-server" % opts,
"/etc/badvpn/badvpn-server", use_sudo=True)
sudo("ln -s /etc/badvpn/badvpn-server /etc/default/badvpn-server")
"""Create cert database"""
put("%(path)s/ca.pem" % opts, "~/")
sudo("certutil -d sql:%(cert_folder)s -N" % opts)
sudo('certutil -d sql:%(cert_folder)s -A -t "CT,," -n "vpnca" -i ~/ca.pem' % opts)
def deploy_scripts():
"""
Upload update_index and setup a cron job
"""
script_dir = join(env.directory, "scripts")
cuisine.dir_ensure(script_dir)
upload_template("deploy/update_index.sh", script_dir, context=env)
def install_gunicorn(self):
self.run_virtualenv('pip -q install gunicorn')
with mode_sudo():
dir_ensure('/var/log/gunicorn/',
owner=self.user_name,
group=self.group_name,
)
if self.util.get_package_manager() == 'apt':
operations.put(
'gunicorn.conf',
'/etc/init/',
use_sudo=True, mode=644)
with mode_sudo():
file_attribs('/etc/init/gunicorn.conf',
mode=700,
owner='root',
group='root',
)
with settings(warn_only=True):
sed('/etc/init/gunicorn.conf',
'\{virtualenv\}',
self.virtualenv_dir,
use_sudo = True,
)
operations.put(
'gunicorn-launcher.sh',
self.virtualenv_dir + '/bin/',
use_sudo=True, mode=750)
with mode_sudo():
file_attribs(self.virtualenv_dir + '/bin/gunicorn-launcher.sh',
mode=700,
owner=self.user_name,
group=self.group_name,
)
with settings(warn_only=True):
sed(self.virtualenv_dir + '/bin/gunicorn-launcher.sh',
'\{virtualenv\}',
self.virtualenv_dir,
use_sudo = True,
)
sed(self.virtualenv_dir + '/bin/gunicorn-launcher.sh',
'\{project\}',
self.www_dir,
use_sudo = True,
)
# install gevent. This is non-critical and might fail so we go to
# warn-only mode
with settings(warn_only=True):
package_ensure('libevent-dev')
self.run_virtualenv('pip -q install gevent')
# # TODO: add -k gevent to gunicorn launcher script
# TODO
upstart_ensure('gunicorn')
def gunicorn_process_ensure(path, template, config, key_env, venv_path='.venv'):
with virtualenv(path, venv_path):
python_package_ensure('gunicorn')
run("cp %s %s" % (template, config))
file_update(config, lambda x: text_template(x,key_env))
dir_ensure('%s/logs' % path)
dir_ensure('%s/run' % path)
def _run(module,
logger,
task,
completed_tasks,
from_command_line=False,
args=None,
kwargs=None):
"""
@type module: module
@type logging: Logger
@type task: Task
@type completed_tasts: set Task
@rtype: set Task
@return: Updated set of completed tasks after satisfying all dependencies.
"""
# Satsify dependencies recursively. Maintain set of completed tasks so each
# task is only performed once.
input_artifacts = {}
for dependency in task.dependencies:
_run(module, logger, dependency, completed_tasks)
input_artifacts[dependency.name] = artifact_file(
dependency.name, completed_tasks[dependency.name])
# Perform current task, if need to.
if from_command_line or task.name not in completed_tasks:
if task.ignored:
logger.info("Ignoring task \"%s\"" % task.name)
cs = 'IGNORE'
else:
logger.info(yellow("Starting task \"%s\"" % task.name))
try:
# Run task.
cs = checksum(module, input_artifacts, task.func,
task.watched_sources, args, kwargs)
if cuisine.dir_exists(artifact_file(task.name, cs)):
logger.info("Nothing changed")
else:
cuisine.dir_remove(artifact_dir(task.name))
cuisine.dir_ensure(artifact_dir(task.name))
cuisine.dir_remove(tmp_file(task.name))
cuisine.dir_ensure(tmp_file(task.name))
State.input_artifacts = input_artifacts
State.output_artifact = task.name
logger.info("Going to write an artifact with checksum " +
cs)
task(*(args or []), **(kwargs or {}))
cuisine.run("mv " + tmp_file(task.name) + " " +
artifact_file(task.name, cs))
except:
logger.critical("Error in task \"%s\"" % task.name)
logger.critical("Aborting build")
raise
logger.info("Completed task " + task.name + " artifact path: " +
artifact_file(task.name, cs))
completed_tasks[task.name] = cs
def run_remote_background_process(p_fab_api,
p_command_to_run,
p_out_file,
p_err_file,
p_log_fun,
p_shell=True,
p_pty=False):
p_log_fun('FUN_ENTER', 'gf_fabric_utils.run_remote_background_process()')
p_log_fun('INFO', 'p_command_to_run:%s' % (p_command_to_run))
cuisine.dir_ensure(os.path.dirname(p_out_file), recursive=True)
cuisine.file_ensure(p_out_file)
cuisine.dir_ensure(os.path.dirname(p_err_file), recursive=True)
cuisine.file_ensure(p_err_file)
#nohup is a POSIX command to ignore the HUP (hangup) signal,
#enabling the command to keep running after the user who
#issues the command has logged out. The HUP (hangup) signal is
#by convention the way a terminal warns depending processes of logout.
#streams stdout/stderr to the out_file/err_file
command_str = 'nohup %s >%s 2>%s </dev/null &' % (
p_command_to_run, p_out_file, p_err_file or '&1')
out = p_fab_api.run(command_str, p_shell, p_pty)
return out
def initialize_postgres():
"""Initialize the main database."""
version = sudo("psql --version | grep -ro '[8-9].[0-9]'")
conf_dir_prefix = "/etc/postgresql/%s/" % version
# temporarily allow root access from localhost
sudo('mv /etc/postgresql/%s/main/pg_hba.conf /etc/postgresql/%s/main/pg_hba.conf.bak' % (version, version))
sudo('echo "local all postgres ident" > /etc/postgresql/%s/main/pg_hba.conf' % version)
sudo('cat /etc/postgresql/%s/main/pg_hba.conf.bak >> /etc/postgresql/%s/main/pg_hba.conf' % (version, version))
sudo('service postgresql-%s restart || /etc/init.d/postgresql restart ' % version)
# set password
password = prompt('Enter a new database password for user `postgres`:')
sudo('psql template1 -c "ALTER USER postgres with encrypted password \'%s\';"' % password, user='******')
# configure daily dumps of all databases
with mode_sudo():
dir_ensure('/var/backups/postgresql', recursive=True)
sudo("echo 'localhost:*:*:postgres:%s' > /root/.pgpass" % password)
sudo('chmod 600 /root/.pgpass')
sudo("echo '0 7 * * * pg_dumpall --username postgres --file /var/backups/postgresql/postgresql_$(date +%%Y-%%m-%%d).dump' > /etc/cron.d/pg_dump")
# remove temporary root access
comment('/etc/postgresql/%s/main/pg_hba.conf' % version, 'local all postgres ident', use_sudo=True)
sudo('service postgresql%s restart || /etc/init.d/postgresql restart' % version)
def setup_erlang(version='R15B03'):
packages_install('gcc gcc-c++ make openssl-devel ncurses-devel')
cuisine.dir_ensure('src')
with cd('src'):
upload_erlang(version)
run('tar xvfz otp_src_{0}.tar.gz'.format(version))
with cd('otp_src_{0}'.format(version)):
run('./configure --prefix=/opt/erlang/{0} \
--enable-threads \
--enable-smp-support \
--enable-halfword-emulator \
--enable-m64-build \
--disable-native-libs \
--disable-sctp \
--enable-kernel-poll \
--disable-hipe \
--without-javac'.format(version))
run('make')
sudo('make install')
run("echo 'export PATH=/opt/erlang/R15B03/bin:$PATH' >> .bashrc")
run('source .bashrc')
def install_nagios_from_source():
'''
Downloads Nagios source code and installs Nagios on VM.
'''
cuisine.dir_ensure('/usr/local/src')
sudo('mkdir -p /etc/httpd/conf.d/')
cuisine.dir_ensure('/etc/httpd/conf.d/')
run('cd /usr/local/src')
if not nagios_downloaded():
sudo(
'wget http://prdownloads.sourceforge.net/sourceforge/nagios/nagios-3.4.1.tar.gz'
)
sudo('tar xzf nagios-3.4.1.tar.gz')
sudo(
'cd ~/nagios && ./configure --with-command-group=nagcmd && make all && make install'
)
sudo(
'cd ~/nagios && make install-init && make install-config && make install-commandmode'
)
sudo('cd ~/nagios && make install-webconf')
sudo(
'cp /etc/httpd/conf.d/nagios.conf /etc/apache2/conf-available/nagios.conf'
)
sudo(
'cp /etc/httpd/conf.d/nagios.conf /etc/apache2/conf-enabled/nagios.conf'
)
sudo('rm -rf /etc/apache2/conf-available/nagios3.conf')
sudo('rm -rf /etc/apache2/conf-enabled/nagios3.conf')
def ensure():
git.ensure_git()
# Make sure the user exists
user_ensure('acme')
group_ensure('acme')
group_user_ensure('acme', 'acme')
# Allow the acme user to reload nginx when updating certs
files.append('/etc/sudoers',
'acme ALL=(root) NOPASSWD: {reload_nginx}'.format(
reload_nginx=_reload_nginx()),
use_sudo=True)
# Make sure acme.sh is installed for the acme user
# if not path.has('acme.sh', user='******'): Does not exist--it's imported from bash
if not files.exists('/home/acme/.acme.sh'):
git.ensure_clone_github('Neilpang/acme.sh',
'/home/acme/acme.sh',
user='******')
with cd("/home/acme/acme.sh"):
sudo("./acme.sh --install", user='******')
dir_ensure(well_known_base)
sudo(
"chown acme:acme {well_known} && chmod 755 {well_known}".format(
well_known=well_known_base)
) # Can't change attributes without sudo in a sticky (not write) directory... annoying
util.put_file("config/certs/lets-encrypt-x3-cross-signed.pem",
"/etc/ssl/certs/lets-encrypt-x3-cross-signed.pem",
user='******',
mode='0644')
def cloudstack_init(fqdn=None):
puts(green('CloudStack Initialize'))
if fqdn is None:
abort('Please set FQDN\n\tex) $ fab .... rolename:"fqdn"')
# 石川さんごめんなさい
sudo('sed -i -e "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config')
sudo('setenforce permissive')
# Repository for CloudStack
repository = '[cloudstack]\nname=cloudstack\n'
repository += 'baseurl=http://cloudstack.apt-get.eu/rhel/4.2/\n'
repository += 'enabled=1\n'
repository += 'gpgcheck=0\n'
cuisine.file_write('/etc/yum.repos.d/CloudStack.repo', repository)
# Setting FQDN
if not fqdn in cuisine.file_read('/etc/hosts'):
sudo('sed -i -e "s/localhost/' + fqdn + ' localhost/" /etc/hosts')
# NTP
install_package('ntp')
cuisine.upstart_ensure('ntpd')
sudo('chkconfig ntpd on')
download_and_upload('ntp/%s-ntp.conf', '/etc/ntp.conf', abort_flg=False)
puts(green('Success'))
#--------------------------------------#
# NFS #
#--------------------------------------#
def nfs(directory):
puts(green('Setup NFS'))
if fqdn is None:
abort('Please set Directory\n\tex) $ fab .... nfsrolename:"/etc/hogehoge"')
# Install
install_package('nfs-utils')
# Start
for name in [ 'rpcbind', 'nfs' ]:
cuisine.upstart_ensure(name)
sudo('chkconfig %s on' % name)
# Create Directory
cuisine.dir_ensure(directory, recursive=True)
# Setting /etc/exports
download_and_upload('nfs/%s-exports', '/etc/exports')
sudo('exportfs -a')
# Setting /etc/sysconfig/nfs
download_and_upload('nfs/%s-nfs', '/etc/sysconfig/nfs')
puts(green('Success'))
def put_csr(csr, user='******'):
with mode_sudo():
dir_ensure('/etc/ssl/csr', mode='1777')
csr_name = csr.split("/")[-1]
return util.put_file(csr,
'/etc/ssl/csr/' + csr_name,
mode='0644',
user=user)[0]
def solr_ensure(project_path, venv_path='.venv'):
with mode_sudo():
package_ensure('openjdk-7-jdk libxml2-dev libxslt1-dev python-dev')
dir_ensure('/usr/java')
file_link('/usr/lib/jvm/java-7-openjdk-amd64', '/usr/java/default')
package_ensure('solr-tomcat')
with virtualenv(project_path, venv_path):
python_package_ensure('pysolr lxml cssselect')
def put_key(key, user='******'):
with mode_sudo():
dir_ensure('/etc/ssl/private', mode='1777')
key_name = key.split("/")[-1]
return util.put_file(key,
'/etc/ssl/private/' + key_name,
mode='0640',
user=user)[0]
def apply(self, computer):
cuisine.package_ensure('nginx-extras')
cuisine.dir_ensure('/srv/www/', mode='755')
tmp_env = {"name": computer}
self.push_package_file_set('nginx', computer, tmp_env)
cuisine.sudo("service nginx restart")
def install_webserver():
""" Install Python """
dir_ensure('/opt/webserver')
with cd('/opt/webserver'):
run('git clone %s buildout.webserver' % (env.git_repo))
with cd('/opt/webserver/buildout.webserver'):
run('virtualenv .')
run('bin/pip install zc.buildout')
run('bin/buildout -c deployment.cfg')
def build():
remote_build_dir = config['config'][env.host_string]['remote_build_dir']
if not dir_exists(remote_build_dir):
dir_ensure(remote_build_dir, recursive=True)
dir_ensure(remote_build_dir + '/build')
build_surface()
def virtualenv_ensure(project_path, venv_path='.venv', packages_file='requirements.txt', restart=False):
package_ensure('python-dev python-pip python-virtualenv')
with virtualenv(project_path, venv_path):
dir_ensure('downloads')
if restart:
run('rm -rf %s' % venv_path)
if not dir_exists(venv_path):
run('virtualenv --no-site-packages --distribute %s' % venv_path)
run('pip install --download-cache downloads -r ' + packages_file)
def build():
remote_build_dir = config['config'][env.host_string]['remote_build_dir']
if not dir_exists(remote_build_dir):
dir_ensure(remote_build_dir, recursive=True)
dir_ensure(remote_build_dir + '/build')
build_hive()
def rsync(repo_url, repo_dir, refspec='master', home='.', base_dir='git', local_tmpdir='/tmp', save_history=False, do_delete=True, check_hostkey=True):
"""
Does a git clone locally first then rsync to remote.
:param repo_url: *required* str; url of the git repo
:param repo_dir: *required* str; dir name of the repo clone
:param refspec: str; the git refspec to checkout for deploy (can be a branch, tag, git hash, etc.)
:param home: str; home directory to deploy the code to.
:param base_dir: str; dir name relative to ``home``.
:param local_tmpdir: str; where the local clone + checkout will be located
:param save_history: bool; if True, then the history of every deploys is tracked, for rollback purposes later.
:param do_delete: bool; if True, then rsync parameter --delete-during will be added
:param check_hostkey: bool; if True, then ssh option StrictHostKeyChecking is enabled
Problem statement: How do we ensure that code from a git repository gets deployed
uniformly, efficiently across all remote hosts.
This is one solution that uses ``rsync`` to push code (just as fabric is push-based).
Git returns to being a code/config repository and is not required in the destination hosts.
Another advantage of this approach is when not all destination hosts have access to the
git repository. If ssh public key auth is used in the repo (e.g. gitolite, Github), each
destination server may then require either: (a) identical ssh keys or (b) provision each
destination server in the repo. Both have maintenance and security issues.
All git operations are done on a separate ``local_tmpdir``, and not on a checkout where
the fabfile is located. Everytime this function is invoked, the ``repo_url`` is always
ensured to be fresh (``git clone + git fetch``). This means only commits fetched from the
``repo_url`` can be deployed.
"""
# ensure git + rsync is available locally
local('which git')
local('which rsync')
# resolve user,host,port for rsh string
user, host, port = normalize(env.host_string)
# ensure the temp paths are ready
local_user = local('whoami', capture=True)
clone_basepath_local = os.path.join(local_tmpdir, local_user, 'deploy', host, user, str(port), 'git')
local('mkdir -p %s' % clone_basepath_local)
# prepare remote path strings
clone_basepath_remote = os.path.join(home, base_dir)
cuisine.dir_ensure(clone_basepath_remote)
# prepare history (for recovery)
hist = git.GitHistory()
if save_history:
remote_hist_path = git.get_remote_git_history_path(home)
try:
tmphist = git.load_remote_git_history(remote_hist_path, local_user=local_user)
if tmphist is not None:
hist = tmphist
except Exception, e:
error("Warning: Unable to load history file %s: %s" % (remote_hist_path, e ))
def install_bacula_client():
"""Install and configure Bacula backup client, which listens for
instructions from Bacula master and backups critical data
when told to do so."""
apt_get('bacula-fd')
# this folder is needed
with mode_sudo():
dir_ensure('/var/spool/bacula', recursive=True)
def apache_site_ensure(name, config, root_dir=None):
'''Configures and loads an apache site config'''
root_dir = root_dir or join('/var/www', name)
with cuisine.mode_sudo():
cuisine.dir_ensure(root_dir, recursive=True)
cuisine.file_write('/etc/apache2/sites-available/{}'.format(name),
config, sudo=True)
if not cuisine.file_exists('/etc/apache2/sites-enabled/{}'.format(name)):
cuisine.sudo('a2ensite {}'.format(name))
cuisine.sudo('service apache2 reload')
def virtualenv_ensure(project_path, venv_path='.venv', packages_file='requirements.txt', restart=False):
package_ensure('python-dev python-pip python-virtualenv')
with cd(project_path):
dir_ensure('downloads')
if restart:
run('rm -rf %s' % venv_path)
if not dir_exists(venv_path):
run('virtualenv --no-site-packages --distribute %s' % venv_path)
with virtualenv(project_path, venv_path):
run('pip install --download-cache downloads -r ' + packages_file)
def put_cert(cert, user='******'):
with mode_sudo():
dir_ensure('/etc/ssl/certs', mode='1777')
if user is None:
user = '******'
cert_name = cert.split("/")[-1]
return util.put_file(cert,
'/etc/ssl/certs/' + cert_name,
mode='0644',
user=user)[0]
def install_nagios_plugins_from_source():
'''
Downloads Nagios plugins source code and installs Nagios plugins on VMs.
'''
cuisine.dir_ensure('/usr/local/src')
run('cd /usr/local/src')
if not nagios_plugins_downloaded():
sudo('wget http://nagios-plugins.org/download/nagios-plugins-2.0.3.tar.gz')
sudo('tar xzf nagios-plugins-2.0.3.tar.gz')
sudo('cd ~/nagios-plugins-2.0.3 && ./configure --with-nagios-user=nagios --with-nagios-group=nagios && make && make install')
def utils_gen_self_signed_cert():
with cuisine.mode_sudo():
cuisine.dir_ensure("/svr/ssl/", recursive=True)
# cuisine.dir_attribs('/svr/ssl/', mode="0400")
sudo(
"openssl req -new -days 365 -nodes -out /svr/ssl/{site_name}.pem -keyout /svr/ssl/{site_name}.key -newkey rsa:2048".format(
site_name=env.site_name
)
)
def _dovecot(database_password):
select_package("apt")
package_ensure(["dovecot-imapd", "dovecot-lmtpd", "dovecot-pgsql", "dovecot-sieve", "dovecot-managesieved"]) # On debian will automatically be enabled
crypto.put_cert('config/certs/imap.za3k.com.pem')
crypto.put_key('config/keys/imap.za3k.com.key')
put('config/dovecot/dovecot.conf', '/etc/dovecot/dovecot.conf', mode='644')
put('config/dovecot/dovecot-sql.conf', '/etc/dovecot/dovecot-sql.conf', mode='600')
_replace('/etc/dovecot/dovecot-sql.conf', 'DOVECOT_DATABASE_PASSWORD', database_password)
dir_ensure("/etc/dovecot/sieve.d")
run("chown vmail:vmail /etc/dovecot/sieve.d")
def install_nrpe_plugin_from_source():
'''
Downloads NRPE plugin source code and installs NRPE plugin on VM.
'''
cuisine.package_ensure_apt('libssl-dev')
cuisine.dir_ensure('/usr/local/src')
run('cd /usr/local/src')
if not nrpe_plugins_downloaded():
sudo('wget http://sourceforge.net/projects/nagios/files/nrpe-2.x/nrpe-2.15/nrpe-2.15.tar.gz')
sudo('tar xzf nrpe-2.15.tar.gz')
sudo('cd ~/nrpe-2.15 && ./configure --with-ssl=/usr/bin/openssl --with-ssl-lib=/usr/lib/x86_64-linux-gnu && make all && make install-plugin')
def create_deploy():
'''
Create a deploy repos on remote server
'''
cuisine.dir_ensure("deploy/lgw.git")
with cd("deploy/lgw.git"):
run("git init --bare")
cuisine.file_ensure("hooks/post-receive")
if not files.contains("hooks/post-receive", POST_RECEIVE):
files.append("hooks/post-receive", POST_RECEIVE)
run('chmod +x hooks/post-receive')
def ensure():
"""Ensure nginx is installed"""
select_package("apt")
already_installed = package_ensure(["nginx"]) # On debian will automatically be enabled
ensure_sites_available()
put('config/nginx/nginx.conf', '/etc/nginx', use_sudo=True)
put('config/nginx/fastcgi_params', '/etc/nginx', use_sudo=True)
crypto.ensure_dhparams('/etc/ssl/dhparams-nginx.pem')
with mode_sudo():
dir_ensure("/var/www", mode='1777') # make sure anyone can add a site
return already_installed
def install_nrpe_plugin_from_source():
'''
Downloads NRPE plugin source code and installs NRPE plugin on VM.
'''
cuisine.package_ensure_apt('libssl-dev')
cuisine.dir_ensure('/usr/local/src')
cuisine.sudo('cd /usr/local/src')
if not nrpe_plugins_downloaded():
cuisine.sudo('wget http://sourceforge.net/projects/nagios/files/nrpe-2.x/nrpe-2.15/nrpe-2.15.tar.gz')
cuisine.sudo('tar xzf nrpe-2.15.tar.gz')
cuisine.run('cd ~/nrpe-2.15 && ./configure --with-ssl=/usr/bin/openssl --with-ssl-lib=/usr/lib/x86_64-linux-gnu && make all && make install-plugin')
def install():
""" Install ngix packages """
puts(green('-> Installing nginx'))
cuisine.package_ensure('nginx')
puts(green('-> Creating apps user'))
cuisine.user_ensure('apps')
puts(green('-> Creating dirs for web apps'))
cuisine.dir_ensure('/home/apps/www')
cuisine.dir_ensure('/home/apps/logs')
def cpanm_bin_installed(home='/tmp'):
cuisine.select_package(option='yum')
cuisine.package_ensure('perl-devel')
binpath = '%s/.deploy/bin' % home
cpanm = '%s/cpanm' % binpath
if not cuisine.file_exists(cpanm):
cuisine.dir_ensure(binpath, recursive=True, mode=755)
cuisine.package_ensure('curl')
run('curl -L http://cpanmin.us > %s' % cpanm)
run('chmod 755 %s' % cpanm)
cuisine.file_exists(cpanm)
return cpanm
def checkout_project(self):
with mode_sudo():
dir_ensure(self.repo_dir,
owner=self.user_name,
group=self.group_name,
recursive=True,
)
with cd(self.repo_dir):
sudo(r'chown %s:%s .' % (self.user_name, self.group_name))
sudo(r'ls -d * > /dev/null 2>&1 || %s clone -q %s .'
% (self.vcs, self.repo_url),
user=self.user_name)
def dir_conf():
notify('Creating the working directory structure.')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_WORKSPACE'])
cuisine.dir_ensure(MACHINE['DIR_ENVIRONMENTS'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_PROJECTS'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_SSL'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
cuisine.dir_ensure(MACHINE['DIR_LOGS'], recursive=True, mode=MACHINE['DIR_MODE'],
owner=KEY, group=MACHINE['OWNER_GROUP'])
def _mount_iso(self):
"""
Mout base image...
"""
print("Mounting base image...")
time.sleep(1)
with mode_sudo():
dir_ensure(self.base_iso_dir, recursive=True)
output = sudo("df -a")
if not self.base_iso_dir in output:
cmd = ['mount', '-o','loop', self.base_iso, self.base_iso_dir]
sudo(' '.join(cmd))
def testAttribs(self):
tmpdir = tempfile.mkdtemp()
try:
dir1_path = os.path.join(tmpdir, 'dir1')
cuisine.dir_ensure(dir1_path)
file1_path = os.path.join(dir1_path, 'file1')
cuisine.file_write(file1_path, 'test', mode='666')
cuisine.dir_attribs(tmpdir, mode=755, recursive=True)
attribs = cuisine.file_attribs_get(dir1_path)
self.assertEqual(attribs.get('mode'), '755')
finally:
cuisine.dir_remove(tmpdir, recursive=True)
def install(version, target):
if not target:
abort('Task argument "target" is required.')
current_dir = os.path.dirname(__file__)
project_path = current_dir
pkg_version = setuptools_get_version(project_path)
pkg_name = setuptools_get_name(project_path)
if not version:
version = pkg_version
print('Installing package {0}, version {1} to target {2}.'.format(
*list(map(yellow, [pkg_name, version, target]))))
if env.confirm:
response = ask('Proceed (y/n)? ', ('y', 'n'))
else:
response = 'y'
if response == 'y':
# From filesystem
#source_package = '~/install/PatZilla/PatZilla-{version}.tar.gz'.format(version=version)
# From PyPI
source_package = 'patzilla=={version}'.format(version=version)
source_config = './patzilla/config/production.ini.tpl'
target_path = os.path.join(INSTALLATION_PATH, 'sites', target)
dir_ensure(target_path, recursive=True)
venv_path = target_path + '/.venv2'
#if not file_is_file(source_package):
# abort('Source package does not exist: ' + source_package)
if not file_is_dir(target_path):
abort('Target path does not exist: ' + target_path)
if not file_is_dir(venv_path):
run('virtualenv --no-site-packages "{0}"'.format(venv_path))
#setup_package('which', venv_path)
# TODO: put these packages to a more convenient location
setup_package(source_package, venv_path)
upload_config(source_config, target_path)
restart_service(target)
else:
print(yellow('Skipped package install due to user request.'))
def repo_rpm_init():
''' initialize package repo '''
package_ensure('createrepo')
for dist, package_format in PACKAGE_FORMAT.items():
if package_format == 'rpm':
dir_ensure('{0}/archive/{1}'.format(env.repo_rpm_root, dist), recursive=True)
for component in RPM_COMPONENTS:
for arch in RPM_ARCHS:
path = pj(env.repo_rpm_root, dist, component, arch)
dir_ensure(path, recursive=True)
run('createrepo {}'.format(path))
def add_nagios_user():
'''
Adds Nagios user and groups to VMs and sets the right permissions.
'''
cuisine.group_ensure('nagios')
cuisine.user_ensure('nagios')
cuisine.group_user_ensure('nagios', 'nagios')
sudo('mkdir -p /usr/local/nagios')
sudo('mkdir -p /usr/local/nagios/libexec')
cuisine.dir_ensure('/usr/local/nagios')
cuisine.dir_ensure('/usr/local/nagios/libexec')
sudo('chown nagios.nagios /usr/local/nagios')
sudo('chown -R nagios.nagios /usr/local/nagios/libexec')
def install_nagios_plugins_from_source():
'''
Downloads Nagios plugins source code and installs Nagios plugins on VMs.
'''
cuisine.dir_ensure('/usr/local/src')
run('cd /usr/local/src')
if not nagios_plugins_downloaded():
sudo(
'wget http://nagios-plugins.org/download/nagios-plugins-2.0.3.tar.gz'
)
sudo('tar xzf nagios-plugins-2.0.3.tar.gz')
sudo(
'cd ~/nagios-plugins-2.0.3 && ./configure --with-nagios-user=nagios --with-nagios-group=nagios && make && make install'
)
def repo_rpm_init():
''' initialize package repo '''
package_ensure('createrepo')
for dist, package_format in PACKAGE_FORMAT.items():
if package_format == 'rpm':
dir_ensure('{0}/archive/{1}'.format(env.repo_rpm_root, dist),
recursive=True)
for component in RPM_COMPONENTS:
for arch in RPM_ARCHS:
path = pj(env.repo_rpm_root, dist, component, arch)
dir_ensure(path, recursive=True)
run('createrepo {}'.format(path))
def create_virtualenv():
"""Create virtualenv for project."""
site = get_project_name()
version = get_config()['version']
virtualenv_dir = "{}/{}/virtualenv".format(SITES_DIR, site)
if cuisine.dir_exists(virtualenv_dir + "/bin"):
fab.puts("virtualenv for {0} already exists".format(site))
return
with cuisine.mode_sudo():
cuisine.dir_ensure(virtualenv_dir, recursive=True)
venv_bin = _python_bin_path(version, 'virtualenv')
fab.sudo("{venv_bin} {virtualenv_dir}".format(
venv_bin=venv_bin, virtualenv_dir=virtualenv_dir))
def _dovecot(database_password):
select_package("apt")
package_ensure([
"dovecot-imapd", "dovecot-lmtpd", "dovecot-pgsql", "dovecot-sieve",
"dovecot-managesieved"
]) # On debian will automatically be enabled
crypto.put_cert('config/certs/imap.za3k.com.pem')
crypto.put_key('config/keys/imap.za3k.com.key')
put('config/dovecot/dovecot.conf', '/etc/dovecot/dovecot.conf', mode='644')
put('config/dovecot/dovecot-sql.conf',
'/etc/dovecot/dovecot-sql.conf',
mode='600')
_replace('/etc/dovecot/dovecot-sql.conf', 'DOVECOT_DATABASE_PASSWORD',
database_password)
dir_ensure("/etc/dovecot/sieve.d")
run("chown vmail:vmail /etc/dovecot/sieve.d")
def install_bacula_master():
"""Install and configure Bacula Master."""
# Official repos only have version 5.0.1, we need 5.0.3
sudo('add-apt-repository ppa:mario-sitz/ppa')
sudo('apt-get update')
sudo(
'apt-get -yq install bacula-console bacula-director-pgsql bacula-sd-pgsql'
)
# folder and files that are expected to be there
with mode_sudo():
dir_ensure('/etc/bacula/clients/')
sudo('touch /etc/bacula/clients/remove_me_once_deployed.conf')
sudo('chown -R bacula /etc/bacula/clients/')
configure_bacula_master()
def project_layout():
""" Makes project directories """
env.user = RTD_USER
env.password = RTD_PASS
dir_ensure("/opt/rtd/apps/readthedocs", recursive=True)
dir_ensure("/opt/rtd/htdocs")
dir_ensure("/opt/rtd/tmp")
dir_ensure("/opt/rtd/logs")
def ensure():
select_package("apt")
package_ensure(["znc"]) # On debian will automatically be enabled
user_ensure('znc')
group_ensure('znc')
group_user_ensure('znc', 'znc')
dir_ensure("/var/znc", mode='755')
dir_ensure("/var/znc/configs", mode='755')
run("chown znc:znc /var/znc")
util.put("/srv/znc.conf", "/var/znc/configs", user="******", mode="600")
util.put("config/keys/znc.pem", "/var/znc", user="******", mode="600")
util.put("config/znc/modules", "/var/znc", user="******", mode="755")
run("cp /var/znc/modules/*.so /usr/lib/znc")
systemd.add_unit("config/systemd/znc.service")
run("systemctl enable znc")
run("systemctl restart znc")
def add_nagios_user():
'''
Adds Nagios user and group and sets correct file permissions.
'''
cuisine.group_ensure('nagcmd')
cuisine.group_ensure('nagios')
cuisine.user_ensure('nagios')
cuisine.group_user_ensure('nagios', 'nagios')
cuisine.group_user_ensure('nagcmd', 'nagios')
cuisine.user_ensure('www-data')
cuisine.group_user_ensure('nagcmd', 'www-data')
sudo('mkdir -p /usr/local/nagios')
sudo('mkdir -p /usr/local/nagios/libexec')
cuisine.dir_ensure('/usr/local/nagios')
cuisine.dir_ensure('/usr/local/nagios/libexec')
sudo('chown nagios.nagcmd /usr/local/nagios')
sudo('chown -R nagios.nagcmd /usr/local/nagios/libexec')
def _dkim_milter():
select_package("apt")
package_ensure("opendkim", "opendkim-tools")
put("config/dkim/opendkim.conf", "/etc", mode='644')
dir_ensure("/etc/opendkim")
run("chmod 755 /etc/opendkim")
put("config/dkim/KeyTable", "/etc/opendkim", mode='644')
put("config/dkim/SigningTable", "/etc/opendkim", mode='644')
put("config/dkim/TrustedHosts", "/etc/opendkim", mode='644')
put("config/dkim/opendkim", "/etc/default", mode='644')
run("mkdir -p /etc/opendkim/keys/za3k.com && chmod 755 /etc/opendkim/keys && chmod 755 /etc/opendkim/keys/za3k.com"
)
put("/srv/keys/dkim/za3k.com/default.private",
"/etc/opendkim/keys/za3k.com",
mode='600')
put("/srv/keys/dkim/za3k.com/default.txt",
"/etc/opendkim/keys/za3k.com",
mode='644')
run("chown opendkim:opendkim -R /etc/opendkim")
def deploy_bundle(local_path,
deploy_path,
file_name='bunle',
branch='master',
remote='bundle'):
"""
deploys Git bundle and setup repository
usage from command line:
fab deploy_bundle:<local-path>,<deploy-path>
"""
mode_local()
local_bundle_dir = os.path.join(os.path.join(local_path, 'temp'))
local_bundle_file = os.path.join(local_bundle_dir, file_name)
bundle_file_names = git_bundle(local_path, local_bundle_file)
if not bundle_file_names:
return
mode_remote()
remote_bundle_dir = os.path.join(deploy_path, 'temp')
dir_ensure(remote_bundle_dir)
main_bundle_file = None
for bundle_file_name in bundle_file_names:
local_bundle_file = os.path.join(local_bundle_dir, bundle_file_name)
remote_bundle_file = os.path.join(remote_bundle_dir, bundle_file_name)
if not main_bundle_file:
main_bundle_file = remote_bundle_file
file_upload(remote_bundle_file, local_bundle_file)
git_unbundle(main_bundle_file,
deploy_path,
branch,
force=True,
remote=remote)
def build(args):
"""
Build the specified module with specified arguments.
@type module: module
@type args: list of arguments
"""
# Build the command line.
parser = _create_parser()
#No args passed.
#if not args: #todo: execute default task.
# parser.print_help()
# print("\n\n"+_CREDIT_LINE)
# exit
# Parse arguments.
args = parser.parse_args(args)
cuisine.mode_local()
if args.version:
print('brick_wall_build %s' % __version__)
sys.exit(0)
#load build file as a module
if not path.isfile(args.file):
print("Build file '%s' does not exist. Please specify a buld file\n" %
args.file)
parser.print_help()
sys.exit(1)
module = imp.load_source(
path.splitext(path.basename(args.file))[0], args.file)
cuisine.dir_ensure(State.tmp_path)
cuisine.dir_ensure(State.artifact_path)
# Run task and all its dependencies.
if args.list_tasks:
print_tasks(module, args.file)
elif not args.tasks:
if not _run_default_task(module):
parser.print_help()
print("\n")
print_tasks(module, args.file)
else:
_run_from_task_names(module, args.tasks)
