def set_maintanance_mode(self, on=True):
settings_local_path = self.project_path + 'settings_local.py'
if on:
uncomment(settings_local_path, r'MAINTENANCE_MODE.*')
else:
comment(settings_local_path, r'MAINTENANCE_MODE.*')
def locales(names):
"""
Require the list of locales to be available.
"""
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
if distrib_id() == "Archlinux":
run_as_root('locale-gen')
else:
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
def setup_demo_config():
"""
Modifies the Eureka instance to comply with the demo instance requirements
--> Memcached + FastCGI + SSL/HTTPS
"""
assert env.hosts
with cd(env.application_dir):
eureka_egg_dir = run(
'./bin/python -c "import pkg_resources; '
'print pkg_resources.get_distribution(\'eureka\').location"')
# ==== HTTPD config ==== #
apache_config = os.path.join(eureka_egg_dir, 'etc', 'httpd.conf')
# HTTPS instead of HTTP
files.sed(
filename=apache_config,
before='<VirtualHost \*:80>',
after='<VirtualHost \*:443>',
)
# Enable SSL (for HTTPS)
files.uncomment(apache_config, 'SSLEngine .*')
with cd('{}/contrib'.format(eureka_egg_dir)):
# Deactivate password change
run('patch --forward "../eureka/domain/models.py" < demo/models_no_pwd_change.diff')
# Restore snapshot every night
run('crontab -l | {{ grep -v "restore_snapshot" ; echo "0 0 * * * {}/sh/restore_snapshot.sh"; }} | crontab -'.format(env.application_dir))
# Restore snapshot
run('./sh/restore_snapshot.sh')
def locales(names):
"""
Require the list of locales to be available.
"""
if distrib_id() == "Ubuntu":
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
run_as_root('touch %s' % config_file)
else:
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
if distrib_id() == "Archlinux":
run_as_root('locale-gen')
else:
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
def locales(names):
"""
Require the list of locales to be available.
"""
if distrib_id() == "Ubuntu":
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
run_as_root('touch %s' % config_file)
else:
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
family = distrib_family()
if family == 'debian':
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
elif family in ['arch', 'gentoo']:
run_as_root('locale-gen')
else:
raise UnsupportedFamily(supported=['debian', 'arch', 'gentoo'])
def install_hadoop():
# Hadoop
sudo('mkdir -p /opt')
if not exists(os.path.join(HADOOP_INSTALL, 'bin/hadoop')):
put(install_file(HADOOP_TARFILE), os.path.join('/opt', HADOOP_TARFILE), use_sudo=True)
with cd('/opt'):
sudo(cmd('tar zxf %s', HADOOP_TARFILE))
sudo(cmd('rm %s', os.path.join('/opt', HADOOP_TARFILE)))
sudo(cmd('chown -R hadoop.hadoop %s', HADOOP_INSTALL))
# Hadoop config files
put('files/core-site.xml', '%s/etc/hadoop/core-site.xml' % (HADOOP_INSTALL,), use_sudo=True)
put('files/hdfs-site.xml', '%s/etc/hadoop/hdfs-site.xml' % (HADOOP_INSTALL,), use_sudo=True)
put('files/yarn-site.xml', '%s/etc/hadoop/yarn-site.xml' % (HADOOP_INSTALL,), use_sudo=True)
put('files/mapred-site.xml', '%s/etc/hadoop/mapred-site.xml' % (HADOOP_INSTALL,), use_sudo=True)
put('files/masters', '%s/etc/hadoop/masters' % (HADOOP_INSTALL,), use_sudo=True)
upload_template('files/slaves', '%s/etc/hadoop/slaves' % (HADOOP_INSTALL,), context={'slaves_list': '\n'.join(SLAVES)}, use_sudo=True)
hadoop_env = '%s/etc/hadoop/hadoop-env.sh' % (HADOOP_INSTALL,)
sed(hadoop_env, '^export JAVA_HOME=.*', 'export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")', use_sudo=True)
uncomment(hadoop_env, 'export\s+HADOOP_HEAPSIZE=', use_sudo=True)
sed(hadoop_env, '^export\s+HADOOP_HEAPSIZE=.*', 'export HADOOP_HEAPSIZE=256', use_sudo=True)
if not contains(hadoop_env, '^export HADOOP_DATANODE_OPTS=.*-client', escape=False, use_sudo=True):
sed(hadoop_env, '^export HADOOP_DATANODE_OPTS="(.*)"$', 'export HADOOP_DATANODE_OPTS="\\1 -client"', use_sudo=True)
sudo(cmd('ln -sf %s /opt/hadoop', HADOOP_INSTALL))
for f in ['start-all.sh', 'stop-all.sh', 'dfs-format.sh', 'clear-dfs.sh', 'nuke-dfs.sh', 'halt-all.sh', 'hdfs-balance.sh']:
put('files/' + f, 'bin/' + f.replace('.sh', ''), mode=0755)
# HDFS directories
sudo('mkdir -p -m 0750 /hadoop/tmp && chown hadoop:hadoop /hadoop/tmp')
sudo('mkdir -p /hadoop/namenode && chown hadoop:hadoop /hadoop/namenode')
sudo('mkdir -p /hadoop/datanode && chown hadoop:hadoop /hadoop/datanode')
def setup_base():
uncomment('/etc/apt/sources.list', '# deb http', use_sudo=True)
uncomment('/etc/apt/sources.list', '# deb-src http', use_sudo=True)
apt_update()
apt_get('python-software-properties')
ppas_path = os.path.join(DEPLOY_PATH, 'ppas.txt')
if os.path.exists(ppas_path):
add_apt_repo(*slurp(ppas_path))
apt_update()
packages_path = os.path.join(DEPLOY_PATH, 'packages.txt')
if os.path.exists(packages_path):
apt_get(*slurp(packages_path))
custom_packages_path = os.path.join(DEPLOY_PATH, 'packages')
custom_packages_search = os.path.join(custom_packages_path, '*.deb')
if os.path.exists(custom_packages_path):
with cd('/tmp'):
for i in glob.glob(custom_packages_search):
put(i, os.path.basename(i))
dpkg_install(os.path.basename(i))
run('rm %s' % os.path.basename(i))
easy_install('pip', use_sudo=True)
pip('virtualenv', use_sudo=True)
sudo('mkdir -p /apps')
def _recommission_host(host):
files.uncomment(
'/etc/nginx/sites-available/{}_commcare'.format(env.deploy_env),
'server[ ]+{}'.format(host),
use_sudo=True,
)
_check_and_reload_nginx()
def _setup_base_packages():
files.uncomment("/etc/apt/sources.list", "deb .*universe", use_sudo=True)
sudo("wget -q -ct0 'http://packages.ourserver.com/ubuntu/dists/lucid/ourrepo/ourrepo.list' -O /etc/apt/sources.list.d/ourrepo.list")
sudo("$(apt-key list | grep 'Our Key' > /dev/null) || apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DEADBEEF")
_full_upgrade()
def nginx_setup(secure=False):
uncomment('/etc/nginx/nginx.conf',
'server_names_hash_bucket_size\s+64',
use_sudo=True)
nginx_conf1 = '/etc/nginx/sites-available/%s.conf' % env.webuser
src = 'conf/nginx.{}conf.template'.format('secure.' if secure else '')
upload_template(
src, nginx_conf1,
{
'host': env.vhost,
'redirect_host': env.redirect_host,
'dir': env.code_dir,
'port': env.gunicorn_port,
},
use_sudo=True,
use_jinja=True,
)
run('sudo rm -f /etc/nginx/sites-enabled/default')
nginx_conf2 = '/etc/nginx/sites-enabled/%s.conf' % env.webuser
run('sudo ln -fs %s %s' % (nginx_conf1, nginx_conf2))
run('sudo nginx -t')
run('sudo service nginx start')
run('sudo service nginx reload')
def base():
"Installs & configures Git / EtcKeeper / DenyHosts"
with settings(hide('stdout')):
# Require
require.deb.packages([
'etckeeper',
'git',
'denyhosts',
])
# git
sudo('git config --global user.name "DevOp"')
sudo('git config --global user.email admin@{}'.format(env.host))
# etckeeper
conf_file = '/etc/etckeeper/etckeeper.conf'
comment(conf_file, 'VCS="bzr"', use_sudo=True, backup='')
uncomment(conf_file, 'VCS="git"', use_sudo=True, backup='')
sudo('etckeeper init')
sudo('etckeeper commit "Initial commit."')
# denyhosts
conf_file = '/etc/denyhosts.conf'
comment(conf_file, 'BLOCK_SERVICE\s*=\s*sshd', use_sudo=True, backup='')
uncomment(conf_file, 'BLOCK_SERVICE\s*=\s*ALL', use_sudo=True, backup='')
sed(conf_file, 'DENY_THRESHOLD_INVALID\s*=\s*5'
, 'DENY_THRESHOLD_INVALID = 3', use_sudo=True, backup='')
sed(conf_file, 'DENY_THRESHOLD_VALID\s*=\s*10'
, 'DENY_THRESHOLD_VALID = 3', use_sudo=True, backup='')
sed(conf_file, 'DENY_THRESHOLD_ROOT\s*=\s*1'
, 'DENY_THRESHOLD_ROOT = 3', use_sudo=True, backup='')
sudo('etckeeper commit "Auto-config DenyHosts.conf."')
def install_supervisor():
require("environment", "project_root", "virtualenv_root", "sudo_user", provided_by="setup_env")
# we don't install supervisor in the virtualenv since we want it to be able to run systemwide.
sudo("pip install supervisor" % env, pty=True, shell=True)
# create the standard conf file
sudo("echo_supervisord_conf > /tmp/supervisord.conf" % env)
sudo("mv /tmp/supervisord.conf /etc/supervisord.conf")
# uncomment the include directive in supervisord.conf so we can point it to our supervisor conf
uncomment("/etc/supervisord.conf", "\;\\[include\\]", use_sudo=True, char=";", backup=".bak")
sudo("echo 'files = %(supervisor_conf_root)s/*.conf' >> /etc/supervisord.conf" % env)
init_temp_path = "/tmp/supervisor_init.tmp"
put(env.supervisor_init_path, init_temp_path)
sudo("chown root %s" % init_temp_path)
sudo("chgrp root %s" % init_temp_path)
sudo("chmod +x %s" % init_temp_path)
sudo("mv %s /etc/init.d/supervisord" % init_temp_path)
sudo("chmod +x /etc/init.d/supervisord")
if env.os == "ubuntu":
sudo("update-rc.d supervisord defaults")
elif env.os == "redhat":
sudo("chkconfig --add supervisord")
sudo("service supervisord start")
# update supervisor instance
_supervisor_command("update")
def locales(names):
"""
Require the list of locales to be available.
"""
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True)
append(config_file, locale, use_sudo=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
sudo('dpkg-reconfigure --frontend=noninteractive locales')
def nginx_setup():
nginx_conf1 = '/etc/nginx/sites-available/%s.conf' % env.webuser
nginx_conf2 = '/etc/nginx/sites-enabled/%s.conf' % env.webuser
with cd(env.code_dir):
upload_template('conf/nginx.conf.template',
env.code_dir + 'conf/nginx.conf', {
'host': env.vhost,
'redirect_host': env.redirect_host,
'dir': env.code_dir,
'port': env.gunicorn_port,
},
use_jinja=True)
uncomment('/etc/nginx/nginx.conf',
'server_names_hash_bucket_size\s+64',
use_sudo=True)
# run('sudo rm -f /etc/nginx/sites-enabled/default')
run('sudo ln -fs %sconf/nginx.conf %s' % (env.code_dir, nginx_conf1))
run('sudo ln -fs %s %s' % (nginx_conf1, nginx_conf2))
run('sudo service nginx configtest')
run('sudo service nginx start')
run('sudo service nginx reload')
def install_vpn():
sudo('apt-get -qy install pptpd')
files.append('/etc/pptpd.conf', 'localip 192.168.240.1', use_sudo=True)
files.append('/etc/pptpd.conf', 'remoteip 192.168.240.2-102', use_sudo=True)
files.append('/etc/ppp/pptpd-options', 'ms-dns 8.8.8.8', use_sudo=True)
files.append('/etc/ppp/pptpd-options', 'ms-dns 8.8.4.4', use_sudo=True)
files.append('/etc/ppp/chap-secrets', '%(username)s pptpd %(password)s *' % ({
'username': VPN_USER,
'password': VPN_PASS,
}),
use_sudo=True)
sudo('/etc/init.d/pptpd restart')
files.uncomment('/etc/sysctl.conf', 'net\.ipv4\.ip_forward=1', use_sudo=True)
sudo('sysctl -p')
sudo('iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE')
if not files.contains('/etc/rc.local', 'iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE', use_sudo=True):
files.comment('/etc/rc.local', 'exit 0', use_sudo=True)
files.append('/etc/rc.local', 'iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE', use_sudo=True)
files.append('/etc/rc.local', 'exit 0', use_sudo=True)
def install_passenger():
""" Installs nginx with passenger support """
# add pgp key
print(green('Adding PGP key'))
deb.add_apt_key(keyid='561F9B9CAC40B2F7', keyserver='keyserver.ubuntu.com')
# add https support for apt
utils.deb.install('apt-transport-https')
# ubuntu 12.04 (precise)
cmd = ('echo "deb https://oss-binaries.phusionpassenger.com/apt/passenger '
'precise main" > /etc/apt/sources.list.d/passenger.list')
sudo(cmd)
cmd = 'sudo chmod 600 /etc/apt/sources.list.d/passenger.list'
sudo(cmd)
# update apt index
deb.update_index(quiet=False)
print(green('Installing nginx and passenger'))
utils.deb.install('nginx-full')
utils.deb.install('passenger')
print(green('Activating passenger'))
uncomment('/etc/nginx/nginx.conf', 'passenger_root', use_sudo=True)
uncomment('/etc/nginx/nginx.conf', 'passenger_ruby', use_sudo=True)
def fix_sshd_config():
'''root needs an actual shell, so fix the sshd_config.'''
config_file = '/etc/ssh/sshd_config'
uncomment(config_file, r'^.*PermitRootLogin yes', use_sudo=True)
comment(config_file,
r'^PermitRootLogin forced-commands-only',
use_sudo=True)
def fab_criar_crons(self):
''' Insere tarefa definida em ../cron/cronconf no crontab do servidor '''
crontab_location = '/etc/crontab'
with cd(env.code_root):
if os.path.exists('cron'):
from cron.cronconf import CRONS
import re
sudo('chmod 646 ' + crontab_location)
for cron in CRONS:
if cron['comando_de_projeto'] and not cron['django_management']:
linha_cron = cron['tempo'] + ' ' + cron['usuario'] + ' ' + env.code_root +'/'+ cron['comando']
else:
if cron['comando_de_projeto'] and cron['django_management']:
linha_cron = cron['tempo'] + ' ' + cron['usuario'] + ' /usr/bin/python ' + env.code_root + '/' + cron['comando'] + ' --settings=med_alliance.settings.' + env.ambiente
else:
linha_cron = cron['tempo'] + ' ' + cron['usuario'] + ' ' + cron['comando']
if cron['ligado']:
if not contains(crontab_location, re.escape(linha_cron)):
append(crontab_location, linha_cron, use_sudo=False)
else:
uncomment(crontab_location, re.escape(linha_cron))
else:
if contains(crontab_location, re.escape(linha_cron)):
comment(crontab_location, re.escape(linha_cron))
sudo('chmod 644 ' + crontab_location)
def nginx_setup():
nginx_conf1 = '/etc/nginx/sites-available/%s.conf' % env.webuser
nginx_conf2 = '/etc/nginx/sites-enabled/%s.conf' % env.webuser
with cd(env.code_dir):
upload_template('conf/nginx.conf.template',
env.code_dir + 'conf/nginx.conf',
{
'host': env.vhost,
'redirect_host': env.redirect_host,
'dir': env.code_dir,
'port': env.gunicorn_port,
}, use_jinja=True)
uncomment('/etc/nginx/nginx.conf',
'server_names_hash_bucket_size\s+64',
use_sudo=True)
# run('sudo rm -f /etc/nginx/sites-enabled/default')
run('sudo ln -fs %sconf/nginx.conf %s' % (env.code_dir, nginx_conf1))
run('sudo ln -fs %s %s' % (nginx_conf1, nginx_conf2))
run('sudo service nginx configtest')
run('sudo service nginx start')
run('sudo service nginx reload')
def set_maintanance_mode(self, on=True):
settings_local_path = self.project_path + 'settings_local.py'
if on:
uncomment(settings_local_path, r'MAINTENANCE_MODE.*')
else:
comment(settings_local_path, r'MAINTENANCE_MODE.*')
def setup_demo_config():
"""
Modifies the Eureka instance to comply with the demo instance requirements
--> Memcached + FastCGI + SSL/HTTPS
"""
assert env.hosts
with cd(env.application_dir):
eureka_egg_dir = run(
'./bin/python -c "import pkg_resources; '
'print pkg_resources.get_distribution(\'eureka\').location"')
# ==== HTTPD config ==== #
apache_config = os.path.join(eureka_egg_dir, 'etc', 'httpd.conf')
# HTTPS instead of HTTP
files.sed(
filename=apache_config,
before='<VirtualHost \*:80>',
after='<VirtualHost \*:443>',
)
# Enable SSL (for HTTPS)
files.uncomment(apache_config, 'SSLEngine .*')
with cd('{}/contrib'.format(eureka_egg_dir)):
# Deactivate password change
run('patch --forward "../eureka/domain/models.py" < demo/models_no_pwd_change.diff'
)
# Restore snapshot every night
run('crontab -l | {{ grep -v "restore_snapshot" ; echo "0 0 * * * {}/sh/restore_snapshot.sh"; }} | crontab -'
.format(env.application_dir))
# Restore snapshot
run('./sh/restore_snapshot.sh')
def require_locale(locale, charset):
"""
Set locale
"""
config_file = '/etc/locale.gen'
uncomment(config_file, '%s %s' % (locale, charset))
require.system.default_locale(locale)
run_as_root('locale-gen')
def update():
"""
Updates package list and installs the ones that need updates.
"""
# Activate Ubuntu's "Universe" repositories.
files.uncomment("/etc/apt/sources.list", regex=r"deb.*universe", use_sudo=True)
sudo("apt-get update -y")
sudo("apt-get upgrade -y")
def require_locale(locale, charset):
"""
Set locale
"""
config_file = '/etc/locale.gen'
uncomment(config_file, '%s %s' % (locale, charset))
require.system.default_locale(locale)
run_as_root('locale-gen')
def localesconfig():
"""
Конфигурирование локалей
"""
uncomment('/etc/locale.gen', 'en_US.UTF-8 UTF-8')
uncomment('/etc/locale.gen', 'ru_RU.UTF-8 UTF-8')
run('echo LANG=ru_RU.UTF-8 > /etc/default/locale')
run('dpkg-reconfigure --frontend=noninteractive locales')
def enable_password_authentication():
"""
Enable password authentication
"""
print('Enable password authentications settings')
if contains('/etc/ssh/sshd_config', 'PasswordAuthentication yes'):
uncomment('/etc/ssh/sshd_config',
'PasswordAuthentication yes',
use_sudo=True)
def restrict_ssh(rollback=False):
"""
Set some sensible restrictions in Ubuntu /etc/ssh/sshd_config and restart sshd
UseDNS no #prevents dns spoofing sshd defaults to yes
X11Forwarding no # defaults to no
AuthorizedKeysFile %h/.ssh/authorized_keys
uncomments PasswordAuthentication no and restarts sshd
"""
if not rollback:
if server_state('ssh_restricted'):
print env.host, 'Warning: sshd_config has already been modified. Skipping..'
return False
sshd_config = '/etc/ssh/sshd_config'
if env.verbosity:
print env.host, "RESTRICTING SSH with "+sshd_config
filename = 'sshd_config'
if not exists('/home/%s/.ssh/authorized_keys'% env.user): #do not pass go do not collect $200
print env.host, 'You need to upload_ssh_key first.'
return False
_backup_file(sshd_config)
context = {"HOST_SSH_PORT": env.HOST_SSH_PORT}
upload_template('woven/ssh/sshd_config','/etc/ssh/sshd_config',context=context,use_sudo=True)
# Restart sshd
sudo('/etc/init.d/ssh restart')
# The user can modify the sshd_config file directly but we save
if env.INTERACTIVE and contains('#PasswordAuthentication no','/etc/ssh/sshd_config',use_sudo=True):
c_text = 'Woven will now remove password login from ssh, and use only your ssh key. \n'
c_text = c_text + 'CAUTION: please confirm that you can ssh %s@%s -p%s from a terminal without requiring a password before continuing.\n'% (env.user, env.host, env.port)
c_text += 'If you cannot login, press enter to rollback your sshd_config file'
proceed = confirm(c_text,default=False)
if not env.INTERACTIVE or proceed:
#uncomments PasswordAuthentication no and restarts
uncomment(sshd_config,'#(\s?)PasswordAuthentication(\s*)no',use_sudo=True)
sudo('/etc/init.d/ssh restart')
else: #rollback
print env.host, 'Rolling back sshd_config to default and proceeding without passwordless login'
_restore_file('/etc/ssh/sshd_config', delete_backup=False)
sed('/etc/ssh/sshd_config','Port '+ str(env.DEFAULT_SSH_PORT),'Port '+str(env.HOST_SSH_PORT),use_sudo=True)
sudo('/etc/init.d/ssh restart')
return False
set_server_state('ssh_restricted')
return True
else: #Full rollback
_restore_file('/etc/ssh/sshd_config')
if server_state('ssh_port_changed'):
sed('/etc/ssh/sshd_config','Port '+ str(env.DEFAULT_SSH_PORT),'Port '+str(env.HOST_SSH_PORT),use_sudo=True)
sudo('/etc/init.d/ssh restart')
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_restricted', delete=True)
return True
def upgrade_packages():
"""
Updates package list and upgrades any outdated packages.
"""
# Activate Ubuntu's "Universe" repositories.
files.uncomment('/etc/apt/sources.list', regex=r'deb.*universe',
use_sudo=True)
sudo('apt-get update -y')
sudo('apt-get upgrade -y')
def _hadoop_heap_configure(hadoop_home, new_heap_size=1000, old_heap_size=2000):
"""
Le Quoc Do - SE Group TU Dresden contribution
"""
filename = hadoop_home + '/etc/hadoop/hadoop-env.sh'
before = 'HADOOP_HEAPSIZE=' + str(new_heap_size)
after = 'HADOOP_HEAPSIZE=' + str(old_heap_size)
files.sed(filename, before, after, limit='')
files.uncomment(filename, 'HADOOP_HEAPSIZE')
def debug(param="on"):
"""Toogle DEBUG variable in local_settings.py"""
with cd(env.path):
config_path = '{vhost_path}/config/local_settings.py'.format(**env)
if param == "on":
uncomment(config_path, r'(DEBUG)')
else:
comment(config_path, r'^(DEBUG)')
execute(restart_webserver)
def setup_etckeeper():
if not is_installed('etckeeper'):
sudo('apt-get install git etckeeper -y')
comment('/etc/etckeeper/etckeeper.conf', 'VCS="bzr"', use_sudo=True)
uncomment('/etc/etckeeper/etckeeper.conf', 'VCS="git"', use_sudo=True)
sudo('etckeeper init')
sudo('etckeeper commit -m "Initial commit."')
else:
print("skip, etckeeper is already installed!")
def pg_allow_from(self, ip_ranges, restart=True):
"""Allow external connections from the given IP range."""
self.pg_set_str('listen_addresses', '*')
files.uncomment(self.pg_hba, 'local +replication', use_sudo=True)
for ip_range in ip_ranges:
hostssl_line = 'hostssl all all %s md5' % ip_range
files.append(self.pg_hba, hostssl_line, use_sudo=True)
if restart:
self.pg_cmd('restart')
def create_user(username):
"""fab -H root@host create_user:username"""
password = getpass.getpass('password for %s> ' % username)
run('useradd -m -G www-data -s /bin/bash -p %s %s' % (crypt.crypt(password, 'salt'), username))
local('ssh-copy-id %s' % env.hosts[0])
run('cp /etc/sudoers /tmp/sudoers.new')
append('/tmp/sudoers.new', '%s ALL=(ALL) NOPASSWD:ALL' % username, use_sudo=True)
run('visudo -c -f /tmp/sudoers.new')
run('EDITOR="cp /tmp/sudoers.new" visudo')
uncomment('~%s/.bashrc' % username, '#force_color_prompt=yes')
def _relocate_db():
_run_postgres_command('stop')
postgres_conf = posixpath.join(env.pg_conf_dir, 'postgresql.conf')
uncomment(postgres_conf, 'data_directory', char='#', use_sudo=True)
sed(postgres_conf,'data_directory.*',"""data_directory='%(new_db_location)s'""" % env,use_sudo=True)
init_script = posixpath.join ('/etc','init.d',env.pg_init_name)
sed(init_script,'PGDATA=.*','PGDATA=%(new_db_location)s' % env,use_sudo=True)
sudo('mv %(pg_data_dir)s %(new_db_location)s' % env)
_run_postgres_command('start')
def config_redis():
if can_connect_redis():
return
escaped_socket = REDIS_SOCKET.replace('/', '\/')
uncomment(REDIS_CONF, fr'^#\s*unixsocket {escaped_socket}$', use_sudo=True)
uncomment(REDIS_CONF, r'^#\s*unixsocketperm 700$', use_sudo=True)
sed(REDIS_CONF, 'unixsocketperm 700', 'unixsocketperm 777', use_sudo=True)
sudo('systemctl restart redis-server')
def install_chef_server():
"Installs and configures Chef Server on the controller node"
chef_server_xml = 'files/chef-server.xml'
chef_server_image = 'http://c390813.r13.cf1.rackcdn.com/chef-server.qcow2'
assert os.path.exists(chef_server_xml), 'Chef Server XML definition not found at %s' % chef_server_xml
install_kvm()
sudo('echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections')
sudo('echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections')
sudo('apt-get install -yq iptables-persistent')
puts(green('Downloading Chef Server image from %s' % chef_server_image))
sudo('mkdir /opt/rpcs', warn_only=True)
sudo('wget -qO /opt/rpcs/chef-server.qcow2 %s' % chef_server_image)
sudo('cp -a /opt/rpcs/chef-server.qcow2{,.pristine}')
puts(green('Configuring Chef Server'))
sudo('virsh net-autostart default --disable && virsh net-destroy default && virsh net-undefine default', warn_only=True)
files.uncomment('/etc/sysctl.conf', r'net\.ipv4\.ip_forward=1', use_sudo=True)
sudo('sysctl net.ipv4.ip_forward=1')
put(chef_server_xml, '/opt/rpcs/chef-server.xml', use_sudo=True)
files.append('/etc/network/interfaces', """
auto chefbr0
iface chefbr0 inet static
address 169.254.123.1
netmask 255.255.255.0
bridge_ports none
bridge_fd 0
bridge_stp off
bridge_maxwait 0
""", use_sudo=True)
puts(green('Starting the Chef Server VM'))
sudo('ifup chefbr0')
sudo('virsh define /opt/rpcs/chef-server.xml')
sudo('virsh autostart chef-server && virsh start chef-server')
puts(green('Configuring NAT rules'))
management_network = prompt('What is the MANAGEMENT network range?', default='10.240.0.0/24')
sudo('iptables -t nat -A PREROUTING -s %s -p tcp --dport 4000 -j DNAT --to-dest 169.254.123.2' % management_network)
sudo('iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE')
sudo('service iptables-persistent save')
puts(green('Copying SSH key to Chef Server VM'))
if not files.exists('/root/.ssh/id_rsa', use_sudo=True):
sudo('ssh-keygen -b 2048 -N "" -f /root/.ssh/id_rsa -t rsa -q')
while sudo('ssh-copy-id -i /root/.ssh/id_rsa.pub [email protected]', warn_only=True).return_code is 1:
sleep_time = 10
puts(yellow('Unable to connect to Chef Server VM. Retrying in %ss...' % sleep_time))
time.sleep(sleep_time)
puts(green('Creating Chef validation and controller client keys'))
sudo('mkdir -p /root/.chef', warn_only=True)
sudo('ssh -i /root/.ssh/id_rsa [email protected] "knife client reregister rack -f .chef/rack.pem; knife client reregister chef-validator -f .chef/validation.pem; sudo cp .chef/validation.pem /usr/share/chef-server-api/public; sudo chmod +r /usr/share/chef-server-api/public/validation.pem; yes | knife client delete controller &> /dev/null; knife environment create -d rpcs &>/dev/null; knife client create controller -d -a" | tail -n+2 > /root/.chef/controller.pem')
def pg_allow_replication(self, user, password, ip_ranges, restart=True):
"""Creates a user for replication and enables replication in pg_hba.conf."""
# XXX: does not support differing master/slave pg versions
self.create_db_user(user, password, replication=True)
files.uncomment(self.pg_hba, 'local +replication', use_sudo=True)
for ip_range in ip_ranges:
hostssl_line = 'hostssl replication all %s md5' % ip_range
files.append(self.pg_hba, hostssl_line, use_sudo=True)
if restart:
sudo('service postgresql restart')
def setup_etckeeper():
"""Setup etckeeper tool with git backend"""
if not utils.is_installed('etckeeper'):
utils.apt_get_update()
sudo('apt-get install git etckeeper -y')
comment('/etc/etckeeper/etckeeper.conf', 'VCS="bzr"', use_sudo=True)
uncomment('/etc/etckeeper/etckeeper.conf', 'VCS="git"', use_sudo=True)
sudo('etckeeper init')
sudo('etckeeper commit -m "Initial commit."')
else:
print("skip, etckeeper is already installed!")
def configure_cron():
"""
Configure crontab jobs
"""
print '=== CONFIGURE CRON JOBS ==='
from fabric.contrib.files import uncomment
for language in env.languages:
uncomment(
os.path.join(env.REMOTE_CODEBASE_PATH,
'newsman/config/cron/cron.job'), 'py %s$' % language)
def raid_monitoring(email=None):
"""Configure monitoring of our RAID-1 field. If anything goes wrong,
send an email!"""
opts = dict(email=email or env.get("email") or err("env.email must be set"))
# enable email notifications from mdadm raid monitor
append("/etc/mdadm/mdadm.conf", "MAILADDR %(email)s" % opts, use_sudo=True)
# enable email notification for SMART disk monitoring
sudo("apt-get -yq install smartmontools")
uncomment("/etc/default/smartmontools", "#start_smartd=yes", use_sudo=True)
def enable_key(self, authorized_file, key):
backup = '.%s.bak' % self.conf.current_time
regex = '%s' % re.escape(key)
commented_key = '#' + regex
if files.contains(
authorized_file, commented_key, exact=True, use_sudo=True):
files.uncomment(authorized_file, regex, use_sudo=True,
backup=backup)
else:
files.append(authorized_file, key, use_sudo=True)
def raid_monitoring(email=None):
"""Configure monitoring of our RAID-1 field. If anything goes wrong,
send an email!"""
opts = dict(email=email or env.get('email')
or err('env.email must be set'), )
# enable email notifications from mdadm raid monitor
append('/etc/mdadm/mdadm.conf', 'MAILADDR %(email)s' % opts, use_sudo=True)
# enable email notification for SMART disk monitoring
sudo('apt-get -yq install smartmontools')
uncomment('/etc/default/smartmontools', '#start_smartd=yes', use_sudo=True)
def update():
"""
Updates package list and installs the ones that need updates.
"""
# Activate Ubuntu's "Universe" repositories.
files.uncomment('/etc/apt/sources.list', regex=r'deb.*universe',
use_sudo=True)
print(green('Updating package repositories'))
sudo('apt-get update -y -q=2')
print(green('Upgrading installed packages'))
sudo('DEBIAN_FRONTEND=noninteractive apt-get upgrade '
'-y -q=2 > /dev/null')
def set_rootalias(rootalias, reload_portfix=True):
'''
Set root mail alias
'''
need_sudo = am_not_root()
aliases = '/etc/aliases'
uncomment(aliases, '^root:', use_sudo=need_sudo, backup='.ORIG')
append(aliases, 'root: %s'% rootalias, use_sudo=need_sudo)
_run('newaliases', use_sudo=need_sudo)
if reload_portfix:
_run('service postfix reload', use_sudo=need_sudo)
def update():
"""
Updates package list and installs the ones that need updates.
"""
# Activate Ubuntu's "Universe" repositories.
files.uncomment('/etc/apt/sources.list',
regex=r'deb.*universe',
use_sudo=True)
print(green('Updating package repositories'))
sudo('apt-get update -y -q=2')
print(green('Upgrading installed packages'))
sudo('DEBIAN_FRONTEND=noninteractive apt-get upgrade '
'-y -q=2 > /dev/null')
def create_user(username):
"""fab -H root@host create_user:username"""
password = getpass.getpass('password for %s> ' % username)
run('useradd -m -G www-data -s /bin/bash -p %s %s' %
(crypt.crypt(password, 'salt'), username))
local('ssh-copy-id %s' % env.hosts[0])
run('cp /etc/sudoers /tmp/sudoers.new')
append('/tmp/sudoers.new',
'%s ALL=(ALL) NOPASSWD:ALL' % username,
use_sudo=True)
run('visudo -c -f /tmp/sudoers.new')
run('EDITOR="cp /tmp/sudoers.new" visudo')
uncomment('~%s/.bashrc' % username, '#force_color_prompt=yes')
def update_locale():
"""
Setup en_US.UTF-8 system locale
"""
check_sudo()
check_os()
comment('/etc/locale.gen', r'^[^#]', use_sudo=True)
uncomment('/etc/locale.gen', r'en_US\.UTF\-8', use_sudo=True, backup='')
sudo('locale-gen')
sudo('echo \'LANGUAGE="en_US.UTF-8"\' > /etc/default/locale'
) # will be locale warning. it's ok
sudo('echo \'LANG="en_US.UTF-8"\' >> /etc/default/locale')
sudo('echo \'LC_ALL="en_US.UTF-8"\' >> /etc/default/locale')
disconnect_all()
def restart(full=False):
"""
Reboots neo4j, nginx, and uwsgi.
"""
with virtualenv():
# Only restart neo4j if needed
if full:
local('service neo4j-service stop && service neo4j-service start')
local('service nginx stop && service nginx start')
uncomment('%s/extras/uwsgi/phaidra.ini' % env.directory,
r'daemonize'
)
restart_uwsgi()
def install_rkhunter(email=None):
"""Install and configure RootKit Hunter."""
opts = dict(email=email or env.get('email')
or err('env.email must be set'), )
# install RKHunter
sudo('apt-get -yq install rkhunter')
# send emails on warnings
uncomment('/etc/rkhunter.conf',
'#MAIL-ON-WARNING=me@mydomain root@mydomain',
use_sudo=True)
sed('/etc/rkhunter.conf',
'me@mydomain root@mydomain',
opts['email'],
use_sudo=True)
# ignore some Ubuntu specific files
uncomment('/etc/rkhunter.conf',
'#ALLOWHIDDENDIR=\/dev\/.udev',
use_sudo=True)
uncomment('/etc/rkhunter.conf',
'#ALLOWHIDDENDIR=\/dev\/.static',
use_sudo=True)
uncomment('/etc/rkhunter.conf',
'#ALLOWHIDDENDIR=\/dev\/.initramfs',
use_sudo=True)
# update files properties DB every time you run apt-get install, this
# prevents warnings every time a new version of some package is installed
append('/etc/default/rkhunter',
'# Update file properties database after running apt-get install',
use_sudo=True)
append('/etc/default/rkhunter', 'APT_AUTOGEN="yes"', use_sudo=True)
def install_bare_essentials():
logger = logging.getLogger("%s.install_bare_essentials" % (APP_NAME, ))
logger.debug("entry.")
uncomment(filename=r"/etc/apt/sources.list",
regex="deb http:\/\/archive.canonical.com\/ubuntu.*partner",
use_sudo=True)
uncomment(filename=r"/etc/apt/sources.list",
regex="deb-src http:\/\/archive.canonical.com\/ubuntu.*partner",
use_sudo=True)
sudo("apt-get update")
sudo("yes yes | apt-get upgrade")
sudo(
"yes yes | apt-get install git mercurial build-essential unzip python-software-properties ruby curl python-dev htop vim vim-nox"
)
def base():
"Installs & configures Git / EtcKeeper / DenyHosts"
with settings(hide('stdout')):
# Require
require.deb.packages([
'etckeeper',
'git',
'denyhosts',
])
# git
sudo('git config --global user.name "DevOp"')
sudo('git config --global user.email admin@{}'.format(env.host))
# etckeeper
conf_file = '/etc/etckeeper/etckeeper.conf'
comment(conf_file, 'VCS="bzr"', use_sudo=True, backup='')
uncomment(conf_file, 'VCS="git"', use_sudo=True, backup='')
sudo('etckeeper init')
sudo('etckeeper commit "Initial commit."')
# denyhosts
conf_file = '/etc/denyhosts.conf'
comment(conf_file,
'BLOCK_SERVICE\s*=\s*sshd',
use_sudo=True,
backup='')
uncomment(conf_file,
'BLOCK_SERVICE\s*=\s*ALL',
use_sudo=True,
backup='')
sed(conf_file,
'DENY_THRESHOLD_INVALID\s*=\s*5',
'DENY_THRESHOLD_INVALID = 3',
use_sudo=True,
backup='')
sed(conf_file,
'DENY_THRESHOLD_VALID\s*=\s*10',
'DENY_THRESHOLD_VALID = 3',
use_sudo=True,
backup='')
sed(conf_file,
'DENY_THRESHOLD_ROOT\s*=\s*1',
'DENY_THRESHOLD_ROOT = 3',
use_sudo=True,
backup='')
sudo('etckeeper commit "Auto-config DenyHosts.conf."')
def restrict_ssh(rollback=False):
"""
Set some sensible restrictions in Ubuntu /etc/ssh/sshd_config and restart sshd
UseDNS no #prevents dns spoofing sshd defaults to yes
X11Forwarding no # defaults to no
AuthorizedKeysFile %h/.ssh/authorized_keys
uncomments PasswordAuthentication no and restarts sshd
"""
if not rollback:
if server_state('ssh_restricted'):
return False
sshd_config = '/etc/ssh/sshd_config'
if env.verbosity:
print env.host, "RESTRICTING SSH with "+sshd_config
filename = 'sshd_config'
if not exists('/home/%s/.ssh/authorized_keys'% env.user): #do not pass go do not collect $200
print env.host, 'You need to upload_ssh_key first.'
return False
_backup_file(sshd_config)
context = {"HOST_SSH_PORT": env.HOST_SSH_PORT}
upload_template('woven/ssh/sshd_config','/etc/ssh/sshd_config',context=context,use_sudo=True)
# Restart sshd
sudo('/etc/init.d/ssh restart')
# The user can modify the sshd_config file directly but we save
proceed = True
if not env.key_filename and (env.DISABLE_SSH_PASSWORD or env.INTERACTIVE) and contains('/etc/ssh/sshd_config','#PasswordAuthentication no',use_sudo=True):
print "WARNING: You may want to test your node ssh login at this point ssh %s@%s -p%s"% (env.user, env.host, env.port)
c_text = 'Would you like to disable password login and use only ssh key authentication'
proceed = confirm(c_text,default=False)
if not env.INTERACTIVE or proceed or env.DISABLE_SSH_PASSWORD:
#uncomments PasswordAuthentication no and restarts
uncomment(sshd_config,'#(\s?)PasswordAuthentication(\s*)no',use_sudo=True)
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_restricted')
return True
else: #Full rollback
_restore_file('/etc/ssh/sshd_config')
if server_state('ssh_port_changed'):
sed('/etc/ssh/sshd_config','Port '+ str(env.DEFAULT_SSH_PORT),'Port '+str(env.HOST_SSH_PORT),use_sudo=True)
sudo('/etc/init.d/ssh restart')
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_restricted', delete=True)
return True
def add(user, pub_id_rsa_file):
with settings(user='******'):
if exists('/home/{}'.format(user)):
run('deluser --remove-home --quiet {}'.format(user))
adduser_flags = (
'--disabled-password',
'--gecos "{} user,,,"'.format(user),
'--quiet',
)
run('adduser {} {}'.format(' '.join(adduser_flags), user))
run('mkdir --mode=0700 /home/{}/.ssh'.format(user))
put(pub_id_rsa_file, '/home/{}/.ssh/authorized_keys'.format(user))
run('chown --recursive {0}: /home/{0}/.ssh'.format(user))
with settings(user=user):
uncomment('.profile', 'umask 022', backup='')
def enable_key(self, authorized_file, key):
backup = '.%s.bak' % self.conf.current_time
regex = '%s' % re.escape(key)
commented_key = '#' + regex
if files.contains(authorized_file,
commented_key,
exact=True,
use_sudo=True):
files.uncomment(authorized_file,
regex,
use_sudo=True,
backup=backup)
else:
files.append(authorized_file, key, use_sudo=True)
def postgres_conf():
uncomment('/etc/postgresql/9.1/main/postgresql.conf',
'listen_addresses',
use_sudo=True,
char='#',
backup='.bak')
sudo('passwd postgres')
with sudo('su - postgres'):
run('psql')
run('CREATE EXTENSION adminpack;')
#
# get out of postgresq shell here
#
run('createuser ' + KEY)
run('exit')
run('createdb ' + KEY)
def configure_settings():
"""
Modify settings file according to the server
"""
print '=== CONFIGURE REPO SETTINGS ==='
from fabric.contrib.files import uncomment
uncomment(
os.path.join(env.REMOTE_CODEBASE_PATH, 'newsman/config/settings.py'),
env.host)
uncomment(
os.path.join(env.REMOTE_CODEBASE_PATH, 'newsman/config/settings.py'),
env.user)
run("cp %s %s" %
(os.path.join(env.REMOTE_CODEBASE_PATH, 'newsman/config/settings.py'),
os.path.join(env.REMOTE_CODEBASE_PATH, 'newsman/public/settings.py')))
def _relocate_db():
_run_postgres_command('stop')
postgres_conf = posixpath.join(env.pg_conf_dir, 'postgresql.conf')
uncomment(postgres_conf, 'data_directory', char='#', use_sudo=True)
sed(postgres_conf,
'data_directory.*',
"""data_directory='%(new_db_location)s'""" % env,
use_sudo=True)
init_script = posixpath.join('/etc', 'init.d', env.pg_init_name)
sed(init_script,
'PGDATA=.*',
'PGDATA=%(new_db_location)s' % env,
use_sudo=True)
sudo('mv %(pg_data_dir)s %(new_db_location)s' % env)
_run_postgres_command('start')