def generate_virtualenv(connection, site=None, python_bin="python"):
"""
Creates or rebuilds a site's virtualenv.
@@@TODO muliple envs for one site, aka predictable rollbacks.
Usage:
fab generate_virtualenv:dev,sitename
fab appnode generate_virtualenv:prod,sitename
"""
if site is None:
sites = [site["name"] for site in settings.SITES]
else:
sites = [site]
for site in sites:
if connection == "dev" and not contains(
"{0}{1}".format(settings.SITES_ROOT, "virtualenv_dev.log"),
site):
log_name = "virtualenv_dev.log"
local("rm -rf {0}{1}/".format(settings.VIRTUALENVS_LOCAL_ROOT,
site))
with lcd(settings.VIRTUALENVS_LOCAL_ROOT):
local(
"virtualenv --no-site-packages --distribute --python={0} {1}"
.format(python_bin, site))
with lcd("{0}{1}".format(settings.SITES_LOCAL_ROOT, site)):
local("ln -sf {0}{1}/lib/python{2}/site-packages".format(
settings.VIRTUALENVS_LOCAL_ROOT, site,
settings.PYTHON_VERSION))
local("echo 'cd {0}{1}/{1}' >> {2}{1}/bin/postactivate".format(
settings.SITES_LOCAL_ROOT, site,
settings.VIRTUALENVS_LOCAL_ROOT))
symlink_packages("dev", site)
pip_requirements("dev", site)
elif connection == "prod" and not contains(
"{0}{1}".format(settings.SITES_ROOT, "virtualenv_prod.log"),
site):
log_name = "virtualenv_prod.log"
run("rm -rf {0}{1}/".format(settings.VIRTUALENVS_ROOT, site))
with cd(settings.VIRTUALENVS_ROOT):
run("virtualenv --no-site-packages --distribute {0}".format(
site))
if not exists("{0}{1}/site-packages".format(
settings.SITES_ROOT, site)):
with cd("{0}{1}".format(settings.SITES_ROOT, site)):
run("ln -sf {0}{1}/lib/python{2}/site-packages".format(
settings.VIRTUALENVS_ROOT, site,
settings.PYTHON_VERSION))
append(
"{0}{1}/bin/postactivate".format(settings.VIRTUALENVS_ROOT,
site), "{0}{1}/{1}".format(
settings.SITES_ROOT,
site,
))
symlink_packages("prod", site)
pip_requirements("prod", site)
else:
print("Bad connection type. Use ``dev`` or ``prod``.")
append("{0}{1}".format(settings.SITES_ROOT, log_name), site)
print(green("sucessfully built virtualenv for for {0}".format(site)))
run("rm {0}{1}".format(settings.SITES_ROOT, "virtualenv.log"))
print(green("sucessfully built virtualenvs for all sites!".format(site)))
def _update_settings(source_folder, site_name, user, db_name, db_user, db_pass,
db_host, db_port):
virtualenv_folder = '/home/%s/.virtualenvs/tdd/bin/' % (user, )
postactivate = virtualenv_folder + 'postactivate'
predeactivate = virtualenv_folder + 'predeactivate'
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
key = ''.join(random.SystemRandom().choice(chars) for _ in range(50))
if not contains(postactivate, "DJANGO_SECRET_KEY"):
append(postactivate, "export DJANGO_SECRET_KEY='%s'" % (key, ))
append(postactivate, "export DJANGO_DEBUG=False")
append(postactivate, "export DB_NAME=%s" % (db_name, ))
append(postactivate, "export DB_USER=%s" % (db_user, ))
append(postactivate, "export DB_PASSWORD=%s" % (db_pass, ))
append(postactivate, "export DB_HOST=%s" % (db_host, ))
append(postactivate, "export DB_PORT=%s" % (db_port, ))
if not contains(predeactivate, "DJANGO_SECRET_KEY"):
append(predeactivate, "unset DJANGO_SECRET_KEY")
append(predeactivate, "unset DJANGO_DEBUG")
append(predeactivate, "unset DB_NAME")
append(predeactivate, "unset DB_USER")
append(predeactivate, "unset DB_PASSWORD")
append(predeactivate, "unset DB_HOST")
append(predeactivate, "unset DB_PORT")
settings_path = source_folder + '/superlists/settings.py'
sed(settings_path, 'DOMAIN = "localhost"', 'DOMAIN = "%s"' % (site_name, ))
sed(settings_path, 'SECURE_SSL_REDIRECT = False',
'SECURE_SSL_REDIRECT = True')
sed(settings_path, 'SESSION_COOKIE_SECURE = False',
'SESSION_COOKIE_SECURE = True')
sed(settings_path, 'CSRF_COOKIE_SECURE = False',
'CSRF_COOKIE_SECURE = True')
def install_puppetmaster():
install_puppetbase()
sudo("apt-get install -y puppetmaster-passenger")
sudo("apt-get install -y git")
sudo("mkdir -p /opt")
if not contains("/etc/puppet/puppet.conf", "manifest =", use_sudo=True):
append(
"/etc/puppet/puppet.conf",
"manifest = $confdir/environments/$environment/manifests/site.pp",
use_sudo=True)
if not contains("/etc/puppet/puppet.conf", "modulepath =", use_sudo=True):
append("/etc/puppet/puppet.conf",
"modulepath = $confdir/environments/$environment/modules",
use_sudo=True)
if not contains("/etc/puppet/puppet.conf", "hiera_config =",
use_sudo=True):
append("/etc/puppet/puppet.conf",
"hiera_config = $confdir/hiera.yaml",
use_sudo=True)
if not exists("/etc/puppet/autosign.conf", use_sudo=True):
append("/etc/puppet/autosign.conf",
'*.%s.compute.internal' % region,
use_sudo=True)
if exists("/opt/dose"):
update_puppetmaster()
else:
with cd("/opt"):
sudo("git clone https://github.com/leanovate/dose.git")
sudo("cp /opt/dose/vagrant/hiera/hiera.yaml /etc/puppet")
sudo("ln -s /opt/dose/vagrant/hiera /etc/puppet/hiera")
if not exists("/etc/puppet/environments/microzon"):
sudo("mkdir -p /etc/puppet/environments/microzon")
with cd("/etc/puppet/environments/microzon"):
sudo("ln -s /opt/dose/vagrant/modules")
sudo("ln -s /opt/dose/vagrant/manifests")
def _develop():
"""
Provision the environment for ecalendar
"""
if not contains('/etc/default/locale', 'en_US.UTF-8'):
sed('/etc/default/locale', 'en_US', 'en_US.UTF-8', use_sudo=True, shell=True)
# sources.list
if not contains('/etc/apt/sources.list', 'mirrors.163'):
sed('/etc/apt/sources.list', 'us\.archive\.ubuntu', 'mirrors.163', use_sudo=True, shell=True)
deb.update_index(quiet=False)
require.mysql.server(password='******')
# build-essential
require.deb.packages([
'libxml2-dev',
'libxslt1-dev',
'build-essential',
'python-dev',
'cmake',
'libmysqlclient-dev',
'libssl-dev'
])
require.deb.packages([
'python-distribute',
'python-pip'
])
with cd ('/vagrant'):
sudo('easy_install -U distribute')
sudo('python setup.py develop', shell=True)
def postgresql_net_access():
with cd('/etc/postgresql/8.4/main/'):
change = False
if not contains(
'pg_hba.conf',
'local all all password',
use_sudo=True):
sudo(
"sed -i 's/local all all ident/local all all password/g' pg_hba.conf",
user='******')
change = True
print(green('Accés local PostgreSQL pour Django'))
if not contains('pg_hba.conf', '0.0.0.0'):
# FIX : teste rate toujours , pourquoi ? - mais pas de double écriture grace à append()
append('pg_hba.conf',
'host all all 0.0.0.0/0 md5',
use_sudo=True)
change = True
print(green('Accés à PostgreSQL via interfaces IP externes'))
if not contains(
'postgresql.conf', 'listen_addresses = \'\*\'', use_sudo=True):
sudo(
'echo "listen_addresses = \'*\'"|cat - postgresql.conf > /tmp/out && mv /tmp/out postgresql.conf'
)
change = True
print(green('PostgreSQL écoute sur toutes les interfaces IP'))
if change: sudo('/etc/init.d/postgresql restart')
def disable_password_authentication():
""" Disables password authentication. """
configuration_file = '/etc/ssh/sshd_config'
# ensure pubkey authentication is enabled
if not contains(configuration_file, '^PubkeyAuthentication yes',
escape=False):
# patterns
before = '^#?PubkeyAuthentication.*$'
after = 'PubkeyAuthentication yes'
sed(configuration_file, before, after, use_sudo=True)
print(green('Pubkey authentication enabled.'))
else:
print(green('Pubkey authentication already enabled.'))
# disable password authentication
if not contains(configuration_file, '^PasswordAuthentication no',
escape=False):
# patterns
before = '^#?PasswordAuthentication.*$'
after = 'PasswordAuthentication no'
sed(configuration_file, before, after, use_sudo=True)
print(green('Password authentication disabled.'))
else:
print(green('Password authentication already disabled.'))
def apache_nginx():
'''Apache + mod_wsgi pour Django avec Nginx en proxy'''
icanhaz.deb.packages(['apache2', 'libapache2-mod-wsgi'])
with cd('/etc/apache2/'):
if not contains('ports.conf', '127.0.0.1', use_sudo=True):
sed('ports.conf',
'NameVirtualHost \\*:80',
'NameVirtualHost 127.0.0.1:80',
use_sudo=True)
sed('ports.conf',
'Listen 80',
'Listen 127.0.0.1:80',
use_sudo=True)
print(green('/etc/apache2/ports.conf updated'))
with cd('/etc/apache2/'):
if not contains('apache2.conf', 'ServerName localhost', use_sudo=True):
sudo(
"echo 'ServerName localhost'|cat - apache2.conf > /tmp/out && mv /tmp/out apache2.conf"
)
sudo("sed -i 's/KeepAlive On/KeepAlive Off/g' apache2.conf")
print(green('/etc/apache2/apache2.conf updated'))
#sudo("apache2ctl graceful") #plante de toute façon sans virtualhosts
pretty_apt(['nginx'])
with cd('/etc/nginx/'):
if not contains('nginx.conf', 'worker_processes 2;', use_sudo=True):
sudo(
"sed -i 's/worker_processes 4;/worker_processes 2;/g' nginx.conf"
)
print(green('/etc/nginx/nginx.conf updated'))
def _update_settings(source_folder, site_name,
user, db_name,
db_user, db_pass,
db_host, db_port):
virtualenv_folder = '/home/%s/.virtualenvs/tdd/bin/' % (user,)
postactivate = virtualenv_folder + 'postactivate'
predeactivate = virtualenv_folder + 'predeactivate'
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
key = ''.join(random.SystemRandom().choice(chars) for _ in range(50))
if not contains(postactivate, "DJANGO_SECRET_KEY"):
append(postactivate, "export DJANGO_SECRET_KEY='%s'" % (key,))
append(postactivate, "export DJANGO_DEBUG=False")
append(postactivate, "export DB_NAME=%s" % (db_name,))
append(postactivate, "export DB_USER=%s" % (db_user,))
append(postactivate, "export DB_PASSWORD=%s" % (db_pass,))
append(postactivate, "export DB_HOST=%s" % (db_host,))
append(postactivate, "export DB_PORT=%s" % (db_port,))
if not contains(predeactivate, "DJANGO_SECRET_KEY"):
append(predeactivate, "unset DJANGO_SECRET_KEY")
append(predeactivate, "unset DJANGO_DEBUG")
append(predeactivate, "unset DB_NAME")
append(predeactivate, "unset DB_USER")
append(predeactivate, "unset DB_PASSWORD")
append(predeactivate, "unset DB_HOST")
append(predeactivate, "unset DB_PORT")
settings_path = source_folder + '/superlists/settings.py'
sed(settings_path, 'DOMAIN = "localhost"', 'DOMAIN = "%s"' % (site_name,))
sed(settings_path, 'SECURE_SSL_REDIRECT = False', 'SECURE_SSL_REDIRECT = True')
sed(settings_path, 'SESSION_COOKIE_SECURE = False', 'SESSION_COOKIE_SECURE = True')
sed(settings_path, 'CSRF_COOKIE_SECURE = False', 'CSRF_COOKIE_SECURE = True')
def fab_criar_crons(self):
''' Insere tarefa definida em ../cron/cronconf no crontab do servidor '''
crontab_location = '/etc/crontab'
with cd(env.code_root):
if os.path.exists('cron'):
from cron.cronconf import CRONS
import re
sudo('chmod 646 ' + crontab_location)
for cron in CRONS:
if cron['comando_de_projeto'] and not cron['django_management']:
linha_cron = cron['tempo'] + ' ' + cron['usuario'] + ' ' + env.code_root +'/'+ cron['comando']
else:
if cron['comando_de_projeto'] and cron['django_management']:
linha_cron = cron['tempo'] + ' ' + cron['usuario'] + ' /usr/bin/python ' + env.code_root + '/' + cron['comando'] + ' --settings=med_alliance.settings.' + env.ambiente
else:
linha_cron = cron['tempo'] + ' ' + cron['usuario'] + ' ' + cron['comando']
if cron['ligado']:
if not contains(crontab_location, re.escape(linha_cron)):
append(crontab_location, linha_cron, use_sudo=False)
else:
uncomment(crontab_location, re.escape(linha_cron))
else:
if contains(crontab_location, re.escape(linha_cron)):
comment(crontab_location, re.escape(linha_cron))
sudo('chmod 644 ' + crontab_location)
def setup_server(mirror=''):
project_settings = get_settings()
projects = build_projects_vars()
project = projects[
'development'] # could use any environment as key user is always the same
if mirror == 'y':
mirror_url = project_settings.MIRROR_URL
else:
mirror_url = ''
for p in project_settings.UBUNTU_PACKAGES:
sudo('apt-get -y install %s' % p)
sudo('pip install pip --upgrade %s' % mirror_url)
for p in project_settings.PIP_PACKAGES:
sudo('sudo pip install %s %s' % (p, mirror_url))
# fixes Warning: cannot find svn location for distribute==0.6.16dev-r0
sudo('pip install distribute --upgrade %s' % mirror_url)
fix_venv_permission()
for file in ('.bash_profile', '.bashrc'):
if not contains('/home/%s/%s' %
(project['user'], file), 'export WORKON_HOME'):
run('echo "export WORKON_HOME=$HOME/.virtualenvs" >> /home/%s/%s' %
(project['user'], file))
if not contains('/home/%s/%s' % (project['user'], file),
'source /usr/local/bin/virtualenvwrapper.sh'):
run('echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/%s/%s'
% (project['user'], file))
def disable_password_authentication():
""" Disables password authentication. """
configuration_file = '/etc/ssh/sshd_config'
# ensure pubkey authentication is enabled
if not contains(
configuration_file, '^PubkeyAuthentication yes', escape=False):
# patterns
before = '^#?PubkeyAuthentication.*$'
after = 'PubkeyAuthentication yes'
sed(configuration_file, before, after, use_sudo=True)
print(green('Pubkey authentication enabled.'))
else:
print(green('Pubkey authentication already enabled.'))
# disable password authentication
if not contains(
configuration_file, '^PasswordAuthentication no', escape=False):
# patterns
before = '^#?PasswordAuthentication.*$'
after = 'PasswordAuthentication no'
sed(configuration_file, before, after, use_sudo=True)
print(green('Password authentication disabled.'))
else:
print(green('Password authentication already disabled.'))
def install_puppetmaster():
install_puppetbase()
sudo("apt-get install -y puppetmaster-passenger")
sudo("apt-get install -y git")
sudo("mkdir -p /opt")
if not contains("/etc/puppet/puppet.conf", "manifest =", use_sudo=True):
append("/etc/puppet/puppet.conf", "manifest = $confdir/environments/$environment/manifests/site.pp", use_sudo=True)
if not contains("/etc/puppet/puppet.conf", "modulepath =", use_sudo=True):
append("/etc/puppet/puppet.conf", "modulepath = $confdir/environments/$environment/modules", use_sudo=True)
if not contains("/etc/puppet/puppet.conf", "hiera_config =", use_sudo=True):
append("/etc/puppet/puppet.conf", "hiera_config = $confdir/hiera.yaml", use_sudo=True)
if not exists("/etc/puppet/autosign.conf", use_sudo=True):
append("/etc/puppet/autosign.conf", '*.%s.compute.internal' % region, use_sudo=True)
if exists("/opt/dose"):
update_puppetmaster()
else:
with cd("/opt"):
sudo("git clone https://github.com/leanovate/dose.git")
sudo("cp /opt/dose/vagrant/hiera/hiera.yaml /etc/puppet")
sudo("ln -s /opt/dose/vagrant/hiera /etc/puppet/hiera")
if not exists("/etc/puppet/environments/microzon"):
sudo("mkdir -p /etc/puppet/environments/microzon")
with cd("/etc/puppet/environments/microzon"):
sudo("ln -s /opt/dose/vagrant/modules")
sudo("ln -s /opt/dose/vagrant/manifests")
def _check_remote_keys():
local_home_dir = local('echo $HOME', capture=True)
local_ssh_dir = os.path.join(local_home_dir, '.ssh')
local_public_key_path = os.path.join(local_ssh_dir, 'id_rsa.pub')
local_private_key_path = os.path.join(local_ssh_dir, 'id_rsa')
remote_home_dir = run('echo $HOME')
remote_ssh_dir = os.path.join(remote_home_dir, '.ssh')
remote_public_key_path = os.path.join(remote_ssh_dir, 'id_rsa.pub')
remote_private_key_path = os.path.join(remote_ssh_dir, 'id_rsa')
remote_private_known_hosts_path = os.path.join(remote_ssh_dir, 'known_hosts')
if not exists(remote_ssh_dir):
_debug('Creading remote dir {}...'.format(remote_ssh_dir))
run('mkdir -p {}'.format(remote_ssh_dir))
_debug('Creating empty file {}...'.format(remote_private_known_hosts_path))
run('touch {}'.format(remote_private_known_hosts_path))
with cd(remote_ssh_dir):
public_key = open(local_public_key_path, 'r').read()
private_key = open(local_private_key_path, 'r').read()
_debug('Checking if public key is in file {}...'.format(remote_public_key_path))
if not contains(remote_public_key_path, public_key):
_debug('Appending public key in file {}...'.format(remote_public_key_path))
append(remote_public_key_path, public_key)
_debug('Checking if private key is in file {}...'.format(remote_private_key_path))
if not contains(remote_private_key_path, private_key):
_debug('Appending private key in file {}...'.format(remote_private_key_path))
append(remote_private_key_path, private_key)
run('chmod 644 {}'.format(remote_public_key_path))
run('chmod 600 {}'.format(remote_private_key_path))
_debug('Checking if {} is in file {}...'.format(env.hosts[0], remote_private_known_hosts_path))
if not contains(remote_private_known_hosts_path, env.hosts[0]):
_debug('Appending {} in file {}...'.format(env.hosts[0], remote_private_known_hosts_path))
run('ssh-keyscan {} >> {}'.format(env.hosts[0], remote_private_known_hosts_path))
def _execute_aptget():
with cd('/'):
if not exists('/swap.img'):
run('apt-get update')
run('apt-get -y install python-pip')
run('pip install virtualenv virtualenvwrapper')
run('apt-get -y install python3 python3-pip build-essential python3-dev git nginx supervisor libncurses5-dev')
run('apt-get -y install vim')
run('apt-get -y install libjpeg62-turbo-dev libfreetype6-dev libtiff5-dev liblcms2-dev libwebp-dev tk8.6-dev libjpeg-dev')
run('apt-get -y install wkhtmltopdf xvfb')
run('apt-get -y install htop')
if not contains('/etc/security/limits.conf', '65536'):
# print LIMITS_FILE_CONTENT
append('/etc/security/limits.conf', LIMITS_FILE_CONTENT)
run('pip3 install --upgrade pip')
if not contains('/root/.bashrc', 'WORKON_HOME'):
# print BASHRC_FILE_CONTENT
append('/root/.bashrc', BASHRC_FILE_CONTENT)
if not exists('/swap.img'):
run('lsb_release -a')
run('dd if=/dev/zero of=/swap.img bs=1024k count=2000')
run('mkswap /swap.img')
run('swapon /swap.img')
run('echo "/swap.img none swap sw 0 0" >> /etc/fstab')
def setup_server(mirror=''):
project_settings = get_settings()
projects = build_projects_vars()
project = projects['development'] # could use any environment as key user is always the same
if mirror == 'y':
mirror_url = project_settings.MIRROR_URL
else:
mirror_url = ''
for p in project_settings.UBUNTU_PACKAGES:
sudo('apt-get -y install %s' % p)
sudo('pip install pip --upgrade %s' % mirror_url)
for p in project_settings.PIP_PACKAGES:
sudo('sudo pip install %s %s' % (p, mirror_url))
# fixes Warning: cannot find svn location for distribute==0.6.16dev-r0
sudo('pip install distribute --upgrade %s' % mirror_url)
fix_venv_permission()
for file in ('.bash_profile', '.bashrc'):
if not contains('/home/%s/%s' % (project['user'], file), 'export WORKON_HOME'):
run('echo "export WORKON_HOME=$HOME/.virtualenvs" >> /home/%s/%s' % (project['user'], file))
if not contains('/home/%s/%s' % (project['user'], file), 'source /usr/local/bin/virtualenvwrapper.sh'):
run('echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/%s/%s' % (project['user'], file))
def _set_multi_value_configs(file_name, edits_list):
"""
Edits MultiStrOpt entries in a config file. Currently only supports adding
a new parameter.
:param file_name: the config file name
:param edits_list: the list of edits to make
:return: None
"""
from fabric.contrib.files import contains, sed
if exists(file_name, use_sudo=True, verbose=True):
print(green("\tEditing %s" % file_name))
for config_entry in edits_list:
# hopefully match all forms of key = [other_val] val [other_val]
# while avoiding key = [other_val] xvalx [other_val]
# pylint: disable=W1401
empty_setting_regex = '^%s[\s]*=[\s]*$' % \
(config_entry['parameter'])
setting_regex = '^[\s]*%s[\s]*=.*(?<=\s|=)%s(?!\S).*$' % \
(config_entry['parameter'], config_entry['value'])
empty_setting_exists = contains(file_name,
empty_setting_regex,
escape=False)
setting_exists = contains(file_name, setting_regex, escape=False)
if not setting_exists and empty_setting_exists:
print(
green("\tReplacing empty %s entry" %
(config_entry['parameter'])))
sed(file_name,
'^%s[\s]*=[\s]*$' % (config_entry['parameter']),
'%s = %s' %
(config_entry['parameter'], config_entry['value']),
backup='.gsbak')
# we have our own backup, so delete the one that sed made
run("rm %s.gsbak" % file_name)
elif not setting_exists:
# add a new line to the appropriate section
print(
green("\tAdding new %s entry" %
(config_entry['parameter'])))
sed(file_name,
'^\[%s\][\s]*$' % (config_entry['section']),
'\[%s\]\\n%s = %s' %
(config_entry['section'], config_entry['parameter'],
config_entry['value']),
backup='.gsbak')
# we have our own backup, so delete the one that sed made
run("rm %s.gsbak" % file_name)
else:
print(
green("\tNo changes required for %s" %
(config_entry['parameter'])))
else:
raise IOError("File not found: %s" % file_name)
def test_enable_password_auth(sshd_config):
from fabtools.ssh import enable_password_auth
enable_password_auth(sshd_config=SSHD_CONFIG)
with quiet():
assert contains(SSHD_CONFIG, 'PasswordAuthentication yes', exact=True)
assert not contains(SSHD_CONFIG, 'PasswordAuthentication no', exact=True)
def _sysctl():
path = '/etc/sysctl.conf'
if not contains(path, 'vm.overcommit_memory = 1'):
sudo('echo -e "vm.overcommit_memory = 1" >> %s' % path)
if not contains(path, 'net.core.somaxconn = 65535'):
sudo('echo -e "net.core.somaxconn = 65535" >> %s' % path)
if not contains(path, 'fs.file-max = 6553560'):
sudo('echo -e "fs.file-max = 6553560" >> %s' % path)
def _set_multi_value_configs(file_name, edits_list):
"""
Edits MultiStrOpt entries in a config file. Currently only supports adding
a new parameter.
:param file_name: the config file name
:param edits_list: the list of edits to make
:return: None
"""
import re
from fabric.contrib.files import sed, contains
if exists(file_name, use_sudo=True, verbose=True):
print(green("\tEditing %s" % file_name))
for config_entry in edits_list:
# hopefully match all forms of key = [other_val] val [other_val]
# while avoiding key = [other_val] xvalx [other_val]
# pylint: disable=W1401
empty_setting_regex = '^\<%s\>[:space:]*=[:space]*$' % \
(config_entry['parameter'])
setting_regex = '^\<%s\>[ \t]*=.*\<%s\>.*$' % \
(config_entry['parameter'], config_entry['value'])
empty_setting_exists = contains(
file_name, empty_setting_regex, escape=False)
setting_exists = contains(
file_name, setting_regex, escape=False)
if not setting_exists and empty_setting_exists:
print(green("\tReplacing empty %s entry" %
(config_entry['parameter'])))
sed(file_name,
'^%s[\s]*=[\s]*$' % (config_entry['parameter']),
'%s = %s' % (config_entry['parameter'],
config_entry['value']),
backup='.gsbak')
# we have our own backup, so delete the one that sed made
run("rm %s.gsbak" % file_name)
elif not setting_exists:
# add a new line to the appropriate section
print(green("\tAdding new %s entry" %
(config_entry['parameter'])))
sed(file_name,
'^\[%s\][\s]*$' % (config_entry['section']),
'\[%s\]\\n%s = %s' % (config_entry['section'],
config_entry['parameter'],
config_entry['value']),
backup='.gsbak')
# we have our own backup, so delete the one that sed made
run("rm %s.gsbak" % file_name)
else:
print(green("\tNo changes required for %s" %
(config_entry['parameter'])))
else:
raise IOError("File not found: %s" % file_name)
def setup_website():
"""Initialise or update the git clone.
e.g. to update the server
fab -H 10.1.1.0:8697 remote setup_website
or if you have configured env.hosts, simply
fab remote setup_website
"""
_all()
fabtools.require.deb.package('libapache2-mod-wsgi')
# Find out if the wsgi user exists and create it if needed e.g.
fabtools.require.user(
env.wsgi_user,
create_group=env.wsgi_user,
system=True,
comment='System user for running the wsgi process under')
if not exists(env.webdir):
sudo('mkdir -p %s' % env.plugin_repo_path)
sudo('chown %s.%s %s' % (env.user, env.user, env.webdir))
# Clone and replace tokens in apache conf
conf_file = ('%s/resources/server_config/apache/%s.apache.conf' %
(env.code_path, env.repo_alias))
run('cp %(conf_file)s.templ %(conf_file)s' % {'conf_file': conf_file})
replace_tokens(conf_file)
with cd('/etc/apache2/sites-available/'):
if exists('%s.apache.conf' % env.repo_alias):
sudo('a2dissite %s.apache.conf' % env.repo_alias)
fastprint('Removing old apache2 conf', False)
sudo('rm %s.apache.conf' % env.repo_alias)
sudo('ln -s %s .' % conf_file)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, env.repo_site_name):
append(hosts, '127.0.0.1 %s' % env.repo_site_name, use_sudo=True)
if not contains(hosts, 'www.' + env.repo_site_name):
append(hosts,
'127.0.0.1 %s' % 'www.' + env.repo_site_name,
use_sudo=True)
# Make sure mod rewrite is enabled
sudo('a2enmod rewrite')
# Enable the vhost configuration
sudo('a2ensite %s.apache.conf' % env.repo_alias)
# Check if apache configs are ok - script will abort if not ok
sudo('/usr/sbin/apache2ctl configtest')
sudo('a2dissite default')
fabtools.require.service.restarted('apache2')
def test_disable_root_login(sshd_config):
from burlap.ssh import disable_root_login
disable_root_login(sshd_config=SSHD_CONFIG)
with quiet():
assert contains(SSHD_CONFIG, 'PermitRootLogin no', exact=True)
assert not contains(SSHD_CONFIG, 'PermitRootLogin yes', exact=True)
def test_disable_password_auth(sshd_config):
from burlap.ssh import disable_password_auth
disable_password_auth(sshd_config=SSHD_CONFIG)
with quiet():
assert contains(SSHD_CONFIG, 'PasswordAuthentication no', exact=True)
assert not contains(SSHD_CONFIG, 'PasswordAuthentication yes', exact=True)
def test_enable_root_login(sshd_config):
from fabtools.ssh import enable_root_login
enable_root_login(sshd_config=SSHD_CONFIG)
with quiet():
assert contains(SSHD_CONFIG, 'PermitRootLogin yes', exact=True)
assert not contains(SSHD_CONFIG, 'PermitRootLogin no', exact=True)
def test_enable_password_auth(sshd_config):
from fabtools.ssh import enable_password_auth
enable_password_auth(sshd_config=SSHD_CONFIG)
with quiet():
assert contains(SSHD_CONFIG, 'PasswordAuthentication yes', exact=True)
assert not contains(
SSHD_CONFIG, 'PasswordAuthentication no', exact=True)
def test_enable_root_login(sshd_config):
from fabtools.ssh import enable_root_login
enable_root_login(sshd_config=SSHD_CONFIG)
with quiet():
assert contains(SSHD_CONFIG, 'PermitRootLogin yes', exact=True)
assert not contains(SSHD_CONFIG, 'PermitRootLogin no', exact=True)
def install_virtualenv():
run('sudo -H pip install virtualenv')
run('sudo -H pip install virtualenvwrapper')
mkdir('$HOME/.virtualenvs')
if not contains('$HOME/.bashrc', 'export WORKON_HOME'):
append('$HOME/.bashrc', 'export WORKON_HOME=$HOME/.virtualenvs')
if not contains('$HOME/.bashrc', 'virtualenvwrapper.sh'):
append('$HOME/.bashrc', 'source /usr/local/bin/virtualenvwrapper.sh')
run('source ~/.bashrc')
def virtualenv_setup():
'''setup virtualenv'''
print(yellow('Environnement virtuel et dossier "projects"...'))
require.python.package('virtualenv', use_sudo=True)
require.python.package('virtualenvwrapper', use_sudo=True)
#require.python.package('virtualenvwrapper.django',use_sudo=True)
print(green('Virtualenv installé.'))
if not 'www-data' in run('echo | groups %(user)s' % env):
sudo('usermod -a -G www-data %(user)s' % env)
print(green('Utilisateur %(user)s ajouté au groupe "www-data".' % env))
if not exists('projects/'):
run('mkdir projects .python-eggs .virtualenvs')
sudo('chown %(user)s:www-data .python-eggs' % env)
sudo('chgrp -R www-data projects/')
sudo('chmod -R 2750 projects/')
print(green('Dossier "projects" créé.'))
# sur .bashrc et pas .bashrc
# + fix pour https://bitbucket.org/dhellmann/virtualenvwrapper/issue/62/hooklog-permissions
run('touch .bash_login')
if not contains('.bash_login', '. .bashrc'):
append('.bash_login', 'if [ $USER == %(user)s ]; then' % env)
if not contains('.bashrc', 'WORKON_HOME'):
append('.bashrc', 'if [ $USER == %(user)s ]; then' % env)
append('.bashrc', ' export WORKON_HOME=$HOME/.virtualenvs')
append('.bashrc', ' export PROJECT_HOME=$HOME/projects')
append('.bashrc',
' export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python')
append(
'.bashrc',
' export VIRTUALENVWRAPPER_VIRTUALENV=/usr/local/bin/virtualenv'
)
append('.bashrc', ' source /usr/local/bin/virtualenvwrapper.sh')
append('.bashrc', 'fi')
append('.bash_profile', 'if [ $USER == %(user)s ]; then' % env)
append('.bash_profile', ' export WORKON_HOME=$HOME/.virtualenvs')
append('.bash_profile', ' export PROJECT_HOME=$HOME/projects')
append('.bash_profile',
' export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python')
append(
'.bash_profile',
' export VIRTUALENVWRAPPER_VIRTUALENV=/usr/local/bin/virtualenv'
)
append('.bash_profile',
' source /usr/local/bin/virtualenvwrapper.sh')
append('.bash_profile', 'fi')
run('source .bashrc')
print(green('Virtualenv et Virtualenvwrapper configurés.'))
# stop warning from bitbucket https://bitbucket.org/site/master/issue/2780/getting-warning-while-using-https-and-ssh
if not contains('.hgrc', 'bitbucket.org'):
append('.hgrc', '[hostfingerprints]')
append(
'.hgrc',
'bitbucket.org = 24:9c:45:8b:9c:aa:ba:55:4e:01:6d:58:ff:e4:28:7d:2a:14:ae:3b'
)
def glance_config():
"""
Add docker container format
:return: None
"""
conf = '/etc/glance/glance-api.conf'
if files.contains(conf, '#container_formats') or files.contains(conf, '# container_formats'):
files.sed(conf, '.*container_formats.*', '', use_sudo=True)
files.sed(conf, '\[DEFAULT\]', '\[DEFAULT\]\\ncontainer_formats = ami,ari,aki,bare,ovf,docker', use_sudo=True)
sudo('service glance-api restart')
def environnement():
print(yellow('Environnements virtuels et dossier projects...'))
icanhaz.python.package('virtualenv', use_sudo=True)
icanhaz.python.package('virtualenvwrapper', use_sudo=True)
#icanhaz.python.package('virtualenvwrapper.django',use_sudo=True)
print(green('Virtualenv installé.'))
if not 'www-data' in run('echo | groups %(user)s' % env):
sudo('usermod -a -G www-data %(user)s' % env)
print(green('Utilisateur %(user)s ajouté au groupe "www-data".' % env))
if not exists('projects/'):
run('mkdir projects .python-eggs .virtualenvs')
sudo('chown %(user)s:www-data .python-eggs' % env)
sudo('chgrp -R www-data projects/')
sudo('chmod -R 2750 projects/')
print(green('Dossier "projects" créé.'))
# sur .bashrc et pas .bashrc
# + fix pour https://bitbucket.org/dhellmann/virtualenvwrapper/issue/62/hooklog-permissions
if not contains('.bashrc', 'WORKON_HOME'):
append('.bashrc', 'if [ $USER == %(user)s ]; then' % env)
append('.bashrc', ' export WORKON_HOME=$HOME/.virtualenvs')
append('.bashrc', ' export PROJECT_HOME=$HOME/projects')
append('.bashrc',
' export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python')
append(
'.bashrc',
' export VIRTUALENVWRAPPER_VIRTUALENV=/usr/local/bin/virtualenv'
)
append('.bashrc', ' source /usr/local/bin/virtualenvwrapper.sh')
append('.bashrc', 'fi')
#append('.bash_profile','if [ -f ~/.bashrc ]; then') #fabric source .bash_profile, pas .bashrc
#append('.bash_profile',' source ~/.bashrc')
#append('.bash_profile','fi')
append('.bash_profile', 'if [ $USER == %(user)s ]; then' % env)
append('.bash_profile', ' export WORKON_HOME=$HOME/.virtualenvs')
append('.bash_profile', ' export PROJECT_HOME=$HOME/projects')
append('.bash_profile',
' export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python')
append(
'.bash_profile',
' export VIRTUALENVWRAPPER_VIRTUALENV=/usr/local/bin/virtualenv'
)
append('.bash_profile',
' source /usr/local/bin/virtualenvwrapper.sh')
append('.bash_profile', 'fi')
run('source .bashrc')
print(green('Virtualenv et Virtualenvwrapper configurés.'))
# stop warning from bitbucket https://bitbucket.org/site/master/issue/2780/getting-warning-while-using-https-and-ssh
if not contains('.hgrc', 'bitbucket.org'):
append('.hgrc', '[hostfingerprints]')
append(
'.hgrc',
'bitbucket.org = 81:2b:08:90:dc:d3:71:ee:e0:7c:b4:75:ce:9b:6c:48:94:56:a1:fe'
)
def assert_manifest_file_content(node_name, software_name, version, action, group):
file_path = calculate_path(group=group, node_name=node_name)
assert_specific_manifest_file_exist(group=group, node_name=node_name)
node_string = "node '{}'".format(node_name)
class_string = "'{}::{}':".format(software_name, action.lower())
version_string = "'{}'".format(version)
assert_true(files.contains(file_path, node_string))
assert_true(files.contains(file_path, class_string))
assert_true(files.contains(file_path, version_string))
def _add_ONE(big_cut):
''' big_cut = 账号:姓名 '''
env.exclude_hosts = ['192.168.100.57']
_check_host()
with cd('%s' % shell_dir):
if contains('%s' % shell_name, big_cut):
print green("大客户 %s 已经存在。" % big_cut)
else:
run ('sed -i \'s/=(/=(%s /\' %s ' % (big_cut, shell_name))
if contains('%s' % shell_name, big_cut):
print green("大客户添加成功。")
else:
print red("大客户添加失败。")
def _delete_ONE(big_cut):
''' big_cut = 账号:姓名 '''
env.exclude_hosts = ['192.168.100.57']
_check_host()
with cd('%s' % shell_dir):
if not contains('%s' % shell_name, big_cut):
print green("大客户 %s 不存在。" % big_cut)
else:
run ('sed -i \'s/%s //\' %s ' % (big_cut, shell_name))
if not contains('%s' % shell_name, big_cut):
print green("大客户删除成功。")
else:
print red("大客户删除失败。")
def assert_manifest_file_content(node_name, software_name, version, action, group):
file_path = calculate_path(group=group, node_name=node_name)
assert_specific_manifest_file_exist(group=group, node_name=node_name)
node_string = "node '{}'".format(node_name)
class_string = "'{}::{}':".format(software_name, action)
version_string = "version"
assert_true(files.contains(file_path, node_string))
assert_true(files.contains(file_path, class_string))
assert_true(files.contains(file_path, version_string))
assert_true(files.contains(file_path, version))
def apache_setup():
'''Config générale Apache + mod_wsgi sans media proxy'''
print(yellow('Configuration d’Apache...'))
pretty_apt(['apache2', 'libapache2-mod-wsgi'])
#virer le site par défaut
with cd('/etc/apache2/'):
if not contains('apache2.conf', 'ServerName localhost', use_sudo=True):
if not contains('apache2.conf', 'ServerName %(domain)s' % env, use_sudo=True):
sudo("echo 'ServerName %(domain)s'|cat - apache2.conf > /tmp/out && mv /tmp/out apache2.conf" % env)
with cd('/etc/apache2/sites-enabled/'):
if exists('000-default'):
sudo('rm 000-default')
print(green('Site par défaut d’Apache supprimé'))
def xfs_create(vg,lv,size,mp,blksize=4096):
lv_create(vg,lv,size)
if not files.exists(mp):
sudo('mkdir -p {0}'.format(mp))
if not files.exists(BY_LABEL%lv):
sudo('mkfs.xfs -f -l size=128m,lazy-count=1 -L {1} -b size={2} /dev/{0}/{1}'.format(vg,lv,blksize))
#sudo('mkfs.xfs -f -l lazy-count=1 -L {1} -b size={2} /dev/{0}/{1}'.format(vg,lv,blksize))
if not files.contains(FSTAB,mp):
files.append(FSTAB,'/dev/{0}/{1} {2} xfs logbufs=8,logbsize=256k,osyncisosync,nobarrier,largeio,noatime,nodiratime,inode64,allocsize=512m 0 0'.format(vg,lv,mp))
if not files.contains(MOUNTS,mp):
sudo('mount {0}'.format(mp))
def disable_selinux():
""" disables selinux """
from fabric.contrib.files import sed, contains
if contains(filename='/etc/selinux/config',
text='SELINUX=enforcing'):
sed('/etc/selinux/config',
'SELINUX=enforcing', 'SELINUX=disabled', use_sudo=True)
if contains(filename='/etc/selinux/config',
text='SELINUXTYPE=enforcing'):
sed('/etc/selinux/config',
'SELINUXTYPE=enforcing', 'SELINUX=targeted', use_sudo=True)
def init_head_node():
init_instance()
install_package('torque-server torque-scheduler torque-client')
# for debugging
install_package('sendmail mutt')
if env.cc_interface == 'public':
instance_hostname = get_public_instance_hostname()
instance_ip = get_public_instance_ip()
#this one is for the scheduler, if using the public interface
if not files.contains(instance_hostname, '/etc/hosts') and not files.contains('127.0.1.1', '/etc/hosts'):
files.append('127.0.1.1 '+instance_hostname, '/etc/hosts', use_sudo=True)
elif env.cc_interface == 'private':
instance_hostname = get_public_instance_hostname()
instance_ip = get_private_instance_ip()
else:
# TODO error
return -1
sudo('rm -f /etc/torque/server_name')
files.append(instance_hostname, '/etc/torque/server_name', use_sudo=True)
# TODO: workaround for Debian bug #XXXXXX
sudo('/etc/init.d/torque-server stop')
time.sleep(6)
sudo('/etc/init.d/torque-server start')
# TODO: workaround for Debian bug #XXXXXX, also catch return code with echo
sudo ('echo `killall pbs_sched`')
time.sleep(2)
sudo('/etc/init.d/torque-scheduler start')
# $SUDO /etc/init.d/torque-server restart
# $SUDO /etc/init.d/torque-scheduler restart
sudo('qmgr -c "s s scheduling=true"')
sudo('qmgr -c "c q batch queue_type=execution"')
sudo('qmgr -c "s q batch started=true"')
sudo('qmgr -c "s q batch enabled=true"')
sudo('qmgr -c "s q batch resources_default.nodes=1"')
sudo('qmgr -c "s q batch resources_default.walltime=3600"')
# had to set this for MPI, TODO: double check
sudo('qmgr -c "s q batch resources_min.nodes=1"')
sudo('qmgr -c "s s default_queue=batch"')
# let all nodes submit jobs, not only the server
sudo('qmgr -c "s s allow_node_submit=true"')
def add_ebs(size_gb, path, iops=None):
""" add an EBS device """
if contains('/etc/fstab', path):
_info('/etc/fstab already contains an entry for ' + path)
return False
ec2 = boto.connect_ec2(env.aws_key, env.aws_secret)
# get ec2 metadata
zone = _get_ec2_metadata('placement/availability-zone')
instance_id = _get_ec2_metadata('instance-id')
# create and attach drive
volume = ec2.create_volume(size_gb, zone, volume_type='io1' if iops else 'standard', iops=iops)
# figure out where drive should be mounted
letters = 'fghijklmnopqrstuvw'
for letter in letters:
drv = '/dev/xvd{}'.format(letter)
# skip this letter if already mounted
if contains('/proc/partitions', 'xvd{}'.format(letter)):
continue
# attach the drive, replacing xv with sd b/c of amazon quirk
time.sleep(10)
volume.attach(instance_id, drv.replace('xv', 's'))
# break if we suceeded
break
else:
# only executed if we didn't break
abort('unable to mount drive')
# TODO: ensure drive is cleaned up
ec2.create_tags([volume.id], {'Name': '{} for {}'.format(path, instance_id)})
_info('waiting for {}...'.format(drv))
while not exists(drv):
time.sleep(1)
# format and mount the drive
sudo('mkfs.xfs {}'.format(drv))
append('/etc/fstab', '{0} {1} xfs defaults 0 0'.format(drv, path), use_sudo=True)
# make & mount
sudo('mkdir -p {}'.format(path))
sudo('mount {}'.format(path))
return True
def disable_selinux():
""" disables selinux """
from fabric.contrib.files import sed, contains
if contains(filename='/etc/selinux/config', text='SELINUX=enforcing'):
sed('/etc/selinux/config',
'SELINUX=enforcing',
'SELINUX=disabled',
use_sudo=True)
if contains(filename='/etc/selinux/config', text='SELINUXTYPE=enforcing'):
sed('/etc/selinux/config',
'SELINUXTYPE=enforcing',
'SELINUX=targeted',
use_sudo=True)
def luchizz_shell():
"""Customize the bash prompt and behavior for a more stylish experience"""
# Load the luchizz bashrc script
global LUCHIZZ_DIR
luchizz_profile = os.path.join(LUCHIZZ_DIR,
'files/profile/luchizz-profile.sh')
with open(luchizz_profile, 'r') as f:
luchizz_profile = f.read()
# Installing default bash changes for newly created users
# FIXME for what the hell is used this folder?
# currently this is causing issues if you connect to localhost debug needed
# new users seems to rely only on /etc/skel/.bashrc
# ~files = put('./files/profile/*', '/etc/profile.d/', use_sudo=True)
# ~for f in files:
# ~sudo('chown root: {}'.format(f))
# ~sudo('chmod 644 {}'.format(f))
# Update the skel file
if not contains('/etc/skel/.bashrc', 'luchizz'):
append('/etc/skel/.bashrc', luchizz_profile, use_sudo=True)
# Set huge history for newly created users
sed('/etc/skel/.bashrc', 'HISTSIZE=.*', 'HISTSIZE=1000000', use_sudo=True)
sed('/etc/skel/.bashrc', 'HISTFILESIZE=.*', 'HISTFILESIZE=1000000',
use_sudo=True)
# Appending bash changes to current users and root
homes = utils.listdir_fullpath('/home')
homes.append('/root')
for u in homes:
bashrc_file = os.path.join(u, '.bashrc')
if not exists(bashrc_file, use_sudo=True):
continue
sed(bashrc_file, 'HISTSIZE=.*', 'HISTSIZE=1000000', use_sudo=True)
sed(bashrc_file, 'HISTFILESIZE=.*', 'HISTFILESIZE=1000000',
use_sudo=True)
if not contains(bashrc_file, 'luchizz'):
append(bashrc_file, luchizz_profile, use_sudo=True)
# Alternate mappings for "page up" and "page down" to search the history
# uncomment the following lines in /etc/inputrc
# "\e[5~": history-search-backward
# "\e[6~": history-search-forward
uncomment('/etc/inputrc', 'history-search-forward', use_sudo=True)
uncomment('/etc/inputrc', 'history-search-backward', use_sudo=True)
# Enable vim syntax
uncomment('/etc/vim/vimrc', 'syntax on', char='"', use_sudo=True)
def create_venv(venv_name=None, requirements_file=None, work_online=None):
"""Makes a python3.6 venv.
"""
venv_name = venv_name or env.venv_name
requirements_file = requirements_file or env.requirements_file
if not exists(env.venv_dir):
run(f'mkdir {env.venv_dir}')
if exists(os.path.join(env.venv_dir, venv_name)):
run('rm -rf {path}'.format(path=os.path.join(env.venv_dir, venv_name)))
with cd(env.venv_dir):
run('python3.6 -m venv --clear --copies {venv_name} {path}'.format(
path=os.path.join(env.venv_dir, venv_name), venv_name=venv_name),
warn_only=True)
text = 'workon () {{ source {activate}; }}'.format(
activate=os.path.join(env.venv_dir, '"$@"', 'bin', 'activate'))
if not contains(env.bash_profile, text):
append(env.bash_profile, text)
if work_online:
run(f'source {activate_venv()} && pip install -U pip setuptools wheel ipython')
with cd(env.project_repo_root):
run(f'source {activate_venv()} && pip install -U -r {env.requirements_file}')
else:
pip_install_from_cache('pip')
pip_install_from_cache('setuptools')
pip_install_from_cache('wheel')
pip_install_from_cache('ipython')
pip_install_requirements_from_cache()
def test_contains_performs_case_insensitive_search(self):
"""
contains() should perform a case-insensitive search when passed `case_sensitive=False`
"""
with hide("everything"):
result = contains("/etc/apache2/apache2.conf", "include Other.CONF", use_sudo=True, case_sensitive=False)
assert result == True
def test_contains_performs_case_sensitive_search(self):
"""
contains() should perform a case-sensitive search by default.
"""
with hide("everything"):
result = contains("/etc/apache2/apache2.conf", "Include other.conf", use_sudo=True)
assert result == True
def setup_kiosk():
""" set up kiosk parts
based on https://www.danpurdy.co.uk/web-development/raspberry-pi-kiosk-screen-tutorial/
or
http://www.raspberry-projects.com/pi/pi-operating-systems/raspbian/gui/auto-run-browser-on-startup
"""
with hide("running", "stderr"):
#@xscreensaver -no-splash
comment("/etc/xdg/lxsession/LXDE-pi/autostart", "@xscreensaver -no-splash", use_sudo=True)
append("/etc/xdg/lxsession/LXDE-pi/autostart",
"@xset s off", use_sudo=True, escape=True)
append("/etc/xdg/lxsession/LXDE-pi/autostart",
"@xset -dpms", use_sudo=True, escape=True)
append("/etc/xdg/lxsession/LXDE-pi/autostart",
"@xset s noblank", use_sudo=True, escape=True)
append("/etc/xdg/lxsession/LXDE-pi/autostart",
"""@sed -i 's/"exited_cleanly": false/"exited_cleanly": true/' ~/.config/chromium/Default/Preferences""",
use_sudo=True, escape=True)
#auto start
if not contains("/etc/xdg/lxsession/LXDE-pi/autostart",
"@chromium --noerrdialogs --kiosk http://www.page-to.display --incognito",
use_sudo=True, escape=True):
append("/etc/xdg/lxsession/LXDE-pi/autostart",
"@chromium --noerrdialogs --kiosk http://dashingdemo.herokuapp.com/sample --incognito",
use_sudo=True, escape=True)
def exclude_in_yum(rpm=None):
''' add exclusion list in /etc/yum.conf '''
if not rpm:
abort("Expected argument rpm")
if contains('/etc/yum.conf',
'^exclude',
exact=True,
use_sudo=True,
escape=False):
print 'env.host', ' exclude already exists in /etc/yum.conf, updating '
sed('/etc/yum.conf',
before='^exclude.*$',
after='&' + ' ' + rpm,
limit='',
use_sudo=True,
backup='.bak',
flags='',
shell=False)
else:
print env.host, ' inserting exclude in /etc/yum.conf '
append('/etc/yum.conf',
'exclude=' + rpm,
use_sudo=True,
partial=False,
escape=True,
shell=False)
def _configure_postgresql(env, delete_main_dbcluster=False):
""" This method is intended for cleaning up the installation when
PostgreSQL is installed from a package. Basically, when PostgreSQL
is installed from a package, it creates a default database cluster
and splits the config file away from the data.
This method can delete the default database cluster that was automatically
created when the package is installed. Deleting the main database cluster
also has the effect of stopping the auto-start of the postmaster server at
machine boot. The method adds all of the PostgreSQL commands to the PATH.
"""
pg_ver = sudo("dpkg -s postgresql | grep Version | cut -f2 -d':'")
pg_ver = pg_ver.strip()[:3] # Get first 3 chars of the version since that's all that's used for dir name
got_ver = False
while(not got_ver):
try:
pg_ver = float(pg_ver)
got_ver = True
except Exception:
print(red("Problems trying to figure out PostgreSQL version."))
pg_ver = raw_input(red("Enter the correct one (eg, 9.1; not 9.1.3): "))
if delete_main_dbcluster:
env.safe_sudo('pg_dropcluster --stop %s main' % pg_ver, user='******')
# Not sure why I ever added this to gvl, doesn't seem needed. -John
#_put_installed_file_as_user("postgresql-%s.conf" % env.postgres_version, "/etc/postgresql/%s/main/postgresql.conf" % env.postgres_version, user='******')
exp = "export PATH=/usr/lib/postgresql/%s/bin:$PATH" % pg_ver
if not contains('/etc/bash.bashrc', exp):
append('/etc/bash.bashrc', exp, use_sudo=True)
def _configure_postgresql(env, delete_main_dbcluster=False):
""" This method is intended for cleaning up the installation when
PostgreSQL is installed from a package. Basically, when PostgreSQL
is installed from a package, it creates a default database cluster
and splits the config file away from the data.
This method can delete the default database cluster that was automatically
created when the package is installed. Deleting the main database cluster
also has the effect of stopping the auto-start of the postmaster server at
machine boot. The method adds all of the PostgreSQL commands to the PATH.
"""
pg_ver = sudo("dpkg -s postgresql | grep Version | cut -f2 -d':'")
pg_ver = pg_ver.strip()[:3] # Get first 3 chars of the version since that's all that's used for dir name
got_ver = False
while(not got_ver):
try:
pg_ver = float(pg_ver)
got_ver = True
except Exception:
print(red("Problems trying to figure out PostgreSQL version."))
pg_ver = raw_input(red("Enter the correct one (eg, 9.1; not 9.1.3): "))
if delete_main_dbcluster:
env.safe_sudo('pg_dropcluster --stop %s main' % pg_ver, user='******')
# Not sure why I ever added this to gvl, doesn't seem needed. -John
#_put_installed_file_as_user("postgresql-%s.conf" % env.postgres_version, "/etc/postgresql/%s/main/postgresql.conf" % env.postgres_version, user='******')
exp = "export PATH=/usr/lib/postgresql/%s/bin:$PATH" % pg_ver
if not contains('/etc/bash.bashrc', exp):
append('/etc/bash.bashrc', exp, use_sudo=True)
def update_docs_site(site_name):
"""Initialise a docs site where we host test pdf."""
code_path = os.path.join(env.repo_path, env.repo_alias)
local_path = '%s/scripts/test-build-repo' % code_path
if not exists(env.inasafe_docs_path):
sudo('mkdir -p %s' % env.docs_path)
sudo('chown %s.%s %s' % (env.user, env.user, env.docs_path))
env.run('cp %s/plugin* %s' % (local_path, env.plugin_repo_path))
env.run('cp %s/icon* %s' % (code_path, env.plugin_repo_path))
env.run(
'cp %(local_path)s/docs.conf.templ '
'%(local_path)s/%(site_name)s.conf' % {
'local_path': local_path,
'site_name': site_name})
sed('%s/inasafe-test.conf' % local_path, '[SITE_NAME]', site_name)
with cd('/etc/apache2/sites-available/'):
if exists('inasafe-docs.conf'):
sudo('a2dissite inasafe-docs.conf')
fastprint('Removing old apache2 conf', False)
sudo('rm inasafe-docs.conf')
sudo('ln -s %s/inasafe-docs.conf .' % local_path)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, site_name):
append(hosts, '127.0.0.1 %s' % site_name, use_sudo=True)
sudo('a2ensite %s.conf' % site_name)
sudo('service apache2 reload')
def test_contains_checks_only_succeeded_flag(self):
"""
contains() should return False on bad grep even if stdout isn't empty
"""
with hide('everything'):
result = contains('/file.txt', 'text', use_sudo=True)
assert result == False
def add_head_to_worker(head_ip):
head_hostname = get_hostname_from_ip(head_ip)
if not files.contains(head_hostname, '/etc/hosts'):
files.append(head_ip+' '+head_hostname, '/etc/hosts', use_sudo=True)
sudo('rm -f /etc/torque/server_name')
files.append(head_hostname, '/etc/torque/server_name', use_sudo=True)
def configure_supervisor(distro):
"""
Configure supervisor for running our WebVirtMgr Django Gunicorn Server
"""
if distro in ["Debian", "Ubuntu"]:
user = "******"
require.supervisor.process(
"webvirtmgr",
command=
"/usr/bin/python /var/www/webvirtmgr/manage.py run_gunicorn -c\
/var/www/webvirtmgr/conf/gunicorn.conf.py",
directory="/var/www/webvirtmgr",
user=user,
stdout_logfile="/var/log/supervisor/webvirtmgr.log",
autostart=True,
autorestart=True,
redirect_stderr=True
)
elif distro in ["CentOS", "RHEL", "Fedora"]:
# first, ensure supervisord is running!
with settings(warn_only=True):
require.service.restart("supervisord")
supervisord = "/etc/supervisord.conf"
if not contains(supervisord, "[program:webvirtmgr]"):
f = open("templates/webvirtmgr.ini")
content = f.read()
f.close()
append(supervisord, content, use_sudo=True)
reload_config()
def install_plugins(*plugins):
install_autojump = False
destination_folder = '~/.zsh'
if not exists(destination_folder):
run('mkdir -p %s' % destination_folder)
plugins_file = '{}/plugins.zsh'.format(destination_folder)
if not exists(plugins_file):
upload_template('{}/templates/oh-my-zsh-plugins'.format(ROOT_FOLDER),
'{}/plugins.zsh'.format(destination_folder),
context={})
plugins_to_install = []
for plugin in plugins:
if plugin == "autojump":
install_autojump = True
if not contains(plugins_file, plugin):
plugins_to_install.append(plugin)
if plugins_to_install:
sed(plugins_file, '\)', ' {})'.format(' '.join(plugins_to_install)))
if install_autojump:
utils.os_commands.install('autojump')
def create_app_user():
#sudo("sudo locale-gen UTF-8")
user_exists = run("id -u hadoop", warn_only=True)
if user_exists.return_code == 1:
sudo("useradd hadoop --password hadoop -d /home/hadoop -s /bin/bash")
if not exists("/home/hadoop/.ssh"):
sudo("mkdir -p /home/hadoop/.ssh")
sudo("chown -R hadoop /home/hadoop")
bash_login_content = """
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
"""
_replace_file_content("/home/hadoop/.bash_login", bash_login_content)
with settings(sudo_user='******'):
if not exists('/home/hadoop/.ssh/id_rsa'):
sudo('ssh-keygen -t rsa -P "" -f /home/hadoop/.ssh/id_rsa')
sudo("cat /home/hadoop/.ssh/id_rsa.pub >> /home/hadoop/.ssh/authorized_keys")
sudo("chmod 0600 /home/hadoop/.ssh/authorized_keys")
sudo("ssh-keyscan -H localhost >> /home/hadoop/.ssh/known_hosts")
sudo("ssh-keyscan -H 0.0.0.0 >> /home/hadoop/.ssh/known_hosts")
if not exists("/home/hadoop/.bashrc"):
sudo("touch /home/hadoop/.bashrc")
if not contains("/home/hadoop/.bashrc", "export HADOOP_HOME=/usr/local/lib/hadoop"):
append("/home/hadoop/.bashrc", APP_USER_SETTINGS, use_sudo=True)
def test_contains_checks_only_succeeded_flag(self):
"""
contains() should return False on bad grep even if stdout isn't empty
"""
with hide("everything"):
result = contains("/file.txt", "text", use_sudo=True)
assert result == False
def _add_user(username, uid=None):
""" Add user with username to the system """
if not contains('/etc/passwd', "%s:" % username):
uid_str = "--uid %s" % uid if uid else ""
sudo('useradd -d /home/%s --create-home --shell /bin/bash '
'-c"Galaxy-required user" %s --user-group %s' %
(username, uid_str, username))
def what_os():
with settings(warn_only=True):
_require_target()
if getattr(env, 'host_os_map', None) is None:
# prior use case of setting a env.remote_os
# did not work when doing multiple hosts with different os!
# Need to keep state per host!
env.host_os_map = defaultdict(lambda: '')
if env.host_os_map[env.host_string] == '':
print 'Testing operating system type...'
if (files.exists('/etc/lsb-release', verbose=True)
and files.contains(text='DISTRIB_ID=Ubuntu',
filename='/etc/lsb-release')):
remote_os = 'ubuntu'
print(
'Found lsb-release and contains "DISTRIB_ID=Ubuntu", '
'this is an Ubuntu System.')
elif files.exists('/etc/redhat-release', verbose=True):
remote_os = 'redhat'
print 'Found /etc/redhat-release, this is a RedHat system.'
else:
print 'System OS not recognized! Aborting.'
exit()
env.host_os_map[env.host_string] = remote_os
return env.host_os_map[env.host_string]
def initialise_qgis_plugin_repo():
"""Initialise a QGIS plugin repo where we host test builds."""
_all()
fabtools.require.deb.package('libapache2-mod-wsgi')
code_path = os.path.join(env.repo_path, env.repo_alias)
local_path = '%s/scripts/test-build-repo' % code_path
if not exists(env.plugin_repo_path):
sudo('mkdir -p %s' % env.plugin_repo_path)
sudo('chown %s.%s %s' % (env.user, env.user, env.plugin_repo_path))
run('cp %s/plugin* %s' % (local_path, env.plugin_repo_path))
run('cp %s/icon* %s' % (code_path, env.plugin_repo_path))
run('cp %(local_path)s/inasafe-test.conf.templ '
'%(local_path)s/inasafe-test.conf' % {'local_path': local_path})
sed('%s/inasafe-test.conf' % local_path,
'inasafe-test.linfiniti.com',
env.repo_site_name)
with cd('/etc/apache2/sites-available/'):
if exists('inasafe-test.conf'):
sudo('a2dissite inasafe-test.conf')
fastprint('Removing old apache2 conf', False)
sudo('rm inasafe-test.conf')
sudo('ln -s %s/inasafe-test.conf .' % local_path)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, 'inasafe-test'):
append(hosts, '127.0.0.1 %s' % env.repo_site_name, use_sudo=True)
sudo('a2ensite inasafe-test.conf')
sudo('service apache2 reload')
def supervisord_config(base_dir, work_dir, pyenv):
l = [supervisord_config_template.format(work_dir=work_dir)]
for module in ['dispatch', 'server']:
l.append(program_template.format(
package='task_les',
module=module,
python_prefix='/usr/local/pyenv/versions/{}'.format(pyenv),
work_dir=work_dir
))
for p in program_list:
i = p.find(' ')
if i >= 0:
path = p[:i]
args = p[i:]
else:
path = p
args = ''
sp = path.split('/')
run('mkdir -p {}/{}/log'.format(base_dir, sp[0]))
l.append(program_template_1.format(
package=sp[0],
module=sp[-1],
python_prefix='/usr/local/pyenv/versions/{}'.format(pyenv),
work_dir='{}/{}'.format(base_dir, sp[0]),
path='{}/{}.pyc'.format(base_dir, path),
args=args))
put(StringIO.StringIO('\n'.join(l)), '{}/etc/supervisord.conf'.format(work_dir))
c = '/usr/local/pyenv/versions/{}/bin/supervisord -c {}/etc/supervisord.conf'.format(pyenv, work_dir)
if not contains('/etc/rc.local', c):
append('/etc/rc.local', c)
def _add_user(username, uid=None):
""" Add user with username to the system """
if not contains('/etc/passwd', "%s:" % username):
uid_str = "--uid %s" % uid if uid else ""
sudo('useradd -d /home/%s --create-home --shell /bin/bash ' \
'-c"Galaxy-required user" %s --user-group %s' % \
(username, uid_str, username))
def initialise_docs_site():
"""Initialise an InaSAFE docs sote where we host test pdf."""
all()
fabtools.require.deb.package('libapache2-mod-wsgi')
code_path = os.path.join(env.repo_path, env.repo_alias)
local_path = '%s/scripts/test-build-repo' % code_path
if not exists(env.inasafe_docs_path):
sudo('mkdir -p %s' % env.inasafe_docs_path)
sudo('chown %s.%s %s' % (env.user, env.user, env.inasafe_docs_path))
run('cp %s/plugin* %s' % (local_path, env.plugin_repo_path))
run('cp %s/icon* %s' % (code_path, env.plugin_repo_path))
run('cp %(local_path)s/inasafe-test.conf.templ '
'%(local_path)s/inasafe-test.conf' % {'local_path': local_path})
sed('%s/inasafe-test.conf' % local_path,
'inasafe-test.linfiniti.com',
env.repo_site_name)
with cd('/etc/apache2/sites-available/'):
if exists('inasafe-docs.conf'):
sudo('a2dissite inasafe-docs.conf')
fastprint('Removing old apache2 conf', False)
sudo('rm inasafe-docs.conf')
sudo('ln -s %s/inasafe-docs.conf .' % local_path)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, 'inasafe-docs'):
append(hosts, '127.0.0.1 %s' % env.repo_site_name, use_sudo=True)
sudo('a2ensite inasafe-docs.conf')
sudo('service apache2 reload')
