def postfix_configure():
'Configure postfix'
config = env.config[env.host_string]['postfix']
with watch('/etc/mailname') as mailname:
fabtools.require.file('/etc/mailname', config['mailname'])
with watch('/etc/postfix/main.cf') as main_cf:
for (key, value) in config['main.cf'].items():
if isinstance(value, list):
ensure_directive('/etc/postfix/main.cf', key, *value)
else:
ensure_directive('/etc/postfix/main.cf', key, value)
with watch('/etc/postfix/master.cf') as master_cf:
if not contains('/etc/postfix/master.cf', 'policy-spf'):
append('/etc/postfix/master.cf', config['master.cf'])
if not exists('/etc/postfix/virtual'):
run('touch /etc/postfix/virtual')
with watch('/etc/postfix/virtual') as virtual:
fabtools.require.file('/etc/postfix/virtual', config['virtual'])
if virtual.changed:
run('postmap /etc/postfix/virtual')
if any([mailname.changed, main_cf.changed, master_cf.changed, virtual.changed]):
run('systemctl restart postfix.service')
def process(name, **kwargs):
"""
Require a supervisor process
"""
from fabtools import require
require.deb.package('supervisor')
require.service.started('supervisor')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
with watch(filename, True, reload_config):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def deb_update_testing():
"Upgrade from stable to testing"
with watch([ '/etc/apt/sources.list',
'/etc/apt/sources.list.d/stable.list',
'/etc/apt/sources.list.d/stable-updates.list',
'/etc/apt/sources.list.d/unstable.list',
'/etc/apt/sources.list.d/testing.list',
'/etc/apt/sources.list.d/testing-updates.list' ]) as sources_list_d:
if contains('/etc/apt/sources.list', '^deb', escape=False):
comment('/etc/apt/sources.list', '^deb', use_sudo=True)
components = ['main', 'contrib', 'non-free']
deb.source('stable', 'http://ftp.nl.debian.org/debian/', 'stable', *components)
deb.source('stable-updates', 'http://security.debian.org/', 'stable/updates', *components)
deb.source('testing', 'http://ftp.nl.debian.org/debian/', 'testing', *components)
deb.source('testing-updates', 'http://security.debian.org/', 'testing/updates', *components)
deb.source('unstable', 'http://ftp.nl.debian.org/debian/', 'unstable', *components)
fabtools.require.file('/etc/apt/preferences.d/testing', 'Package: *\nPin: release a=testing\nPin-Priority: 901\n')
fabtools.require.file('/etc/apt/preferences.d/stable', 'Package: *\nPin: release a=stable\nPin-Priority: -2\n')
fabtools.require.file('/etc/apt/preferences.d/unstable', 'Package: *\nPin: release a=unstable\nPin-Priority: -1\n')
if sources_list_d.changed:
run('DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTED=none apt-get'
' -o Dpkg::Options::="--force-confnew"'
' --force-yes'
' -fuy'
' --quiet'
' dist-upgrade')
def database(name,
owner,
template='template0',
encoding='UTF8',
locale='en_US.UTF-8'):
"""
Require a PostgreSQL database.
::
from fabtools import require
require.postgres.database('myapp', owner='dbuser')
"""
if not database_exists(name):
with watch('/etc/locale.gen') as locales:
require_locale(locale)
if locales.changed:
restarted(_service_name())
create_database(name,
owner,
template=template,
encoding=encoding,
locale=locale)
def locales(names):
"""
Require the list of locales to be available.
"""
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
if distrib_id() == "Archlinux":
run_as_root('locale-gen')
else:
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
def locales(names):
"""
Require the list of locales to be available.
"""
if distrib_id() == "Ubuntu":
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
run_as_root('touch %s' % config_file)
else:
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
family = distrib_family()
if family == 'debian':
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
elif family in ['arch', 'gentoo']:
run_as_root('locale-gen')
else:
raise UnsupportedFamily(supported=['debian', 'arch', 'gentoo'])
def process(name, **kwargs):
"""
Require a supervisor process
"""
from fabtools import require
require.deb.package("supervisor")
require.service.started("supervisor")
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault("autorestart", "true")
params.setdefault("redirect_stderr", "true")
# Build config file from parameters
lines = []
lines.append("[program:%(name)s]" % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = "/etc/supervisor/conf.d/%(name)s.conf" % locals()
with watch(filename, True, reload_config):
require.file(filename, contents="\n".join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == "STOPPED":
start_process(name)
def locales(names):
"""
Require the list of locales to be available.
"""
if distrib_id() == "Ubuntu":
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
run_as_root('touch %s' % config_file)
else:
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
if distrib_id() == "Archlinux":
run_as_root('locale-gen')
else:
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
def process(name, **kwargs):
"""
Require a supervisor process
"""
from fabtools import require
require.deb.package('supervisor')
require.service.started('supervisor')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
with watch(filename, callback=update_config, use_sudo=True):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def locales(names):
"""
Require the list of locales to be available.
"""
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True)
append(config_file, locale, use_sudo=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
sudo('dpkg-reconfigure --frontend=noninteractive locales')
def instance(name, version=VERSION, **kwargs):
"""
Require a Redis instance to be running
The instance will be managed using supervisord.
"""
from fabtools import require
installed_from_source(version)
require.directory('/etc/redis', use_sudo=True, owner='redis')
require.directory('/var/db/redis', use_sudo=True, owner='redis')
require.directory('/var/log/redis', use_sudo=True, owner='redis')
require.directory('/var/run/redis', use_sudo=True, owner='redis')
# Required for background saving
with settings(warn_only=True):
require.system.sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', '127.0.0.1')
params.setdefault('port', '6379')
params.setdefault('logfile', '/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dbfilename', '/var/db/redis/redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
context = dict(need_restart=False)
def on_change():
context['need_restart'] = True
with watch(config_filename, True, on_change):
require.file(config_filename, contents='\n'.join(lines),
use_sudo=True, owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require.supervisor.process(process_name,
user='******',
directory='/var/run/redis',
command="%(redis_server)s %(config_filename)s" % locals())
if context['need_restart']:
fabtools.supervisor.restart_process(process_name)
def process(name, **kwargs):
"""
Require a supervisor process to be running.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisord documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the supervisord defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from fabtools import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisord documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from fabtools import require
require.deb.package("supervisor")
require.service.started("supervisor")
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault("autorestart", "true")
params.setdefault("redirect_stderr", "true")
# Build config file from parameters
lines = []
lines.append("[program:%(name)s]" % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = "/etc/supervisor/conf.d/%(name)s.conf" % locals()
with watch(filename, callback=update_config, use_sudo=True):
require.file(filename, contents="\n".join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == "STOPPED":
start_process(name)
def test_flag_is_not_set_when_watched_file_is_not_modified(watched_file):
from fabtools.files import watch
with watch('watched') as f:
pass
assert not f.changed
def test_flag_is_not_set_when_watched_file_is_not_modified(watched_file):
from fabtools.files import watch
with watch('watched') as f:
pass
assert not f.changed
def nfs_configure():
with watch('/etc/exports') as exports:
for n in env.config[env.host_string]['nfs']['exports']:
line = '{0:16} {1}({2})'.format(n['path'], n['net'], ','.join(n['options']))
if not contains('/etc/exports', line):
append('/etc/exports', line)
if exports.changed:
run('systemctl restart nfs-kernel-server')
def process(name, **kwargs):
"""
Require a supervisor process to be running.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisor documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the ``supervisor`` defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from fabtools import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisor documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from fabtools import require
require.deb.package('supervisor')
require.service.started('supervisor')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
with watch(filename, callback=update_config, use_sudo=True):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def test_flag_is_set_when_watched_file_is_modified(watched_file):
from fabtools.files import watch
from fabtools.require import file as require_file
with watch('watched') as f:
require_file('watched', contents='bbb')
assert f.changed
def test_flag_is_set_when_watched_file_is_modified(watched_file):
from fabtools.files import watch
from fabtools.require import file as require_file
with watch('watched') as f:
require_file('watched', contents='bbb')
assert f.changed
def deb_set_hostname():
"Sets the right hostname based on config"
current = run('hostname', quiet=True)
wanted = env.config[env.host_string]['hostname']
if current != wanted:
with watch([ '/etc/hostname',
'/etc/hosts' ]) as etc_host:
sed('/etc/hostname', current, wanted)
sed('/etc/hosts', current, wanted)
if etc_host.changed:
run('hostname {0}'.format(wanted))
def prepare_boot():
"""
Install boot
"""
# Prepare mkinitcpio
config_file = '/mnt/etc/mkinitcpio.conf'
comment(config_file, '^HOOKS')
with watch(config_file):
append(config_file, 'HOOKS="base udev autodetect modconf block keyboard lvm2 encrypt filesystems fsck"')
run_on_archroot('mkinitcpio -p linux')
# Prepare syslinux
config_file = '/mnt/boot/syslinux/syslinux.cfg'
with watch(config_file):
sed(config_file, 'root=/dev/sda3', 'root=/dev/vg/root')
run_on_archroot('syslinux-install_update -iam')
# Configure home encrypted mount
config_file = '/mnt/etc/crypttab'
with watch(config_file):
append(config_file, 'home /dev/vg/home', 'root=/dev/vg/root')
def prepare_boot():
"""
Install boot
"""
# Prepare mkinitcpio
config_file = '/mnt/etc/mkinitcpio.conf'
comment(config_file, '^HOOKS')
with watch(config_file):
append(config_file, 'HOOKS="base udev autodetect modconf block keyboard lvm2 encrypt filesystems fsck"')
run_on_archroot('mkinitcpio -p linux')
# Prepare syslinux
config_file = '/mnt/boot/syslinux/syslinux.cfg'
with watch(config_file):
sed(config_file, 'root=/dev/sda3', 'root=/dev/vg/root')
run_on_archroot('syslinux-install_update -iam')
# Configure home encrypted mount
config_file = '/mnt/etc/crypttab'
with watch(config_file):
append(config_file, 'home /dev/vg/home', 'root=/dev/vg/root')
def source(name, uri, distribution, *components):
"""
Require a package source
"""
path = '/etc/apt/sources.list.d/%(name)s.list' % locals()
components = ' '.join(components)
source_line = 'deb %(uri)s %(distribution)s %(components)s\n' % locals()
def on_update():
puts('Added APT repository: %s' % source_line)
update_index()
with watch(path, _callable=on_update):
fabtools.require.file(path=path, contents=source_line, use_sudo=True)
def deb_install_systemd():
"Install systemd and update grub config"
if not deb.is_installed('systemd'):
deb.package('systemd')
with watch('/etc/default/grub') as etc_default_grub:
sed('/etc/default/grub',
'GRUB_CMDLINE_LINUX_DEFAULT="quiet"',
'GRUB_CMDLINE_LINUX_DEFAULT="quiet init=/bin/systemd"')
if etc_default_grub.changed:
run('update-grub2')
reboot()
run('/bin/true')
def nginx_configure():
'Configure nginx servers'
for site in env.config[env.host_string]['nginx']['sites']:
if not exists('/etc/nginx/sites-enabled/' + site):
run('touch /etc/nginx/sites-enabled/' + site)
with watch([ '/etc/nginx/sites-enabled/' + site
for site in env.config[env.host_string]['nginx']['sites'] ]) as sites:
for (site, content) in env.config[env.host_string]['nginx']['sites'].items():
fabtools.require.file('/etc/nginx/sites-enabled/' + site, contents=content)
if sites.changed:
run('nginx -t')
run('systemctl restart nginx')
def uwsgi_create_uwsgi_apps():
'Enable and start sockets for uWSGI apps'
for app in env.config[env.host_string]['uwsgi']:
if not exists('/etc/systemd/system/sockets.target.wants/uwsgi@{name}.socket'.format(**app)):
run('systemctl enable uwsgi@{name}.socket'.format(**app))
run('systemctl start uwsgi@{name}.socket'.format(**app))
config = yaml.dump(app['config'], default_flow_style=False)
if not exists('/etc/uwsgi/apps-enabled/{name}.yaml'.format(**app)):
run('touch /etc/uwsgi/apps-enabled/{name}.yaml'.format(**app))
with watch('/etc/uwsgi/apps-enabled/{name}.yaml'.format(**app)) as config_yaml:
fabtools.require.file('/etc/uwsgi/apps-enabled/{name}.yaml'.format(**app), contents=config)
if config_yaml.changed:
run('systemctl stop uwsgi@{name}.service'.format(**app))
def test_callback_is_not_called_when_watched_file_is_not_modified(watched_file):
from fabtools.files import watch
from fabtools.require import file as require_file
try:
with watch('watched', callback=partial(require_file, 'modified2')):
pass
assert not is_file('modified2')
finally:
run('rm -f modified2')
def test_callback_is_called_when_watched_file_is_modified(watched_file):
from fabtools.files import watch
from fabtools.require import file as require_file
try:
with watch('watched', callback=partial(require_file, 'modified1')):
require_file('watched', contents='bbb')
assert is_file('modified1')
finally:
run('rm -f modified1')
def source(name, uri, distribution, *components):
"""
Require a package source
"""
path = '/etc/apt/sources.list.d/%(name)s.list' % locals()
components = ' '.join(components)
source_line = 'deb %(uri)s %(distribution)s %(components)s\n' % locals()
def on_update():
puts('Added APT repository: %s' % source_line)
update_index()
with watch(path, _callable=on_update):
fabtools.require.file(path=path, contents=source_line, use_sudo=True)
def sysctl(key, value, persist=True):
"""
Require a kernel parameter to have a specific value
"""
if get_sysctl(key) != value:
set_sysctl(key, value)
if persist:
from fabtools import require
filename = '/etc/sysctl.d/60-%s.conf' % key
def on_change():
sudo('service procps start')
with watch(filename, True, on_change):
require.file(filename,
contents='%(key)s = %(value)s\n' % locals(),
use_sudo=True)
def sysctl(key, value, persist=True):
"""
Require a kernel parameter to have a specific value.
"""
if get_sysctl(key) != value:
set_sysctl(key, value)
if persist:
from fabtools import require
filename = '/etc/sysctl.d/60-%s.conf' % key
with watch(filename, use_sudo=True) as config:
require.file(filename,
contents='%(key)s = %(value)s\n' % locals(),
use_sudo=True)
if config.changed:
sudo('service procps start')
def require_yaourt_configuration():
"""
Add a yaourt configuration
"""
config_file = '/etc/pacman.conf'
with watch(config_file) as config:
append(config_file, '[archlinuxfr]')
append(config_file, 'SigLevel = Never')
append(config_file, 'Server = http://repo.archlinux.fr/%s' % env.arch)
if env.arch == 'x86_64':
append(config_file, '[multilib]')
append(config_file, 'Include = /etc/pacman.d/mirrorlist # multilib')
if config.changed:
arch.update_index()
def deb_upgrade(dist=False):
'Upgrade everything'
with watch('/var/lib/apt/periodic/update-success-stamp') as update:
deb.uptodate_index(max_age=3600)
if update.changed:
run('DEBIAN_FRONTED=noninteractive APT_LISTCHANGES_FRONTEND=none apt-get'
' -o Dpkg::Options::="--force-confnew"'
' --force-yes'
' -fuy'
' --quiet'
' upgrade')
run('DEBIAN_FRONTED=noninteractive APT_LISTCHANGES_FRONTEND=none apt-get'
' -o Dpkg::Options::="--force-confnew"'
' --force-yes'
' -fuy'
' --quiet'
' autoremove')
def require_yaourt_configuration():
"""
Add a yaourt configuration
"""
config_file = '/etc/pacman.conf'
with watch(config_file) as config:
append(config_file, '[archlinuxfr]')
append(config_file, 'SigLevel = Never')
append(config_file, 'Server = http://repo.archlinux.fr/%s' % env.arch)
if env.arch == 'x86_64':
append(config_file, '[multilib]')
append(config_file, 'Include = /etc/pacman.d/mirrorlist # multilib')
if config.changed:
arch.update_index()
def firewall(zones=None, interfaces=None, policy=None, rules=None,
routestopped=None, masq=None):
"""
Ensure that a firewall is configured.
Example::
from fabtools.shorewall import *
from fabtools import require
# We need a firewall with some custom rules
require.shorewall.firewall(
rules=[
Ping(),
SSH(),
HTTP(),
HTTPS(),
SMTP(),
rule(port=1234, source=hosts(['example.com'])),
]
)
"""
family = distrib_family()
if family != 'debian':
raise UnsupportedFamily(supported=['debian'])
require_deb_package('shorewall')
with watch(CONFIG_FILES) as config:
_zone_config(zones)
_interfaces_config(interfaces)
_policy_config(policy)
_rules_config(rules)
_routestopped_config(routestopped)
_masq_config(masq)
if config.changed:
puts("Shorewall configuration changed")
if is_started():
restart('shorewall')
with settings(hide('running'), shell_env()):
sed('/etc/default/shorewall', 'startup=0', 'startup=1', use_sudo=True)
def _locales_generic(names, config_file, command):
supported = supported_locales()
_check_for_unsupported_locales(names, supported)
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
charset_from_name = dict(supported)
for name in names:
charset = charset_from_name[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
if config.changed:
run_as_root(command)
def firewall(zones=None, interfaces=None, policy=None, rules=None, routestopped=None, masq=None):
"""
Ensure that a firewall is configured.
Example::
from fabtools.shorewall import *
from fabtools import require
# We need a firewall with some custom rules
require.shorewall.firewall(
rules=[
Ping(),
SSH(),
HTTP(),
HTTPS(),
SMTP(),
rule(port=1234, source=hosts(['example.com'])),
]
)
"""
family = distrib_family()
if family != "debian":
raise UnsupportedFamily(supported=["debian"])
require_deb_package("shorewall")
with watch(CONFIG_FILES) as config:
_zone_config(zones)
_interfaces_config(interfaces)
_policy_config(policy)
_rules_config(rules)
_routestopped_config(routestopped)
_masq_config(masq)
if config.changed:
puts("Shorewall configuration changed")
if is_started():
restart("shorewall")
with settings(hide("running"), shell_env()):
sed("/etc/default/shorewall", "startup=0", "startup=1", use_sudo=True)
def sysctl(key, value, persist=True):
"""
Require a kernel parameter to have a specific value.
"""
if get_sysctl(key) != value:
set_sysctl(key, value)
if persist:
from fabtools.require import file as require_file
filename = '/etc/sysctl.d/60-%s.conf' % key
with watch(filename, use_sudo=True) as config:
require_file(filename,
contents='%(key)s = %(value)s\n' % locals(),
use_sudo=True)
if config.changed:
if distrib_family() == 'debian':
run_as_root('service procps start')
def sysctl(key, value, persist=True):
"""
Require a kernel parameter to have a specific value.
"""
if get_sysctl(key) != value:
set_sysctl(key, value)
if persist:
from fabtools.require import file as require_file
filename = '/etc/sysctl.d/60-%s.conf' % key
with watch(filename, use_sudo=True) as config:
require_file(filename,
contents='%(key)s = %(value)s\n' % locals(),
use_sudo=True)
if config.changed:
if distrib_family() == 'debian':
run_as_root('service procps start')
def source(name, uri, distribution, *components):
"""
Require a package source.
::
from fabtools import require
# Official MongoDB packages
require.deb.source('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen')
"""
path = '/etc/apt/sources.list.d/%(name)s.list' % locals()
components = ' '.join(components)
source_line = 'deb %(uri)s %(distribution)s %(components)s\n' % locals()
with watch(path) as config:
fabtools.require.file(path=path, contents=source_line, use_sudo=True)
if config.changed:
puts('Added APT repository: %s' % source_line)
update_index()
def source(name, uri, distribution, *components):
"""
Require a package source.
::
from fabtools import require
# Official MongoDB packages
require.deb.source('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen')
"""
path = "/etc/apt/sources.list.d/%(name)s.list" % locals()
components = " ".join(components)
source_line = "deb %(uri)s %(distribution)s %(components)s\n" % locals()
with watch(path) as config:
fabtools.require.file(path=path, contents=source_line, use_sudo=True)
if config.changed:
puts("Added APT repository: %s" % source_line)
update_index()
def _update_ssh_setting(sshd_config, name, value):
"""
Update a yes/no setting in the SSH config file
"""
with watch(sshd_config) as config_file:
with shell_env():
# First try to change existing setting
sed(sshd_config,
r'^(\s*#\s*)?%s\s+(yes|no)' % name,
'%s %s' % (name, value),
use_sudo=True)
# Then append setting if it's still missing
_append(sshd_config, '%s %s' % (name, value), use_sudo=True)
if config_file.changed and is_running('ssh'):
restart('ssh')
def firewall(zones=None, interfaces=None, policy=None, rules=None, routestopped=None):
"""
Require a firewall
"""
package('shorewall')
def on_change():
puts("Shorewall configuration changed")
if is_started():
restart('shorewall')
with watch(CONFIG_FILES, False, on_change):
_zone_config(zones)
_interfaces_config(interfaces)
_policy_config(policy)
_rules_config(rules)
_routestopped_config(routestopped)
with settings(hide('running')):
sed('/etc/default/shorewall', 'startup=0', 'startup=1', use_sudo=True)
def _update_ssh_setting(sshd_config, name, value):
"""
Update a yes/no setting in the SSH config file
"""
with watch(sshd_config) as config_file:
# First try to change existing setting
sed(sshd_config,
r'^(\s*#\s*)?%s\s+(yes|no)' % name,
'%s %s' % (name, value),
use_sudo=True)
# Then append setting if it's still missing
_append(sshd_config,
'%s %s' % (name, value),
use_sudo=True)
if config_file.changed and is_running('ssh'):
restart('ssh')
def database(name, owner, template='template0', encoding='UTF8',
locale='en_US.UTF-8'):
"""
Require a PostgreSQL database.
::
from fabtools import require
require.postgres.database('myapp', owner='dbuser')
"""
if not database_exists(name):
with watch('/etc/locale.gen') as locales:
require_locale(locale)
if locales.changed:
restarted(_service_name())
create_database(name, owner, template=template, encoding=encoding,
locale=locale)
def firewall(zones=None, interfaces=None, policy=None, rules=None,
routestopped=None, masq=None):
"""
Ensure that a firewall is configured.
Example::
from fabtools.shorewall import *
from fabtools import require
# We need a firewall with some custom rules
require.shorewall.firewall(
rules=[
Ping(),
SSH(),
HTTP(),
HTTPS(),
SMTP(),
rule(port=1234, source=hosts(['example.com'])),
]
)
"""
package('shorewall')
with watch(CONFIG_FILES) as config:
_zone_config(zones)
_interfaces_config(interfaces)
_policy_config(policy)
_rules_config(rules)
_routestopped_config(routestopped)
_masq_config(masq)
if config.changed:
puts("Shorewall configuration changed")
if is_started():
restart('shorewall')
with settings(hide('running')):
sed('/etc/default/shorewall', 'startup=0', 'startup=1', use_sudo=True)
def _locales_generic(names, config_file, command):
supported = supported_locales()
_check_for_unsupported_locales(names, supported)
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
charset_from_name = dict(supported)
for name in names:
charset = charset_from_name[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file,
locale,
use_sudo=True,
partial=True,
shell=True)
if config.changed:
run_as_root(command)
def locales(names):
"""
Require the list of locales to be available
"""
config_file = '/var/lib/locales/supported.d/local'
def regenerate():
sudo('dpkg-reconfigure locales')
# Regenerate locales if config file changes
with watch(config_file, True, regenerate):
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
append(config_file, locale, use_sudo=True)
else:
warn('Unsupported locale name "%s"' % name)
def locales(names):
"""
Require the list of locales to be available
"""
config_file = '/var/lib/locales/supported.d/local'
def regenerate():
sudo('dpkg-reconfigure locales')
# Regenerate locales if config file changes
with watch(config_file, True, regenerate):
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
append(config_file, locale, use_sudo=True)
else:
warn('Unsupported locale name "%s"' % name)
def supervisor_process(service):
_name = service['name']
args = service['args']
service_name = '%s_%s' % (env.app, _name)
lines = ['[program:%(service_name)s]' % locals()]
stderr_logfile = join(env.log_path, _name + '_supervisor_error.log')
stdout_logfile = join(env.log_path, _name + '_supervisor_access.log')
if service['framework'] == 'django':
# wsgi_app = args.get('wsgi_app', _name)
wsgi_app = '%s.wsgi:application' % env.app
command = '%s run gunicorn -c guniconfig.py %s %s' % (
env.pipenv_path, wsgi_app, args['port'])
if service['framework'] == 'flask':
command = '%s run flask run -p %s' % (env.pipenv_path, args['port'])
elif service['framework'] == 'celery':
command = '%s run celery -A %s worker --loglevel=info -E --concurrency=10' % ( # NOQA
env.pipenv_path, env.app)
elif service['framework'] == 'celery_beat':
command = '%s run celery -A %s beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler' % ( # NOQA
env.pipenv_path, env.app)
params = dict(
command=command,
directory=env.app_path,
stderr_logfile=stderr_logfile,
environment=env.shell_envs_supervisor,
stdout_logfile=stdout_logfile,
autorestart=args.get('autorestart', 'true'),
user=env.user,
redirect_stderr=args.get('redirect_stderr', 'true'),
# user=env.app_user, # FIXME: Services should start as a system user
)
for key, value in sorted(params.items()):
lines.append('%s=%s' % (key, value))
filename = '/etc/supervisor/conf.d/%(service_name)s.conf' % locals()
with watch(filename, callback=supervisor.update_config, use_sudo=True):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
def instance(name, version=VERSION, **kwargs):
"""
Require a Redis instance to be running
The instance will be managed using supervisord.
"""
from fabtools import require
installed_from_source(version)
require.directory('/etc/redis', use_sudo=True, owner='redis')
require.directory('/var/db/redis', use_sudo=True, owner='redis')
require.directory('/var/log/redis', use_sudo=True, owner='redis')
require.directory('/var/run/redis', use_sudo=True, owner='redis')
# Required for background saving
require.system.sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', '127.0.0.1')
params.setdefault('port', '6379')
params.setdefault('logfile',
'/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dbfilename',
'/var/db/redis/redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
need_restart = False
def on_change():
need_restart = True
with watch(config_filename, True, on_change):
require.file(config_filename,
contents='\n'.join(lines),
use_sudo=True,
owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require.supervisor.process(process_name,
user='******',
directory='/var/run/redis',
command="%(redis_server)s %(config_filename)s" %
locals())
if need_restart:
fabtools.supervisor.restart_process(process_name)
def instance(name, version=VERSION, bind='127.0.0.1', port=6379, **kwargs):
"""
Require a Redis instance to be running.
The required Redis version will be automatically installed using
:py:func:`fabtools.require.redis.installed_from_source` if needed.
You can specify the IP address and port on which to listen to using the
*bind* and *port* parameters.
.. warning::
Redis is designed to be accessed by trusted clients inside trusted
environments. It is usually not a good idea to expose the Redis
instance directly to the internet. Therefore, with the default
settings, the Redis instance will only listen to local clients.
If you want to make your Redis instance accessible to other servers
over an untrusted network, you should probably add some firewall rules
to restrict access. For example: ::
from fabtools import require
from fabtools.shorewall import Ping, SSH, hosts, rule
# The computers that will need to talk to the Redis server
REDIS_CLIENTS = [
'web1.example.com',
'web2.example.com',
]
# The Redis server port
REDIS_PORT = 6379
# Setup a basic firewall
require.shorewall.firewall(
rules=[
Ping(),
SSH(),
rule(port=REDIS_PORT, source=hosts(REDIS_CLIENTS)),
]
)
# Make the Redis instance listen on all interfaces
require.redis.instance('mydb', bind='0.0.0.0', port=REDIS_PORT)
.. seealso:: `Redis Security <http://redis.io/topics/security>`_
You can also use any valid Redis configuration directives as extra
keyword arguments. For directives that can be repeated on multiple
lines (such as ``save``), you can supply a list of values.
The instance will be managed using supervisord, as a process named
``redis_{name}``, running as the ``redis`` user.
::
from fabtools import require
from fabtools.supervisor import process_status
require.redis.instance('mydb')
print process_status('redis_mydb')
.. seealso:: :ref:`supervisor_module` and
:ref:`require_supervisor_module`
The default settings enable persistence using periodic RDB snapshots
saved in the `/var/db/redis` directory.
You may want to use AOF persistence instead: ::
require.redis.instance('mydb', appendonly='yes', save=[])
In certain situations, you may want to disable persistence completely: ::
require.redis.instance('cache', port=6380, save=[])
.. seealso:: `Redis Persistence <http://redis.io/topics/persistence>`_
"""
from fabtools.require import directory as require_directory
from fabtools.require import file as require_file
from fabtools.require.supervisor import process as require_process
from fabtools.require.system import sysctl as require_sysctl
installed_from_source(version)
require_directory('/etc/redis', use_sudo=True, owner='redis')
require_directory('/var/db/redis', use_sudo=True, owner='redis')
require_directory('/var/log/redis', use_sudo=True, owner='redis')
# Required for background saving
with settings(warn_only=True):
require_sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', bind)
params.setdefault('port', port)
params.setdefault(
'logfile', '/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dir', '/var/db/redis')
params.setdefault('dbfilename', 'redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
with watch(config_filename, use_sudo=True) as config:
require_file(config_filename, contents='\n'.join(lines),
use_sudo=True, owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require_process(
process_name,
user='******',
directory='/var/run',
command="%(redis_server)s %(config_filename)s" % locals(),
)
# Restart if needed
if config.changed:
fabtools.supervisor.restart_process(process_name)
def instance(name, version=VERSION, **kwargs):
"""
Require a Redis instance to be running.
The instance will be managed using supervisord, as a process named
``redis_{name}``, running as the ``redis`` user.
::
from fabtools import require
from fabtools.supervisor import process_status
require.redis.installed_from_source()
require.redis.instance('db1', port='6379')
require.redis.instance('db2', port='6380')
print process_status('redis_db1')
print process_status('redis_db2')
.. seealso:: :ref:`supervisor_module` and
:ref:`require_supervisor_module`
"""
from fabtools.require import directory as require_directory
from fabtools.require import file as require_file
from fabtools.require.supervisor import process as require_process
from fabtools.require.system import sysctl as require_sysctl
installed_from_source(version)
require_directory('/etc/redis', use_sudo=True, owner='redis')
require_directory('/var/db/redis', use_sudo=True, owner='redis')
require_directory('/var/log/redis', use_sudo=True, owner='redis')
require_directory('/var/run/redis', use_sudo=True, owner='redis')
# Required for background saving
with settings(warn_only=True):
require_sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', '127.0.0.1')
params.setdefault('port', '6379')
params.setdefault('logfile',
'/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dbfilename',
'/var/db/redis/redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
with watch(config_filename, use_sudo=True) as config:
require_file(config_filename,
contents='\n'.join(lines),
use_sudo=True,
owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require_process(
process_name,
user='******',
directory='/var/run/redis',
command="%(redis_server)s %(config_filename)s" % locals(),
)
# Restart if needed
if config.changed:
fabtools.supervisor.restart_process(process_name)
def process(name, use_pip=False, **kwargs):
"""
Require a supervisor process to be running. Installs supervisor
from the default system package manager unless ``use_pip`` is
truthy.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisor documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the ``supervisor`` defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from fabtools import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisor documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from fabtools.require import file as require_file
from fabtools.require.python import package as require_python_package
from fabtools.require.deb import package as require_deb_package
from fabtools.require.rpm import package as require_rpm_package
from fabtools.require.arch import package as require_arch_package
from fabtools.require.service import started as require_started
# configure installation. override default package installation w/ use_pip
family = distrib_family()
if family == 'debian':
require_package = require_deb_package
package_name = 'supervisor'
daemon_name = 'supervisor'
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
elif family == 'redhat':
require_package = require_rpm_package
package_name = 'supervisor'
daemon_name = 'supervisord'
filename = '/etc/supervisord.d/%(name)s.ini' % locals()
elif family == 'arch':
require_package = require_arch_package
package_name = 'supervisor'
daemon_name = 'supervisord'
filename = '/etc/supervisor.d/%(name)s.ini' % locals()
else:
raise UnsupportedFamily(supported=['debian', 'redhat', 'arch'])
if use_pip:
require_package = require_python_package
package_name = 'supervisor'
# install supervisor and make sure its started
require_package(package_name)
require_started(daemon_name)
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
with watch(filename, callback=update_config, use_sudo=True):
require_file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
