def uptodate_index(quiet=True, max_age=86400):
"""
Require an up-to-date package index.
This will update the package index (using ``apt-get update``) if the last
update occured more than *max_age* ago.
*max_age* can be specified either as an integer (a value in seconds),
or as a dictionary whose keys are units (``seconds``, ``minutes``,
``hours``, ``days``, ``weeks``, ``months``) and values are integers.
The default value is 1 hour.
Examples: ::
from burlap import require
# Update index if last time was more than 1 day ago
require.deb.uptodate_index(max_age={'day': 1})
# Update index if last time was more than 1 hour and 30 minutes ago
require.deb.uptodate_index(max_age={'hour': 1, 'minutes': 30})
"""
from burlap.require import file as require_file
require_file('/etc/apt/apt.conf.d/15burlap-update-stamp',
contents='''\
APT::Update::Post-Invoke-Success {"touch /var/lib/apt/periodic/burlap-update-success-stamp 2>/dev/null || true";};
''',
use_sudo=True)
if system.time() - last_update_time() > _to_seconds(max_age):
update_index(quiet=quiet)
def uptodate_index(quiet=True, max_age=86400):
"""
Require an up-to-date package index.
This will update the package index (using ``apt-get update``) if the last
update occured more than *max_age* ago.
*max_age* can be specified either as an integer (a value in seconds),
or as a dictionary whose keys are units (``seconds``, ``minutes``,
``hours``, ``days``, ``weeks``, ``months``) and values are integers.
The default value is 1 hour.
Examples: ::
from burlap import require
# Update index if last time was more than 1 day ago
require.deb.uptodate_index(max_age={'day': 1})
# Update index if last time was more than 1 hour and 30 minutes ago
require.deb.uptodate_index(max_age={'hour': 1, 'minutes': 30})
"""
from burlap.require import file as require_file
require_file('/etc/apt/apt.conf.d/15burlap-update-stamp', contents='''\
APT::Update::Post-Invoke-Success {"touch /var/lib/apt/periodic/burlap-update-success-stamp 2>/dev/null || true";};
''', use_sudo=True)
if system.time() - last_update_time() > _to_seconds(max_age):
update_index(quiet=quiet)
def test_flag_is_set_when_watched_file_is_modified(watched_file):
from burlap.files import watch
from burlap.require import file as require_file
with watch('watched') as f:
require_file('watched', contents='bbb')
assert f.changed
def test_flag_is_set_when_watched_file_is_modified(watched_file):
from burlap.files import watch
from burlap.require import file as require_file
with watch('watched') as f:
require_file('watched', contents='bbb')
assert f.changed
def test_run_query_without_supplying_the_password(mysql_server, mysql_user):
from burlap.mysql import query
username, password = mysql_user
try:
require_file('.my.cnf', contents="[mysql]\npassword={0}".format(password))
with settings(mysql_user=username):
query('select 2;', use_sudo=False)
finally:
run('rm -f .my.cnf')
def test_require_file_from_url():
"""
Require that a file exists, whose contents should come from a URL
"""
from burlap.require import file as require_file
try:
require_file(url='http://www.google.com/robots.txt')
assert is_file('robots.txt')
finally:
run('rm -f robots.txt')
def test_callback_is_called_when_watched_file_is_modified(watched_file):
from burlap.files import watch
from burlap.require import file as require_file
try:
with watch('watched', callback=partial(require_file, 'modified1')):
require_file('watched', contents='bbb')
assert is_file('modified1')
finally:
run('rm -f modified1')
def test_require_file_from_url():
"""
Require that a file exists, whose contents should come from a URL
"""
from burlap.require import file as require_file
try:
require_file(url='http://www.google.com/robots.txt')
assert is_file('robots.txt')
finally:
run('rm -f robots.txt')
def test_run_query_without_supplying_the_password(mysql_server, mysql_user):
from burlap.mysql import query
username, password = mysql_user
try:
require_file('.my.cnf',
contents="[mysql]\npassword={0}".format(password))
with settings(mysql_user=username):
query('select 2;', use_sudo=False)
finally:
run('rm -f .my.cnf')
def test_callback_is_called_when_watched_file_is_modified(watched_file):
from burlap.files import watch
from burlap.require import file as require_file
try:
with watch('watched', callback=partial(require_file, 'modified1')):
require_file('watched', contents='bbb')
assert is_file('modified1')
finally:
run('rm -f modified1')
def test_require_file():
"""
Require that a file exists, whose contents should come from a URL
"""
from burlap.require import file as require_file
try:
require_file('foo')
assert is_file('foo')
assert run('cat foo') == ''
finally:
run('rm -f foo')
def test_require_file():
"""
Require that a file exists, whose contents should come from a URL
"""
from burlap.require import file as require_file
try:
require_file('foo')
assert is_file('foo')
assert run('cat foo') == ''
finally:
run('rm -f foo')
def test_file_changes_ownership():
from burlap.files import owner
from burlap.require.files import file as require_file
try:
run('touch foo')
assert owner('foo') == env.user
require_file('foo', use_sudo=True)
assert owner('foo') == 'root'
finally:
run_as_root('rm -f foo')
def test_file_changes_ownership():
from burlap.files import owner
from burlap.require.files import file as require_file
try:
run('touch foo')
assert owner('foo') == env.user
require_file('foo', use_sudo=True)
assert owner('foo') == 'root'
finally:
run_as_root('rm -f foo')
def installed_from_source(version=VERSION):
"""
Require Redis to be installed from source.
The compiled binaries will be installed in ``/opt/redis-{version}/``.
"""
from burlap.require import directory as require_directory
from burlap.require import file as require_file
from burlap.require import user as require_user
from burlap.require.deb import packages as require_deb_packages
from burlap.require.rpm import packages as require_rpm_packages
family = distrib_family()
if family == 'debian':
require_deb_packages([
'build-essential',
])
elif family == 'redhat':
require_rpm_packages([
'gcc',
'make',
])
require_user('redis', home='/var/lib/redis', system=True)
require_directory('/var/lib/redis', owner='redis', use_sudo=True)
dest_dir = '/opt/redis-%(version)s' % locals()
require_directory(dest_dir, use_sudo=True, owner='redis')
if not is_file('%(dest_dir)s/redis-server' % locals()):
with cd('/tmp'):
# Download and unpack the tarball
tarball = 'redis-%(version)s.tar.gz' % locals()
url = _download_url(version) + tarball
require_file(tarball, url=url)
run('tar xzf %(tarball)s' % locals())
# Compile and install binaries
with cd('redis-%(version)s' % locals()):
run('make')
for filename in BINARIES:
run_as_root('cp -pf src/%(filename)s %(dest_dir)s/' %
locals())
run_as_root('chown redis: %(dest_dir)s/%(filename)s' %
locals())
def test_empty_file_has_correct_permissions():
from burlap.files import owner, group, mode
from burlap.require.files import file as require_file
try:
run_as_root('touch foo')
require_file('bar', use_sudo=True)
assert owner('foo') == owner('bar')
assert group('foo') == group('bar')
assert mode('foo') == mode('bar')
finally:
run_as_root('rm -f foo bar')
def test_file_with_contents_has_correct_permissions():
from burlap.files import owner, group, mode
from burlap.require.files import file as require_file
try:
run_as_root('echo "something" > foo')
require_file('bar', contents='something', use_sudo=True)
assert owner('foo') == owner('bar')
assert group('foo') == group('bar')
assert mode('foo') == mode('bar')
finally:
run_as_root('rm -f foo bar')
def allow_sudo_user(setup_package):
"""
Fix sudo config if needed
Some Vagrant boxes come with a too restrictive sudoers config
and only allow the vagrant user to run commands as root.
"""
from burlap.require import file as require_file
require_file(
'/etc/sudoers.d/burlap',
contents="vagrant ALL=(ALL) NOPASSWD:ALL\n",
owner='root',
mode='440',
use_sudo=True,
)
def allow_sudo_user(setup_package):
"""
Fix sudo config if needed
Some Vagrant boxes come with a too restrictive sudoers config
and only allow the vagrant user to run commands as root.
"""
from burlap.require import file as require_file
require_file(
'/etc/sudoers.d/burlap',
contents="vagrant ALL=(ALL) NOPASSWD:ALL\n",
owner='root',
mode='440',
use_sudo=True,
)
def test_file_with_contents_has_correct_permissions():
from burlap.files import owner, group, mode
from burlap.require.files import file as require_file
try:
run_as_root('echo "something" > foo')
require_file('bar', contents='something', use_sudo=True)
assert owner('foo') == owner('bar')
assert group('foo') == group('bar')
assert mode('foo') == mode('bar')
finally:
run_as_root('rm -f foo bar')
def test_empty_file_has_correct_permissions():
from burlap.files import owner, group, mode
from burlap.require.files import file as require_file
try:
run_as_root('touch foo')
require_file('bar', use_sudo=True)
assert owner('foo') == owner('bar')
assert group('foo') == group('bar')
assert mode('foo') == mode('bar')
finally:
run_as_root('rm -f foo bar')
def installed_from_source(version=VERSION):
"""
Require Redis to be installed from source.
The compiled binaries will be installed in ``/opt/redis-{version}/``.
"""
from burlap.require import directory as require_directory
from burlap.require import file as require_file
from burlap.require import user as require_user
from burlap.require.deb import packages as require_deb_packages
from burlap.require.rpm import packages as require_rpm_packages
family = distrib_family()
if family == 'debian':
require_deb_packages([
'build-essential',
])
elif family == 'redhat':
require_rpm_packages([
'gcc',
'make',
])
require_user('redis', home='/var/lib/redis', system=True)
require_directory('/var/lib/redis', owner='redis', use_sudo=True)
dest_dir = '/opt/redis-%(version)s' % locals()
require_directory(dest_dir, use_sudo=True, owner='redis')
if not is_file('%(dest_dir)s/redis-server' % locals()):
with cd('/tmp'):
# Download and unpack the tarball
tarball = 'redis-%(version)s.tar.gz' % locals()
url = _download_url(version) + tarball
require_file(tarball, url=url)
run('tar xzf %(tarball)s' % locals())
# Compile and install binaries
with cd('redis-%(version)s' % locals()):
run('make')
for filename in BINARIES:
run_as_root('cp -pf src/%(filename)s %(dest_dir)s/' % locals())
run_as_root('chown redis: %(dest_dir)s/%(filename)s' % locals())
def test_require_file_from_string():
"""
Require that a file exists, whose contents should be this string
"""
from burlap.require import file as require_file
try:
bar_contents = "This is the contents of the bar file"
require_file('bar', contents=bar_contents)
assert is_file('bar')
assert run('cat bar') == bar_contents
finally:
run('rm -f bar')
def default_locale(name):
"""
Require the locale to be the default.
"""
from burlap.require import file as require_file
# Ensure the locale is available
locale(name)
# Make it the default
contents = 'LANG="%s"\n' % name
if distrib_family() == 'arch':
config_file = '/etc/locale.conf'
else:
config_file = '/etc/default/locale'
require_file(config_file, contents, use_sudo=True)
def test_require_file_from_string():
"""
Require that a file exists, whose contents should be this string
"""
from burlap.require import file as require_file
try:
bar_contents = "This is the contents of the bar file"
require_file('bar', contents=bar_contents)
assert is_file('bar')
assert run('cat bar') == bar_contents
finally:
run('rm -f bar')
def default_locale(name):
"""
Require the locale to be the default.
"""
from burlap.require import file as require_file
# Ensure the locale is available
locale(name)
# Make it the default
contents = 'LANG="%s"\n' % name
if distrib_family() == 'arch':
config_file = '/etc/locale.conf'
else:
config_file = '/etc/default/locale'
require_file(config_file, contents, use_sudo=True)
def test_install_dependencies_from_package_json_file(nodejs, testdir):
from burlap.nodejs import install_dependencies, package_version, uninstall_package
with cd(testdir):
require_file('package.json', contents=json.dumps({
'name': 'nodetest',
'version': '1.0.0',
'dependencies': {
'underscore': '1.4.2'
}
}))
install_dependencies()
assert is_file('node_modules/underscore/underscore.js')
assert package_version('underscore', local=True) == '1.4.2'
uninstall_package('underscore', local=True)
def sysctl(key, value, persist=True):
"""
Require a kernel parameter to have a specific value.
"""
if get_sysctl(key) != value:
set_sysctl(key, value)
if persist:
from burlap.require import file as require_file
filename = '/etc/sysctl.d/60-%s.conf' % key
with watch(filename, use_sudo=True) as config:
require_file(filename,
contents='%(key)s = %(value)s\n' % locals(),
use_sudo=True)
if config.changed:
if distrib_family() == 'debian':
with settings(warn_only=True):
run_as_root('service procps start')
def test_install_dependencies_from_package_json_file(nodejs, testdir):
from burlap.nodejs import install_dependencies, package_version, uninstall_package
with cd(testdir):
require_file('package.json',
contents=json.dumps({
'name': 'nodetest',
'version': '1.0.0',
'dependencies': {
'underscore': '1.4.2'
}
}))
install_dependencies()
assert is_file('node_modules/underscore/underscore.js')
assert package_version('underscore', local=True) == '1.4.2'
uninstall_package('underscore', local=True)
def sysctl(key, value, persist=True):
"""
Require a kernel parameter to have a specific value.
"""
if get_sysctl(key) != value:
set_sysctl(key, value)
if persist:
from burlap.require import file as require_file
filename = '/etc/sysctl.d/60-%s.conf' % key
with watch(filename, use_sudo=True) as config:
require_file(filename,
contents='%(key)s = %(value)s\n' % locals(),
use_sudo=True)
if config.changed:
if distrib_family() == 'debian':
with settings(warn_only=True):
run_as_root('service procps start')
def test_require_file_from_local_file():
"""
Require that a file exists, whose contents should be this local file
"""
from burlap.require import file as require_file
try:
baz_contents = "This is the contents of the bar file"
fd, filename = mkstemp()
tmp_file = os.fdopen(fd, 'w')
tmp_file.write(baz_contents)
tmp_file.close()
require_file('baz', source=filename)
assert is_file('baz')
assert run('cat baz') == baz_contents
finally:
os.remove(filename)
run('rm -f baz')
def test_require_file_from_local_file():
"""
Require that a file exists, whose contents should be this local file
"""
from burlap.require import file as require_file
try:
baz_contents = "This is the contents of the bar file"
fd, filename = mkstemp()
tmp_file = os.fdopen(fd, 'w')
tmp_file.write(baz_contents)
tmp_file.close()
require_file('baz', source=filename)
assert is_file('baz')
assert run('cat baz') == baz_contents
finally:
os.remove(filename)
run('rm -f baz')
def source(name, uri, distribution, *components):
"""
Require a package source.
::
from burlap import require
# Official MongoDB packages
require.deb.source('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen')
"""
from burlap.require import file as require_file
path = '/etc/apt/sources.list.d/%(name)s.list' % locals()
components = ' '.join(components)
source_line = 'deb %(uri)s %(distribution)s %(components)s\n' % locals()
with watch(path) as config:
require_file(path=path, contents=source_line, use_sudo=True)
if config.changed:
puts('Added APT repository: %s' % source_line)
update_index()
def source(name, uri, distribution, *components):
"""
Require a package source.
::
from burlap import require
# Official MongoDB packages
require.deb.source('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen')
"""
from burlap.require import file as require_file
path = '/etc/apt/sources.list.d/%(name)s.list' % locals()
components = ' '.join(components)
source_line = 'deb %(uri)s %(distribution)s %(components)s\n' % locals()
with watch(path) as config:
require_file(path=path, contents=source_line, use_sudo=True)
if config.changed:
puts('Added APT repository: %s' % source_line)
update_index()
def install_from_source(version=DEFAULT_VERSION, checkinstall=False):
"""
Install Node JS from source.
If *checkinstall* is ``True``, a distribution package will be built.
::
import burlap
# Install Node.js
burlap.nodejs.install_nodejs()
.. note:: This function may not work for old versions of Node.js.
"""
from burlap.require.deb import packages as require_deb_packages
from burlap.require.rpm import packages as require_rpm_packages
from burlap.require import file as require_file
family = distrib_family()
if family == 'debian':
packages = [
'build-essential',
'libssl-dev',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_deb_packages(packages)
elif family == 'redhat':
packages = [
'gcc',
'gcc-c++',
'make',
'openssl-devel',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_rpm_packages(packages)
filename = 'node-v%s.tar.gz' % version
foldername = filename[0:-7]
require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % {
'version': version,
'filename': filename,
})
run('tar -xzf %s' % filename)
with cd(foldername):
run('./configure')
run('make -j%d' % (cpus() + 1))
if checkinstall:
run_as_root(
'checkinstall -y --pkgname=nodejs --pkgversion=%(version) '
'--showinstall=no make install' % locals())
else:
run_as_root('make install')
run('rm -rf %(filename)s %(foldername)s' % locals())
def process(name, **kwargs):
"""
Require a supervisor process to be running.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisor documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the ``supervisor`` defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from burlap import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisor documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from burlap.require import file as require_file
from burlap.require.deb import package as require_deb_package
from burlap.require.rpm import package as require_rpm_package
from burlap.require.arch import package as require_arch_package
from burlap.require.service import started as require_started
family = distrib_family()
if family == 'debian':
require_deb_package('supervisor')
require_started('supervisor')
elif family == 'redhat':
require_rpm_package('supervisord')
require_started('supervisord')
elif family == 'arch':
require_arch_package('supervisor')
require_started('supervisord')
else:
raise UnsupportedFamily(supported=['debian', 'redhat', 'arch'])
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
if family == 'debian':
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
elif family == 'redhat':
filename = '/etc/supervisord.d/%(name)s.ini' % locals()
elif family == 'arch':
filename = '/etc/supervisor.d/%(name)s.ini' % locals()
with watch(filename, callback=update_config, use_sudo=True):
require_file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def install_from_source(version=DEFAULT_VERSION, checkinstall=False):
"""
Install Node JS from source.
If *checkinstall* is ``True``, a distribution package will be built.
::
import burlap
# Install Node.js
burlap.nodejs.install_nodejs()
.. note:: This function may not work for old versions of Node.js.
"""
from burlap.require.deb import packages as require_deb_packages
from burlap.require.rpm import packages as require_rpm_packages
from burlap.require import file as require_file
family = distrib_family()
if family == 'debian':
packages = [
'build-essential',
'libssl-dev',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_deb_packages(packages)
elif family == 'redhat':
packages = [
'gcc',
'gcc-c++',
'make',
'openssl-devel',
'python',
]
if checkinstall:
packages.append('checkinstall')
require_rpm_packages(packages)
filename = 'node-v%s.tar.gz' % version
foldername = filename[0:-7]
require_file(url='http://nodejs.org/dist/v%(version)s/%(filename)s' % {
'version': version,
'filename': filename,
})
run('tar -xzf %s' % filename)
with cd(foldername):
run('./configure')
run('make -j%d' % (cpus() + 1))
if checkinstall:
run_as_root('checkinstall -y --pkgname=nodejs --pkgversion=%(version) '
'--showinstall=no make install' % locals())
else:
run_as_root('make install')
run('rm -rf %(filename)s %(foldername)s' % locals())
def watched_file():
from burlap.require import file as require_file
filename = 'watched'
require_file('watched', contents='aaa')
return filename
def watched_file():
from burlap.require import file as require_file
filename= 'watched'
require_file('watched', contents='aaa')
return filename
def install_from_source(path=DEFAULT_INSTALLATION_PATH,
version=DEFAULT_VERSION,
mirror=DEFAULT_MIRROR,
overwrite=False):
"""
Install Tomcat from source.
::
import burlap
# Install Tomcat
burlap.tomcat.install_from_source(version='6.0.36')
"""
from burlap.require import file as require_file
from burlap.require.files import directory as require_directory
# Tokenize version into parts
version_tokens = version.split('.')
version_major = version_tokens[0]
# Parse the filename and folder
file_name = 'apache-tomcat-%s.tar.gz' % version
folder_name = 'apache-tomcat-%s' % version
# Build the distribution in /tmp
with cd('/tmp'):
# Make sure we have the tarball downloaded.
if not is_file(os.path.join('/tmp/', file_name)):
# Otherwise, download the tarball based on our mirror and version.
tomcat_url = '%s/dist/tomcat/tomcat-%s/v%s/bin/%s' % (mirror, version_major, version, file_name)
# Ensure the file has been downloaded
require_file(url=tomcat_url)
# Extract the file
run('tar -xzf %s' % file_name)
# Handle possibility of existing path
if is_dir(path):
if overwrite is False:
# Raise exception as we don't want to overwrite
raise OSError("Path %s already exists and overwrite not set." % path)
else:
# Otherwise, backup the tomcat path
backup_installation_path = path + ".backup"
if is_dir(backup_installation_path):
run_as_root("rm -rf %s" % backup_installation_path)
run_as_root("mv %s %s" % (path, backup_installation_path))
"""
After all that, let's ensure we have the installation path setup
properly and place the install.
"""
require_directory(path, mode='755', use_sudo=True)
run_as_root('mv %s/* %s' % (folder_name, path))
# Now cleanup temp.
run("rm -rf %s*" % file_name)
# Finally, configure and start Tomcat
configure_tomcat(path, overwrite=overwrite)
start_tomcat()
def test_require_file_in_guest_context_manager(container):
from burlap.openvz import guest
with guest(container):
require_file('/tmp/foo')
assert is_file('/tmp/foo')
def instance(name, version=VERSION, bind='127.0.0.1', port=6379, **kwargs):
"""
Require a Redis instance to be running.
The required Redis version will be automatically installed using
:py:func:`burlap.require.redis.installed_from_source` if needed.
You can specify the IP address and port on which to listen to using the
*bind* and *port* parameters.
.. warning::
Redis is designed to be accessed by trusted clients inside trusted
environments. It is usually not a good idea to expose the Redis
instance directly to the internet. Therefore, with the default
settings, the Redis instance will only listen to local clients.
If you want to make your Redis instance accessible to other servers
over an untrusted network, you should probably add some firewall rules
to restrict access. For example: ::
from burlap import require
from burlap.shorewall import Ping, SSH, hosts, rule
# The computers that will need to talk to the Redis server
REDIS_CLIENTS = [
'web1.example.com',
'web2.example.com',
]
# The Redis server port
REDIS_PORT = 6379
# Setup a basic firewall
require.shorewall.firewall(
rules=[
Ping(),
SSH(),
rule(port=REDIS_PORT, source=hosts(REDIS_CLIENTS)),
]
)
# Make the Redis instance listen on all interfaces
require.redis.instance('mydb', bind='0.0.0.0', port=REDIS_PORT)
.. seealso:: `Redis Security <http://redis.io/topics/security>`_
You can also use any valid Redis configuration directives as extra
keyword arguments. For directives that can be repeated on multiple
lines (such as ``save``), you can supply a list of values.
The instance will be managed using supervisord, as a process named
``redis_{name}``, running as the ``redis`` user.
::
from burlap import require
from burlap.supervisor import process_status
require.redis.instance('mydb')
print(process_status('redis_mydb'))
.. seealso:: :ref:`supervisor_module` and
:ref:`require_supervisor_module`
The default settings enable persistence using periodic RDB snapshots
saved in the `/var/db/redis` directory.
You may want to use AOF persistence instead: ::
require.redis.instance('mydb', appendonly='yes', save=[])
In certain situations, you may want to disable persistence completely: ::
require.redis.instance('cache', port=6380, save=[])
.. seealso:: `Redis Persistence <http://redis.io/topics/persistence>`_
"""
from burlap.require import directory as require_directory
from burlap.require import file as require_file
from burlap.require.supervisor import process as require_process
from burlap.require.system import sysctl as require_sysctl
installed_from_source(version)
require_directory('/etc/redis', use_sudo=True, owner='redis')
require_directory('/var/db/redis', use_sudo=True, owner='redis')
require_directory('/var/log/redis', use_sudo=True, owner='redis')
# Required for background saving
with settings(warn_only=True):
require_sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', bind)
params.setdefault('port', port)
params.setdefault('logfile',
'/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dir', '/var/db/redis')
params.setdefault('dbfilename', 'redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
with watch(config_filename, use_sudo=True) as config:
require_file(config_filename,
contents='\n'.join(lines),
use_sudo=True,
owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require_process(
process_name,
user='******',
directory='/var/run',
command="%(redis_server)s %(config_filename)s" % locals(),
)
# Restart if needed
if config.changed:
burlap.supervisor.restart_process(process_name)
def instance(name, version=VERSION, bind='127.0.0.1', port=6379, **kwargs):
"""
Require a Redis instance to be running.
The required Redis version will be automatically installed using
:py:func:`burlap.require.redis.installed_from_source` if needed.
You can specify the IP address and port on which to listen to using the
*bind* and *port* parameters.
.. warning::
Redis is designed to be accessed by trusted clients inside trusted
environments. It is usually not a good idea to expose the Redis
instance directly to the internet. Therefore, with the default
settings, the Redis instance will only listen to local clients.
If you want to make your Redis instance accessible to other servers
over an untrusted network, you should probably add some firewall rules
to restrict access. For example: ::
from burlap import require
from burlap.shorewall import Ping, SSH, hosts, rule
# The computers that will need to talk to the Redis server
REDIS_CLIENTS = [
'web1.example.com',
'web2.example.com',
]
# The Redis server port
REDIS_PORT = 6379
# Setup a basic firewall
require.shorewall.firewall(
rules=[
Ping(),
SSH(),
rule(port=REDIS_PORT, source=hosts(REDIS_CLIENTS)),
]
)
# Make the Redis instance listen on all interfaces
require.redis.instance('mydb', bind='0.0.0.0', port=REDIS_PORT)
.. seealso:: `Redis Security <http://redis.io/topics/security>`_
You can also use any valid Redis configuration directives as extra
keyword arguments. For directives that can be repeated on multiple
lines (such as ``save``), you can supply a list of values.
The instance will be managed using supervisord, as a process named
``redis_{name}``, running as the ``redis`` user.
::
from burlap import require
from burlap.supervisor import process_status
require.redis.instance('mydb')
print(process_status('redis_mydb'))
.. seealso:: :ref:`supervisor_module` and
:ref:`require_supervisor_module`
The default settings enable persistence using periodic RDB snapshots
saved in the `/var/db/redis` directory.
You may want to use AOF persistence instead: ::
require.redis.instance('mydb', appendonly='yes', save=[])
In certain situations, you may want to disable persistence completely: ::
require.redis.instance('cache', port=6380, save=[])
.. seealso:: `Redis Persistence <http://redis.io/topics/persistence>`_
"""
from burlap.require import directory as require_directory
from burlap.require import file as require_file
from burlap.require.supervisor import process as require_process
from burlap.require.system import sysctl as require_sysctl
installed_from_source(version)
require_directory('/etc/redis', use_sudo=True, owner='redis')
require_directory('/var/db/redis', use_sudo=True, owner='redis')
require_directory('/var/log/redis', use_sudo=True, owner='redis')
# Required for background saving
with settings(warn_only=True):
require_sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', bind)
params.setdefault('port', port)
params.setdefault('logfile', '/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dir', '/var/db/redis')
params.setdefault('dbfilename', 'redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
with watch(config_filename, use_sudo=True) as config:
require_file(config_filename, contents='\n'.join(lines),
use_sudo=True, owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require_process(
process_name,
user='******',
directory='/var/run',
command="%(redis_server)s %(config_filename)s" % locals(),
)
# Restart if needed
if config.changed:
burlap.supervisor.restart_process(process_name)
def process(name, **kwargs):
"""
Require a supervisor process to be running.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisor documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the ``supervisor`` defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from burlap import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisor documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from burlap.require import file as require_file
from burlap.require.deb import package as require_deb_package
from burlap.require.rpm import package as require_rpm_package
from burlap.require.arch import package as require_arch_package
from burlap.require.service import started as require_started
family = distrib_family()
if family == 'debian':
require_deb_package('supervisor')
require_started('supervisor')
elif family == 'redhat':
require_rpm_package('supervisord')
require_started('supervisord')
elif family == 'arch':
require_arch_package('supervisor')
require_started('supervisord')
else:
raise UnsupportedFamily(supported=['debian', 'redhat', 'arch'])
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
if family == 'debian':
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
elif family == 'redhat':
filename = '/etc/supervisord.d/%(name)s.ini' % locals()
elif family == 'arch':
filename = '/etc/supervisor.d/%(name)s.ini' % locals()
with watch(filename, callback=update_config, use_sudo=True):
require_file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def install_from_source(path=DEFAULT_INSTALLATION_PATH,
version=DEFAULT_VERSION,
mirror=DEFAULT_MIRROR,
overwrite=False):
"""
Install Tomcat from source.
::
import burlap
# Install Tomcat
burlap.tomcat.install_from_source(version='6.0.36')
"""
from burlap.require import file as require_file
from burlap.require.files import directory as require_directory
# Tokenize version into parts
version_tokens = version.split('.')
version_major = version_tokens[0]
# Parse the filename and folder
file_name = 'apache-tomcat-%s.tar.gz' % version
folder_name = 'apache-tomcat-%s' % version
# Build the distribution in /tmp
with cd('/tmp'):
# Make sure we have the tarball downloaded.
if not is_file(os.path.join('/tmp/', file_name)):
# Otherwise, download the tarball based on our mirror and version.
tomcat_url = '%s/dist/tomcat/tomcat-%s/v%s/bin/%s' % (
mirror, version_major, version, file_name)
# Ensure the file has been downloaded
require_file(url=tomcat_url)
# Extract the file
run('tar -xzf %s' % file_name)
# Handle possibility of existing path
if is_dir(path):
if overwrite is False:
# Raise exception as we don't want to overwrite
raise OSError("Path %s already exists and overwrite not set." %
path)
else:
# Otherwise, backup the tomcat path
backup_installation_path = path + ".backup"
if is_dir(backup_installation_path):
run_as_root("rm -rf %s" % backup_installation_path)
run_as_root("mv %s %s" % (path, backup_installation_path))
"""
After all that, let's ensure we have the installation path setup
properly and place the install.
"""
require_directory(path, mode='755', use_sudo=True)
run_as_root('mv %s/* %s' % (folder_name, path))
# Now cleanup temp.
run("rm -rf %s*" % file_name)
# Finally, configure and start Tomcat
configure_tomcat(path, overwrite=overwrite)
start_tomcat()
def sshd_config(request):
require_file(SSHD_CONFIG, contents=dedent(request.param))