def update_filters():
changes = []
# Generate desired state as enabled_name => source_name
config = blueprint.get('config', {})
filters = {
'{}-{}.conf'.format(str(weight).zfill(2), conf): "{}.conf".format(conf)
for weight, conf in config.iteritems()
}
# Get current state
with silent():
enabled_filters = run('ls {}'.format(conf_enabled_path)).split()
# Disable extra services
if blueprint.get('auto_disable_conf', True):
for link in set(enabled_filters) - set(filters.keys()):
info('Disabling conf: {}', link)
changes.append(link)
with silent(), sudo(), cd(conf_enabled_path):
debian.rm(link)
# Enable services
for target in set(filters.keys()) - set(enabled_filters):
source = os.path.join(conf_available_path, filters[target])
info('Enabling conf: {}', target)
changes.append(source)
with silent(), sudo(), cd(conf_enabled_path):
debian.ln(source, target)
return changes
def update_modules(beat):
changes = []
# Get desired state
desired_modules = set(blueprint.get('{}.modules'.format(beat), []))
# Get current state
with silent():
module_files = run('find /etc/{}/modules.d -iname "*.yml"'.format(beat)).split()
enabled_modules = {os.path.basename(module).split('.')[0] for module in module_files}
# Disable extra services
for extra in enabled_modules - desired_modules:
info('Disabling {} module: {}', beat, extra)
changes.append(extra)
with silent(), sudo():
run('{} modules disable {}'.format(beat, extra))
# Enable services
for missing in desired_modules - enabled_modules:
info('Enabling {} module: {}', beat, missing)
changes.append(missing)
with silent(), sudo():
run('{} modules enable {}'.format(beat, missing))
return changes
def dump(schema=None):
"""
Dump and download all configured, or given, schemas.
:param schema: Specific shema to dump and download.
"""
if not schema:
schemas = blueprint.get('schemas', {}).keys()
for i, schema in enumerate(schemas, start=1):
print("{i}. {schema}".format(i=i, schema=schema))
valid_indices = '[1-{}]+'.format(len(schemas))
schema_choice = prompt('Select schema to dump:',
default='1',
validate=valid_indices)
schema = schemas[int(schema_choice) - 1]
with sudo('postgres'):
now = datetime.now().strftime('%Y-%m-%d')
output_file = '/tmp/{}_{}.backup'.format(schema, now)
filename = os.path.basename(output_file)
options = dict(format='tar', output_file=output_file, schema=schema)
info('Dumping schema {}...', schema)
run('pg_dump -c -F {format} -f {output_file} {schema}'.format(
**options))
info('Downloading dump...')
local_file = '~/{}'.format(filename)
files.get(output_file, local_file)
with sudo(), silent():
debian.rm(output_file)
info('New smoking hot dump at {}', local_file)
def create_app_root():
from .project import app_root
with sudo():
# Create global apps root
root_path = app_root()
debian.mkdir(root_path, recursive=True)
def configure():
"""
Enable/disable configured programs
"""
with sudo():
# Upload templates
uploads = blueprint.upload("init/", "/etc/init/")
uploads.extend(blueprint.upload("supervisord.conf", "/etc/"))
# Disable previously enabled programs not configured programs-enabled
changes = []
programs = blueprint.get("programs") or []
auto_disable = blueprint.get("auto_disable_programs", True)
if auto_disable:
with silent():
enabled_program_links = run("ls {}".format(programs_enabled_path)).split()
for link in enabled_program_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in programs and link_name not in programs:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable programs from settings
for program in programs:
changed = enable(program, do_reload=False)
changes.append(changed)
### Reload supervisor if new templates or any program has been enabled/disabled
if uploads or any(changes):
reload()
def configure():
"""
Configure PureFTP
"""
with sudo():
# Echo configurations
setup_config()
for user in blueprint.get('users'):
username, password = user['username'], user['password']
if 'homedir' in user:
user_home = user['homedir']
else:
user_home = os.path.join(ftp_root, username)
passwd_path = '/etc/pure-ftpd/pureftpd.passwd'
with settings(warn_only=True):
if files.exists(passwd_path) and run('pure-pw show {}'.format(
username)).return_code == 0:
continue
debian.mkdir(user_home, owner=ftp_user, group=ftp_group)
prompts = {
'Password: '******'Enter it again: ': password
}
with settings(prompts=prompts):
run('pure-pw useradd {} -u {} -g {} -d {}'.format(username, ftp_user, ftp_group,
user_home))
run('pure-pw mkdb')
restart()
def install():
with sudo():
info('Installing Ruby v1.9.3')
debian.apt_get('install', 'ruby1.9.3')
info('Installing Bundler')
gem('install', 'bundler')
def service(name, action, check_status=True):
c = fabric.context_managers
with sudo('root'), c.settings(c.hide('running', 'stdout', 'stderr',
'warnings'),
warn_only=True):
info('Service: {} {}', name, action)
if check_status:
output = run('service {} status'.format(name),
pty=False,
combine_stderr=True)
if output.return_code != 0:
puts(indent(magenta(output)))
return
elif action in output:
puts(
indent('...has status {}'.format(
magenta(output[len(name) + 1:]))))
return
output = run('service {} {}'.format(name, action),
pty=False,
combine_stderr=True)
if output.return_code != 0:
puts(indent(magenta(output)))
def install_solr():
with sudo():
version = blueprint.get("version")
version_tuple = tuple(map(int, version.split(".")))
archive = "solr-{}.tgz".format(version)
if version_tuple < (4, 1, 0):
archive = "apache-{}".format(archive)
url = "https://archive.apache.org/dist/lucene/solr/{}/{}".format(version, archive)
with cd("/tmp"):
info("Download {} ({})", "Solr", version)
run("wget {}".format(url))
info("Extracting archive...")
with silent():
run("tar xzf {}".format(archive))
solr_version_dir = os.path.splitext(archive)[0]
solr_version_path = os.path.join("/usr", "share", solr_version_dir)
debian.chmod(solr_version_dir, 755, "solr", "solr", recursive=True)
if files.exists(solr_version_path):
info("Found same existing version, removing it...")
debian.rm(solr_version_path, recursive=True)
debian.mv(solr_version_dir, "/usr/share/")
debian.ln(solr_version_path, solr_home)
debian.rm(archive)
def configure():
"""
Configure nginx and enable/disable sites
"""
with sudo():
# Upload templates
context = {'num_cores': debian.nproc()}
uploads = blueprint.upload('./', nginx_root, context)
# Disable previously enabled sites not configured sites-enabled
changes = []
sites = blueprint.get('sites')
auto_disable_sites = blueprint.get('auto_disable_sites', True)
if auto_disable_sites:
with silent():
enabled_site_links = run(
'ls {}'.format(sites_enabled_path)).split()
for link in enabled_site_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in sites and link_name not in sites:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable sites from settings
for site in sites:
changed = enable(site, do_reload=False)
changes.append(changed)
### Reload nginx if new templates or any site has been enabled/disabled
if uploads or any(changes):
reload()
def enable(site, do_reload=True):
"""
Enable site
:param site: Site to enable
:param do_reload: Reload nginx service
:return: Got enabled?
"""
enabled = False
site = site if site.endswith(
'.conf') or site == 'default' else '{}.conf'.format(site)
with sudo():
available_site = os.path.join(sites_available_path, site)
if not files.exists(available_site):
warn('Invalid site: {}'.format(site))
else:
with cd(sites_enabled_path):
if not files.exists(site):
info('Enabling site: {}', site)
with silent():
debian.ln(available_site, site)
enabled = True
if do_reload:
reload()
return enabled
def install_solr():
with sudo():
version = blueprint.get('version')
version_tuple = tuple(map(int, version.split('.')))
archive = 'solr-{}.tgz'.format(version)
if version_tuple < (4, 1, 0):
archive = 'apache-{}'.format(archive)
url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive)
with cd('/tmp'):
info('Download {} ({})', 'Solr', version)
run('wget {}'.format(url))
info('Extracting archive...')
with silent():
run('tar xzf {}'.format(archive))
solr_version_dir = os.path.splitext(archive)[0]
solr_version_path = os.path.join('/usr', 'share', solr_version_dir)
debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True)
if files.exists(solr_version_path):
info('Found same existing version, removing it...')
debian.rm(solr_version_path, recursive=True)
debian.mv(solr_version_dir, '/usr/share/')
debian.ln(solr_version_path, solr_home)
debian.rm(archive)
def install_user():
with sudo():
user.create_service_user('solr', home=solr_home)
# Home dir will be created by solr installation (symbolic link)
debian.mkdir('/var/lib/solr', mode=755, owner='solr', group='solr')
debian.mkdir('/var/log/solr', mode=755, owner='solr', group='solr')
def setup_schemas(drop=False):
"""
Create database schemas and grant user privileges
:param drop: Drop existing schemas before creation
"""
schemas = blueprint.get('schemas', {})
with sudo('postgres'):
for schema, config in schemas.iteritems():
user, password = config['user'], config.get('password')
info('Creating user {}', user)
if password:
_client_exec("CREATE ROLE %(user)s WITH PASSWORD '%(password)s' LOGIN",
user=user, password=password)
else:
_client_exec("CREATE ROLE %(user)s LOGIN", user=user)
if drop:
info('Droping schema {}', schema)
_client_exec('DROP DATABASE %(name)s', name=schema)
info('Creating schema {}', schema)
_client_exec('CREATE DATABASE %(name)s', name=schema)
info('Granting user {} to schema {}'.format(user, schema))
_client_exec("GRANT ALL PRIVILEGES ON DATABASE %(schema)s to %(user)s",
schema=schema, user=user)
for ext in blueprint.get('extensions', []):
info('Creating extension {}'.format(ext))
_client_exec("CREATE EXTENSION IF NOT EXISTS %(ext)s", ext=ext, schema=schema)
def install():
with sudo():
from blues import java
java.install()
version = blueprint.get('version', '1.0')
info('Adding apt repository for {} version {}', 'elasticsearch',
version)
repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format(
version)
debian.add_apt_repository(repository)
info('Adding apt key for', repository)
debian.add_apt_key(
'http://packages.elasticsearch.org/GPG-KEY-elasticsearch')
debian.apt_get('update')
# Install elasticsearch (and java)
info('Installing {} version {}', 'elasticsearch', version)
debian.apt_get('install', 'elasticsearch')
# Install plugins
plugins = blueprint.get('plugins', [])
for plugin in plugins:
info('Installing elasticsearch "{}" plugin...', plugin)
install_plugin(plugin)
# Enable on boot
debian.add_rc_service('elasticsearch', priorities='defaults 95 10')
def setup():
"""
Setup Logstash server
"""
from .elasticsearch import add_elastic_repo
with sudo():
branch = blueprint.get('branch', '6.x')
add_elastic_repo(branch)
version = blueprint.get('version', 'latest')
info('Installing {} version {}', 'logstash', version)
package = 'logstash' + ('={}'.format(version) if version != 'latest' else '')
debian.apt_get('install', package)
# Enable on boot
debian.add_rc_service('logstash')
# prep custom folders
debian.mkdir(conf_available_path)
debian.mkdir(conf_enabled_path)
# Install plugins
plugins = blueprint.get('plugins', [])
for plugin in plugins:
info('Installing logstash "{}" plugin...', plugin)
install_plugin(plugin)
configure()
def install_user():
with sudo():
user.create_service_user("solr", home=solr_home)
# Home dir will be created by solr installation (symbolic link)
debian.mkdir("/var/lib/solr", mode=755, owner="solr", group="solr")
debian.mkdir("/var/log/solr", mode=755, owner="solr", group="solr")
def install():
with sudo():
from blues import java
java.install()
version = blueprint.get('version', '1.0')
info('Adding apt repository for {} version {}', 'elasticsearch', version)
repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format(version)
debian.add_apt_repository(repository)
info('Adding apt key for', repository)
debian.add_apt_key('http://packages.elasticsearch.org/GPG-KEY-elasticsearch')
debian.apt_get('update')
# Install elasticsearch (and java)
info('Installing {} version {}', 'elasticsearch', version)
debian.apt_get('install', 'elasticsearch')
# Install plugins
plugins = blueprint.get('plugins', [])
for plugin in plugins:
info('Installing elasticsearch "{}" plugin...', plugin)
install_plugin(plugin)
# Enable on boot
debian.add_rc_service('elasticsearch', priorities='defaults 95 10')
def add_apt_ppa(name, accept=True, src=False):
with sudo(), fabric.context_managers.cd('/etc/apt/sources.list.d'):
source_list = '%s-%s.list' % (name.replace('/', '-'), lbs_codename())
if not fabric.contrib.files.exists(source_list):
add_apt_repository('ppa:{}'.format(name), accept=accept, src=src)
apt_get('update')
def generate_pgtune_conf(role='db'):
"""
Run pgtune and create pgtune.conf
:param role: Which fabric role to place local pgtune.conf template under
"""
conf_path = postgres_root('postgresql.conf')
with sudo(), silent():
output = run('pgtune -T Web -i {}'.format(conf_path)).strip()
def parse(c):
lines = [l for l in c.splitlines() if '# pgtune' in l]
for line in lines:
try:
comment = line.index('#')
line = line[:comment]
except ValueError:
pass
clean = lambda s: s.strip('\n\r\t\'" ')
key, _, value = line.partition('=')
key, value = map(clean, (key, value))
if key:
yield key, value or None
tune_conf = dict(parse(output))
tune_conf.update(blueprint.get('pgtune', {}))
tune_conf = '\n'.join((' = '.join(item)) for item in tune_conf.iteritems())
conf_dir = os.path.join(os.path.dirname(env['real_fabfile']), 'templates', role, 'postgres')
conf_path = os.path.join(conf_dir, 'pgtune.conf')
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
with open(conf_path, 'w+') as f:
f.write(tune_conf)
def install():
with sudo():
info('Adding apt repository for {}', 'docker')
repository = '[arch=amd64] https://download.docker.com/linux/ubuntu {} stable'.format(
debian.lsb_codename())
debian.add_apt_repository(repository)
gpg_key = 'https://download.docker.com/linux/ubuntu/gpg'
info('Adding apt key from {}', gpg_key)
debian.add_apt_key(gpg_key)
debian.apt_get_update()
# Install Docker
version = blueprint.get('version', 'latest')
info('Target version {}', version)
package = 'docker-ce' + ('={}'.format(version) if version != 'latest' else '')
info('Installing {}', package)
debian.apt_get('install', package)
if debian.lsb_release() == '14.04':
blueprint.upload('./docker.default', '/etc/default/docker')
else:
blueprint.upload('./docker.service',
'/etc/systemd/system/docker.service')
debian.ln(
'/etc/systemd/system/docker.service',
'/etc/systemd/system/multi-user.target.wants/docker.service',
)
debian.systemd_daemon_reload()
def install_solr():
with sudo():
version = blueprint.get('version')
version_tuple = tuple(map(int, version.split('.')))
archive = 'solr-{}.tgz'.format(version)
if version_tuple < (4, 1, 0):
archive = 'apache-{}'.format(archive)
url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive)
with cd('/tmp'):
info('Download {} ({})', 'Solr', version)
run('wget {}'.format(url))
info('Extracting archive...')
with silent():
run('tar xzf {}'.format(archive))
solr_version_dir = os.path.splitext(archive)[0]
solr_version_path = os.path.join('/usr', 'share', solr_version_dir)
debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True)
if files.exists(solr_version_path):
info('Found same existing version, removing it...')
debian.rm(solr_version_path, recursive=True)
debian.mv(solr_version_dir, '/usr/share/')
debian.ln(solr_version_path, solr_home)
debian.rm(archive)
def install():
with sudo():
info("Installing Ruby v1.9.3")
debian.apt_get("install", "ruby1.9.3")
info("Installing Bundler")
gem("install", "bundler")
def setup():
"""
Install nfs-common
"""
with sudo():
debian.apt_get('install', 'nfs-common')
configure()
def enable(program, do_reload=True):
"""
Enable program.
:param program: Program to enable
:param do_reload: Reload supervisor
:return: Got enabled?
"""
enabled = False
program = program if program.endswith(".conf") or program == "default" else "{}.conf".format(program)
with sudo():
available_program = os.path.join(programs_available_path, program)
if not files.exists(available_program):
warn("Invalid program: {}".format(program))
else:
with cd(programs_enabled_path):
if not files.exists(program):
info("Enabling program: {}", program)
with silent():
debian.ln(available_program, program)
enabled = True
if do_reload:
reload()
return enabled
def setup():
"""
Install nfs-common
"""
with sudo():
debian.apt_get('install', 'nfs-common')
configure()
def install_system_dependencies():
"""
Install system wide packages that application depends on.
"""
with sudo(), silent():
info('Install system dependencies')
system_dependencies = blueprint.get('system_dependencies')
if system_dependencies:
dependencies = []
repositories = []
ppa_dependencies = []
for dependency in system_dependencies:
dep, _, rep = dependency.partition('@')
if rep:
if rep not in repositories:
repositories.append(rep)
ppa_dependencies.append(dep)
elif dep not in dependencies:
dependencies.append(dep)
debian.apt_get_update()
debian.apt_get('install', *dependencies)
if repositories:
for repository in repositories:
debian.add_apt_repository(repository, src=True)
debian.apt_get_update()
debian.apt_get('install', *ppa_dependencies)
def install_project_user():
"""
Create project user and groups.
Create user home dir.
Disable ssh host checking.
Create log dir.
"""
from .project import project_home
with sudo():
info('Install application user')
username = blueprint.get('project')
home_path = project_home()
# Setup groups for project user
project_user_groups = ['app-data', 'www-data']
for group in project_user_groups:
debian.groupadd(group, gid_min=10000)
# Get UID for project user
user.create_system_user(username, groups=project_user_groups,
home=home_path)
# Create application log path
application_log_path = os.path.join('/var', 'log', username)
debian.mkdir(application_log_path, group='app-data', mode=1775)
# Configure ssh for github
user.set_strict_host_checking(username, 'github.com')
def install():
with sudo():
debian.apt_get('install', 'pure-ftpd', 'openssl')
# Create ftp user
user.create_service_user(ftp_user, groups=[ftp_group])
# Create ftp root dir
debian.mkdir(ftp_root, mode=1770, owner=ftp_user, group=ftp_group)
# Set up symlinks
debian.ln('/etc/pure-ftpd/conf/PureDB', '/etc/pure-ftpd/auth/PureDB')
# Enable TLS
run('echo 1 > /etc/pure-ftpd/conf/TLS')
key_path = '/etc/ssl/private/pure-ftpd.pem'
if not files.exists(key_path):
prompts = {
'Country Name (2 letter code) [AU]:': '',
'State or Province Name (full name) [Some-State]:': '',
'Locality Name (eg, city) []:': '',
'Organization Name (eg, company) [Internet Widgits Pty Ltd]:': '',
'Organizational Unit Name (eg, section) []:': '',
'Common Name (e.g. server FQDN or YOUR name) []:': '',
'Email Address []:': ''
}
with settings(prompts=prompts):
run('openssl req -x509 -nodes -newkey rsa:2048 -keyout {0} -out {0}'.format(
key_path))
debian.chmod(key_path, 600)
def install_forwarder():
"""
TODO: Build from github
- wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz
- gunzip <go>
- mv go /usr/local/
- apt-get install unzip make ruby ruby-dev
- wget https://github.com/elasticsearch/logstash-forwarder/archive/master.zip
- unzip <forwarder>
- cd <forwarder>
- go build
- gem install fpm
- make deb
- dpkg -i <forwarder>
"""
with sudo():
info('Adding apt repository for {}', 'logstash forwarder')
debian.add_apt_repository(
'http://packages.elasticsearch.org/logstashforwarder/debian stable main'
)
info('Installing {}', 'logstash forwarder')
debian.apt_get('update')
debian.apt_get('install', 'logstash-forwarder')
# Upload init script
blueprint.upload('forwarder/init.d/logstash-forwarder', '/etc/init.d/')
debian.chmod('/etc/init.d/logstash-forwarder', mode=755)
# Enable on boot
debian.add_rc_service('logstash-forwarder')
def enable(conf, weight, do_restart=True):
"""
Enable logstash input/output provider
:param conf: Input or output provider config file
:param weight: Weight of provider
:param do_restart: Restart service
:return: Got enabled?
"""
enabled = False
conf = conf if conf.endswith('.conf') else '{}.conf'.format(conf)
with sudo():
available_conf = os.path.join(conf_available_path, conf)
if not files.exists(available_conf):
warn('Invalid conf: {}'.format(conf))
else:
with cd(conf_enabled_path):
weight = str(weight).zfill(2)
conf = '{}-{}'.format(weight, conf)
if not files.exists(conf):
info('Enabling conf: {}', conf)
with silent():
debian.ln(available_conf, conf)
enabled = True
if do_restart:
restart('server')
return enabled
def add_fstab(filesystem=None, mount_point=None, type='auto', options='rw', dump='0', pazz='0'):
"""
Add mount point configuration to /etc/fstab.
If mount point already mounted on different file system then unmount.
:param str filesystem: The partition or storage device to be mounted
:param str mount_point: The mount point where <filesystem> is mounted to
:param str type: The file system type (Default: auto)
:param str options: Mount options of the filesystem (Default: rw)
:param str dump: Used by the dump utility to decide when to make a backup, 0|1 (Default: 0)
:param str pazz: Used by fsck to decide which order filesystems are to be checked (Default: 0)
"""
with sudo():
fstab_line = '{fs} {mount} {type} {options} {dump} {pazz}'.format(
fs=filesystem,
mount=mount_point,
type=type,
options=options,
dump=dump,
pazz=pazz
)
validate_boot_options(options)
# Add mount to /etc/fstab if not already there (?)
with silent():
output = run('cat /etc/fstab')
fstab = output.stdout
if fstab_line not in fstab.split('\n'): # TODO: Handle comments
info('Adding fstab: {} on {}', filesystem, mount_point)
fabric.contrib.files.append('/etc/fstab', fstab_line, use_sudo=True)
# Unmount any previous mismatching mount point
mounted_file_system = get_mount(mount_point)
if mounted_file_system and mounted_file_system != filesystem:
unmount(mount_point)
def install_project_user():
"""
Create project user and groups.
Create user home dir.
Disable ssh host checking.
Create log dir.
"""
with sudo():
info('Install application user')
username = blueprint.get('project')
home_path = project_home()
# Setup groups for project user
project_user_groups = ['app-data', 'www-data']
for group in project_user_groups:
debian.groupadd(group, gid_min=10000)
# Get UID for project user
user.create_system_user(username, groups=project_user_groups, home=home_path)
# Configure ssh for github
user.set_strict_host_checking(username, 'github.com')
# Create application log path
application_log_path = os.path.join('/var', 'log', username)
debian.mkdir(application_log_path, group='app-data', mode=1775)
def install():
with sudo():
debian.apt_get('install', 'pure-ftpd', 'openssl')
# Create ftp user
user.create_service_user(ftp_user, groups=[ftp_group])
# Create ftp root dir
debian.mkdir(ftp_root, mode=1770, owner=ftp_user, group=ftp_group)
# Set up symlinks
debian.ln('/etc/pure-ftpd/conf/PureDB', '/etc/pure-ftpd/auth/PureDB')
# Enable TLS
run('echo 1 > /etc/pure-ftpd/conf/TLS')
key_path = '/etc/ssl/private/pure-ftpd.pem'
if not files.exists(key_path):
prompts = {
'Country Name (2 letter code) [AU]:': '',
'State or Province Name (full name) [Some-State]:': '',
'Locality Name (eg, city) []:': '',
'Organization Name (eg, company) [Internet Widgits Pty Ltd]:':
'',
'Organizational Unit Name (eg, section) []:': '',
'Common Name (e.g. server FQDN or YOUR name) []:': '',
'Email Address []:': ''
}
with settings(prompts=prompts):
run('openssl req -x509 -nodes -newkey rsa:2048 -keyout {0} -out {0}'
.format(key_path))
debian.chmod(key_path, 600)
def configure_infra():
with sudo():
info('Adding license key to config')
context = {"newrelic_key": blueprint.get('newrelic_key', None)}
blueprint.upload('newrelic-infra.yml',
'/etc/newrelic-infra.yml',
context=context)
def configure():
"""
Configure nginx and enable/disable sites
"""
with sudo():
# Upload templates
context = {
'num_cores': debian.nproc()
}
uploads = blueprint.upload('./', nginx_root, context)
# Disable previously enabled sites not configured sites-enabled
changes = []
sites = blueprint.get('sites')
auto_disable_sites = blueprint.get('auto_disable_sites', True)
if auto_disable_sites:
with silent():
enabled_site_links = run('ls {}'.format(sites_enabled_path)).split()
for link in enabled_site_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in sites and link_name not in sites:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable sites from settings
for site in sites:
changed = enable(site, do_reload=False)
changes.append(changed)
### Reload nginx if new templates or any site has been enabled/disabled
if uploads or any(changes):
reload()
def install():
with sudo():
# TODO: Change to java.install (openjdk)?
debian.add_apt_ppa('webupd8team/java')
debian.debconf_set_selections(
'shared/accepted-oracle-license-v1-1 select true',
'shared/accepted-oracle-license-v1-1 seen true')
version = blueprint.get('version', '1.0')
info('Adding apt repository for {} version {}', 'elasticsearch',
version)
repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format(
version)
debian.add_apt_repository(repository)
info('Adding apt key for', repository)
debian.add_apt_key(
'http://packages.elasticsearch.org/GPG-KEY-elasticsearch')
debian.apt_get('update')
# Install elasticsearch (and java)
info('Installing {} version {}', 'elasticsearch', version)
debian.apt_get('install', 'oracle-java7-installer', 'elasticsearch')
# Enable on boot
debian.add_rc_service('elasticsearch', priorities='defaults 95 10')
def enable(program, do_reload=True):
"""
Enable program.
:param program: Program to enable
:param do_reload: Reload supervisor
:return: Got enabled?
"""
enabled = False
program = program if program.endswith(
'.conf') or program == 'default' else '{}.conf'.format(program)
with sudo():
available_program = os.path.join(programs_available_path, program)
if not files.exists(available_program):
warn('Invalid program: {}'.format(program))
else:
with cd(programs_enabled_path):
if not files.exists(program):
info('Enabling program: {}', program)
with silent():
debian.ln(available_program, program)
enabled = True
if do_reload:
reload()
return enabled
def setup_schemas(drop=False):
"""
Create database schemas and grant user privileges
:param drop: Drop existing schemas before creation
"""
schemas = blueprint.get('schemas', {})
with sudo('postgres'):
for schema, config in schemas.iteritems():
user, password = config['user'], config.get('password')
info('Creating user {}', user)
if password:
_client_exec(
"CREATE ROLE %(user)s WITH PASSWORD '%(password)s' LOGIN",
user=user,
password=password)
else:
_client_exec("CREATE ROLE %(user)s LOGIN", user=user)
if drop:
info('Droping schema {}', schema)
_client_exec('DROP DATABASE %(name)s', name=schema)
info('Creating schema {}', schema)
_client_exec('CREATE DATABASE %(name)s', name=schema)
info('Granting user {} to schema {}'.format(user, schema))
_client_exec(
"GRANT ALL PRIVILEGES ON DATABASE %(schema)s to %(user)s",
schema=schema,
user=user)
def enable(conf, weight, do_restart=True):
"""
Enable logstash input/output provider
:param conf: Input or output provider config file
:param weight: Weight of provider
:param do_restart: Restart service
:return: Got enabled?
"""
enabled = False
conf = conf if conf.endswith(".conf") else "{}.conf".format(conf)
with sudo():
available_conf = os.path.join(conf_available_path, conf)
if not files.exists(available_conf):
warn("Invalid conf: {}".format(conf))
else:
with cd(conf_enabled_path):
weight = str(weight).zfill(2)
conf = "{}-{}".format(weight, conf)
if not files.exists(conf):
info("Enabling conf: {}", conf)
with silent():
debian.ln(available_conf, conf)
enabled = True
if do_restart:
restart("server")
return enabled
def configure():
"""
Upload vassals
"""
with sudo():
# Upload templates
blueprint.upload('init/', '/etc/init/')
def install_forwarder():
"""
TODO: Build from github
- wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz
- gunzip <go>
- mv go /usr/local/
- apt-get install unzip make ruby ruby-dev
- wget https://github.com/elasticsearch/logstash-forwarder/archive/master.zip
- unzip <forwarder>
- cd <forwarder>
- go build
- gem install fpm
- make deb
- dpkg -i <forwarder>
"""
with sudo():
info("Adding apt repository for {}", "logstash forwarder")
debian.add_apt_repository("http://packages.elasticsearch.org/logstashforwarder/debian stable main")
info("Installing {}", "logstash forwarder")
debian.apt_get("update")
debian.apt_get("install", "logstash-forwarder")
# Upload init script
blueprint.upload("forwarder/init.d/logstash-forwarder", "/etc/init.d/")
debian.chmod("/etc/init.d/logstash-forwarder", mode=755)
# Enable on boot
debian.add_rc_service("logstash-forwarder")
def install():
from refabric.context_managers import sudo
from .debian import add_apt_ppa, apt_get
with sudo():
add_apt_ppa('vbernat/haproxy-1.5', src=True)
apt_get('install', 'haproxy')
def install():
with sudo():
# Generate a root password and save it in root home
root_conf_path = '/root/.my.cnf'
if not fabric.contrib.files.exists(root_conf_path):
root_pw = generate_password()
blueprint.upload('root_my.cnf', '/root/.my.cnf', {'password': root_pw})
debian.chmod('/root/.my.cnf', mode=600)
else:
# TODO: use fabric.operations.get instead of cat when up to date with upstream
with silent():
output = run('cat {}'.format(root_conf_path))
fd = StringIO(output)
config_parser = ConfigParser.RawConfigParser()
config_parser.readfp(fd)
root_pw = config_parser.get('client', 'password')
# Install external PPA
info('Adding apt key for {}', __name__)
run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A")
info('Adding apt repository for {}', __name__)
debian.add_apt_repository('http://repo.percona.com/apt trusty main')
debian.apt_get('update')
# Percona/MySQL base dependencies
dependencies = (
'percona-server-server',
'percona-server-client',
'libmysqlclient-dev',
'mysqltuner'
)
# Configure debconf to autoset root password on installation prompts
server_package = dependencies[0]
debian.debconf_communicate('PURGE', server_package)
with silent():
debian.debconf_set_selections(
'{}/root_password password {}'.format(server_package, root_pw),
'{}/root_password_again password {}'.format(server_package, root_pw)
)
# Install package
info('Installing {}', __name__)
debian.apt_get('install', *dependencies)
debian.debconf_communicate('PURGE', server_package)
# Auto-answer mysql_secure_installation prompts
prompts = {
'Enter current password for root (enter for none): ': root_pw,
'Change the root password? [Y/n] ': 'n',
'Remove anonymous users? [Y/n] ': 'Y',
'Disallow root login remotely? [Y/n] ': 'Y',
'Remove test database and access to it? [Y/n] ': 'Y',
'Reload privilege tables now? [Y/n] ': 'Y'
}
# Run mysql_secure_installation to remove test-db and remote root login
with settings(prompts=prompts):
run('mysql_secure_installation')
def configure():
"""
Enable/disable configured programs
"""
with sudo():
# Upload templates
uploads = blueprint.upload('init/', '/etc/init/')
uploads.extend(blueprint.upload('supervisord.conf', '/etc/'))
# Disable previously enabled programs not configured programs-enabled
changes = []
programs = blueprint.get('programs') or []
auto_disable = blueprint.get('auto_disable_programs', True)
if auto_disable:
with silent():
enabled_program_links = run(
'ls {}'.format(programs_enabled_path)).split()
for link in enabled_program_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in programs and link_name not in programs:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable programs from settings
for program in programs:
changed = enable(program, do_reload=False)
changes.append(changed)
### Reload supervisor if new templates or any program has been enabled/disabled
if uploads or any(changes):
reload()
def enable(site, do_reload=True):
"""
Enable site
:param site: Site to enable
:param do_reload: Reload nginx service
:return: Got enabled?
"""
enabled = False
site = site if site.endswith('.conf') or site == 'default' else '{}.conf'.format(site)
with sudo():
available_site = os.path.join(sites_available_path, site)
if not files.exists(available_site):
warn('Invalid site: {}'.format(site))
else:
with cd(sites_enabled_path):
if not files.exists(site):
info('Enabling site: {}', site)
with silent():
debian.ln(available_site, site)
enabled = True
if do_reload:
reload()
return enabled
def configure():
"""
Configure PureFTP
"""
with sudo():
# Echo configurations
setup_config()
for user in blueprint.get('users'):
username, password = user['username'], user['password']
if 'homedir' in user:
user_home = user['homedir']
else:
user_home = os.path.join(ftp_root, username)
passwd_path = '/etc/pure-ftpd/pureftpd.passwd'
with settings(warn_only=True):
if files.exists(passwd_path) and run(
'pure-pw show {}'.format(username)).return_code == 0:
continue
debian.mkdir(user_home, owner=ftp_user, group=ftp_group)
prompts = {'Password: '******'Enter it again: ': password}
with settings(prompts=prompts):
run('pure-pw useradd {} -u {} -g {} -d {}'.format(
username, ftp_user, ftp_group, user_home))
run('pure-pw mkdb')
restart()
def add_apt_ppa(name, accept=True, src=False):
with sudo(), fabric.context_managers.cd('/etc/apt/sources.list.d'):
source_list = '%s-%s.list' % (name.replace('/', '-'), lsb_codename())
if not fabric.contrib.files.exists(source_list):
add_apt_repository('ppa:{}'.format(name), accept=accept, src=src)
apt_get_update()
def install_user():
with sudo():
user.create_service_user('solr', home=solr_home)
debian.mkdir(solr_home, mode=755, owner='solr', group='solr')
debian.mkdir('/var/lib/solr', mode=755, owner='solr', group='solr')
debian.mkdir('/var/log/solr', mode=755, owner='solr', group='solr')
def install():
with sudo():
packages = [
'wkhtmltopdf', 'xvfb', 'xfonts-100dpi', 'xfonts-75dpi',
'xfonts-cyrillic'
]
debian.apt_get('install', *packages)
def install():
with sudo():
info('Installing Ruby v1.9.3')
debian.apt_get('install', 'ruby1.9.3')
info('Installing Bundler')
gem('install', 'bundler')
def dump(schema=None, ignore_tables=''):
"""
Dump and download a schema.
:param schema: Specific shema to dump and download.
:param ignore_tables: Tables to skip, separated by | (pipe)
"""
if not schema:
schemas = blueprint.get('schemas', {}).keys()
for i, schema in enumerate(schemas, start=1):
print("{i}. {schema}".format(i=i, schema=schema))
valid_indices = '[1-{}]+'.format(len(schemas))
schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices)
schema = schemas[int(schema_choice)-1]
now = datetime.now().strftime('%Y-%m-%d')
output_file = '/tmp/{}_{}.backup.gz'.format(schema, now)
filename = os.path.basename(output_file)
info('Dumping schema {}...', schema)
extra_args = []
for table in ignore_tables.split('|'):
extra_args.append('--ignore-table={}.{}'.format(schema, table))
dump_cmd = 'mysqldump {} {} | gzip > {}'.format(schema, ' '.join(extra_args), output_file)
run('sudo su root -c "{}"'.format(dump_cmd))
info('Downloading dump...')
local_file = '~/%s' % filename
fabric.contrib.files.get(output_file, local_file)
with sudo(), silent():
debian.rm(output_file)
info('New smoking hot dump at {}', local_file)
def locale_gen(locale, utf8=True):
locale_gen_cmd = 'locale-gen {}'.format(locale)
locale_gen_utf8_cmd = locale_gen_cmd + '.UTF-8'
with sudo():
run(locale_gen_cmd)
if utf8:
run(locale_gen_utf8_cmd)
def configure():
"""
Configure wkhtmltopdf
"""
destination = '/usr/local/bin/wkhtmltopdf.sh'
blueprint.upload('wkhtmltopdf.sh', destination)
with sudo():
debian.chmod(destination, '+x')
def flush():
"""
Delete all cached keys
"""
info('Flushing Memcached...')
with sudo(), silent():
run('echo "flush_all" | /bin/netcat -q 2 127.0.0.1 11211')
info('Down the drain!')
def install():
with sudo():
info('Install python dependencies')
debian.apt_get('install', 'python-dev', 'python-setuptools')
run('easy_install pip')
run('touch {}'.format(pip_log_file))
debian.chmod(pip_log_file, mode=777)
pip('install', 'setuptools', '--upgrade')
def export(path, host, options='rw,async,no_root_squash,no_subtree_check'):
config_line = '%s\t\t%s(%s)' % (path, host, options)
with sudo(), cd('/etc'):
if not files.contains('exports', config_line):
info('Exporting: {}', path)
files.append('exports', config_line)
return True
return False
def install_virtualenv():
"""
Create a project virtualenv.
"""
with sudo():
virtualenv.install()
with sudo_project():
virtualenv.create(virtualenv_path())
def install_system_dependencies():
"""
Install system wide packages that application depends on.
"""
with sudo():
info('Install system dependencies')
dependencies = blueprint.get('system_dependencies')
if dependencies:
debian.apt_get('install', *dependencies)