def install_solr():
with sudo():
version = blueprint.get("version")
version_tuple = tuple(map(int, version.split(".")))
archive = "solr-{}.tgz".format(version)
if version_tuple < (4, 1, 0):
archive = "apache-{}".format(archive)
url = "https://archive.apache.org/dist/lucene/solr/{}/{}".format(version, archive)
with cd("/tmp"):
info("Download {} ({})", "Solr", version)
run("wget {}".format(url))
info("Extracting archive...")
with silent():
run("tar xzf {}".format(archive))
solr_version_dir = os.path.splitext(archive)[0]
solr_version_path = os.path.join("/usr", "share", solr_version_dir)
debian.chmod(solr_version_dir, 755, "solr", "solr", recursive=True)
if files.exists(solr_version_path):
info("Found same existing version, removing it...")
debian.rm(solr_version_path, recursive=True)
debian.mv(solr_version_dir, "/usr/share/")
debian.ln(solr_version_path, solr_home)
debian.rm(archive)
def dump(schema=None, ignore_tables=''):
"""
Dump and download a schema.
:param schema: Specific shema to dump and download.
:param ignore_tables: Tables to skip, separated by | (pipe)
"""
if not schema:
schemas = blueprint.get('schemas', {}).keys()
for i, schema in enumerate(schemas, start=1):
print("{i}. {schema}".format(i=i, schema=schema))
valid_indices = '[1-{}]+'.format(len(schemas))
schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices)
schema = schemas[int(schema_choice)-1]
now = datetime.now().strftime('%Y-%m-%d')
output_file = '/tmp/{}_{}.backup.gz'.format(schema, now)
filename = os.path.basename(output_file)
info('Dumping schema {}...', schema)
extra_args = []
for table in ignore_tables.split('|'):
extra_args.append('--ignore-table={}.{}'.format(schema, table))
dump_cmd = 'mysqldump {} {} | gzip > {}'.format(schema, ' '.join(extra_args), output_file)
run('sudo su root -c "{}"'.format(dump_cmd))
info('Downloading dump...')
local_file = '~/%s' % filename
fabric.contrib.files.get(output_file, local_file)
with sudo(), silent():
debian.rm(output_file)
info('New smoking hot dump at {}', local_file)
def install_solr():
with sudo():
version = blueprint.get('version')
version_tuple = tuple(map(int, version.split('.')))
archive = 'solr-{}.tgz'.format(version)
if version_tuple < (4, 1, 0):
archive = 'apache-{}'.format(archive)
url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive)
with cd('/tmp'):
info('Download {} ({})', 'Solr', version)
run('wget {}'.format(url))
info('Extracting archive...')
with silent():
run('tar xzf {}'.format(archive))
solr_version_dir = os.path.splitext(archive)[0]
solr_version_path = os.path.join('/usr', 'share', solr_version_dir)
debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True)
if files.exists(solr_version_path):
info('Found same existing version, removing it...')
debian.rm(solr_version_path, recursive=True)
debian.mv(solr_version_dir, '/usr/share/')
debian.ln(solr_version_path, solr_home)
debian.rm(archive)
def install():
with sudo():
# Generate a root password and save it in root home
root_conf_path = '/root/.my.cnf'
if not fabric.contrib.files.exists(root_conf_path):
root_pw = generate_password()
blueprint.upload('root_my.cnf', '/root/.my.cnf', {'password': root_pw})
debian.chmod('/root/.my.cnf', mode=600)
else:
# TODO: use fabric.operations.get instead of cat when up to date with upstream
with silent():
output = run('cat {}'.format(root_conf_path))
fd = StringIO(output)
config_parser = ConfigParser.RawConfigParser()
config_parser.readfp(fd)
root_pw = config_parser.get('client', 'password')
# Install external PPA
info('Adding apt key for {}', __name__)
run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A")
info('Adding apt repository for {}', __name__)
debian.add_apt_repository('http://repo.percona.com/apt trusty main')
debian.apt_get('update')
# Percona/MySQL base dependencies
dependencies = (
'percona-server-server',
'percona-server-client',
'libmysqlclient-dev',
'mysqltuner'
)
# Configure debconf to autoset root password on installation prompts
server_package = dependencies[0]
debian.debconf_communicate('PURGE', server_package)
with silent():
debian.debconf_set_selections(
'{}/root_password password {}'.format(server_package, root_pw),
'{}/root_password_again password {}'.format(server_package, root_pw)
)
# Install package
info('Installing {}', __name__)
debian.apt_get('install', *dependencies)
debian.debconf_communicate('PURGE', server_package)
# Auto-answer mysql_secure_installation prompts
prompts = {
'Enter current password for root (enter for none): ': root_pw,
'Change the root password? [Y/n] ': 'n',
'Remove anonymous users? [Y/n] ': 'Y',
'Disallow root login remotely? [Y/n] ': 'Y',
'Remove test database and access to it? [Y/n] ': 'Y',
'Reload privilege tables now? [Y/n] ': 'Y'
}
# Run mysql_secure_installation to remove test-db and remote root login
with settings(prompts=prompts):
run('mysql_secure_installation')
def install_solr():
with sudo():
version = blueprint.get('version')
version_tuple = tuple(map(int, version.split('.')))
archive = 'solr-{}.tgz'.format(version)
if version_tuple < (4, 1, 0):
archive = 'apache-{}'.format(archive)
url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive)
with cd('/tmp'):
info('Download {} ({})', 'Solr', version)
run('wget {}'.format(url))
info('Extracting archive...')
with silent():
run('tar xzf {}'.format(archive))
solr_version_dir = os.path.splitext(archive)[0]
solr_version_path = os.path.join('/usr', 'share', solr_version_dir)
debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True)
if files.exists(solr_version_path):
info('Found same existing version, removing it...')
debian.rm(solr_version_path, recursive=True)
debian.mv(solr_version_dir, '/usr/share/')
debian.ln(solr_version_path, solr_home)
debian.rm(archive)
def update_modules(beat):
changes = []
# Get desired state
desired_modules = set(blueprint.get('{}.modules'.format(beat), []))
# Get current state
with silent():
module_files = run('find /etc/{}/modules.d -iname "*.yml"'.format(beat)).split()
enabled_modules = {os.path.basename(module).split('.')[0] for module in module_files}
# Disable extra services
for extra in enabled_modules - desired_modules:
info('Disabling {} module: {}', beat, extra)
changes.append(extra)
with silent(), sudo():
run('{} modules disable {}'.format(beat, extra))
# Enable services
for missing in desired_modules - enabled_modules:
info('Enabling {} module: {}', beat, missing)
changes.append(missing)
with silent(), sudo():
run('{} modules enable {}'.format(beat, missing))
return changes
def dump(schema=None):
"""
Dump and download all configured, or given, schemas.
:param schema: Specific shema to dump and download.
"""
if not schema:
schemas = blueprint.get('schemas', {}).keys()
for i, schema in enumerate(schemas, start=1):
print("{i}. {schema}".format(i=i, schema=schema))
valid_indices = '[1-{}]+'.format(len(schemas))
schema_choice = prompt('Select schema to dump:',
default='1',
validate=valid_indices)
schema = schemas[int(schema_choice) - 1]
with sudo('postgres'):
now = datetime.now().strftime('%Y-%m-%d')
output_file = '/tmp/{}_{}.backup'.format(schema, now)
filename = os.path.basename(output_file)
options = dict(format='tar', output_file=output_file, schema=schema)
info('Dumping schema {}...', schema)
run('pg_dump -c -F {format} -f {output_file} {schema}'.format(
**options))
info('Downloading dump...')
local_file = '~/{}'.format(filename)
files.get(output_file, local_file)
with sudo(), silent():
debian.rm(output_file)
info('New smoking hot dump at {}', local_file)
def send_deploy_event():
newrelic_key = blueprint.get('newrelic_key', None)
app_name = blueprint.get('app_name', None)
if newrelic_key and app_name:
url = 'https://api.newrelic.com/deployments.xml'
headers = {'x-api-key': newrelic_key}
with cd(python_path()):
tag = run('git describe --tags')
commit_hash = run('git rev-parse HEAD')
deployer = git.get_local_commiter()
payload = {
'deployment[app_name]': app_name,
# 'application_id': '1234567',
'deployment[description]': tag,
'deployment[revision]': commit_hash,
# 'deployment[changelog]': changes,
'deployment[user]': deployer,
}
response = requests.post(url, data=payload, headers=headers)
info(response.text)
else:
info('No key found')
#http://gc-taylor.com/blog/2013/02/11/fabric-task-for-notifying-new-relic-code-deploy/#sthash.5AnhN3An.sfju
def install():
with sudo():
info('Install python dependencies')
debian.apt_get('install', 'python-dev', 'python-setuptools')
run('easy_install pip')
run('touch {}'.format(pip_log_file))
debian.chmod(pip_log_file, mode=777)
pip('install', 'setuptools', '--upgrade')
def flush():
"""
Delete all cached keys
"""
info('Flushing Memcached...')
with sudo(), silent():
run('echo "flush_all" | /bin/netcat -q 2 127.0.0.1 11211')
info('Down the drain!')
def install():
with sudo():
info('Install python dependencies')
debian.apt_get('install', 'python-dev', 'python-setuptools')
run('easy_install pip')
run('touch {}'.format(pip_log_file))
debian.chmod(pip_log_file, mode=777)
pip('install', 'setuptools', '--upgrade')
def pip_sync(manifest, quiet=False):
pip('install', 'pip-tools', quiet=quiet)
options = ""
if quiet:
options += ' -q'
run('pip-sync{} {}'.format(options, manifest), pty=False)
def install():
with sudo():
# Generate a root password and save it in root home
root_conf_path = '/root/.my.cnf'
if not fabric.contrib.files.exists(root_conf_path):
root_pw = generate_password()
blueprint.upload('root_my.cnf', '/root/.my.cnf',
{'password': root_pw})
debian.chmod('/root/.my.cnf', mode=600)
else:
# TODO: use fabric.operations.get instead of cat when up to date with upstream
with silent():
output = run('cat {}'.format(root_conf_path))
fd = StringIO(output)
config_parser = ConfigParser.RawConfigParser()
config_parser.readfp(fd)
root_pw = config_parser.get('client', 'password')
# Install external PPA
info('Adding apt key for {}', __name__)
run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A"
)
info('Adding apt repository for {}', __name__)
debian.add_apt_repository('http://repo.percona.com/apt trusty main')
debian.apt_get('update')
# Percona/MySQL base dependencies
dependencies = ('percona-server-server', 'percona-server-client',
'libmysqlclient-dev', 'mysqltuner')
# Configure debconf to autoset root password on installation prompts
server_package = dependencies[0]
debian.debconf_communicate('PURGE', server_package)
with silent():
debian.debconf_set_selections(
'{}/root_password password {}'.format(server_package, root_pw),
'{}/root_password_again password {}'.format(
server_package, root_pw))
# Install package
info('Installing {}', __name__)
debian.apt_get('install', *dependencies)
debian.debconf_communicate('PURGE', server_package)
# Auto-answer mysql_secure_installation prompts
prompts = {
'Enter current password for root (enter for none): ': root_pw,
'Change the root password? [Y/n] ': 'n',
'Remove anonymous users? [Y/n] ': 'Y',
'Disallow root login remotely? [Y/n] ': 'Y',
'Remove test database and access to it? [Y/n] ': 'Y',
'Reload privilege tables now? [Y/n] ': 'Y'
}
# Run mysql_secure_installation to remove test-db and remote root login
with settings(prompts=prompts):
run('mysql_secure_installation')
def create_server_ssl_cert():
with sudo():
info('Generating SSL certificate...')
debian.mkdir('/etc/pki/tls/certs')
debian.mkdir('/etc/pki/tls/private')
with cd('/etc/pki/tls'):
key = 'private/logstash-forwarder.key'
crt = 'certs/logstash-forwarder.crt'
run('openssl req -x509 -batch -nodes -days 3650 -newkey rsa:2048 -keyout {} -out {}'.format(key, crt))
def configure():
"""
Configure newrelic server
"""
with sudo():
info('Adding license key to config')
newrelic_key = blueprint.get('newrelic_key', None)
run('nrsysmond-config --set license_key={}'.format(newrelic_key))
def ctl(command=None):
"""
Run rabbitmqctl with given command
:param command: Control command to execute
"""
if not command:
abort('No command given, $ fab rabbitmq.ctl:stop_app')
with sudo():
run('rabbitmqctl {}'.format(command))
def reload(self, vassals=None):
"""
Touch reload specified vassals
:param vassals: Vassals to reload
"""
for vassal_ini in vassals or self.list_vassals():
vassal_ini_path = os.path.join(self.get_config_path(), vassal_ini)
with sudo(), silent():
run('touch {}'.format(vassal_ini_path))
def reload(self, vassals=None):
"""
Touch reload specified vassals
:param vassals: Vassals to reload
"""
for vassal_ini in vassals or self.list_vassals():
vassal_ini_path = os.path.join(self.get_config_path(), vassal_ini)
with sudo(), silent():
run('touch {}'.format(vassal_ini_path))
def ctl(command=None):
"""
Run rabbitmqctl with given command
:param command:
:return:
"""
if not command:
abort('No command given, $ fab rabbitmq.ctl:stop_app')
with sudo():
run('rabbitmqctl {}'.format(command))
def top(vassal_name=None):
"""
Launch uwsgitop for vassal stats socket
:param vassal_name: The vassal to show stats for (Default: project setting)
"""
# TODO: fix missing output
with sudo(), hide_prefix():
vassal = vassal_name or blueprint.get('project')
stats_path = os.path.join(tmpfs_path, '{}-stats.sock'.format(vassal))
run('uwsgitop {}'.format(stats_path))
def create(path):
options = ''
if python.requested_version() >= (3,):
options += ' -p /usr/bin/python3'
if not files.exists(path):
info('Creating virtualenv: {}', path)
run('virtualenv{options} {}'.format(path, options=options))
else:
info('Virtualenv already exists: {}', path)
def fifo(vassal_name, command):
"""
Issue FIFO commands to a vassal.
:param vassal_name: The vassal to command
:param command: The FIFO command to issue
See: http://uwsgi-docs.readthedocs.org/en/latest/MasterFIFO.html
"""
fifo_file = '/run/uwsgi/fifo-{}'.format(vassal_name)
with sudo(), silent():
run('echo {} > {}'.format(command, fifo_file))
def reload(vassal_path=None):
"""
Reload uwsgi or reload specific vassal @ path, via touch.
:param vassal_path: The absolute path to vassal ini to reload. If not given, the uwsgi service will reload
"""
if not vassal_path:
debian.service('uwsgi', 'reload', check_status=False)
else:
vassal_name = os.path.splitext(os.path.basename(vassal_path))[0]
with sudo(), silent():
info('Reloading {} uWSGI vassal', vassal_name)
run('touch {}'.format(vassal_path))
def reload(vassal_path=None):
"""
Reload uwsgi or reload specific vassal @ path, via touch.
:param vassal_path: The absolute path to vassal ini to reload. If not given, the uwsgi service will reload
"""
if not vassal_path:
debian.service('uwsgi', 'reload', check_status=False)
else:
vassal_name = os.path.splitext(os.path.basename(vassal_path))[0]
with sudo():
info('Reloading {} uWSGI vassal', vassal_name)
run('touch {}'.format(vassal_path))
def install():
with sudo():
info('Downloading wowza')
version = blueprint.get('wowza_version', '4.1.2')
binary = 'WowzaStreamingEngine-{}.deb.bin'.format(version)
version_path = version.replace('.', '-')
url = 'http://www.wowza.com/downloads/WowzaStreamingEngine-{}/{}'.format(
version_path, binary)
run('wget -P /tmp/ {url}'.format(url=url))
debian.chmod('/tmp/{}'.format(binary), '+x')
info('Installing wowza')
run('/tmp/{}'.format(binary))
def install():
with sudo():
info('Downloading kibana')
version = blueprint.get('version', '3.1.2')
tar_file = 'kibana-{}.tar.gz'.format(version)
run('wget -P /tmp/ https://download.elasticsearch.org/kibana/kibana/{f}'.format(f=tar_file))
# Extract and soft link kibana in web root
web_root = '/srv/www/'
debian.mkdir(web_root, mode=1775, owner='www-data', group='www-data')
run('tar xzf /tmp/{f} -C {web_root}'.format(f=tar_file, web_root=web_root))
src_root = os.path.join(web_root, 'kibana-{version}'.format(version=version))
debian.chown(src_root, owner='www-data', group='www-data', recursive=True)
debian.ln(src_root, '/srv/www/kibana')
def install_testing():
package_name = 'rabbitmq-server'
debian.debconf_set_selections('%s rabbitmq-server/upgrade_previous note' % package_name)
with sudo():
info('Adding apt key for {}', package_name)
run("apt-key adv --keyserver pgp.mit.edu --recv-keys 0x056E8E56")
info('Adding apt repository for {}', package_name)
debian.add_apt_repository('http://www.rabbitmq.com/debian/ testing main')
debian.apt_get_update()
info('Installing {}', package_name)
debian.apt_get('install', package_name)
def install():
with sudo():
info('Downloading wowza')
version = blueprint.get('wowza_version', '4.1.2')
binary = 'WowzaStreamingEngine-{}.deb.bin'.format(version)
version_path = version.replace('.', '-')
url = 'http://www.wowza.com/downloads/WowzaStreamingEngine-{}/{}'.format(version_path,
binary)
run('wget -P /tmp/ {url}'.format(url=url))
debian.chmod('/tmp/{}'.format(binary), '+x')
info('Installing wowza')
run('/tmp/{}'.format(binary))
def create_server_ssl_cert():
with sudo():
info("Generating SSL certificate...")
debian.mkdir("/etc/pki/tls/certs")
debian.mkdir("/etc/pki/tls/private")
with cd("/etc/pki/tls"):
hostname = debian.hostname()
key = "private/logstash-forwarder.key"
crt = "certs/logstash-forwarder.crt"
run(
"openssl req -x509 -batch -nodes -days 3650 -newkey rsa:2048 "
"-keyout {} "
"-out {} "
'-subj "/CN={}"'.format(key, crt, hostname)
)
def configure():
"""
Configure nginx and enable/disable sites
"""
with sudo():
# Upload templates
context = {
'num_cores': debian.nproc()
}
uploads = blueprint.upload('./', nginx_root, context)
# Disable previously enabled sites not configured sites-enabled
changes = []
sites = blueprint.get('sites')
auto_disable_sites = blueprint.get('auto_disable_sites', True)
if auto_disable_sites:
with silent():
enabled_site_links = run('ls {}'.format(sites_enabled_path)).split()
for link in enabled_site_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in sites and link_name not in sites:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable sites from settings
for site in sites:
changed = enable(site, do_reload=False)
changes.append(changed)
### Reload nginx if new templates or any site has been enabled/disabled
if uploads or any(changes):
reload()
def generate_pgtune_conf(role='db'):
"""
Run pgtune and create pgtune.conf
:param role: Which fabric role to place local pgtune.conf template under
"""
conf_path = postgres_root('postgresql.conf')
with sudo(), silent():
output = run('pgtune -T Web -i {}'.format(conf_path)).strip()
def parse(c):
lines = [l for l in c.splitlines() if '# pgtune' in l]
for line in lines:
try:
comment = line.index('#')
line = line[:comment]
except ValueError:
pass
clean = lambda s: s.strip('\n\r\t\'" ')
key, _, value = line.partition('=')
key, value = map(clean, (key, value))
if key:
yield key, value or None
tune_conf = dict(parse(output))
tune_conf.update(blueprint.get('pgtune', {}))
tune_conf = '\n'.join((' = '.join(item)) for item in tune_conf.iteritems())
conf_dir = os.path.join(os.path.dirname(env['real_fabfile']), 'templates', role, 'postgres')
conf_path = os.path.join(conf_dir, 'pgtune.conf')
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
with open(conf_path, 'w+') as f:
f.write(tune_conf)
def clone(url, branch=None, repository_path=None, **kwargs):
"""
Clone repository and branch.
:param url: Git url to clone
:param branch: Branch to checkout
:param repository_path: Destination
:param kwargs: Not used but here for easier kwarg passing
:return: (destination, got_cloned bool)
"""
repository = parse_url(url, branch=branch)
name = repository['name']
branch = repository['branch']
cloned = False
if not repository_path:
repository_path = os.path.join('.', name)
if not files.exists(os.path.join(repository_path, '.git')):
info('Cloning {}@{} into {}', url, branch, repository_path)
with silent('warnings'):
cmd = 'git clone -b {branch} {remote} {name}'.format(branch=branch, remote=url, name=name)
output = run(cmd)
if output.return_code != 0:
warn('Failed to clone repository "{}", probably permission denied!'.format(name))
cloned = None
else:
cloned = True
else:
info('Git repository already cloned: {}', name)
return repository_path, cloned
def configure_server(config, auto_disable_conf=True, **context):
context.setdefault('use_ssl', True)
context.setdefault('elasticsearch_host', '127.0.0.1')
uploads = blueprint.upload('./server/', '/etc/logstash/', context)
# Disable previously enabled conf not configured through config in settings
changes = []
if auto_disable_conf:
with silent():
enabled_conf_links = run('ls {}'.format(conf_enabled_path)).split()
conf_prospects = [
'{}-{}.conf'.format(str(weight).zfill(2), conf)
for weight, conf in config.iteritems()
]
for link in enabled_conf_links:
if link not in conf_prospects:
changed = disable(link, do_restart=False)
changes.append(changed)
# Enable conf from settings
for weight, conf in config.iteritems():
changed = enable(conf, weight, do_restart=False)
changes.append(changed)
return bool(uploads or any(changes))
def reset(branch, repository_path=None, **kwargs):
"""
Fetch, reset, clean and checkout repository branch.
:return: commit
"""
commit = None
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path):
name = os.path.basename(repository_path)
info('Resetting git repository: {}@{}', name, branch)
with silent('warnings'):
commands = [
'git fetch origin', # Fetch branches and tags
'git reset --hard HEAD', # Make hard reset to HEAD
'git clean -fdx', # Remove untracked files pyc, xxx~ etc
'git checkout HEAD', # Checkout HEAD
'git reset refs/remotes/origin/{} --hard'.format(branch) # Reset to branch
]
output = run(' && '.join(commands))
if output.return_code != 0:
warn('Failed to reset repository "{}", probably permission denied!'.format(name))
else:
output = output.split(os.linesep)[-1][len('HEAD is now at '):]
commit = output.split()[0]
info('HEAD is now at: {}', output)
return commit
def install_testing():
package_name = 'rabbitmq-server'
debian.debconf_set_selections('%s rabbitmq-server/upgrade_previous note' %
package_name)
with sudo():
info('Adding apt key for {}', package_name)
run("apt-key adv --keyserver pgp.mit.edu --recv-keys 0x056E8E56")
info('Adding apt repository for {}', package_name)
debian.add_apt_repository(
'http://www.rabbitmq.com/debian/ testing main')
debian.apt_get('update')
info('Installing {}', package_name)
debian.apt_get('install', package_name)
def configure():
"""
Enable/disable configured programs
"""
with sudo():
# Upload templates
uploads = blueprint.upload("init/", "/etc/init/")
uploads.extend(blueprint.upload("supervisord.conf", "/etc/"))
# Disable previously enabled programs not configured programs-enabled
changes = []
programs = blueprint.get("programs") or []
auto_disable = blueprint.get("auto_disable_programs", True)
if auto_disable:
with silent():
enabled_program_links = run("ls {}".format(programs_enabled_path)).split()
for link in enabled_program_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in programs and link_name not in programs:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable programs from settings
for program in programs:
changed = enable(program, do_reload=False)
changes.append(changed)
### Reload supervisor if new templates or any program has been enabled/disabled
if uploads or any(changes):
reload()
def reset(branch, repository_path=None, **kwargs):
"""
Fetch, reset, clean and checkout repository branch.
:return: commit
"""
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path):
name = os.path.basename(repository_path)
info('Resetting git repository: {}@{}', name, branch)
commands = [
'git fetch origin', # Fetch branches and tags
'git reset --hard HEAD', # Make hard reset to HEAD
'git clean -fdx', # Remove untracked files pyc, xxx~ etc
'git checkout HEAD', # Checkout HEAD
'git reset refs/remotes/origin/{} --hard'.format(
branch) # Reset to branch
]
with silent():
output = run(' && '.join(commands))
output = output.split(os.linesep)[-1].lstrip('HEAD is now at ')
commit = output.split()[0]
info('HEAD is now at: {}', output)
return commit
def version():
if not hasattr(version, 'version'):
version_string = run('python --version').stdout
_, ver = version_string.split(' ')
version.version = tuple(map(int, ver.split('.')))
return version.version
def info(scope=''):
"""
Get runtime information from redis itself
"""
with silent(), hide_prefix():
output = api.run('redis-cli info ' + scope)
api.info(output)
def configure():
"""
Configure nginx and enable/disable sites
"""
with sudo():
# Upload templates
context = {'num_cores': debian.nproc()}
uploads = blueprint.upload('./', nginx_root, context)
# Disable previously enabled sites not configured sites-enabled
changes = []
sites = blueprint.get('sites')
auto_disable_sites = blueprint.get('auto_disable_sites', True)
if auto_disable_sites:
with silent():
enabled_site_links = run(
'ls {}'.format(sites_enabled_path)).split()
for link in enabled_site_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in sites and link_name not in sites:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable sites from settings
for site in sites:
changed = enable(site, do_reload=False)
changes.append(changed)
### Reload nginx if new templates or any site has been enabled/disabled
if uploads or any(changes):
reload()
def log(repository_path=None, commit='HEAD', count=1, path=None):
"""
Get log for repository and optional commit range.
:param repository_path: Repository path
:param commit: Commit to log, ex HEAD..origin
:param path: Path or file to log
:return: [(<commit>, <comment>), ...]
"""
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path), silent():
cmd = 'git log --pretty=oneline {}'.format(commit)
if count:
cmd += ' -{}'.format(count)
if path:
cmd += ' -- {}'.format(path)
output = run(cmd, pty=False)
git_log = output.stdout.strip()
git_log = [
col.strip() for row in git_log.split('\n')
for col in row.split(' ', 1) if col
]
git_log = zip(git_log[::2], git_log[1::2])
return git_log
def configure():
"""
Enable/disable configured programs
"""
with sudo():
# Upload templates
uploads = blueprint.upload('init/', '/etc/init/')
uploads.extend(blueprint.upload('supervisord.conf', '/etc/'))
# Disable previously enabled programs not configured programs-enabled
changes = []
programs = blueprint.get('programs') or []
auto_disable = blueprint.get('auto_disable_programs', True)
if auto_disable:
with silent():
enabled_program_links = run(
'ls {}'.format(programs_enabled_path)).split()
for link in enabled_program_links:
link_name = os.path.splitext(link)[0] # Without extension
if link not in programs and link_name not in programs:
changed = disable(link, do_reload=False)
changes.append(changed)
### Enable programs from settings
for program in programs:
changed = enable(program, do_reload=False)
changes.append(changed)
### Reload supervisor if new templates or any program has been enabled/disabled
if uploads or any(changes):
reload()
def update_filters():
changes = []
# Generate desired state as enabled_name => source_name
config = blueprint.get('config', {})
filters = {
'{}-{}.conf'.format(str(weight).zfill(2), conf): "{}.conf".format(conf)
for weight, conf in config.iteritems()
}
# Get current state
with silent():
enabled_filters = run('ls {}'.format(conf_enabled_path)).split()
# Disable extra services
if blueprint.get('auto_disable_conf', True):
for link in set(enabled_filters) - set(filters.keys()):
info('Disabling conf: {}', link)
changes.append(link)
with silent(), sudo(), cd(conf_enabled_path):
debian.rm(link)
# Enable services
for target in set(filters.keys()) - set(enabled_filters):
source = os.path.join(conf_available_path, filters[target])
info('Enabling conf: {}', target)
changes.append(source)
with silent(), sudo(), cd(conf_enabled_path):
debian.ln(source, target)
return changes
def pip(command, *options, **kwargs):
# TODO: change pip log location, per env? per user?
# Perhaps we should just remove the log_file argument and let pip put it
# where it belongs.
info('Running pip {} {}', command, ' '.join(options))
bin = kwargs.pop('bin',
'pip3' if requested_version() >= (3,)
else 'pip2')
quiet = kwargs.pop('quiet', False)
cmd = ('{pip} {command} {options} {verbosity} '
'--exists-action=s --log={log_file} --log-file={log_file}')
run(cmd.format(pip=bin,
command=command,
options=' '.join(options),
verbosity='-v' if not quiet else '-q',
log_file=pip_log_file), pty=False)
def clone(url, branch=None, repository_path=None, **kwargs):
repository = parse_url(url, branch=branch)
name = repository['name']
branch = repository['branch']
if not repository_path:
repository_path = os.path.join('.', name)
if not files.exists(os.path.join(repository_path, '.git')):
info('Cloning {}@{} into {}', url, branch, repository_path)
cmd = 'git clone -b {branch} {remote} {name}'.format(branch=branch,
remote=url,
name=name)
run(cmd)
else:
info('Git repository already cloned: {}', name)
return repository_path
def manage(cmd=''):
"""
Run django management command
"""
if not cmd:
cmd = prompt('Enter django management command:')
with sudo_project(), cd(python_path()), virtualenv.activate(
virtualenv_path()), hide_prefix():
return run('python {manage} {cmd}'.format(cmd=cmd,
manage=blueprint.get(
'manage', 'manage.py')))
def install():
with sudo():
info('Downloading kibana')
version = blueprint.get('version', '3.1.0')
tar_file = 'kibana-{}.tar.gz'.format(version)
run('wget -P /tmp/ https://download.elasticsearch.org/kibana/kibana/{f}'
.format(f=tar_file))
# Extract and soft link kibana in web root
web_root = '/srv/www/'
debian.mkdir(web_root, mode=1775, owner='www-data', group='www-data')
run('tar xzf /tmp/{f} -C {web_root}'.format(f=tar_file,
web_root=web_root))
src_root = os.path.join(web_root,
'kibana-{version}'.format(version=version))
debian.chown(src_root,
owner='www-data',
group='www-data',
recursive=True)
debian.ln(src_root, '/srv/www/kibana')
def install_solr():
with sudo():
version = blueprint.get('version')
version_tuple = tuple(map(int, version.split('.')))
archive = 'solr-{}.tgz'.format(version)
if version_tuple < (4, 1, 0):
archive = 'apache-{}'.format(archive)
url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(
version, archive)
with cd('/tmp'):
info('Download {} ({})', 'Solr', version)
run('wget {}'.format(url))
info('Extracting archive...')
with silent():
run('tar xzf {}'.format(archive))
debian.mv(os.path.splitext(archive)[0], solr_home)
debian.chmod(solr_home, 755, 'solr', 'solr', recursive=True)
debian.rm(archive)
def setup_shared_memory():
"""
http://leopard.in.ua/2013/09/05/postgresql-sessting-shared-memory/
"""
sysctl_path = '/etc/sysctl.conf'
shmmax_configured = files.contains(sysctl_path, 'kernel.shmmax')
shmall_configured = files.contains(sysctl_path, 'kernel.shmall')
if not any([shmmax_configured, shmall_configured]):
page_size = debian.page_size()
phys_pages = debian.phys_pages()
shmall = phys_pages / 2
shmmax = shmall * page_size
shmmax_str = 'kernel.shmmax = {}'.format(shmmax)
shmall_str = 'kernel.shmall = {}'.format(shmall)
with sudo():
files.append(sysctl_path, shmmax_str, partial=True)
files.append(sysctl_path, shmall_str, partial=True)
run('sysctl -p')
info('Added **{}** to {}', shmmax_str, sysctl_path)
info('Added **{}** to {}', shmall_str, sysctl_path)
def current_tag(repository_path=None):
"""
Get most recent tag
:param repository_path: Repository path
:return: The most recent tag
"""
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path), silent():
output = run('git describe --long --tags --dirty --always', pty=False)
# 20141114.1-306-g72354ae-dirty
return output.strip().rsplit('-', 2)[0]
def dump(schema=None, ignore_tables=''):
"""
Dump and download a schema.
:param schema: Specific shema to dump and download.
:param ignore_tables: Tables to skip, separated by | (pipe)
"""
if not schema:
schemas = blueprint.get('schemas', {}).keys()
for i, schema in enumerate(schemas, start=1):
print("{i}. {schema}".format(i=i, schema=schema))
valid_indices = '[1-{}]+'.format(len(schemas))
schema_choice = prompt('Select schema to dump:',
default='1',
validate=valid_indices)
schema = schemas[int(schema_choice) - 1]
now = datetime.now().strftime('%Y-%m-%d')
output_file = '/tmp/{}_{}.backup.gz'.format(schema, now)
filename = os.path.basename(output_file)
info('Dumping schema {}...', schema)
extra_args = []
for table in ignore_tables.split('|'):
extra_args.append('--ignore-table={}.{}'.format(schema, table))
dump_cmd = 'mysqldump {} {} | gzip > {}'.format(schema,
' '.join(extra_args),
output_file)
run('sudo su root -c "{}"'.format(dump_cmd))
info('Downloading dump...')
local_file = '~/%s' % filename
fabric.contrib.files.get(output_file, local_file)
with sudo(), silent():
debian.rm(output_file)
info('New smoking hot dump at {}', local_file)
def get_commit(repository_path=None, short=False):
"""
Get current checked out commit for cloned repository path.
:param repository_path: Repository path
:param short: Format git commit hash in short (7) format
:return: Commit hash
"""
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path), silent():
output = run('git rev-parse HEAD')
commit = output.strip()
if short:
commit = commit[:7]
return commit
def diff_stat(repository_path=None, commit='HEAD^', path=None):
"""
Get diff stats for path.
:param repository_path: Repository path
:param commit: Commit to diff against, ex 12345..67890
:param path: Path or file to diff
:return: tuple(num files changed, num insertions, num deletions)
"""
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path), silent():
# Example output (note leading space):
# 719 files changed, 104452 insertions(+), 29309 deletions(-)
# 1 file changed, 1 insertion(+)
output = run('git diff --shortstat {} -- {}'.format(commit, path),
pty=False)
parts = output.strip().split(', ') if output else []
changed, insertions, deletions = 0, 0, 0
for part in parts:
match = re.match(r'^\s*(\d+)\s+(.+)$', part)
if not match:
raise ValueError('no regex match for {!r} in {!r}'.format(
part, output))
n, label = match.groups()
if label.endswith('(+)'):
insertions = int(n)
elif label.endswith('(-)'):
deletions = int(n)
elif label.endswith('changed'):
changed = int(n)
else:
raise ValueError('unexpected git output')
return changed, insertions, deletions
def generate_pgtune_conf(role='db'):
"""
Run pgtune and create pgtune.conf
:param role: Which fabric role to place local pgtune.conf template under
"""
conf_path = postgres_root('postgresql.conf')
with sudo(), silent():
output = run('pgtune -T Web -i {}'.format(conf_path)).strip()
def parse(c):
lines = [l for l in c.splitlines() if '# pgtune' in l]
for line in lines:
try:
comment = line.index('#')
line = line[:comment]
except ValueError:
pass
clean = lambda s: s.strip('\n\r\t\'" ')
key, _, value = line.partition('=')
key, value = map(clean, (key, value))
if key:
yield key, value or None
tune_conf = dict(parse(output))
tune_conf.update(blueprint.get('pgtune', {}))
tune_conf = '\n'.join(
(' = '.join(item)) for item in tune_conf.iteritems())
conf_dir = os.path.join(os.path.dirname(env['real_fabfile']),
'templates', role, 'postgres')
conf_path = os.path.join(conf_dir, 'pgtune.conf')
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
with open(conf_path, 'w+') as f:
f.write(tune_conf)
def clone(url, branch=None, repository_path=None, **kwargs):
"""
Clone repository and branch.
:param url: Git url to clone
:param branch: Branch to checkout
:param repository_path: Destination
:param kwargs: Not used but here for easier kwarg passing
:return: (destination, got_cloned bool)
"""
repository = parse_url(url, branch=branch)
name = repository['name']
branch = repository['branch']
cloned = False
if not repository_path:
repository_path = os.path.join('.', name)
if not files.exists(os.path.join(repository_path, '.git')):
info('Cloning {}@{} into {}', url, branch, repository_path)
with silent('warnings'):
cmd = 'git clone -b {branch} {remote} {name}'.format(branch=branch,
remote=url,
name=name)
output = run(cmd)
if output.return_code != 0:
warn(
'Failed to clone repository "{}", probably permission denied!'.
format(name))
cloned = None
else:
cloned = True
else:
info('Git repository already cloned: {}', name)
return repository_path, cloned
def reset(branch, repository_path=None, **kwargs):
"""
Fetch, reset, clean and checkout repository branch.
:return: commit
"""
commit = None
if not repository_path:
repository_path = debian.pwd()
with cd(repository_path):
name = os.path.basename(repository_path)
info('Resetting git repository: {}@{}', name, branch)
with silent('warnings'):
commands = [
'git fetch origin', # Fetch branches and tags
'git reset --hard HEAD', # Make hard reset to HEAD
'git clean -fdx', # Remove untracked files pyc, xxx~ etc
'git checkout HEAD', # Checkout HEAD
'git reset refs/remotes/origin/{} --hard'.format(
branch) # Reset to branch
]
output = run(' && '.join(commands))
if output.return_code != 0:
warn(
'Failed to reset repository "{}", probably permission denied!'.
format(name))
else:
output = output.split(os.linesep)[-1][len('HEAD is now at '):]
commit = output.split()[0]
info('HEAD is now at: {}', output)
return commit
