def _install_package(self):
functions.execute_on_host('utils.install_package',
package_name='epel-release')
installed = functions.execute_on_host('utils.install_package',
package_name='redis')
if installed:
sudo('chkconfig redis on')
def _install_package(self):
functions.execute_on_host('utils.install_package',
package_name='epel-release')
installed = functions.execute_on_host('utils.install_package',
package_name='redis')
if installed:
sudo('chkconfig redis on')
def update(self, master=None, full_sync=False):
if not master:
master = self._get_master()
functions.execute_on_host('postgres.update_slave',
master=master,
full_sync=full_sync)
def _install_package(self):
functions.execute_on_host('utils.install_package',
package_name='haproxy')
# Some package versions don't include user
with settings(warn_only=True):
sudo("groupadd haproxy")
sudo("useradd -g haproxy -s /usr/bin/false haproxy")
def link_and_restart(code_hash=None):
"""
Update server links and restart this project.
Takes an optional code_hash argument
"""
role = env.host_roles.get(env.host_string)
if role:
task_name = "servers.{0}.update_code_links".format(
env.role_name_map.get(role))
else:
raise Exception("Don't know how to deploy this host")
functions.execute_on_host(task_name, code_hash=code_hash)
if env.host_string == env.all_hosts[-1]:
roles = {}
for x in env.all_hosts:
r = env.host_roles.get(x)
if not r in roles:
roles[r] = []
roles[r].append(x)
for r, v in roles.items():
task_name = "servers.{0}.restart_services".format(
env.role_name_map.get(r))
execute(task_name, hosts=v)
def run(self, section=None):
"""
"""
functions.execute_on_host('utils.install_package',
package_name='pgbouncer')
self._setup_parameter('%s/pgbouncer.ini' % self.config_dir,
**self.config)
if not section:
section = 'db-server'
username = self._get_username(section)
self._get_passwd(username)
# postgres should be the owner of these config files
sudo('chown -R postgres:postgres %s' % self.config_dir)
# pgbouncer won't run smoothly without these directories
sudo('mkdir -p /var/run/pgbouncer')
sudo('mkdir -p /var/log/pgbouncer')
sudo('chown postgres:postgres /var/run/pgbouncer')
sudo('chown postgres:postgres /var/log/pgbouncer')
# start pgbouncer
pgbouncer_control_file = '/etc/default/pgbouncer'
sudo("sed -i 's/START=0/START=1/' %s" % pgbouncer_control_file)
sudo('service pgbouncer start')
def _setup_logging(self):
with settings(warn_only=True):
functions.execute_on_host('utils.install_package',
package_name='rsyslog')
sudo('svcadm disable system/system-log')
update_conf = False
with settings(warn_only=True):
result = run("grep haproxy {0}".format(self.rsyslog_conf))
if result.return_code:
update_conf = True
if update_conf:
lines = [
"$ModLoad imudp.so", "$UDPServerRun 514",
"$UDPServerAddress 127.0.0.1",
"local1.* -{0}".format(self.logfile), "& ~"
]
start = int(
run('grep -n "ModLoad imsolaris" {0} | cut -f1 -d:'.format(
self.rsyslog_conf)))
for line in lines:
start = start + 1
sudo("sed -i '{0}i{1}' {2}".format(start, line,
self.rsyslog_conf))
sudo('logadm -C 3 -p1d -c -w {0} -z 1'.format(self.logfile))
functions.execute_on_host('utils.start_or_restart',
name='rsyslog',
host=[env.host_string])
def _install_package(self):
installed = functions.execute_on_host('utils.install_package',
package_name='postgresql')
if installed:
sudo('update-rc.d postgresql defaults')
functions.execute_on_host('utils.install_package',
package_name='postgresql-contrib')
def redis_setup(self):
"""
The redis that comes with collectd uses credis.
That doesn't work with newer redis versions.
So we install hiredis and compile this version source
and copy the plugin in.
"""
path = self._get_collectd_headers()
context = self._plugin_context('redis')
functions.execute_on_host('hiredis.setup')
run('wget --no-check-certificate {0}'.format(context['plugin_url']))
run("gcc -DHAVE_CONFIG_H -I{0} -I{0}core -Wall -Werror -g -O2 -fPIC -DPIC -o redis.o -c redis.c"
.format(path))
run('gcc -shared redis.o {0} -Wl,-lhiredis -Wl,-soname -Wl,redis.so -o redis.so'
.format(self._gcc_share_args()))
sudo('cp redis.so {0}'.format(self.plugin_path))
self._add_to_types([
'blocked_clients value:GAUGE:0:U',
'changes_since_last_save value:GAUGE:0:U',
'pubsub value:GAUGE:0:U',
'expired_keys value:GAUGE:0:U',
])
run('rm redis.*')
self.render_plugin_configs('redis', self.get_template_context())
def link_and_restart(code_hash=None):
"""
Update server links and restart this project.
Takes an optional code_hash argument
"""
role = env.host_roles.get(env.host_string)
if role:
task_name = "servers.{0}.update_code_links".format(
env.role_name_map.get(role))
else:
raise Exception("Don't know how to deploy this host")
functions.execute_on_host(task_name, code_hash=code_hash)
if env.host_string == env.all_hosts[-1]:
roles = {}
for x in env.all_hosts:
r = env.host_roles.get(x)
if not r in roles:
roles[r] = []
roles[r].append(x)
for r, v in roles.items():
task_name = "servers.{0}.restart_services".format(
env.role_name_map.get(r))
execute(task_name, hosts=v)
def _update_firewalls(self):
if self.setup_firewall:
functions.execute_on_host('firewall.setup')
# Update any section where this section appears
for section in env.config_object.server_sections():
if self.config_section in env.config_object.get_list(section,
env.config_object.ALLOWED_SECTIONS) and env.roledefs[section]:
functions.execute_on_platform('firewall.setup', hosts=env.roledefs[section])
def setup(self):
sudo("mkdir -p {0}".format(self.pid_dir))
sudo('chown -R %s:%s %s' % (self.user, self.group, self.pid_dir))
functions.execute_on_host('python.setup', packages=['django-celery'])
self.upload_templates()
path = self._setup_logs()
self._setup_rotate(path)
self._setup_service()
def setup(self):
sudo("mkdir -p {0}".format(self.pid_dir))
sudo('chown -R %s:%s %s' % (self.user, self.group, self.pid_dir))
functions.execute_on_host('python.setup', packages=['django-celery'])
self.upload_templates()
path = self._setup_logs()
self._setup_rotate(path)
self._setup_service()
def _update_firewalls(self):
if self.setup_firewall:
functions.execute_on_host('firewall.setup')
# Update any section where this section appears
for section in env.config_object.server_sections():
if self.config_section in env.config_object.get_list(
section, env.config_object.ALLOWED_SECTIONS
) and env.roledefs[section]:
functions.execute_on_platform('firewall.setup',
hosts=env.roledefs[section])
def _add_package(self):
installed = functions.execute_on_host('utils.install_package', package_name='snmpd')
if installed:
sudo('sed -i "/^# deb.*multiverse/ s/^# //" /etc/apt/sources.list')
functions.execute_on_host('utils.install_package',
package_name='snmp-mibs-downloader', update=True)
sudo('update-rc.d snmpd defaults')
sudo('echo "" > {0}'.format(self.remote_config_path))
sudo('service snmpd start')
def _install_plugin(self, plugin, context, **kwargs):
task_name = 'collectd.{0}_setup'.format(plugin)
task = functions.get_task_instance(task_name)
package_names = self.package_names
if task:
functions.execute_on_host(task_name, **kwargs)
else:
if package_names and plugin in package_names:
self._add_package(package_names[plugin])
self.render_plugin_configs(plugin, context, **kwargs)
def _install_plugin(self, plugin, context, **kwargs):
task_name = 'collectd.{0}_setup'.format(plugin)
task = functions.get_task_instance(task_name)
package_names = self.package_names
if task:
functions.execute_on_host(task_name, **kwargs)
else:
if package_names and plugin in package_names:
self._add_package(package_names[plugin])
self.render_plugin_configs(plugin, context, **kwargs)
def _install_package(self):
pk_version = self.db_version.replace('.', '')
functions.execute_on_host('utils.install_package',
package_name="postgresql{0}-server".format(pk_version),
remote=self.package_path)
functions.execute_on_host('utils.install_package',
package_name="postgresql{0}-contrib".format(pk_version))
postgres_conf = os.path.join(self.config_dir, 'postgresql.conf')
self._override_pgdata()
if not exists(postgres_conf, use_sudo=True):
sudo("service postgresql-%s initdb" % self.db_version)
sudo('chkconfig postgresql-%s on' % self.db_version)
def get_task_context(self):
context = super(FirewallSetup, self).get_task_context()
role = env.host_roles.get(env.host_string)
if not role:
raise Exception("Unknown role for %s" % env.host_string)
context['internal_interface'] = functions.execute_on_host(
'utils.get_interface', internal=True)
context['external_interface'] = functions.execute_on_host(
'utils.get_interface', internal=False)
context['tcp_lines'] = TCPOptions(env.config_object).get_config_list(
role, context['internal_interface'], context['external_interface'])
context['udp_lines'] = UDPOptions(env.config_object).get_config_list(
role, context['internal_interface'], context['external_interface'])
return context
def get_task_context(self):
context = super(FirewallSetup, self).get_task_context()
role = env.host_roles.get(env.host_string)
if not role:
raise Exception("Unknown role for %s" % env.host_string)
context['internal_interface'] = functions.execute_on_host('utils.get_interface', internal=True)
context['external_interface'] = functions.execute_on_host('utils.get_interface', internal=False)
context['tcp_lines'] = TCPOptions(env.config_object).get_config_list(role,
context['internal_interface'],
context['external_interface'])
context['udp_lines'] = UDPOptions(env.config_object).get_config_list(role,
context['internal_interface'],
context['external_interface'])
return context
def _install_package(self):
pk_version = self.db_version.replace('.', '')
functions.execute_on_host(
'utils.install_package',
package_name="postgresql{0}-server".format(pk_version),
remote=self.package_path)
functions.execute_on_host(
'utils.install_package',
package_name="postgresql{0}-contrib".format(pk_version))
postgres_conf = os.path.join(self.config_dir, 'postgresql.conf')
self._override_pgdata()
if not exists(postgres_conf, use_sudo=True):
sudo("service postgresql-%s initdb" % self.db_version)
sudo('chkconfig postgresql-%s on' % self.db_version)
def _update_server(self, branch=None, update_configs=True, link_code=True):
if not branch:
branch = self.git_branch
if env.get('deploy_ready') != branch:
functions.execute_on_host('local.deploy.prep_code', branch=branch)
env.deploy_ready = branch
functions.execute_on_host('local.deploy.deploy_code', branch=branch)
if link_code:
self._link_code()
if update_configs:
self._update_configs()
def run(self, code_hash=None):
"""
Link active to a code directory
Default implementation links deployed code to the active locations
and updates static and purges old code.
"""
if not code_hash:
code_hash = run('tail {0}'.format(
os.path.join(env.base_remote_path, 'updating', CODE_VERSION)))
if not code_hash:
raise Exception("Code hash not found, do a full deploy")
code_dir = os.path.join(env.base_remote_path, 'code', code_hash)
if not exists(code_dir):
raise Exception(
"{0} does not exist, do a full deploy".format(code_dir))
static_dir = functions.execute_on_host(
'nginx.context')['static_location']
self._link_static(code_dir, static_dir)
self._link_active(code_dir)
self._purge(static_dir)
def get_ip_command(interface):
"""
"""
if not interface:
interface = functions.execute_on_host('utils.get_interface')
return 'ifconfig %s | grep inet | grep -v inet6 | cut -d ":" -f 2 | cut -d " " -f 2' % interface
def migrate(self, code_hash):
manager = os.path.join(env.base_remote_path, 'code', code_hash, 'project', 'manage.py')
if exists(manager):
context = functions.execute_on_host('python.context')
run('{0}bin/python {1} migrate'.format(context['location'], manager))
else:
raise Exception("{0} does not exist, do a full deploy".format(code_dir))
def get_ip_command(interface):
"""
"""
if not interface:
interface = functions.execute_on_host('utils.get_interface')
return 'ifconfig %s | grep inet | grep -v inet6 | cut -d ":" -f 2 | cut -d " " -f 2' % interface
def _update_config(self, config_section):
if not env.host_string:
print "env.host_string is None, please specify a host by -H "
sys.exit(1)
self._is_section_exists(config_section)
added = False
cons = env.config_object.get_list(config_section,
env.config_object.CONNECTIONS)
if not env.host_string in cons:
added = True
cons.append(env.host_string)
env.config_object.set_list(config_section,
env.config_object.CONNECTIONS,
cons)
ips = env.config_object.get_list(config_section,
env.config_object.INTERNAL_IPS)
internal_ip = functions.execute_on_host('utils.get_ip', None)
ips.append(internal_ip)
env.config_object.set_list(config_section,
env.config_object.INTERNAL_IPS,
ips)
env.roledefs[config_section].append(env.host_string)
env.host_roles[env.host_string] = config_section
return added
def _update_config(self, config_section):
if not env.host_string:
print "env.host_string is None, please specify a host by -H "
sys.exit(1)
self._is_section_exists(config_section)
added = False
cons = env.config_object.get_list(config_section,
env.config_object.CONNECTIONS)
if not env.host_string in cons:
added = True
cons.append(env.host_string)
env.config_object.set_list(config_section,
env.config_object.CONNECTIONS, cons)
ips = env.config_object.get_list(config_section,
env.config_object.INTERNAL_IPS)
internal_ip = functions.execute_on_host('utils.get_ip', None)
ips.append(internal_ip)
env.config_object.set_list(config_section,
env.config_object.INTERNAL_IPS, ips)
env.roledefs[config_section].append(env.host_string)
env.host_roles[env.host_string] = config_section
return added
def get_ip_command(interface):
"""
get IP address
"""
if not interface:
interface = functions.execute_on_host('utils.get_interface')
return 'ifconfig %s | grep Bcast | cut -d ":" -f 2 | cut -d " " -f 1' % interface
def _setup_service(self, env_value=None):
installed = functions.execute_on_host('utils.install_package', package_name='supervisor')
if installed:
sudo('update-rc.d supervisor defaults')
if self.conf_location:
gunicorn_conf = os.path.join(env.configs_path,
"gunicorn/supervisor_{0}.conf".format(self.gunicorn_name))
sudo('ln -sf {0} {1}'.format(gunicorn_conf, self.conf_location))
def _update_server(self, branch=None, update_configs=True, link_code=True):
if not branch:
branch = self.git_branch
if env.get('deploy_ready') != branch:
functions.execute_on_host('local.deploy.prep_code',
branch=branch)
env.deploy_ready = branch
functions.execute_on_host('local.deploy.deploy_code',
branch=branch)
if link_code:
self._link_code()
if update_configs:
self._update_configs()
def migrate(self, code_hash):
manager = os.path.join(env.base_remote_path, 'code', code_hash,
'project', 'manage.py')
if exists(manager):
context = functions.execute_on_host('python.context')
run('{0}bin/python {1} migrate'.format(context['location'],
manager))
else:
raise Exception(
"{0} does not exist, do a full deploy".format(code_dir))
def _setup_service(self, env_value=None):
# we use supervisor to control gunicorn
installed = functions.execute_on_host('utils.install_package', package_name='supervisor')
if installed:
sudo('update-rc.d supervisor defaults')
if self.conf_location:
celery_conf = os.path.join(env.configs_path,
"celery/supervisor_{0}.conf".format(self.name))
sudo('ln -sf {0} {1}'.format(celery_conf, self.conf_location))
def _get_context(self, role):
""" Return a template context for settings generation.
Returns
-------
out : dict
Template context dictionary.
"""
context = functions.get_role_context(role).get('django', {})
context.update({'nginx': functions.execute_on_host('nginx.context')})
return context
def _get_context(self, role):
""" Return a template context for settings generation.
Returns
-------
out : dict
Template context dictionary.
"""
context = functions.get_role_context(role).get('django', {})
context.update({
'nginx' : functions.execute_on_host('nginx.context')
})
return context
def redis_setup(self):
"""
The redis that comes with collectd uses credis.
That doesn't work with newer redis versions.
So we install hiredis and compile this version source
and copy the plugin in.
"""
path = self._get_collectd_headers()
context = self._plugin_context('redis')
functions.execute_on_host('hiredis.setup')
run('wget --no-check-certificate {0}'.format(context['plugin_url']))
run("gcc -DHAVE_CONFIG_H -I{0} -I{0}core -Wall -Werror -g -O2 -fPIC -DPIC -o redis.o -c redis.c".format(path))
run('gcc -shared redis.o {0} -Wl,-lhiredis -Wl,-soname -Wl,redis.so -o redis.so'.format(self._gcc_share_args()))
sudo('cp redis.so {0}'.format(self.plugin_path))
self._add_to_types([
'blocked_clients value:GAUGE:0:U',
'changes_since_last_save value:GAUGE:0:U',
'pubsub value:GAUGE:0:U',
'expired_keys value:GAUGE:0:U',
])
run('rm redis.*')
self.render_plugin_configs('redis', self.get_template_context())
def deploy(branch=None, update_configs=False, no_restart=False, hosts=None):
"""
Deploy this project.
Internally calls the update task for on the server for each
host. After all are finished calls restart services for
each host.
Takes an optional branch argument that can be used
to deploy a branch other than master.
"""
restart = not no_restart
role = env.host_roles.get(env.host_string)
if role:
task_name = "servers.{0}.update".format(env.role_name_map.get(role))
else:
raise Exception("Don't know how to deploy this host")
functions.execute_on_host(task_name,
branch=branch,
update_configs=update_configs,
link_code=restart)
if env.host_string == env.all_hosts[-1] and restart:
roles = {}
for x in env.all_hosts:
r = env.host_roles.get(x)
if not r in roles:
roles[r] = []
roles[r].append(x)
for r, v in roles.items():
task_name = "servers.{0}.restart_services".format(
env.role_name_map.get(r))
execute(task_name, hosts=v)
def run(self, section=None):
"""
"""
functions.execute_on_host('utils.install_package', package_name='pgbouncer')
self._setup_parameter('%s/pgbouncer.ini' % self.config_dir, **self.config)
if not section:
section = 'db-server'
username = self._get_username(section)
self._get_passwd(username)
# postgres should be the owner of these config files
sudo('chown -R postgres:postgres %s' % self.config_dir)
# pgbouncer won't run smoothly without these directories
sudo('mkdir -p /var/run/pgbouncer')
sudo('mkdir -p /var/log/pgbouncer')
sudo('chown postgres:postgres /var/run/pgbouncer')
sudo('chown postgres:postgres /var/log/pgbouncer')
# start pgbouncer
pgbouncer_control_file = '/etc/default/pgbouncer'
sudo("sed -i 's/START=0/START=1/' %s" %pgbouncer_control_file)
sudo('service pgbouncer start')
def deploy(branch=None, update_configs=False, no_restart=False, hosts=None):
"""
Deploy this project.
Internally calls the update task for on the server for each
host. After all are finished calls restart services for
each host.
Takes an optional branch argument that can be used
to deploy a branch other than master.
"""
restart = not no_restart
role = env.host_roles.get(env.host_string)
if role:
task_name = "servers.{0}.update".format(
env.role_name_map.get(role))
else:
raise Exception("Don't know how to deploy this host")
functions.execute_on_host(task_name, branch=branch,
update_configs=update_configs,
link_code=restart)
if env.host_string == env.all_hosts[-1] and restart:
roles = {}
for x in env.all_hosts:
r = env.host_roles.get(x)
if not r in roles:
roles[r] = []
roles[r].append(x)
for r, v in roles.items():
task_name = "servers.{0}.restart_services".format(
env.role_name_map.get(r))
execute(task_name, hosts=v)
def add_packages(self):
# Makes no sense, but their package doesn't install the bin
with settings(warn_only=True):
functions.execute_on_host('utils.install_package',
package_name='sdc-manta')
functions.execute_on_host('utils.install_package',
package_name='py27-gdbm')
result = run('which mls')
if not result.succeeded:
functions.execute_on_host('utils.install_package',
package_name='scmgit')
sudo('npm install manta -g')
def run(self, code_hash=None):
"""
Link active to a code directory
Default implementation links deployed code to the active locations
and updates static and purges old code.
"""
if not code_hash:
code_hash = run('tail {0}'.format(
os.path.join(env.base_remote_path, 'updating', CODE_VERSION)))
if not code_hash:
raise Exception("Code hash not found, do a full deploy")
code_dir = os.path.join(env.base_remote_path, 'code', code_hash)
if not exists(code_dir):
raise Exception("{0} does not exist, do a full deploy".format(code_dir))
static_dir = functions.execute_on_host('nginx.context')['static_location']
self._link_static(code_dir, static_dir)
self._link_active(code_dir)
self._purge(static_dir)
def _start_db_server(self):
service = "postgresql"
task = "{0}.start_or_restart".format(self.utils)
functions.execute_on_host(task, service)
def _setup_database(self):
functions.execute_on_host('postgres.master_setup',
section=self.config_section)
def _setup_db(self):
master = self._get_master()
functions.execute_on_host('postgres.slave_setup', master=master)
def update(self):
functions.execute_on_host('postgres.update')
def _setup_db(self):
dict = functions.execute_on_host('postgres.master_setup')
def _install_package(self):
installed = functions.execute_on_host('utils.install_package',
package_name='nginx',
remote="http://nginx.org/packages/rhel/6/noarch/RPMS/nginx-release-rhel-6-0.el6.ngx.noarch.rpm")
if installed:
sudo('chkconfig nginx on')
def start(self):
functions.execute_on_host('utils.start_or_restart_supervisor', name=self.name)
def install_package(self):
functions.execute_on_host('utils.install_package',
package_name=self.pkg_name)
def start(self):
functions.execute_on_host('utils.start_or_restart', name='redis')
def _install_package(self):
functions.execute_on_host('utils.install_package',
package_name='redis')
def start(self):
functions.execute_on_host('utils.start_or_restart', name=self.gunicorn_name)
def build_settings(self, code_dir):
functions.execute_on_host('local.deploy.build_settings', code_dir=code_dir)
def start(self):
functions.execute_on_host('utils.start_or_restart_service', name='nginx',
host=[env.host_string])
def migrate_db(code_hash=None):
if env.host_string == env.all_hosts[-1]:
print "migrations should only be run from one host, using {0}".format(env.host_string)
functions.execute_on_host('local.deploy.migrate_db', code_hash=code_hash)