def deploy_api(commit='master'):
now = int(time.time())
require.files.directory(env.deployment_dir(now))
with cd(env.deployment_dir(now)):
run(u'wget {}'.format(env.apitaxi_archive.format(commit)))
run('unzip {}.zip'.format(commit))
if commit != 'master':
run('mv APITaxi-{} APITaxi-master'.format(commit))
with cd(env.apitaxi_dir(now)):
require.python.virtualenv(env.apitaxi_venv_path(now))
with python.virtualenv(env.apitaxi_venv_path(now)):
python.install_pip(use_sudo=False)
require.python.package('uwsgi')
python.install_requirements('requirements.txt')
put(environ['APITAXI_CONFIG_FILE'], env.apitaxi_config_path(now))
with shell_env(APITAXI_CONFIG_FILE=env.apitaxi_config_path(now)):
for i in range(1, 30):
if service.is_running('supervisor'):
break
time.sleep(1)
run('python manage.py db upgrade')
install_admin_user()
deploy_front(now)
deploy_nginx_api_site(now)
if not service.is_running('nginx'):
service.start('nginx')
clean_directories(now)
stop_old_processes(now)
restart_stats_workers(now)
def openerp_rsyslog():
require.files.file(
'/etc/rsyslog.d/20-openerp.conf',
source='files/etc/rsyslog.d/20-openerp.conf',
owner='root',
group='root',
mode=644,
use_sudo = True
)
require.files.file(
'/etc/init.d/openerp',
source='files/etc/init.d/openerp',
owner='root',
group='root',
mode='755',
use_sudo=True
)
require.files.directory(
'/var/log/openerp',
mode=777,
use_sudo=True
)
require.service.restarted('rsyslog')
## we need a full stop and start as we updated /etc/init.d/openerp file
puts('Stopping openerp')
service.stop('openerp')
puts('Starting openerp')
service.start('openerp')
def restart_nginx():
print(green("Restarting Nginx"))
if service.is_running('nginx'):
#service.restart('nginx')
service.reload('nginx')
else:
service.start('nginx')
def install():
"""
Install graphite stack
"""
# Requirement platform
require.deb.packages(["gcc", "python-dev", "python-pip", "python-cairo"])
# Install Graphite
run("mkdir -p /opt/graphite")
run("pip install whisper carbon graphite-web")
# Prepare Carbon
with cd("/opt/graphite/conf/"):
run("cp carbon.conf.example carbon.conf")
run("cp storage-schemas.conf.example storage-schemas.conf")
# Set initd
put("files/carbon", "/etc/init.d/", mode=0755)
run("update-rc.d carbon defaults")
service.start("carbon")
# Install Django
run("pip install django django-tagging gunicorn")
with cd("/opt/graphite/webapp/graphite"):
run("python manage.py syncdb")
# Install server
put("files/local_settings.py", "/opt/graphite/webapps/graphite/")
put("files/graphite", "/etc/init.d/", mode=0755)
run("update-rc.d graphite defaults")
run("python manage.py runserver")
def create_eng_instance(instance):
""" Create a new kraken instance (idempotent)
* Install requirements
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_eng_instance_conf(instance, host)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
if not is_link(kraken_bin):
idempotent_symlink("/usr/bin/kraken",
kraken_bin,
use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER,
bin=kraken_bin))
kraken = "kraken_{}".format(instance.name)
if not service.is_running(kraken):
# TODO test this on systemd machines
if env.use_systemd:
sudo("systemctl enable kraken_{}.service".format(
instance.name))
else:
sudo("update-rc.d kraken_{} defaults".format(
instance.name))
print(
blue(
"INFO: kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running"
.format(instance=instance.name,
server=get_host_addr(env.host_string))))
service.start(kraken)
run("sleep 5") # we wait a bit for the kraken to pop
with settings(warn_only=True):
run("pgrep --list-name --full /srv/kraken/{}/kraken".format(
instance.name))
print(
blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name,
server=get_host_addr(host))))
def livebuild():
pkg('openssh-server live-build python git-core apt-cacher-ng zsh',
'debootstrap')
run('update-rc.d apt-cacher-ng defaults')
service.start('apt-cacher-ng')
require.directory('/var/build/')
for conf in [ '/etc/profile', '/root/.bashrc', '/root/.zshrc' ]:
run('echo "export http_proxy=http://localhost:3142/" >> %s' % conf)
def install_remote_syslog(version='0.13', port='58173'):
""" Installs and configures remote_syslog to send events to papertrail """
package_name = 'remote_syslog_linux_amd64.tar.gz'
folder = 'remote_syslog'
url = ('https://github.com/papertrail/remote_syslog2/releases/download/'
'v{}/{}')
url = url.format(version, package_name)
print(blue('Downloading remote syslog'))
cmd = 'wget {} -O {}'.format(url, package_name)
run(cmd)
print(blue('Unpacking remote syslog'))
cmd = 'tar xvf {}'.format(package_name)
run(cmd)
print(blue('Installing remote syslog'))
cmd = 'cp {}/remote_syslog /usr/local/bin/'.format(folder)
sudo(cmd)
print(blue('Uploading config file'))
config_file = 'remote_syslog.yml'
config_file_path = '{}/templates/{}'.format(ROOT_FOLDER, config_file)
target_dir = '/etc/'
context = {
'user': env.user,
'port': port,
}
upload_template(config_file_path,
target_dir,
context=context,
use_sudo=True)
# change config file owner to root
cmd = 'chown root:root {}{}'.format(target_dir, config_file)
sudo(cmd)
print(blue('Registering at system startup'))
upstart_file = 'remote_syslog.conf'
upstart_file_path = '{}/templates/{}'.format(ROOT_FOLDER, upstart_file)
upstart_dir = '/etc/init/'
put(upstart_file_path, upstart_dir, use_sudo=True, mode=0644)
# change config file owner to root
cmd = 'chown root:root {}{}'.format(upstart_dir, upstart_file)
sudo(cmd)
service.start('remote_syslog')
print(blue('Cleaning installation files'))
cmd = 'rm -rf {} {}'.format(folder, package_name)
run(cmd)
def create_eng_instance(instance):
"""Create a new kraken instance
* Install requirements (idem potem)
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_eng_instance_conf(instance)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
if not is_link(kraken_bin):
files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER,
bin=kraken_bin))
#run("chmod 755 /etc/init.d/kraken_{}".format(instance))
# TODO refactor this and test it on systemd and non-systemd machines
if not env.use_systemd:
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(
blue(
"INFO: Kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running".
format(instance=instance.name,
server=get_host_addr(env.host_string))))
service.start("kraken_{}".format(instance.name))
run("sleep 5") # we wait a bit for the kraken to pop
# test it !
# execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False)
with settings(warn_only=True):
run("pgrep --list-name --full {}".format(instance.name))
print(
blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name,
server=get_host_addr(env.host_string))))
def started(service):
"""
Require a service to be started.
::
from fabtools import require
require.service.started('foo')
"""
if not is_running(service):
start(service)
def install_remote_syslog(version='0.13', port='58173'):
""" Installs and configures remote_syslog to send events to papertrail """
package_name = 'remote_syslog_linux_amd64.tar.gz'
folder = 'remote_syslog'
url = ('https://github.com/papertrail/remote_syslog2/releases/download/'
'v{}/{}')
url = url.format(version, package_name)
print(blue('Downloading remote syslog'))
cmd = 'wget {} -O {}'.format(url, package_name)
run(cmd)
print(blue('Unpacking remote syslog'))
cmd = 'tar xvf {}'.format(package_name)
run(cmd)
print(blue('Installing remote syslog'))
cmd = 'cp {}/remote_syslog /usr/local/bin/'.format(folder)
sudo(cmd)
print(blue('Uploading config file'))
config_file = 'remote_syslog.yml'
config_file_path = '{}/templates/{}'.format(ROOT_FOLDER, config_file)
target_dir = '/etc/'
context = {
'user': env.user,
'port': port,
}
upload_template(config_file_path, target_dir, context=context,
use_sudo=True)
# change config file owner to root
cmd = 'chown root:root {}{}'.format(target_dir, config_file)
sudo(cmd)
print(blue('Registering at system startup'))
upstart_file = 'remote_syslog.conf'
upstart_file_path = '{}/templates/{}'.format(ROOT_FOLDER, upstart_file)
upstart_dir = '/etc/init/'
put(upstart_file_path, upstart_dir, use_sudo=True, mode=0644)
# change config file owner to root
cmd = 'chown root:root {}{}'.format(upstart_dir, upstart_file)
sudo(cmd)
service.start('remote_syslog')
print(blue('Cleaning installation files'))
cmd = 'rm -rf {} {}'.format(folder, package_name)
run(cmd)
def restarted(service):
"""
Require a service to be restarted.
::
from fabtools import require
require.service.restarted('foo')
"""
if is_running(service):
restart(service)
else:
start(service)
def started(service):
"""
Require a service to be started.
::
from fabtools import require
require.service.started('foo')
"""
if not is_running(service):
if using_systemd():
systemd.start(service)
else:
start(service)
def install_haibu():
"""
Setup Haibu Application Manager.
"""
with cd('/home/cozy/cozy-setup'):
cozydo('HOME=/home/cozy npm install')
sudo('cp paas.conf /etc/init/')
if not service.is_running("paas"):
service.start('paas')
else:
service.restart('paas')
print(green("Haibu successfully started"))
def create_eng_instance(instance):
"""Create a new kraken instance
* Install requirements (idem potem)
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require.files.directory(instance.base_destination_dir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
update_eng_instance_conf(instance)
# kraken.ini, pid and binary symlink
if not exists("{}/{}/kraken".format(env.kraken_basedir, instance.name)):
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name)
files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin))
#run("chmod 755 /etc/init.d/kraken_{}".format(instance))
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(blue("INFO: Kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running".format(
instance=instance.name, server=get_host_addr(env.host_string))))
service.start("kraken_{} start".format(instance.name))
run("sleep 5") # we wait a bit for the kraken to pop
# test it !
# execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False)
print("server: {}".format(env.host_string))
run("pgrep --list-name --full {}".format(instance.name))
print(blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name, server=get_host_addr(env.host_string))))
def create_eng_instance(instance):
""" Create a new kraken instance (idempotent)
* Install requirements
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
update_eng_instance_conf(instance, host)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name)
if not is_link(kraken_bin):
idempotent_symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin))
kraken = "kraken_{}".format(instance.name)
if not service.is_running(kraken):
# TODO test this on systemd machines
if env.use_systemd:
sudo("systemctl enable kraken_{}.service".format(instance.name))
else:
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(blue("INFO: kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running"
.format(instance=instance.name, server=get_host_addr(env.host_string))))
service.start(kraken)
run("sleep 5") # we wait a bit for the kraken to pop
with settings(warn_only=True):
run("pgrep --list-name --full /srv/kraken/{}/kraken".format(instance.name))
print(blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name, server=get_host_addr(host))))
def deploy_api():
now = int(time.time())
require.files.directory(env.deployment_dir(now))
with cd(env.deployment_dir(now)):
run('wget https://github.com/openmaraude/APITaxi/archive/master.zip')
run('unzip master.zip')
with cd(env.apitaxi_dir(now)):
require.python.virtualenv(env.apitaxi_venv_path(now))
with python.virtualenv(env.apitaxi_venv_path(now)):
python.install_pip()
require.python.package('uwsgi')
python.install_requirements('requirements.txt')
put(environ['APITAXI_CONFIG_FILE'], env.apitaxi_config_path(now))
run('python manage.py db upgrade')
deploy_nginx_api_site(now)
if not service.is_running('nginx'):
service.start('nginx')
clean_directories(now)
stop_old_processes(now)
def recreate_pvedata(dirsave, size):
"""
Recreate the pve-data with the defined size
"""
# Stop service
service.stop('pvedaemon')
service.stop('vz')
service.stop('qemu-server')
# Backup the /var/lib/vz
run('tar -czf %s/vz.tgz /var/lib/vz' % dirsave)
# Delete the pve-data
run('umount /var/lib/vz')
run('lvremove /dev/pve/data')
# Create the new pve-data
run('lvcreate -n data -L %s pve' % size)
run('mkfs.ext3 /dev/mapper/pve-data')
run('mount /dev/mapper/pve-data /var/lib/vz')
# Restore the Backup /var/lib/vz
run('tar -xzf %s/vz.tgz -C /' % dirsave)
# Start service
service.start('qemu-server')
service.start('vz')
service.start('pvedaemon')
def recreate_pvedata(dirsave, size):
"""
Recreate the pve-data with the defined size
"""
# Stop service
service.stop('pvedaemon')
service.stop('vz')
service.stop('qemu-server')
# Backup the /var/lib/vz
run('tar -czf %s/vz.tgz /var/lib/vz' % dirsave)
# Delete the pve-data
run('umount /var/lib/vz')
run('lvremove /dev/pve/data')
# Create the new pve-data
run('lvcreate -n data -L %s pve' % size)
run('mkfs.ext3 /dev/mapper/pve-data')
run('mount /dev/mapper/pve-data /var/lib/vz')
# Restore the Backup /var/lib/vz
run('tar -xzf %s/vz.tgz -C /' % dirsave)
# Start service
service.start('qemu-server')
service.start('vz')
service.start('pvedaemon')
def install():
"""
Install graphite stack
"""
# Requirement platform
require.deb.packages([
'gcc',
'python-dev',
'python-pip',
'python-cairo'
])
# Install Graphite
run('mkdir -p /opt/graphite')
run('pip install whisper carbon graphite-web')
# Prepare Carbon
with cd('/opt/graphite/conf/'):
run('cp carbon.conf.example carbon.conf')
run('cp storage-schemas.conf.example storage-schemas.conf')
# Set initd
put('files/carbon', '/etc/init.d/', mode=0755)
run('update-rc.d carbon defaults')
service.start('carbon')
# Install Django
run ('pip install django django-tagging gunicorn')
with cd('/opt/graphite/webapp/graphite'):
run('python manage.py syncdb')
# Install server
put('files/local_settings.py', '/opt/graphite/webapps/graphite/')
put('files/graphite', '/etc/init.d/', mode=0755)
run('update-rc.d graphite defaults')
run('python manage.py runserver')
def started():
"""
Ensure that the firewall is started.
"""
if not is_started():
start('shorewall')
def started():
"""
Ensure the firewall is started
"""
if not is_started():
start('shorewall')
def restart_app(service_name=SERVICE_NAME):
if service.is_running(SERVICE_NAME):
service.restart(SERVICE_NAME)
else:
service.start(SERVICE_NAME)
def setup_tyr():
require.deb.packages([
"alembic",
"rabbitmq-server",
'redis-server'])
require.users.user('www-data')
utils.require_directories([env.tyr_base_instances_dir,
env.tyr_basedir,
],
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
utils.require_directory(env.tyr_base_logdir,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='777',
use_sudo=True)
utils.require_directory(env.tyr_base_destination_dir,
is_on_nfs4=False,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
require.files.file(env.tyr_base_logfile,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
require.files.file(env.tyr_logfile_pattern,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
update_tyr_conf()
# we need to deploy the services conf file
_upload_template('tyr/tyr_beat.jinja', env.tyr_beat_service_file, user='******', mode='755',
context={
'env': env
})
_upload_template('tyr/tyr_worker.jinja', env.tyr_worker_service_file, user='******', mode='755',
context={
'env': env
})
if not files.is_dir(env.tyr_migration_dir):
files.symlink('/usr/share/tyr/migrations/', env.tyr_migration_dir, use_sudo=True)
sudo("chown www-data:www-data {}".format(env.tyr_migration_dir))
# we create a symlink for tyr manage_py
tyr_symlink = os.path.join(env.tyr_basedir, 'manage.py')
if not files.is_file(tyr_symlink):
files.symlink('/usr/bin/manage_tyr.py', tyr_symlink, use_sudo=True)
service.start('rabbitmq-server')
service.start('redis-server')
def start(component):
service.start(component)
def setup_tyr():
require.deb.packages(["alembic", "rabbitmq-server", 'redis-server'])
require.users.user('www-data')
utils.require_directories([
env.tyr_base_instances_dir,
env.tyr_basedir,
],
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
utils.require_directory(env.tyr_base_logdir,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='777',
use_sudo=True)
utils.require_directory(env.tyr_base_destination_dir,
is_on_nfs4=False,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
require.files.file(env.tyr_base_logfile,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
require.files.file(env.tyr_logfile_pattern,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
update_tyr_conf()
# we need to deploy the services conf file
_upload_template('tyr/tyr_beat.jinja',
env.tyr_beat_service_file,
user='******',
mode='755',
context={'env': env})
_upload_template('tyr/tyr_worker.jinja',
env.tyr_worker_service_file,
user='******',
mode='755',
context={'env': env})
if not files.is_dir(env.tyr_migration_dir):
files.symlink('/usr/share/tyr/migrations/',
env.tyr_migration_dir,
use_sudo=True)
sudo("chown www-data:www-data {}".format(env.tyr_migration_dir))
# we create a symlink for tyr manage_py
tyr_symlink = os.path.join(env.tyr_basedir, 'manage.py')
if not files.is_file(tyr_symlink):
files.symlink('/usr/bin/manage_tyr.py', tyr_symlink, use_sudo=True)
service.start('rabbitmq-server')
service.start('redis-server')
def install_fail2ban():
require.deb.package(['fail2ban'])
if not service.is_running('fail2ban'):
service.start('fail2ban')
def start_tomcat():
"""
Start the Tomcat service.
"""
start('tomcat')
def start():
service.start(service_name)
def started():
"""
Ensure that the firewall is started.
"""
if not is_started():
start("shorewall")
def aptcacher():
pkg('apt-cacher-ng')
run_as_root('update-rc.d apt-cacher-ng defaults')
service.start('apt-cacher-ng')
def start_tomcat():
"""
Start the Tomcat service.
"""
start('tomcat')
def start():
""" Starts redis daemon """
service.start('redis_6379')
def restart_app(service_name=SERVICE_NAME):
if service.is_running(SERVICE_NAME):
service.restart(SERVICE_NAME)
else:
service.start(SERVICE_NAME)
def restart(component):
if service.is_running(component):
service.restart(component)
else:
service.start(component)
